🚑 Restore files from nextcloud (#423)

* 🚑Modify nextcloud migration to be able to back up files from next cloud

Co-authored-by: Monta <monta@HP-ProBook-445-14-inch-G9-Notebook-PC-505aadfc.localdomain>

tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts

@@ -44,6 +44,7 @@ export class NextcloudMigration {
 
   async migrate(username: string, password: string) {
     const dir = this.createTmpDir(username);
+    // const dir = "/tmp/to_upload"
     try {
       const user = await this.getLDAPUser(username);
       //create user if needed Twake Drive
@@ -113,6 +114,7 @@ export class NextcloudMigration {
   async upload(user: TwakeDriveUser, sourceDirPath: string, parentDirId = "user_" + user.id) {
     const dirsToUpload: Map<string, string> = new Map<string, string>();
     const filesToUpload: string[] = [];
+    const existingFiles: string[] = [];
 
     const parent = await this.driveClient.getDocument(parentDirId);
 
@@ -126,6 +128,9 @@ export class NextcloudMigration {
       const stat = fs.statSync(filePath);
       if (exists(name)) {
         logger.info(`File ${name} already exists`);
+        //find document that we need to replace
+        existingFiles.push(filePath);
+        //skip
       } else {
         if (stat.isFile()) {
           logger.info(`Add file for future upload ${filePath}`);
@@ -136,6 +141,34 @@ export class NextcloudMigration {
         }
       }
     });
+    //check existing files
+    for (const fPath of existingFiles) {
+      logger.debug(`Check existing file ${fPath}`)
+      let name = path.parse(fPath).name;
+      let candidatesWithTheSameName = parent.children.filter(i => i.name.startsWith(name));
+      if (candidatesWithTheSameName.length > 1) {
+        logger.warn("WE HAVE MORE MORE THAN ONE FILE WITH NAME: " + name);
+      } else {
+        const doc = candidatesWithTheSameName[0];
+        if (doc.is_directory) {
+          logger.info(`Directory ${name} exists, try to upload inside`);
+          await this.upload(user, fPath, doc.id);
+        } else {
+          //check that it exists in S3
+          if (await this.driveClient.existsInS3(doc.id)) {
+            logger.info(`File ${name}, docId = ${doc.id} exists in S3`);
+          } else {
+            //if it doesn't exists upload just one file
+            logger.info(`File ${name}, is missing, uploading files`);
+            const response = await this.driveClient.uploadToS3(fPath, doc.id);
+            if (!response && !response.success) {
+              console.log(`Error creating file '${name}' in S3`)
+            }
+          }
+        }
+      }
+    }
+
 
     //upload all files
     logger.debug(`UPLOAD FILES FOR  ${sourceDirPath}`)

tdrive/backend/utils/nextcloud-migration/src/twake_client.ts

@@ -17,6 +17,11 @@ export class TwakeDriveClient {
 
   private readonly config: TwakeClientConfiguration;
 
+  private clientInstance;
+
+  //00000000-0000-4000-0000-000000000000
+  private company = "00000000-0000-4000-0000-000000000000";
+
   constructor(config: TwakeClientConfiguration) {
     this.config = config;
     //remove the trailing '/'
@@ -33,7 +38,7 @@ export class TwakeDriveClient {
     form.append('file', fs.createReadStream(path));
 
     const response
-      = await client.post(`${this.config.url}/internal/services/files/v1/companies/00000000-0000-4000-0000-000000000000/files`, form);
+      = await client.post(`${this.config.url}/internal/services/files/v1/companies/${this.company}/files`, form);
 
     logger.info(response.data);
 
@@ -47,7 +52,7 @@ export class TwakeDriveClient {
 
   async createDirectory(name: string, parent: string) {
     return this.createDocument({
-      company_id: '00000000-0000-4000-0000-000000000000',
+      company_id: this.company,
       name: name,
       parent_id: parent,
       is_directory: true,
@@ -74,7 +79,7 @@ export class TwakeDriveClient {
 
   private async createDocument(item: any, version: any) {
     const response
-      = await (await this.client()).post(`${this.config.url}/internal/services/documents/v1/companies/00000000-0000-4000-0000-000000000000/item`, {
+      = await (await this.client()).post(`${this.config.url}/internal/services/documents/v1/companies/${this.company}/item`, {
       item,
       version,
     });
@@ -85,7 +90,7 @@ export class TwakeDriveClient {
   async getDocument(id: string) {
     logger.info(`Get information for the doc ${id}`);
     const response
-      = await (await this.client()).get(`${this.config.url}/internal/services/documents/v1/companies/00000000-0000-4000-0000-000000000000/item/${id}`, {
+      = await (await this.client()).get(`${this.config.url}/internal/services/documents/v1/companies/${this.company}/item/${id}`, {
     });
     logger.info(response.data);
     return response.data;
@@ -122,12 +127,15 @@ export class TwakeDriveClient {
   }
 
   private async client() {
-    return axios.create({
-      baseURL: this.config.url,
-      headers: {
-        Authorization: `Bearer ${await (this.accessToken())}`,
-      },
-    });
+    if (!this.clientInstance) {
+      this.clientInstance = axios.create({
+        baseURL: this.config.url,
+        headers: {
+          Authorization: `Bearer ${await (this.accessToken())}`,
+        },
+      });
+    }
+    return this.clientInstance;
   }
 
   private async accessToken(): Promise<string> {
@@ -161,6 +169,34 @@ export class TwakeDriveClient {
     }
   };
 
+  async uploadToS3(fPath: string, id: string) {
+    logger.info(`Upload to S3 ${fPath}`);
+    const client = await this.client();
+
+    const form = new FormData();
+    form.append('file', fs.createReadStream(fPath));
+
+    const response
+      = await client.post(`${this.config.url}/internal/services/files/v1/S3/restore/${id}`, form);
+
+    logger.info(response.data);
+
+    return response.data;
+  }
+
+  async existsInS3(id: string): Promise<boolean> {
+    const client = await this.client();
+    const response
+      = await client.get(`${this.config.url}/internal/services/files/v1/S3/exist/${id}`);
+    if (response.status != 200) {
+      logger.warn(`Error receiving S3 status for ${id}`);
+      logger.warn(response.data)
+      return true;
+    } else {
+      logger.info(`Exists ${JSON.stringify(response.data)}`)
+      return response.data?.exist;
+    }
+  }
 }
 
 export type TwakeDriveUser = {