🚑 Restore files from nextcloud (#423)

* 🚑Modify nextcloud migration to be able to back up files from next cloud Co-authored-by: Monta <monta@HP-ProBook-445-14-inch-G9-Notebook-PC-505aadfc.localdomain>
2024-03-07 18:33:08 +01:00
parent eacdb29bb2
commit c9ce1fce43
5 changed files with 165 additions and 11 deletions
@@ -402,8 +402,48 @@ export class FileServiceImpl {
    }
    return data;
  }
-}

+  async checkFileExistsS3(id: string): Promise<any> {
+    try {
+      const doc = await this.documentRepository.findOne({
+        id,
+        company_id: "00000000-0000-4000-0000-000000000000",
+      });
+      const externalId = doc.last_version_cache.file_metadata.external_id;
+      const file = await this.getFile({
+        id: externalId,
+        company_id: "00000000-0000-4000-0000-000000000000",
+      });
+      const exist = await gr.platformServices.storage.exists(getFilePath(file));
+      if (exist) {
+        return { exist: true, file };
+      } else {
+        return { exist: false, file };
+      }
+    } catch (error) {
+      logger.error(`Error while checking file ${id} in S3`, error);
+      return { exist: false, file: null };
+    }
+  }
+
+  async restoreFileS3(id: string, file: Multipart, options: UploadOptions): Promise<any> {
+    try {
+      const result = await this.checkFileExistsS3(id);
+      if (result.exist) {
+        return { success: true };
+      }
+      await gr.platformServices.storage.write(getFilePath(result.file), file.file, {
+        chunkNumber: options.chunkNumber,
+        encryptionAlgo: this.algorithm,
+        encryptionKey: result.file.encryption_key,
+      });
+      return { success: true };
+    } catch (error) {
+      logger.error(`Error while uploading missing file ${id} to S3`, error);
+      return { success: false };
+    }
+  }
+}
 function getFilePath(entity: File): string {
  return `${gr.platformServices.storage.getHomeDir()}/files/${entity.company_id}/${
    entity.user_id || "anonymous"
@@ -105,6 +105,37 @@ export class FileController {

    return { status: deleteResult.deleted ? "success" : "error" };
  }
+
+  async checkFileS3Exists(
+    request: FastifyRequest<{ Params: { company_id: string; id: string } }>,
+  ): Promise<{ isInS3: boolean }> {
+    const params = request.params;
+    return await gr.services.files.checkFileExistsS3(params.id);
+  }
+
+  async restoreFileS3(
+    request: FastifyRequest<{
+      Params: { company_id: string; id: string };
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      Querystring: any;
+    }>,
+  ): Promise<{ resource: PublicFile }> {
+    const params = request.params;
+    let file: null | Multipart = null;
+    if (request.isMultipart()) {
+      file = await request.file();
+    }
+
+    const q = request.query;
+    const options: any = {
+      totalChunks: parseInt(q.resumableTotalChunks || q.total_chunks) || 1,
+      chunkNumber: parseInt(q.resumableChunkNumber || q.chunk_number) || 1,
+    };
+
+    const result = await gr.services.files.restoreFileS3(params.id, file, options);
+
+    return result;
+  }
 }

 function getCompanyExecutionContext(
@@ -53,6 +53,20 @@ const routes: FastifyPluginCallback = (fastify: FastifyInstance, options, next)
    handler: fileController.checkConsistency.bind(fileController),
  });

+  fastify.route({
+    method: "GET",
+    url: "/S3/exist/:id",
+    preValidation: [fastify.authenticate],
+    handler: fileController.checkFileS3Exists.bind(fileController),
+  });
+
+  fastify.route({
+    method: "POST",
+    url: "/S3/restore/:id",
+    preValidation: [fastify.authenticate],
+    handler: fileController.restoreFileS3.bind(fileController),
+  });
+
  next();
 };

@@ -44,6 +44,7 @@ export class NextcloudMigration {

  async migrate(username: string, password: string) {
    const dir = this.createTmpDir(username);
+    // const dir = "/tmp/to_upload"
    try {
      const user = await this.getLDAPUser(username);
      //create user if needed Twake Drive
@@ -113,6 +114,7 @@ export class NextcloudMigration {
  async upload(user: TwakeDriveUser, sourceDirPath: string, parentDirId = "user_" + user.id) {
    const dirsToUpload: Map<string, string> = new Map<string, string>();
    const filesToUpload: string[] = [];
+    const existingFiles: string[] = [];

    const parent = await this.driveClient.getDocument(parentDirId);

@@ -126,6 +128,9 @@ export class NextcloudMigration {
      const stat = fs.statSync(filePath);
      if (exists(name)) {
        logger.info(`File ${name} already exists`);
+        //find document that we need to replace
+        existingFiles.push(filePath);
+        //skip
      } else {
        if (stat.isFile()) {
          logger.info(`Add file for future upload ${filePath}`);
@@ -136,6 +141,34 @@ export class NextcloudMigration {
        }
      }
    });
+    //check existing files
+    for (const fPath of existingFiles) {
+      logger.debug(`Check existing file ${fPath}`)
+      let name = path.parse(fPath).name;
+      let candidatesWithTheSameName = parent.children.filter(i => i.name.startsWith(name));
+      if (candidatesWithTheSameName.length > 1) {
+        logger.warn("WE HAVE MORE MORE THAN ONE FILE WITH NAME: " + name);
+      } else {
+        const doc = candidatesWithTheSameName[0];
+        if (doc.is_directory) {
+          logger.info(`Directory ${name} exists, try to upload inside`);
+          await this.upload(user, fPath, doc.id);
+        } else {
+          //check that it exists in S3
+          if (await this.driveClient.existsInS3(doc.id)) {
+            logger.info(`File ${name}, docId = ${doc.id} exists in S3`);
+          } else {
+            //if it doesn't exists upload just one file
+            logger.info(`File ${name}, is missing, uploading files`);
+            const response = await this.driveClient.uploadToS3(fPath, doc.id);
+            if (!response && !response.success) {
+              console.log(`Error creating file '${name}' in S3`)
+            }
+          }
+        }
+      }
+    }
+

    //upload all files
    logger.debug(`UPLOAD FILES FOR  ${sourceDirPath}`)
@@ -17,6 +17,11 @@ export class TwakeDriveClient {

  private readonly config: TwakeClientConfiguration;

+  private clientInstance;
+
+  //00000000-0000-4000-0000-000000000000
+  private company = "00000000-0000-4000-0000-000000000000";
+
  constructor(config: TwakeClientConfiguration) {
    this.config = config;
    //remove the trailing '/'
@@ -33,7 +38,7 @@ export class TwakeDriveClient {
    form.append('file', fs.createReadStream(path));

    const response
-      = await client.post(`${this.config.url}/internal/services/files/v1/companies/00000000-0000-4000-0000-000000000000/files`, form);
+      = await client.post(`${this.config.url}/internal/services/files/v1/companies/${this.company}/files`, form);

    logger.info(response.data);

@@ -47,7 +52,7 @@ export class TwakeDriveClient {

  async createDirectory(name: string, parent: string) {
    return this.createDocument({
-      company_id: '00000000-0000-4000-0000-000000000000',
+      company_id: this.company,
      name: name,
      parent_id: parent,
      is_directory: true,
@@ -74,7 +79,7 @@ export class TwakeDriveClient {

  private async createDocument(item: any, version: any) {
    const response
-      = await (await this.client()).post(`${this.config.url}/internal/services/documents/v1/companies/00000000-0000-4000-0000-000000000000/item`, {
+      = await (await this.client()).post(`${this.config.url}/internal/services/documents/v1/companies/${this.company}/item`, {
      item,
      version,
    });
@@ -85,7 +90,7 @@ export class TwakeDriveClient {
  async getDocument(id: string) {
    logger.info(`Get information for the doc ${id}`);
    const response
-      = await (await this.client()).get(`${this.config.url}/internal/services/documents/v1/companies/00000000-0000-4000-0000-000000000000/item/${id}`, {
+      = await (await this.client()).get(`${this.config.url}/internal/services/documents/v1/companies/${this.company}/item/${id}`, {
    });
    logger.info(response.data);
    return response.data;
@@ -122,12 +127,15 @@ export class TwakeDriveClient {
  }

  private async client() {
-    return axios.create({
-      baseURL: this.config.url,
-      headers: {
-        Authorization: `Bearer ${await (this.accessToken())}`,
-      },
-    });
+    if (!this.clientInstance) {
+      this.clientInstance = axios.create({
+        baseURL: this.config.url,
+        headers: {
+          Authorization: `Bearer ${await (this.accessToken())}`,
+        },
+      });
+    }
+    return this.clientInstance;
  }

  private async accessToken(): Promise<string> {
@@ -161,6 +169,34 @@ export class TwakeDriveClient {
    }
  };

+  async uploadToS3(fPath: string, id: string) {
+    logger.info(`Upload to S3 ${fPath}`);
+    const client = await this.client();
+
+    const form = new FormData();
+    form.append('file', fs.createReadStream(fPath));
+
+    const response
+      = await client.post(`${this.config.url}/internal/services/files/v1/S3/restore/${id}`, form);
+
+    logger.info(response.data);
+
+    return response.data;
+  }
+
+  async existsInS3(id: string): Promise<boolean> {
+    const client = await this.client();
+    const response
+      = await client.get(`${this.config.url}/internal/services/files/v1/S3/exist/${id}`);
+    if (response.status != 200) {
+      logger.warn(`Error receiving S3 status for ${id}`);
+      logger.warn(response.data)
+      return true;
+    } else {
+      logger.info(`Exists ${JSON.stringify(response.data)}`)
+      return response.data?.exist;
+    }
+  }
 }

 export type TwakeDriveUser = {