✨ Ignore errors during nextcloud migration to be able to import files partially (#579)

* ✨ Ignore errors during nextcloud migration to be able to import files partially for big migrations
* 👷 Fixed Nextcloud migration publishing script
* ✏️ Removed wrong comment
* 🐛 Fixed nextcloud migration, when there is a folder and a file with the same name

tdrive/backend/utils/nextcloud-migration/src/executor.ts

@@ -0,0 +1,81 @@
+import { logger } from "./logger";
+
+type FunctionArgs = any[];
+
+type FunctionStats = {
+  functionName: string;
+  successfulExecutions: number;
+  failedExecutions: number;
+  failedExecutionsArgs: FunctionArgs[];
+};
+
+export class FunctionExecutor {
+  private stats: Map<string, FunctionStats> = new Map();
+
+  async executeWithRetries(
+    fn: (...args: any[]) => Promise<any>,
+    args: FunctionArgs,
+    retries: number = 3,
+    throwError: boolean = false,
+  ): Promise<any> {
+    const functionName = fn.name;
+    let success = false;
+    let executionResult = null;
+
+    for (let attempt = 0; attempt < retries; attempt++) {
+      try {
+        executionResult = await fn(...args);
+        success = true;
+        break;
+      } catch (error) {
+        executionResult = error;
+        logger.info(`Attempt ${attempt + 1} failed for ${functionName}`, error);
+      }
+    }
+
+    if (!success && throwError) throw executionResult;
+
+    let stats = this.stats.get(functionName);
+    if (!stats) {
+      stats = {
+        functionName,
+        successfulExecutions: 0,
+        failedExecutions: 0,
+        failedExecutionsArgs: []
+      };
+      this.stats.set(functionName, stats);
+    }
+
+    if (success) {
+      stats.successfulExecutions++;
+    } else {
+      stats.failedExecutions++;
+      stats.failedExecutionsArgs.push( args );
+    }
+    return executionResult;
+  }
+
+  async getStats() {
+    return this.stats;
+  }
+
+  printStatistics(): void {
+    logger.info("Execution Statistics:");
+    this.stats.forEach((stat) => {
+      logger.info(
+        `Function: ${stat.functionName}, Successful Executions: ${stat.successfulExecutions}, Failed Executions: ${stat.failedExecutions}`
+      );
+    });
+  }
+
+  printFailedExecutions(): void {
+    logger.info("Failed Executions:");
+    this.stats.forEach((execution) => {
+      if (execution.failedExecutionsArgs.length > 0) {
+        logger.info(`   Function: ${execution.functionName}`);
+        execution.failedExecutionsArgs.forEach(args =>
+          logger.info(`       args: ${JSON.stringify(args)}`));
+      }
+    });
+  }
+}

tdrive/backend/utils/nextcloud-migration/src/express_server.ts

@@ -60,8 +60,8 @@ app.post("/", async (req: Request, res: Response) => {
     res.status(400).send("Username and password for nextcloud are required");
   }
   try {
-    await nextcloud.migrate(params.username, params.password, params.dir);
+    const stats = await nextcloud.migrate(params.username, params.password, params.dir);
-    res.status(200).send("Sync DONE ✅");
+    res.status(200).send(JSON.stringify(stats));
   } catch (e) {
     console.error(e)
     res.status(500).send("Error during synchronization:: " + e.message)

tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts

@@ -6,6 +6,7 @@ import { TwakeDriveClient, TwakeDriveUser } from './twake_client';
 import path from 'path';
 import { logger } from "./logger"
 import { User, UserProvider, UserProviderFactory, UserProviderType } from "./user_privider";
+import { FunctionExecutor } from "./executor";
 
 export interface NextcloudMigrationConfiguration {
   shell: {
@@ -36,6 +37,8 @@ export class NextcloudMigration {
 
   driveClient: TwakeDriveClient;
 
+  executor = new FunctionExecutor();
+
   constructor(config: NextcloudMigrationConfiguration) {
     this.config = config;
     this.userProvider = (new UserProviderFactory()).get(config.userProvider, config[config.userProvider]);
@@ -55,6 +58,9 @@ export class NextcloudMigration {
       if(!dir) await this.download(username, password, dirTmp);
       //upload files to the Twake Drive
       await this.upload(driveUser, dirTmp);
+      this.executor.printStatistics();
+      this.executor.printFailedExecutions();
+      return this.executor.getStats();
     } catch (e) {
       console.error('Error downloading files from next cloud', e);
       throw e;
@@ -119,7 +125,9 @@ export class NextcloudMigration {
     const parent = await this.driveClient.getDocument(parentDirId);
 
     const exists = (filename: string) => {
-      return parent.children.filter(i => i.name.startsWith(path.parse(filename).name)).length > 0;
+      let parsedPath = path.parse(filename);
+      let name = `${parsedPath.name}${parsedPath.ext > '' ? parsedPath.ext : ''}`;
+      return parent.children.filter(i => i.name == name).length > 0;
     }
 
     logger.debug(`Reading content of the directory ${sourceDirPath} ...`)
@@ -144,8 +152,9 @@ export class NextcloudMigration {
     //check existing files
     for (const fPath of existingFiles) {
       logger.debug(`Check existing file ${fPath}`)
-      let name = path.parse(fPath).name;
+      let parsedPath = path.parse(fPath);
-      let candidatesWithTheSameName = parent.children.filter(i => i.name.startsWith(name));
+      let name = parsedPath.name + (parsedPath.ext > '' ? parsedPath.ext : '');
+      let candidatesWithTheSameName = parent.children.filter(i => i.name === name);
       if (candidatesWithTheSameName.length > 1) {
         logger.warn("WE HAVE MORE MORE THAN ONE FILE WITH NAME: " + name);
       } else {
@@ -169,19 +178,22 @@ export class NextcloudMigration {
       }
     }
 
-
     //upload all files
     logger.debug(`UPLOAD FILES FOR  ${sourceDirPath}`)
     for (const file of filesToUpload) {
       logger.debug(`Upload file ${file}`)
-      await this.driveClient.createFile(file, parentDirId);
+      await this.executor.executeWithRetries(this.driveClient.createFile.bind(this.driveClient), [file, parentDirId], 3)
+      // await this.driveClient.createFile(file, parentDirId);
     }
 
     logger.debug(`UPLOAD DIRS FOR  ${sourceDirPath}`)
     for (const [name, path] of dirsToUpload) {
       logger.info(`Create directory ${name}`);
-      const dir = await this.driveClient.createDirectory(name, parentDirId)
+      const dir = await this.executor.executeWithRetries(this.driveClient.createDirectory.bind(this.driveClient), [name, parentDirId], 3)
-      await this.upload(user, path, dir.id);
+      // const dir = await this.driveClient.createDirectory(name, parentDirId)
+      if (dir) {
+        await this.upload(user, path, dir.id);
+      }
     }
   }
 

tdrive/backend/utils/nextcloud-migration/test/executor.test.ts

@@ -0,0 +1,43 @@
+import { describe, expect, test } from "@jest/globals";
+import { FunctionExecutor } from "../src/executor";
+
+describe('Function Executor tests', () => {
+
+  const subj = new FunctionExecutor();
+
+  test('Should successfully execute mock function', async () => {
+    //when
+    await subj.executeWithRetries(successFunction, [1, 2], 3);
+    await subj.executeWithRetries(successFunction, [4, 5], 3);
+
+    //then
+    let stats = await subj.getStats();
+    expect(stats?.get("successFunction")?.successfulExecutions).toEqual(2);
+  });
+
+  test('Should successfully gather arguments of the failed executions', async () => {
+    await subj.executeWithRetries(failedFunction, [1, 2], 3);
+
+    //then
+    let stats = await subj.getStats();
+    expect(stats?.get("failedFunction")?.failedExecutions).toBe(1);
+    expect(stats?.get("failedFunction")?.failedExecutionsArgs[0]).toStrictEqual([1, 2]);
+  });
+
+  test('Should successfully gather arguments of the class member', async () => {
+    await subj.executeWithRetries(subj.getStats.bind(this), [1, 2], 3);
+
+    //then
+    let stats = await subj.getStats();
+    expect(stats?.get("bound getStats")?.successfulExecutions).toEqual(1);
+  });
+
+  const successFunction = async (a: number, b: number) => {
+    return a + b;
+  };
+
+  const failedFunction = async () => {
+    throw new Error();
+  };
+
+});