diff --git a/tdrive/backend/utils/nextcloud-migration/src/executor.ts b/tdrive/backend/utils/nextcloud-migration/src/executor.ts new file mode 100644 index 00000000..b3b81c24 --- /dev/null +++ b/tdrive/backend/utils/nextcloud-migration/src/executor.ts @@ -0,0 +1,81 @@ +import { logger } from "./logger"; + +type FunctionArgs = any[]; + +type FunctionStats = { + functionName: string; + successfulExecutions: number; + failedExecutions: number; + failedExecutionsArgs: FunctionArgs[]; +}; + +export class FunctionExecutor { + private stats: Map = new Map(); + + async executeWithRetries( + fn: (...args: any[]) => Promise, + args: FunctionArgs, + retries: number = 3, + throwError: boolean = false, + ): Promise { + const functionName = fn.name; + let success = false; + let executionResult = null; + + for (let attempt = 0; attempt < retries; attempt++) { + try { + executionResult = await fn(...args); + success = true; + break; + } catch (error) { + executionResult = error; + logger.info(`Attempt ${attempt + 1} failed for ${functionName}`, error); + } + } + + if (!success && throwError) throw executionResult; + + let stats = this.stats.get(functionName); + if (!stats) { + stats = { + functionName, + successfulExecutions: 0, + failedExecutions: 0, + failedExecutionsArgs: [] + }; + this.stats.set(functionName, stats); + } + + if (success) { + stats.successfulExecutions++; + } else { + stats.failedExecutions++; + stats.failedExecutionsArgs.push( args ); + } + return executionResult; + } + + async getStats() { + return this.stats; + } + + printStatistics(): void { + logger.info("Execution Statistics:"); + this.stats.forEach((stat) => { + logger.info( + `Function: ${stat.functionName}, Successful Executions: ${stat.successfulExecutions}, Failed Executions: ${stat.failedExecutions}` + ); + }); + } + + printFailedExecutions(): void { + logger.info("Failed Executions:"); + this.stats.forEach((execution) => { + if (execution.failedExecutionsArgs.length > 0) { + logger.info(` Function: ${execution.functionName}`); + execution.failedExecutionsArgs.forEach(args => + logger.info(` args: ${JSON.stringify(args)}`)); + } + }); + } +} \ No newline at end of file diff --git a/tdrive/backend/utils/nextcloud-migration/src/express_server.ts b/tdrive/backend/utils/nextcloud-migration/src/express_server.ts index 51bda38f..acd590b4 100644 --- a/tdrive/backend/utils/nextcloud-migration/src/express_server.ts +++ b/tdrive/backend/utils/nextcloud-migration/src/express_server.ts @@ -60,8 +60,8 @@ app.post("/", async (req: Request, res: Response) => { res.status(400).send("Username and password for nextcloud are required"); } try { - await nextcloud.migrate(params.username, params.password, params.dir); - res.status(200).send("Sync DONE ✅"); + const stats = await nextcloud.migrate(params.username, params.password, params.dir); + res.status(200).send(JSON.stringify(stats)); } catch (e) { console.error(e) res.status(500).send("Error during synchronization:: " + e.message) diff --git a/tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts b/tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts index e63776a3..01aa9204 100644 --- a/tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts +++ b/tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts @@ -6,6 +6,7 @@ import { TwakeDriveClient, TwakeDriveUser } from './twake_client'; import path from 'path'; import { logger } from "./logger" import { User, UserProvider, UserProviderFactory, UserProviderType } from "./user_privider"; +import { FunctionExecutor } from "./executor"; export interface NextcloudMigrationConfiguration { shell: { @@ -36,6 +37,8 @@ export class NextcloudMigration { driveClient: TwakeDriveClient; + executor = new FunctionExecutor(); + constructor(config: NextcloudMigrationConfiguration) { this.config = config; this.userProvider = (new UserProviderFactory()).get(config.userProvider, config[config.userProvider]); @@ -55,6 +58,9 @@ export class NextcloudMigration { if(!dir) await this.download(username, password, dirTmp); //upload files to the Twake Drive await this.upload(driveUser, dirTmp); + this.executor.printStatistics(); + this.executor.printFailedExecutions(); + return this.executor.getStats(); } catch (e) { console.error('Error downloading files from next cloud', e); throw e; @@ -119,7 +125,9 @@ export class NextcloudMigration { const parent = await this.driveClient.getDocument(parentDirId); const exists = (filename: string) => { - return parent.children.filter(i => i.name.startsWith(path.parse(filename).name)).length > 0; + let parsedPath = path.parse(filename); + let name = `${parsedPath.name}${parsedPath.ext > '' ? parsedPath.ext : ''}`; + return parent.children.filter(i => i.name == name).length > 0; } logger.debug(`Reading content of the directory ${sourceDirPath} ...`) @@ -144,8 +152,9 @@ export class NextcloudMigration { //check existing files for (const fPath of existingFiles) { logger.debug(`Check existing file ${fPath}`) - let name = path.parse(fPath).name; - let candidatesWithTheSameName = parent.children.filter(i => i.name.startsWith(name)); + let parsedPath = path.parse(fPath); + let name = parsedPath.name + (parsedPath.ext > '' ? parsedPath.ext : ''); + let candidatesWithTheSameName = parent.children.filter(i => i.name === name); if (candidatesWithTheSameName.length > 1) { logger.warn("WE HAVE MORE MORE THAN ONE FILE WITH NAME: " + name); } else { @@ -169,19 +178,22 @@ export class NextcloudMigration { } } - //upload all files logger.debug(`UPLOAD FILES FOR ${sourceDirPath}`) for (const file of filesToUpload) { logger.debug(`Upload file ${file}`) - await this.driveClient.createFile(file, parentDirId); + await this.executor.executeWithRetries(this.driveClient.createFile.bind(this.driveClient), [file, parentDirId], 3) + // await this.driveClient.createFile(file, parentDirId); } logger.debug(`UPLOAD DIRS FOR ${sourceDirPath}`) for (const [name, path] of dirsToUpload) { logger.info(`Create directory ${name}`); - const dir = await this.driveClient.createDirectory(name, parentDirId) - await this.upload(user, path, dir.id); + const dir = await this.executor.executeWithRetries(this.driveClient.createDirectory.bind(this.driveClient), [name, parentDirId], 3) + // const dir = await this.driveClient.createDirectory(name, parentDirId) + if (dir) { + await this.upload(user, path, dir.id); + } } } diff --git a/tdrive/backend/utils/nextcloud-migration/test/executor.test.ts b/tdrive/backend/utils/nextcloud-migration/test/executor.test.ts new file mode 100644 index 00000000..6137e47c --- /dev/null +++ b/tdrive/backend/utils/nextcloud-migration/test/executor.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, test } from "@jest/globals"; +import { FunctionExecutor } from "../src/executor"; + +describe('Function Executor tests', () => { + + const subj = new FunctionExecutor(); + + test('Should successfully execute mock function', async () => { + //when + await subj.executeWithRetries(successFunction, [1, 2], 3); + await subj.executeWithRetries(successFunction, [4, 5], 3); + + //then + let stats = await subj.getStats(); + expect(stats?.get("successFunction")?.successfulExecutions).toEqual(2); + }); + + test('Should successfully gather arguments of the failed executions', async () => { + await subj.executeWithRetries(failedFunction, [1, 2], 3); + + //then + let stats = await subj.getStats(); + expect(stats?.get("failedFunction")?.failedExecutions).toBe(1); + expect(stats?.get("failedFunction")?.failedExecutionsArgs[0]).toStrictEqual([1, 2]); + }); + + test('Should successfully gather arguments of the class member', async () => { + await subj.executeWithRetries(subj.getStats.bind(this), [1, 2], 3); + + //then + let stats = await subj.getStats(); + expect(stats?.get("bound getStats")?.successfulExecutions).toEqual(1); + }); + + const successFunction = async (a: number, b: number) => { + return a + b; + }; + + const failedFunction = async () => { + throw new Error(); + }; + +});