From c9ce1fce430203adac0cc1348c7c0ed9e96efdcd Mon Sep 17 00:00:00 2001 From: Anton Shepilov Date: Thu, 7 Mar 2024 18:33:08 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=91=20Restore=20files=20from=20nextclo?= =?UTF-8?q?ud=20(#423)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🚑Modify nextcloud migration to be able to back up files from next cloud Co-authored-by: Monta --- .../node/src/services/files/services/index.ts | 42 +++++++++++++- .../services/files/web/controllers/files.ts | 31 ++++++++++ .../node/src/services/files/web/routes.ts | 14 +++++ .../src/nextcloud_migration.ts | 33 +++++++++++ .../nextcloud-migration/src/twake_client.ts | 56 +++++++++++++++---- 5 files changed, 165 insertions(+), 11 deletions(-) diff --git a/tdrive/backend/node/src/services/files/services/index.ts b/tdrive/backend/node/src/services/files/services/index.ts index 145b3ad0..2ef67c4a 100644 --- a/tdrive/backend/node/src/services/files/services/index.ts +++ b/tdrive/backend/node/src/services/files/services/index.ts @@ -402,8 +402,48 @@ export class FileServiceImpl { } return data; } -} + async checkFileExistsS3(id: string): Promise { + try { + const doc = await this.documentRepository.findOne({ + id, + company_id: "00000000-0000-4000-0000-000000000000", + }); + const externalId = doc.last_version_cache.file_metadata.external_id; + const file = await this.getFile({ + id: externalId, + company_id: "00000000-0000-4000-0000-000000000000", + }); + const exist = await gr.platformServices.storage.exists(getFilePath(file)); + if (exist) { + return { exist: true, file }; + } else { + return { exist: false, file }; + } + } catch (error) { + logger.error(`Error while checking file ${id} in S3`, error); + return { exist: false, file: null }; + } + } + + async restoreFileS3(id: string, file: Multipart, options: UploadOptions): Promise { + try { + const result = await this.checkFileExistsS3(id); + if (result.exist) { + return { success: true }; + } + await gr.platformServices.storage.write(getFilePath(result.file), file.file, { + chunkNumber: options.chunkNumber, + encryptionAlgo: this.algorithm, + encryptionKey: result.file.encryption_key, + }); + return { success: true }; + } catch (error) { + logger.error(`Error while uploading missing file ${id} to S3`, error); + return { success: false }; + } + } +} function getFilePath(entity: File): string { return `${gr.platformServices.storage.getHomeDir()}/files/${entity.company_id}/${ entity.user_id || "anonymous" diff --git a/tdrive/backend/node/src/services/files/web/controllers/files.ts b/tdrive/backend/node/src/services/files/web/controllers/files.ts index 7b653105..e919a313 100644 --- a/tdrive/backend/node/src/services/files/web/controllers/files.ts +++ b/tdrive/backend/node/src/services/files/web/controllers/files.ts @@ -105,6 +105,37 @@ export class FileController { return { status: deleteResult.deleted ? "success" : "error" }; } + + async checkFileS3Exists( + request: FastifyRequest<{ Params: { company_id: string; id: string } }>, + ): Promise<{ isInS3: boolean }> { + const params = request.params; + return await gr.services.files.checkFileExistsS3(params.id); + } + + async restoreFileS3( + request: FastifyRequest<{ + Params: { company_id: string; id: string }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + Querystring: any; + }>, + ): Promise<{ resource: PublicFile }> { + const params = request.params; + let file: null | Multipart = null; + if (request.isMultipart()) { + file = await request.file(); + } + + const q = request.query; + const options: any = { + totalChunks: parseInt(q.resumableTotalChunks || q.total_chunks) || 1, + chunkNumber: parseInt(q.resumableChunkNumber || q.chunk_number) || 1, + }; + + const result = await gr.services.files.restoreFileS3(params.id, file, options); + + return result; + } } function getCompanyExecutionContext( diff --git a/tdrive/backend/node/src/services/files/web/routes.ts b/tdrive/backend/node/src/services/files/web/routes.ts index 1ca4f72f..ea659c00 100644 --- a/tdrive/backend/node/src/services/files/web/routes.ts +++ b/tdrive/backend/node/src/services/files/web/routes.ts @@ -53,6 +53,20 @@ const routes: FastifyPluginCallback = (fastify: FastifyInstance, options, next) handler: fileController.checkConsistency.bind(fileController), }); + fastify.route({ + method: "GET", + url: "/S3/exist/:id", + preValidation: [fastify.authenticate], + handler: fileController.checkFileS3Exists.bind(fileController), + }); + + fastify.route({ + method: "POST", + url: "/S3/restore/:id", + preValidation: [fastify.authenticate], + handler: fileController.restoreFileS3.bind(fileController), + }); + next(); }; diff --git a/tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts b/tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts index 7103bac2..f877ad33 100644 --- a/tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts +++ b/tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts @@ -44,6 +44,7 @@ export class NextcloudMigration { async migrate(username: string, password: string) { const dir = this.createTmpDir(username); + // const dir = "/tmp/to_upload" try { const user = await this.getLDAPUser(username); //create user if needed Twake Drive @@ -113,6 +114,7 @@ export class NextcloudMigration { async upload(user: TwakeDriveUser, sourceDirPath: string, parentDirId = "user_" + user.id) { const dirsToUpload: Map = new Map(); const filesToUpload: string[] = []; + const existingFiles: string[] = []; const parent = await this.driveClient.getDocument(parentDirId); @@ -126,6 +128,9 @@ export class NextcloudMigration { const stat = fs.statSync(filePath); if (exists(name)) { logger.info(`File ${name} already exists`); + //find document that we need to replace + existingFiles.push(filePath); + //skip } else { if (stat.isFile()) { logger.info(`Add file for future upload ${filePath}`); @@ -136,6 +141,34 @@ export class NextcloudMigration { } } }); + //check existing files + for (const fPath of existingFiles) { + logger.debug(`Check existing file ${fPath}`) + let name = path.parse(fPath).name; + let candidatesWithTheSameName = parent.children.filter(i => i.name.startsWith(name)); + if (candidatesWithTheSameName.length > 1) { + logger.warn("WE HAVE MORE MORE THAN ONE FILE WITH NAME: " + name); + } else { + const doc = candidatesWithTheSameName[0]; + if (doc.is_directory) { + logger.info(`Directory ${name} exists, try to upload inside`); + await this.upload(user, fPath, doc.id); + } else { + //check that it exists in S3 + if (await this.driveClient.existsInS3(doc.id)) { + logger.info(`File ${name}, docId = ${doc.id} exists in S3`); + } else { + //if it doesn't exists upload just one file + logger.info(`File ${name}, is missing, uploading files`); + const response = await this.driveClient.uploadToS3(fPath, doc.id); + if (!response && !response.success) { + console.log(`Error creating file '${name}' in S3`) + } + } + } + } + } + //upload all files logger.debug(`UPLOAD FILES FOR ${sourceDirPath}`) diff --git a/tdrive/backend/utils/nextcloud-migration/src/twake_client.ts b/tdrive/backend/utils/nextcloud-migration/src/twake_client.ts index 1ad72b64..41ce0652 100644 --- a/tdrive/backend/utils/nextcloud-migration/src/twake_client.ts +++ b/tdrive/backend/utils/nextcloud-migration/src/twake_client.ts @@ -17,6 +17,11 @@ export class TwakeDriveClient { private readonly config: TwakeClientConfiguration; + private clientInstance; + + //00000000-0000-4000-0000-000000000000 + private company = "00000000-0000-4000-0000-000000000000"; + constructor(config: TwakeClientConfiguration) { this.config = config; //remove the trailing '/' @@ -33,7 +38,7 @@ export class TwakeDriveClient { form.append('file', fs.createReadStream(path)); const response - = await client.post(`${this.config.url}/internal/services/files/v1/companies/00000000-0000-4000-0000-000000000000/files`, form); + = await client.post(`${this.config.url}/internal/services/files/v1/companies/${this.company}/files`, form); logger.info(response.data); @@ -47,7 +52,7 @@ export class TwakeDriveClient { async createDirectory(name: string, parent: string) { return this.createDocument({ - company_id: '00000000-0000-4000-0000-000000000000', + company_id: this.company, name: name, parent_id: parent, is_directory: true, @@ -74,7 +79,7 @@ export class TwakeDriveClient { private async createDocument(item: any, version: any) { const response - = await (await this.client()).post(`${this.config.url}/internal/services/documents/v1/companies/00000000-0000-4000-0000-000000000000/item`, { + = await (await this.client()).post(`${this.config.url}/internal/services/documents/v1/companies/${this.company}/item`, { item, version, }); @@ -85,7 +90,7 @@ export class TwakeDriveClient { async getDocument(id: string) { logger.info(`Get information for the doc ${id}`); const response - = await (await this.client()).get(`${this.config.url}/internal/services/documents/v1/companies/00000000-0000-4000-0000-000000000000/item/${id}`, { + = await (await this.client()).get(`${this.config.url}/internal/services/documents/v1/companies/${this.company}/item/${id}`, { }); logger.info(response.data); return response.data; @@ -122,12 +127,15 @@ export class TwakeDriveClient { } private async client() { - return axios.create({ - baseURL: this.config.url, - headers: { - Authorization: `Bearer ${await (this.accessToken())}`, - }, - }); + if (!this.clientInstance) { + this.clientInstance = axios.create({ + baseURL: this.config.url, + headers: { + Authorization: `Bearer ${await (this.accessToken())}`, + }, + }); + } + return this.clientInstance; } private async accessToken(): Promise { @@ -161,6 +169,34 @@ export class TwakeDriveClient { } }; + async uploadToS3(fPath: string, id: string) { + logger.info(`Upload to S3 ${fPath}`); + const client = await this.client(); + + const form = new FormData(); + form.append('file', fs.createReadStream(fPath)); + + const response + = await client.post(`${this.config.url}/internal/services/files/v1/S3/restore/${id}`, form); + + logger.info(response.data); + + return response.data; + } + + async existsInS3(id: string): Promise { + const client = await this.client(); + const response + = await client.get(`${this.config.url}/internal/services/files/v1/S3/exist/${id}`); + if (response.status != 200) { + logger.warn(`Error receiving S3 status for ${id}`); + logger.warn(response.data) + return true; + } else { + logger.info(`Exists ${JSON.stringify(response.data)}`) + return response.data?.exist; + } + } } export type TwakeDriveUser = {