From eacdb29bb2a4f7ee2c96a2a82cc9d96d850874f0 Mon Sep 17 00:00:00 2001 From: Anton Shepilov Date: Thu, 7 Mar 2024 00:07:25 +0100 Subject: [PATCH] =?UTF-8?q?*=20=F0=9F=9A=91=20Check=20files=20that=20are?= =?UTF-8?q?=20missing=20in=20storage=20(#419)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🚑Check files that are missing in storage --- .../storage/connectors/S3/s3-service.ts | 3 +- .../documents/entities/missed-drive-file.ts | 50 +++++++++++++++++++ .../node/src/services/files/services/index.ts | 28 ++++++++--- 3 files changed, 71 insertions(+), 10 deletions(-) create mode 100644 tdrive/backend/node/src/services/documents/entities/missed-drive-file.ts diff --git a/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts b/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts index 400ac403..11182817 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts @@ -2,7 +2,6 @@ import * as Minio from "minio"; import { logger } from "../../../../../../core/platform/framework"; import { Readable } from "stream"; import { StorageConnectorAPI, WriteMetadata } from "../../provider"; -import fs from "fs"; export type S3Configuration = { bucket: string; @@ -89,7 +88,7 @@ export default class S3ConnectorService implements StorageConnectorAPI { break; } } catch (e) { - logger.error(`Error getting information from S3`, e); + logger.error("Error getting information from S3", e); } if (i === tries) { diff --git a/tdrive/backend/node/src/services/documents/entities/missed-drive-file.ts b/tdrive/backend/node/src/services/documents/entities/missed-drive-file.ts new file mode 100644 index 00000000..651298c9 --- /dev/null +++ b/tdrive/backend/node/src/services/documents/entities/missed-drive-file.ts @@ -0,0 +1,50 @@ +import { Type } from "class-transformer"; +import { Column, Entity } from "../../../core/platform/services/database/services/orm/decorators"; + +export const TYPE = "missed_drive_files"; + +@Entity(TYPE, { + primaryKey: ["id"], + type: TYPE, +}) +export class MissedDriveFile { + @Type(() => String) + @Column("id", "uuid", { generator: "uuid" }) + id: string; + + @Type(() => String) + @Column("doc_id", "string") + doc_id: string; + + @Type(() => String) + @Column("file_id", "string") + file_id: string; + + @Type(() => Boolean) + @Column("is_in_trash", "boolean") + is_in_trash: boolean; + + @Type(() => String) + @Column("name", "string") + name: string; + + @Type(() => String) + @Column("description", "string") + description: string; + + @Type(() => Number) + @Column("added", "number") + added: number; + + @Type(() => Number) + @Column("last_modified", "number") + last_modified: number; + + @Type(() => String) + @Column("content_keywords", "string") + user_email: string; + + @Type(() => String) + @Column("creator", "uuid") + creator: string; +} diff --git a/tdrive/backend/node/src/services/files/services/index.ts b/tdrive/backend/node/src/services/files/services/index.ts index 8fa0130c..145b3ad0 100644 --- a/tdrive/backend/node/src/services/files/services/index.ts +++ b/tdrive/backend/node/src/services/files/services/index.ts @@ -23,6 +23,7 @@ import { PreviewFinishedProcessor } from "./preview"; import _ from "lodash"; import User from "../../user/entities/user"; import { DriveFile } from "../../documents/entities/drive-file"; +import { MissedDriveFile } from "../../documents/entities/missed-drive-file"; import { FileVersion } from "../../documents/entities/file-version"; export class FileServiceImpl { @@ -30,6 +31,7 @@ export class FileServiceImpl { repository: Repository; userRepository: Repository; documentRepository: Repository; + missedFileRepository: Repository; versionRepository: Repository; private algorithm = "aes-256-cbc"; private max_preview_file_size = 50000000; @@ -47,6 +49,12 @@ export class FileServiceImpl { DriveFile, )), ]); + await Promise.all([ + (this.missedFileRepository = await gr.database.getRepository( + "missed_drive_files", + MissedDriveFile, + )), + ]); await Promise.all([ (this.versionRepository = await gr.database.getRepository( "drive_file_versions", @@ -362,15 +370,19 @@ export class FileServiceImpl { company_id: "00000000-0000-4000-0000-000000000000", }); const user = await this.userRepository.findOne({ id: doc.creator }); - data.push({ + const missedFile = new MissedDriveFile(); + Object.assign(missedFile, { + id: version.file_metadata.external_id, + added: doc.added, + doc_id: doc.id, file_id: version.file_metadata.external_id, - user: user?.email_canonical, - doc: { - id: version.drive_item_id, - name: doc.name, - }, + creator: user?.id, + name: doc.name, + is_in_trash: doc.is_in_trash, + user_email: user?.email_canonical, }); - console.log(`Missing files:: ${JSON.stringify(data)}`); + await this.missedFileRepository.save(missedFile); + logger.info(`Missing file:: ${JSON.stringify(missedFile)}`); } } catch (e) { logger.warn(`Can't find ${version.file_metadata.external_id} in DB`); @@ -380,12 +392,12 @@ export class FileServiceImpl { } } await Promise.all(jobs); - console.log(`Missing files:: ${JSON.stringify(data)}`); //go to next page page = Pagination.fromPaginable(result.nextPage); } while (page.page_token); } finally { this.checkConsistencyInProgress = false; + logger.info("Scanning for missing file finished."); } } return data;