From 56b76767900ac321fde357bdbd3c532513966423 Mon Sep 17 00:00:00 2001 From: Anton Shepilov Date: Wed, 6 Mar 2024 18:25:37 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=91=20Check=20files=20that=20are=20mis?= =?UTF-8?q?sing=20in=20storage=20(#416)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🚑Check files that are missing in storage --- .../storage/connectors/S3/s3-service.ts | 24 ++++++ .../storage/connectors/local/service.ts | 6 ++ .../core/platform/services/storage/index.ts | 4 + .../platform/services/storage/provider.ts | 7 ++ .../node/src/services/files/services/index.ts | 82 +++++++++++++++++++ .../services/files/web/controllers/files.ts | 5 ++ .../node/src/services/files/web/routes.ts | 6 ++ 7 files changed, 134 insertions(+) diff --git a/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts b/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts index 856ae544..400ac403 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts @@ -2,6 +2,7 @@ import * as Minio from "minio"; import { logger } from "../../../../../../core/platform/framework"; import { Readable } from "stream"; import { StorageConnectorAPI, WriteMetadata } from "../../provider"; +import fs from "fs"; export type S3Configuration = { bucket: string; @@ -77,4 +78,27 @@ export default class S3ConnectorService implements StorageConnectorAPI { } catch (err) {} return false; } + + async exists(path: string): Promise { + logger.trace(`Reading file ... ${path}`); + const tries = 2; + for (let i = 0; i <= tries; i++) { + try { + const stat = await this.client.statObject(this.minioConfiguration.bucket, path); + if (stat?.size > 0) { + break; + } + } catch (e) { + logger.error(`Error getting information from S3`, e); + } + + if (i === tries) { + logger.info(`Unable to get file after ${tries} tries:`); + return false; + } + await new Promise(r => setTimeout(r, 500)); + logger.info(`File ${path} not found in S3 bucket, retrying...`); + } + return true; + } } diff --git a/tdrive/backend/node/src/core/platform/services/storage/connectors/local/service.ts b/tdrive/backend/node/src/core/platform/services/storage/connectors/local/service.ts index 53672bb8..cf0e7495 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/connectors/local/service.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/connectors/local/service.ts @@ -72,4 +72,10 @@ export default class LocalConnectorService implements StorageConnectorAPI { delete(path: string): Promise { return rm(this.getFullPath(path), { recursive: false, force: true }); } + + async exists(path: string): Promise { + logger.trace(`Reading file ... ${path}`); + const fullPath = this.getFullPath(path); + return fs.existsSync(fullPath); + } } diff --git a/tdrive/backend/node/src/core/platform/services/storage/index.ts b/tdrive/backend/node/src/core/platform/services/storage/index.ts index a92a0efe..ccdd461e 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/index.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/index.ts @@ -75,6 +75,10 @@ export default class StorageService extends TdriveService implements return this.homeDir; } + exists(path: string, options?: ReadOptions): Promise { + return this.getConnector().exists(path, options); + } + async write(path: string, stream: Stream, options?: WriteOptions): Promise { try { if (options?.encryptionKey) { diff --git a/tdrive/backend/node/src/core/platform/services/storage/provider.ts b/tdrive/backend/node/src/core/platform/services/storage/provider.ts index 2112405f..00904dba 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/provider.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/provider.ts @@ -43,6 +43,13 @@ export interface StorageConnectorAPI { */ read(path: string, options?: ReadOptions, context?: ExecutionContext): Promise; + /** + * Check that the file is exists + * + * @param path + */ + exists(path: string, options?: ReadOptions, context?: ExecutionContext): Promise; + /** * Remove a path * diff --git a/tdrive/backend/node/src/services/files/services/index.ts b/tdrive/backend/node/src/services/files/services/index.ts index 9766ab03..8fa0130c 100644 --- a/tdrive/backend/node/src/services/files/services/index.ts +++ b/tdrive/backend/node/src/services/files/services/index.ts @@ -11,6 +11,8 @@ import { CrudException, DeleteResult, ExecutionContext, + ListResult, + Pagination, } from "../../../core/platform/framework/api/crud-service"; import gr from "../../global-resolver"; import { @@ -19,16 +21,38 @@ import { } from "../../../services/previews/types"; import { PreviewFinishedProcessor } from "./preview"; import _ from "lodash"; +import User from "../../user/entities/user"; +import { DriveFile } from "../../documents/entities/drive-file"; +import { FileVersion } from "../../documents/entities/file-version"; export class FileServiceImpl { version: "1"; repository: Repository; + userRepository: Repository; + documentRepository: Repository; + versionRepository: Repository; private algorithm = "aes-256-cbc"; private max_preview_file_size = 50000000; + private checkConsistencyInProgress = false; async init(): Promise { try { await Promise.all([(this.repository = await gr.database.getRepository("files", File))]); + await Promise.all([ + (this.userRepository = await gr.database.getRepository("user", User)), + ]); + await Promise.all([ + (this.documentRepository = await gr.database.getRepository( + "drive_files", + DriveFile, + )), + ]); + await Promise.all([ + (this.versionRepository = await gr.database.getRepository( + "drive_file_versions", + FileVersion, + )), + ]); gr.platformServices.messageQueue.processor.addHandler( new PreviewFinishedProcessor(this, this.repository), ); @@ -308,6 +332,64 @@ export class FileServiceImpl { return new DeleteResult("files", fileToDelete, true); } + + async checkConsistency(): Promise { + const data = []; + if (!this.checkConsistencyInProgress) { + this.checkConsistencyInProgress = true; + let result: ListResult; + let page: Pagination = { limitStr: "20" }; + try { + do { + result = await this.versionRepository.find({}, { pagination: page }); + //check that the file exists + const jobs: Promise[] = []; + for (const version of result.getEntities()) { + if (version.file_metadata.external_id) { + const checkFile = async () => { + try { + const file = await this.getFile({ + id: version.file_metadata.external_id, + company_id: "00000000-0000-4000-0000-000000000000", + }); + const exist = await gr.platformServices.storage.exists(getFilePath(file)); + if (exist) { + logger.info(`File ${version.file_metadata.external_id} exists in S3`); + } else { + logger.info(`File ${version.file_metadata.external_id} DOES NOT exists in S3`); + const doc = await this.documentRepository.findOne({ + id: version.drive_item_id, + company_id: "00000000-0000-4000-0000-000000000000", + }); + const user = await this.userRepository.findOne({ id: doc.creator }); + data.push({ + file_id: version.file_metadata.external_id, + user: user?.email_canonical, + doc: { + id: version.drive_item_id, + name: doc.name, + }, + }); + console.log(`Missing files:: ${JSON.stringify(data)}`); + } + } catch (e) { + logger.warn(`Can't find ${version.file_metadata.external_id} in DB`); + } + }; + jobs.push(checkFile.bind(this)()); + } + } + await Promise.all(jobs); + console.log(`Missing files:: ${JSON.stringify(data)}`); + //go to next page + page = Pagination.fromPaginable(result.nextPage); + } while (page.page_token); + } finally { + this.checkConsistencyInProgress = false; + } + } + return data; + } } function getFilePath(entity: File): string { diff --git a/tdrive/backend/node/src/services/files/web/controllers/files.ts b/tdrive/backend/node/src/services/files/web/controllers/files.ts index dab7f21c..7b653105 100644 --- a/tdrive/backend/node/src/services/files/web/controllers/files.ts +++ b/tdrive/backend/node/src/services/files/web/controllers/files.ts @@ -59,6 +59,11 @@ export class FileController { } } + async checkConsistency(request: FastifyRequest, response: FastifyReply): Promise { + const data = await gr.services.files.checkConsistency(); + response.send(data); + } + async thumbnail( request: FastifyRequest<{ Params: { company_id: string; id: string; index: string } }>, response: FastifyReply, diff --git a/tdrive/backend/node/src/services/files/web/routes.ts b/tdrive/backend/node/src/services/files/web/routes.ts index a9e04428..1ca4f72f 100644 --- a/tdrive/backend/node/src/services/files/web/routes.ts +++ b/tdrive/backend/node/src/services/files/web/routes.ts @@ -46,6 +46,12 @@ const routes: FastifyPluginCallback = (fastify: FastifyInstance, options, next) preValidation: [fastify.authenticate], handler: fileController.delete.bind(fileController), }); + fastify.route({ + method: "GET", + url: "/check-files", + preValidation: [fastify.authenticate], + handler: fileController.checkConsistency.bind(fileController), + }); next(); };