* 🚑 Check files that are missing in storage (#419)

* 🚑Check files that are missing in storage

tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts

@@ -2,7 +2,6 @@ import * as Minio from "minio";
 import { logger } from "../../../../../../core/platform/framework";
 import { Readable } from "stream";
 import { StorageConnectorAPI, WriteMetadata } from "../../provider";
-import fs from "fs";
 
 export type S3Configuration = {
   bucket: string;
@@ -89,7 +88,7 @@ export default class S3ConnectorService implements StorageConnectorAPI {
           break;
         }
       } catch (e) {
-        logger.error(`Error getting information from S3`, e);
+        logger.error("Error getting information from S3", e);
       }
 
       if (i === tries) {

tdrive/backend/node/src/services/documents/entities/missed-drive-file.ts

@@ -0,0 +1,50 @@
+import { Type } from "class-transformer";
+import { Column, Entity } from "../../../core/platform/services/database/services/orm/decorators";
+
+export const TYPE = "missed_drive_files";
+
+@Entity(TYPE, {
+  primaryKey: ["id"],
+  type: TYPE,
+})
+export class MissedDriveFile {
+  @Type(() => String)
+  @Column("id", "uuid", { generator: "uuid" })
+  id: string;
+
+  @Type(() => String)
+  @Column("doc_id", "string")
+  doc_id: string;
+
+  @Type(() => String)
+  @Column("file_id", "string")
+  file_id: string;
+
+  @Type(() => Boolean)
+  @Column("is_in_trash", "boolean")
+  is_in_trash: boolean;
+
+  @Type(() => String)
+  @Column("name", "string")
+  name: string;
+
+  @Type(() => String)
+  @Column("description", "string")
+  description: string;
+
+  @Type(() => Number)
+  @Column("added", "number")
+  added: number;
+
+  @Type(() => Number)
+  @Column("last_modified", "number")
+  last_modified: number;
+
+  @Type(() => String)
+  @Column("content_keywords", "string")
+  user_email: string;
+
+  @Type(() => String)
+  @Column("creator", "uuid")
+  creator: string;
+}

tdrive/backend/node/src/services/files/services/index.ts

@@ -23,6 +23,7 @@ import { PreviewFinishedProcessor } from "./preview";
 import _ from "lodash";
 import User from "../../user/entities/user";
 import { DriveFile } from "../../documents/entities/drive-file";
+import { MissedDriveFile } from "../../documents/entities/missed-drive-file";
 import { FileVersion } from "../../documents/entities/file-version";
 
 export class FileServiceImpl {
@@ -30,6 +31,7 @@ export class FileServiceImpl {
   repository: Repository<File>;
   userRepository: Repository<User>;
   documentRepository: Repository<DriveFile>;
+  missedFileRepository: Repository<MissedDriveFile>;
   versionRepository: Repository<FileVersion>;
   private algorithm = "aes-256-cbc";
   private max_preview_file_size = 50000000;
@@ -47,6 +49,12 @@ export class FileServiceImpl {
           DriveFile,
         )),
       ]);
+      await Promise.all([
+        (this.missedFileRepository = await gr.database.getRepository<MissedDriveFile>(
+          "missed_drive_files",
+          MissedDriveFile,
+        )),
+      ]);
       await Promise.all([
         (this.versionRepository = await gr.database.getRepository<FileVersion>(
           "drive_file_versions",
@@ -362,15 +370,19 @@ export class FileServiceImpl {
                       company_id: "00000000-0000-4000-0000-000000000000",
                     });
                     const user = await this.userRepository.findOne({ id: doc.creator });
-                    data.push({
+                    const missedFile = new MissedDriveFile();
+                    Object.assign(missedFile, {
+                      id: version.file_metadata.external_id,
+                      added: doc.added,
+                      doc_id: doc.id,
                       file_id: version.file_metadata.external_id,
-                      user: user?.email_canonical,
+                      creator: user?.id,
-                      doc: {
+                      name: doc.name,
-                        id: version.drive_item_id,
+                      is_in_trash: doc.is_in_trash,
-                        name: doc.name,
+                      user_email: user?.email_canonical,
-                      },
                     });
-                    console.log(`Missing files:: ${JSON.stringify(data)}`);
+                    await this.missedFileRepository.save(missedFile);
+                    logger.info(`Missing file:: ${JSON.stringify(missedFile)}`);
                   }
                 } catch (e) {
                   logger.warn(`Can't find ${version.file_metadata.external_id} in DB`);
@@ -380,12 +392,12 @@ export class FileServiceImpl {
             }
           }
           await Promise.all(jobs);
-          console.log(`Missing files:: ${JSON.stringify(data)}`);
           //go to next page
           page = Pagination.fromPaginable(result.nextPage);
         } while (page.page_token);
       } finally {
         this.checkConsistencyInProgress = false;
+        logger.info("Scanning for missing file finished.");
       }
     }
     return data;