🚑 Check files that are missing in storage (#416)

* 🚑Check files that are missing in storage

tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts

@@ -2,6 +2,7 @@ import * as Minio from "minio";
 import { logger } from "../../../../../../core/platform/framework";
 import { Readable } from "stream";
 import { StorageConnectorAPI, WriteMetadata } from "../../provider";
+import fs from "fs";
 
 export type S3Configuration = {
   bucket: string;
@@ -77,4 +78,27 @@ export default class S3ConnectorService implements StorageConnectorAPI {
     } catch (err) {}
     return false;
   }
+
+  async exists(path: string): Promise<boolean> {
+    logger.trace(`Reading file ... ${path}`);
+    const tries = 2;
+    for (let i = 0; i <= tries; i++) {
+      try {
+        const stat = await this.client.statObject(this.minioConfiguration.bucket, path);
+        if (stat?.size > 0) {
+          break;
+        }
+      } catch (e) {
+        logger.error(`Error getting information from S3`, e);
+      }
+
+      if (i === tries) {
+        logger.info(`Unable to get file after ${tries} tries:`);
+        return false;
+      }
+      await new Promise(r => setTimeout(r, 500));
+      logger.info(`File ${path} not found in S3 bucket, retrying...`);
+    }
+    return true;
+  }
 }

tdrive/backend/node/src/core/platform/services/storage/connectors/local/service.ts

@@ -72,4 +72,10 @@ export default class LocalConnectorService implements StorageConnectorAPI {
   delete(path: string): Promise<void> {
     return rm(this.getFullPath(path), { recursive: false, force: true });
   }
+
+  async exists(path: string): Promise<boolean> {
+    logger.trace(`Reading file ... ${path}`);
+    const fullPath = this.getFullPath(path);
+    return fs.existsSync(fullPath);
+  }
 }

tdrive/backend/node/src/core/platform/services/storage/index.ts

@@ -75,6 +75,10 @@ export default class StorageService extends TdriveService<StorageAPI> implements
     return this.homeDir;
   }
 
+  exists(path: string, options?: ReadOptions): Promise<boolean> {
+    return this.getConnector().exists(path, options);
+  }
+
   async write(path: string, stream: Stream, options?: WriteOptions): Promise<WriteMetadata> {
     try {
       if (options?.encryptionKey) {

tdrive/backend/node/src/core/platform/services/storage/provider.ts

@@ -43,6 +43,13 @@ export interface StorageConnectorAPI {
    */
   read(path: string, options?: ReadOptions, context?: ExecutionContext): Promise<Readable>;
 
+  /**
+   * Check that the file is exists
+   *
+   * @param path
+   */
+  exists(path: string, options?: ReadOptions, context?: ExecutionContext): Promise<boolean>;
+
   /**
    * Remove a path
    *