♻️Added specific Exception to be monitored to one-of storage strategy

tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts

@@ -2,8 +2,10 @@ import * as Minio from "minio";
 import { logger } from "../../../../../../core/platform/framework";
 import { Readable } from "stream";
 import { StorageConnectorAPI, WriteMetadata } from "../../provider";
+import { randomUUID } from "crypto";
 
 export type S3Configuration = {
+  id: string;
   bucket: string;
   region: string;
   endPoint: string;
@@ -17,10 +19,20 @@ export type S3Configuration = {
 export default class S3ConnectorService implements StorageConnectorAPI {
   client: Minio.Client;
   minioConfiguration: S3Configuration;
+  id: string;
 
   constructor(S3Configuration: S3Configuration) {
     this.client = new Minio.Client(S3Configuration);
     this.minioConfiguration = S3Configuration;
+    this.id = this.minioConfiguration.id;
+    if (!this.id) {
+      this.id = randomUUID();
+      logger.info(`Identifier for S3 storage haven't been set, initializing to '${this.id}'`);
+    }
+  }
+
+  getId() {
+    return this.id;
   }
 
   write(path: string, stream: Readable): Promise<WriteMetadata> {

tdrive/backend/node/src/core/platform/services/storage/connectors/local/service.ts

@@ -5,16 +5,28 @@ import { rm } from "fs/promises"; // Do not change the import, this is not the s
 import { StorageConnectorAPI, WriteMetadata } from "../../provider";
 import fs from "fs";
 import { logger } from "../../../../framework/logger";
+import { randomUUID } from "crypto";
 
 export type LocalConfiguration = {
+  id: string;
   path: string;
 };
 
 export default class LocalConnectorService implements StorageConnectorAPI {
+  id: string;
   configuration: LocalConfiguration;
 
   constructor(localConfiguration: LocalConfiguration) {
     this.configuration = localConfiguration;
+    this.id = this.configuration.id;
+    if (!this.id) {
+      this.id = randomUUID();
+      logger.info(`Identifier for local storage haven't been set, initializing to '${this.id}'`);
+    }
+  }
+
+  getId() {
+    return this.id;
   }
 
   write(relativePath: string, stream: Readable): Promise<WriteMetadata> {

tdrive/backend/node/src/core/platform/services/storage/default-storage-strategy.ts

@@ -15,6 +15,10 @@ export class DefaultStorageStrategy implements StorageConnectorAPI {
     this.connector = connector;
   }
 
+  getId() {
+    return this.connector.getId();
+  }
+
   write = (
     path: string,
     stream: Stream,

tdrive/backend/node/src/core/platform/services/storage/exceptions.ts

@@ -0,0 +1,2 @@
+export class FileNotFountException extends Error {}
+export class WriteFileException extends Error {}

tdrive/backend/node/src/core/platform/services/storage/oneof-storage-strategy.ts

@@ -7,6 +7,7 @@ import {
   WriteOptions,
 } from "../storage/provider";
 import { logger } from "../../../platform/framework";
+import { FileNotFountException, WriteFileException } from "./exceptions";
 
 /**
  * OneOfStorageStrategy is responsible for managing multiple storage backends.
@@ -22,8 +23,17 @@ import { logger } from "../../../platform/framework";
  * and make "garbage" collection.
  */
 export class OneOfStorageStrategy implements StorageConnectorAPI {
+  id: string;
+
   constructor(private readonly storages: StorageConnectorAPI[]) {}
 
+  getId() {
+    if (!this.id) {
+      this.id = this.storages.map(s => s.getId()).join("_");
+    }
+    return this.id;
+  }
+
   /**
    * Writes a file to all configured storages.
    * The write operation is considered successful if one of the storage succeed.
@@ -38,7 +48,7 @@ export class OneOfStorageStrategy implements StorageConnectorAPI {
 
     // destroy all streams if there is an error in the input stream
     stream.on("error", err => {
-      logger.error("Error in input stream, destroying all write streams:", err);
+      logger.error(err, "Error in input stream, destroying all write streams");
       passThroughStreams.forEach(stream => stream.destroy(err));
     });
 
@@ -54,19 +64,25 @@ export class OneOfStorageStrategy implements StorageConnectorAPI {
 
     // Write to all storages with error handling
     const writeResults = await Promise.allSettled(
-      this.storages.map(
+      this.storages.map((storage, index) =>
-        (storage, index) => storage.write(path, passThroughStreams[index]),
+        storage.write(path, passThroughStreams[index], options),
-        options,
       ),
     );
 
     // Log all errors and throw if all write operations fail
     const errors = writeResults.filter(result => result.status === "rejected");
     errors.forEach((error, index) => {
-      logger.error(`Error writing to storage ${index}:`, (error as PromiseRejectedResult).reason);
+      const storageId = this.storages[index].getId();
+      logger.error(
+        new OneOfStorageWriteOneFailedException(
+          storageId,
+          `Error writing to storage ${storageId}`,
+          (error as PromiseRejectedResult).reason,
+        ),
+      );
     });
     if (errors.length === this.storages.length) {
-      throw new Error("Write operation failed for all storages");
+      throw new WriteFileException(`Write ${path} failed for all storages`);
     }
 
     const successResult = writeResults.filter(
@@ -89,10 +105,16 @@ export class OneOfStorageStrategy implements StorageConnectorAPI {
           return await storage.read(path, options);
         }
       } catch (err) {
-        logger.error("Fallback storage read failed:", err);
+        logger.error(
+          new OneOfStorageReadOneFailedException(
+            storage.getId(),
+            `Reading ${path} from storage ${storage} failed.`,
+            err,
+          ),
+        );
       }
     }
-    throw new Error(`Error reading ${path} in all the storages.`);
+    throw new FileNotFountException(`Error reading ${path}`);
   };
 
   /**
@@ -121,3 +143,21 @@ export class OneOfStorageStrategy implements StorageConnectorAPI {
     });
   };
 }
+
+/**
+ * Throw when read from one of the storages is filed.
+ */
+class StorageException extends Error {
+  constructor(readonly storageId: string, details: string, error: Error) {
+    super(details, error);
+  }
+}
+
+/**
+ * Throw when read from one of the storages is filed.
+ */
+class OneOfStorageReadOneFailedException extends StorageException {}
+/**
+ * Thrown when write operation to one of the storages is failed.
+ */
+class OneOfStorageWriteOneFailedException extends StorageException {}

tdrive/backend/node/src/core/platform/services/storage/provider.ts

@@ -23,6 +23,12 @@ export type DeleteOptions = {
 };
 
 export interface StorageConnectorAPI {
+  /**
+   * Returns identifier of a storage that should've been set in configuration.
+   *
+   */
+  getId(): string;
+
   /**
    * Write a stream to a path
    *

tdrive/backend/node/src/core/platform/services/storage/storage-service.ts

@@ -20,6 +20,7 @@ import StorageAPI, {
 
 import { OneOfStorageStrategy } from "./oneof-storage-strategy";
 import { DefaultStorageStrategy } from "./default-storage-strategy";
+import { FileNotFountException } from "./exceptions";
 
 type EncryptionConfiguration = {
   secret: string | null;
@@ -101,15 +102,15 @@ export default class StorageService extends TdriveService<StorageAPI> implements
   }
 
   async read(path: string, options?: ReadOptions): Promise<Readable> {
+    if (!(await this.exists(path, options))) {
+      throw new FileNotFountException();
+    }
     try {
       // eslint-disable-next-line @typescript-eslint/no-this-alias
       const self = this;
       const chunks = options?.totalChunks || 1;
       let count = 1;
 
-      // Check if the first chunk or file exists
-      await self._read(options?.totalChunks ? `${path}/chunk${count}` : path);
-
       async function factory(callback: (err?: Error, stream?: Stream) => unknown) {
         if (count > chunks) {
           callback();
@@ -282,6 +283,10 @@ export default class StorageService extends TdriveService<StorageAPI> implements
       this.homeDir = "/tdrive";
     }
   }
+
+  getId() {
+    return this.connector.getId();
+  }
 }
 
 type StorageConfiguration = {

tdrive/backend/node/src/services/previews/services/files/processing/pdf.ts

@@ -36,7 +36,9 @@ export async function convertFromPdf(
       //Just no more page to convert
     }
   } catch (error) {
-    const pdfConversionError = new PDFConversionError("Can't convert file with pdf-image.", { cause: error });
+    const pdfConversionError = new PDFConversionError("Can't convert file with pdf-image.", {
+      cause: error,
+    });
     logger.error(pdfConversionError);
     for (const file of pages) {
       cleanFiles([file]);

tdrive/backend/node/test/e2e/files/storage.oneof.spec.ts

@@ -56,7 +56,7 @@ describe("The OneOf Storage feature", () => {
       expect(filesUpload.id).toBeTruthy();
 
       // expect failed upload
-      expect(writeLocalMock.mock.calls.length).toEqual(1);
+      expect(writeLocalMock.mock.calls.length).toBeGreaterThan(0);
       //expect that file can be downloaded
       const fileDownloadResponse = await platform.app.inject({
         method: "GET",