diff --git a/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts b/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts index d79beaa0..c61fd3e8 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts @@ -2,8 +2,10 @@ import * as Minio from "minio"; import { logger } from "../../../../../../core/platform/framework"; import { Readable } from "stream"; import { StorageConnectorAPI, WriteMetadata } from "../../provider"; +import { randomUUID } from "crypto"; export type S3Configuration = { + id: string; bucket: string; region: string; endPoint: string; @@ -17,10 +19,20 @@ export type S3Configuration = { export default class S3ConnectorService implements StorageConnectorAPI { client: Minio.Client; minioConfiguration: S3Configuration; + id: string; constructor(S3Configuration: S3Configuration) { this.client = new Minio.Client(S3Configuration); this.minioConfiguration = S3Configuration; + this.id = this.minioConfiguration.id; + if (!this.id) { + this.id = randomUUID(); + logger.info(`Identifier for S3 storage haven't been set, initializing to '${this.id}'`); + } + } + + getId() { + return this.id; } write(path: string, stream: Readable): Promise { diff --git a/tdrive/backend/node/src/core/platform/services/storage/connectors/local/service.ts b/tdrive/backend/node/src/core/platform/services/storage/connectors/local/service.ts index cf0e7495..16cf356f 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/connectors/local/service.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/connectors/local/service.ts @@ -5,16 +5,28 @@ import { rm } from "fs/promises"; // Do not change the import, this is not the s import { StorageConnectorAPI, WriteMetadata } from "../../provider"; import fs from "fs"; import { logger } from "../../../../framework/logger"; +import { randomUUID } from "crypto"; export type LocalConfiguration = { + id: string; path: string; }; export default class LocalConnectorService implements StorageConnectorAPI { + id: string; configuration: LocalConfiguration; constructor(localConfiguration: LocalConfiguration) { this.configuration = localConfiguration; + this.id = this.configuration.id; + if (!this.id) { + this.id = randomUUID(); + logger.info(`Identifier for local storage haven't been set, initializing to '${this.id}'`); + } + } + + getId() { + return this.id; } write(relativePath: string, stream: Readable): Promise { diff --git a/tdrive/backend/node/src/core/platform/services/storage/default-storage-strategy.ts b/tdrive/backend/node/src/core/platform/services/storage/default-storage-strategy.ts index 520fa320..c14034ca 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/default-storage-strategy.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/default-storage-strategy.ts @@ -15,6 +15,10 @@ export class DefaultStorageStrategy implements StorageConnectorAPI { this.connector = connector; } + getId() { + return this.connector.getId(); + } + write = ( path: string, stream: Stream, diff --git a/tdrive/backend/node/src/core/platform/services/storage/exceptions.ts b/tdrive/backend/node/src/core/platform/services/storage/exceptions.ts new file mode 100644 index 00000000..f4fb0f7b --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/storage/exceptions.ts @@ -0,0 +1,2 @@ +export class FileNotFountException extends Error {} +export class WriteFileException extends Error {} diff --git a/tdrive/backend/node/src/core/platform/services/storage/oneof-storage-strategy.ts b/tdrive/backend/node/src/core/platform/services/storage/oneof-storage-strategy.ts index 2a88cc2c..b71c7026 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/oneof-storage-strategy.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/oneof-storage-strategy.ts @@ -7,6 +7,7 @@ import { WriteOptions, } from "../storage/provider"; import { logger } from "../../../platform/framework"; +import { FileNotFountException, WriteFileException } from "./exceptions"; /** * OneOfStorageStrategy is responsible for managing multiple storage backends. @@ -22,8 +23,17 @@ import { logger } from "../../../platform/framework"; * and make "garbage" collection. */ export class OneOfStorageStrategy implements StorageConnectorAPI { + id: string; + constructor(private readonly storages: StorageConnectorAPI[]) {} + getId() { + if (!this.id) { + this.id = this.storages.map(s => s.getId()).join("_"); + } + return this.id; + } + /** * Writes a file to all configured storages. * The write operation is considered successful if one of the storage succeed. @@ -38,7 +48,7 @@ export class OneOfStorageStrategy implements StorageConnectorAPI { // destroy all streams if there is an error in the input stream stream.on("error", err => { - logger.error("Error in input stream, destroying all write streams:", err); + logger.error(err, "Error in input stream, destroying all write streams"); passThroughStreams.forEach(stream => stream.destroy(err)); }); @@ -54,19 +64,25 @@ export class OneOfStorageStrategy implements StorageConnectorAPI { // Write to all storages with error handling const writeResults = await Promise.allSettled( - this.storages.map( - (storage, index) => storage.write(path, passThroughStreams[index]), - options, + this.storages.map((storage, index) => + storage.write(path, passThroughStreams[index], options), ), ); // Log all errors and throw if all write operations fail const errors = writeResults.filter(result => result.status === "rejected"); errors.forEach((error, index) => { - logger.error(`Error writing to storage ${index}:`, (error as PromiseRejectedResult).reason); + const storageId = this.storages[index].getId(); + logger.error( + new OneOfStorageWriteOneFailedException( + storageId, + `Error writing to storage ${storageId}`, + (error as PromiseRejectedResult).reason, + ), + ); }); if (errors.length === this.storages.length) { - throw new Error("Write operation failed for all storages"); + throw new WriteFileException(`Write ${path} failed for all storages`); } const successResult = writeResults.filter( @@ -89,10 +105,16 @@ export class OneOfStorageStrategy implements StorageConnectorAPI { return await storage.read(path, options); } } catch (err) { - logger.error("Fallback storage read failed:", err); + logger.error( + new OneOfStorageReadOneFailedException( + storage.getId(), + `Reading ${path} from storage ${storage} failed.`, + err, + ), + ); } } - throw new Error(`Error reading ${path} in all the storages.`); + throw new FileNotFountException(`Error reading ${path}`); }; /** @@ -121,3 +143,21 @@ export class OneOfStorageStrategy implements StorageConnectorAPI { }); }; } + +/** + * Throw when read from one of the storages is filed. + */ +class StorageException extends Error { + constructor(readonly storageId: string, details: string, error: Error) { + super(details, error); + } +} + +/** + * Throw when read from one of the storages is filed. + */ +class OneOfStorageReadOneFailedException extends StorageException {} +/** + * Thrown when write operation to one of the storages is failed. + */ +class OneOfStorageWriteOneFailedException extends StorageException {} diff --git a/tdrive/backend/node/src/core/platform/services/storage/provider.ts b/tdrive/backend/node/src/core/platform/services/storage/provider.ts index 00904dba..d46a554c 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/provider.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/provider.ts @@ -23,6 +23,12 @@ export type DeleteOptions = { }; export interface StorageConnectorAPI { + /** + * Returns identifier of a storage that should've been set in configuration. + * + */ + getId(): string; + /** * Write a stream to a path * diff --git a/tdrive/backend/node/src/core/platform/services/storage/storage-service.ts b/tdrive/backend/node/src/core/platform/services/storage/storage-service.ts index 0427d2d0..90ff1ff8 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/storage-service.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/storage-service.ts @@ -20,6 +20,7 @@ import StorageAPI, { import { OneOfStorageStrategy } from "./oneof-storage-strategy"; import { DefaultStorageStrategy } from "./default-storage-strategy"; +import { FileNotFountException } from "./exceptions"; type EncryptionConfiguration = { secret: string | null; @@ -101,15 +102,15 @@ export default class StorageService extends TdriveService implements } async read(path: string, options?: ReadOptions): Promise { + if (!(await this.exists(path, options))) { + throw new FileNotFountException(); + } try { // eslint-disable-next-line @typescript-eslint/no-this-alias const self = this; const chunks = options?.totalChunks || 1; let count = 1; - // Check if the first chunk or file exists - await self._read(options?.totalChunks ? `${path}/chunk${count}` : path); - async function factory(callback: (err?: Error, stream?: Stream) => unknown) { if (count > chunks) { callback(); @@ -282,6 +283,10 @@ export default class StorageService extends TdriveService implements this.homeDir = "/tdrive"; } } + + getId() { + return this.connector.getId(); + } } type StorageConfiguration = { diff --git a/tdrive/backend/node/src/services/previews/services/files/processing/pdf.ts b/tdrive/backend/node/src/services/previews/services/files/processing/pdf.ts index 6d99acc7..53dddc5a 100644 --- a/tdrive/backend/node/src/services/previews/services/files/processing/pdf.ts +++ b/tdrive/backend/node/src/services/previews/services/files/processing/pdf.ts @@ -36,7 +36,9 @@ export async function convertFromPdf( //Just no more page to convert } } catch (error) { - const pdfConversionError = new PDFConversionError("Can't convert file with pdf-image.", { cause: error }); + const pdfConversionError = new PDFConversionError("Can't convert file with pdf-image.", { + cause: error, + }); logger.error(pdfConversionError); for (const file of pages) { cleanFiles([file]); diff --git a/tdrive/backend/node/test/e2e/files/storage.oneof.spec.ts b/tdrive/backend/node/test/e2e/files/storage.oneof.spec.ts index ae8002f1..8d935108 100644 --- a/tdrive/backend/node/test/e2e/files/storage.oneof.spec.ts +++ b/tdrive/backend/node/test/e2e/files/storage.oneof.spec.ts @@ -56,7 +56,7 @@ describe("The OneOf Storage feature", () => { expect(filesUpload.id).toBeTruthy(); // expect failed upload - expect(writeLocalMock.mock.calls.length).toEqual(1); + expect(writeLocalMock.mock.calls.length).toBeGreaterThan(0); //expect that file can be downloaded const fileDownloadResponse = await platform.app.inject({ method: "GET",