From 873ef8c08cdb6943a5f4d8a28145d17539243907 Mon Sep 17 00:00:00 2001 From: Anton Shepilov Date: Sun, 14 Jan 2024 02:40:02 +0300 Subject: [PATCH] =?UTF-8?q?=F0=9F=8C=9FAdd=20an=20option=20to=20not=20dele?= =?UTF-8?q?te=20files=20from=20S3=20storage=20(#321)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../config/custom-environment-variables.json | 3 +- tdrive/backend/node/config/default.json | 3 +- .../S3/{service.ts => s3-service.ts} | 10 +++- .../core/platform/services/storage/index.ts | 3 +- .../storage/connectors/S3/s3-service.test.ts | 51 +++++++++++++++++++ 5 files changed, 65 insertions(+), 5 deletions(-) rename tdrive/backend/node/src/core/platform/services/storage/connectors/S3/{service.ts => s3-service.ts} (85%) create mode 100644 tdrive/backend/node/test/unit/core/services/storage/connectors/S3/s3-service.test.ts diff --git a/tdrive/backend/node/config/custom-environment-variables.json b/tdrive/backend/node/config/custom-environment-variables.json index d2f35c10..2fc2621e 100644 --- a/tdrive/backend/node/config/custom-environment-variables.json +++ b/tdrive/backend/node/config/custom-environment-variables.json @@ -94,7 +94,8 @@ "port": "STORAGE_S3_PORT", "useSSL": "STORAGE_S3_USE_SSL", "accessKey": "STORAGE_S3_ACCESS_KEY", - "secretKey": "STORAGE_S3_SECRET_KEY" + "secretKey": "STORAGE_S3_SECRET_KEY", + "disableRemove": "STORAGE_S3_DISABLE_REMOVE" } }, "email-pusher": { diff --git a/tdrive/backend/node/config/default.json b/tdrive/backend/node/config/default.json index 44e79d39..0dc9884a 100644 --- a/tdrive/backend/node/config/default.json +++ b/tdrive/backend/node/config/default.json @@ -121,7 +121,8 @@ "port": 9000, "useSSL": false, "accessKey": "ABCD", - "secretKey": "x1yz" + "secretKey": "x1yz", + "disableRemove": false }, "local": { "path": "/tdrive" diff --git a/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/service.ts b/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts similarity index 85% rename from tdrive/backend/node/src/core/platform/services/storage/connectors/S3/service.ts rename to tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts index 8659ceb7..0d13fd84 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/service.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts @@ -11,6 +11,7 @@ export type S3Configuration = { useSSL: boolean; accessKey: string; secretKey: string; + disableRemove: boolean; }; export default class S3ConnectorService implements StorageConnectorAPI { @@ -66,8 +67,13 @@ export default class S3ConnectorService implements StorageConnectorAPI { async remove(path: string): Promise { try { - await this.client.removeObject(this.minioConfiguration.bucket, path); - return true; + if (this.minioConfiguration.disableRemove) { + logger.info(`File ${path} wasn't removed, file removal is disabled in configuration`); + return true; + } else { + await this.client.removeObject(this.minioConfiguration.bucket, path); + return true; + } } catch (err) {} return false; } diff --git a/tdrive/backend/node/src/core/platform/services/storage/index.ts b/tdrive/backend/node/src/core/platform/services/storage/index.ts index f4afefed..cacda65f 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/index.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/index.ts @@ -3,7 +3,7 @@ import { Stream, Readable } from "stream"; import Multistream from "multistream"; import { Consumes, logger, TdriveService } from "../../framework"; import LocalConnectorService, { LocalConfiguration } from "./connectors/local/service"; -import S3ConnectorService from "./connectors/S3/service"; +import S3ConnectorService from "./connectors/S3/s3-service"; import StorageAPI, { DeleteOptions, ReadOptions, @@ -47,6 +47,7 @@ export default class StorageService extends TdriveService implements useSSL: Boolean(this.configuration.get("S3.useSSL")), accessKey: this.configuration.get("S3.accessKey"), secretKey: this.configuration.get("S3.secretKey"), + disableRemove: this.configuration.get("S3.disableRemove"), }); } else { logger.info("Using 'local' connector for storage."); diff --git a/tdrive/backend/node/test/unit/core/services/storage/connectors/S3/s3-service.test.ts b/tdrive/backend/node/test/unit/core/services/storage/connectors/S3/s3-service.test.ts new file mode 100644 index 00000000..39397b33 --- /dev/null +++ b/tdrive/backend/node/test/unit/core/services/storage/connectors/S3/s3-service.test.ts @@ -0,0 +1,51 @@ +import S3ConnectorService, { + S3Configuration +} from "../../../../../../../src/core/platform/services/storage/connectors/S3/s3-service"; +import { jest } from "@jest/globals"; + +describe("The S3ConnectorService", () => { + + beforeEach(async () => { + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test("remove shouldnt remove file if the option is disabled", async () => { + //given + const subj: S3ConnectorService = new S3ConnectorService({ + disableRemove: true, + endPoint: "play.min.io", + port: 9000 + } as S3Configuration); + const removeSpy = jest.spyOn(subj.client, "removeObject"); + + //when + const result = await subj.remove("path"); + + //then + expect(result).toBeTruthy(); + expect(removeSpy).not.toHaveBeenCalled(); + }); + + test("remove should call remove if the option is not disabled", async () => { + //given + const subj: S3ConnectorService = new S3ConnectorService({ + disableRemove: false, + endPoint: "play.min.io", + port: 9000, + bucket: "testbucket" + } as S3Configuration); + const removeSpy = jest.spyOn(subj.client, "removeObject"); + removeSpy.mockReturnThis(); + + //when + const result = await subj.remove("path"); + + //then + expect(removeSpy).toHaveBeenCalled(); + expect(removeSpy).toBeCalledWith("testbucket", "path"); + }); + +}); \ No newline at end of file