✨Resilience strategy to read and write only to one storage

tdrive/backend/node/test/e2e/files/load_test_config.ts

@@ -0,0 +1,8 @@
+// @ts-ignore
+import path from "path";
+// @ts-ignore
+import config from "config";
+
+const ourConfigDir = path.join(__dirname, 'oneof-storage');
+let configs = config.util.loadFileConfigs(ourConfigDir);
+config.util.extendDeep(config, configs);

tdrive/backend/node/test/e2e/files/oneof-storage/runtime.json

@@ -0,0 +1,21 @@
+{
+
+  "storage": {
+    "secret": "0ea28a329df23220fa814e005bfb671c",
+    "iv": "1234abcd00000000",
+    "strategy": "oneof",
+    "oneof": [
+      {
+        "type": "local",
+        "local": {
+          "path": "-tmp/storage1"
+        }
+      },
+      {
+        "type": "local",
+        "local": {
+          "path": "/tmp/storage2"
+        }
+      }]
+  }
+}

tdrive/backend/node/test/e2e/files/storage.oneof.spec.ts

@@ -0,0 +1,71 @@
+import "./load_test_config"
+import "reflect-metadata";
+import { afterAll, beforeAll, afterEach, describe, expect, it } from "@jest/globals";
+import { init, TestPlatform } from "../setup";
+import UserApi from "../common/user-api";
+import LocalConnectorService from "../../../src/core/platform/services/storage/connectors/local/service"
+import { StorageConnectorAPI } from "../../../src/core/platform/services/storage/provider";
+
+describe("The OneOf Storage feature", () => {
+  const url = "/internal/services/files/v1";
+  let platform: TestPlatform;
+  let helpers: UserApi;
+
+  beforeAll(async () => {
+    platform = await init({
+      services: ["webserver", "database", "storage", "files", "previews"],
+    });
+    helpers = await UserApi.getInstance(platform);
+  }, 300000000);
+
+  afterEach(async () => {
+    jest.restoreAllMocks();
+  });
+
+  afterAll(async () => {
+    await platform?.tearDown();
+    platform = null;
+  });
+
+  describe("On file upload", () => {
+
+    it("should fail an upload POST when ALL backend storage fails", async () => {
+      const thrower = () => {
+        throw new Error("<Mock error done on purpose on upload to storage (for the E2E test)>");
+      };
+      const writeLocalMock = jest.spyOn(LocalConnectorService.prototype, "write").mockRejectedValue("Error");
+
+      // expect(response.statusCode).toBe(500);
+      await expect(helpers.uploadRandomFile()).rejects.toThrow("Error code: 500");
+
+      // expect(response.statusCode).toBe(500);
+      expect(writeLocalMock.mock.calls.length).toEqual(2);
+    });
+
+    it("should successfully upload file when one backend storage fails", async () => {
+      const thrower = () => {
+        throw new Error("<Mock error done on purpose on upload to storage (for the E2E test)>");
+      };
+
+      const connectors = (platform.storage.getConnector() as any).storages as Array<StorageConnectorAPI>;
+
+      const writeLocalMock = jest.spyOn(connectors[0], "write").mockRejectedValue("Error");
+
+      // expect(response.statusCode).toBe(200);
+      const filesUpload = await helpers.uploadRandomFile();
+      expect(filesUpload.id).toBeTruthy();
+
+      // expect failed upload
+      expect(writeLocalMock.mock.calls.length).toEqual(1);
+      //expect that file can be downloaded
+      const fileDownloadResponse = await platform.app.inject({
+        method: "GET",
+        url: `${url}/companies/${platform.workspace.company_id}/files/${filesUpload.id}/download`,
+      });
+      expect(fileDownloadResponse).toBeTruthy();
+      expect(fileDownloadResponse.statusCode).toBe(200);
+
+    });
+
+  });
+});

tdrive/backend/node/test/unit/core/services/storage/oneof-storage-strategy.test.ts

@@ -0,0 +1,184 @@
+import { Readable, PassThrough } from 'stream';
+import { OneOfStorageStrategy } from "../../../../../src/core/platform/services/storage/oneof-storage-strategy"; // Adjust the import path as necessary
+import {StorageConnectorAPI } from "../../../../../src/core/platform/services/storage/provider"; // Adjust the import path as necessary
+
+function mockStream() {
+  const stream = new Readable();
+  stream.push("Test data");
+  stream.push(null);
+  return stream;
+}
+
+function mockStorage(size: number) {
+  return {
+    write: jest.fn().mockResolvedValue({ size: size }),
+    read: jest.fn(),
+    exists: jest.fn(),
+    remove: jest.fn().mockResolvedValue(true)
+  } as unknown as StorageConnectorAPI;
+}
+
+describe('OneOfStorageStrategy', () => {
+  let storage1: StorageConnectorAPI;
+  let storage2: StorageConnectorAPI;
+  let storage3: StorageConnectorAPI;
+  let oneOfStorageStrategy: OneOfStorageStrategy;
+
+  beforeEach(() => {
+    // Mocking two storage backends
+    storage1 = mockStorage(100);
+    storage2 =  mockStorage(200);
+    storage3 =  mockStorage(300);
+
+    oneOfStorageStrategy = new OneOfStorageStrategy([storage1, storage2, storage3]);
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('write', () => {
+    it('should write to all configured storages', async () => {
+      const path = 'test/path';
+      const stream = mockStream();
+
+      const result = await oneOfStorageStrategy.write(path, stream);
+
+      expect(storage1.write).toHaveBeenCalled();
+      expect(storage2.write).toHaveBeenCalled();
+      expect(result.size).toBe(100); // Assuming the first storage returned this size
+    });
+
+    it('should throw an error if all storage writes fail', async () => {
+      storage1.write = jest.fn().mockRejectedValue(new Error('Write failed'));
+      storage2.write = jest.fn().mockRejectedValue(new Error('Write failed'));
+      storage3.write = jest.fn().mockRejectedValue(new Error('Write failed'));
+
+      const path = 'test/path';
+      const stream = mockStream();
+      await expect(oneOfStorageStrategy.write(path, stream)).rejects.toThrow('Write operation failed for all storages');
+    });
+
+    it('should succeed if one storage write fails but the other succeeds', async () => {
+      //given
+      storage1.write = jest.fn().mockRejectedValue(new Error('Write failed'));
+      storage2.write = jest.fn().mockResolvedValue({ size: 200 });
+      storage3.write = jest.fn().mockRejectedValue(new Error('Write failed'));
+
+      const path = 'test/path';
+      const stream = mockStream();
+
+      //when
+      const result = await oneOfStorageStrategy.write(path, stream);
+
+      //then
+      expect(result.size).toBe(200); // Assuming the second storage returned this size
+    });
+
+    it('should verify that all data was written to the storage stream', async () => {
+      const data = "Test data";
+      const stream = new Readable();
+      stream.push(data);
+      stream.push(null);
+
+      storage1.write = jest.fn().mockImplementation((_, stream) => {
+        return new Promise((resolve, reject) => {
+          let writtenData = '';
+          stream.on('data', chunk => {
+            writtenData += chunk;
+          });
+          stream.on('end', () => {
+            if (writtenData === data) {
+              resolve({ size: writtenData.length });
+            } else {
+              reject(new Error('Data mismatch'));
+            }
+          });
+        });
+      });
+
+      const result = await oneOfStorageStrategy.write("test/path", stream);
+
+      expect(result.size).toBe(data.length);
+    });
+  });
+
+  describe('read', () => {
+    it('should read from the first storage that has the file', async () => {
+      const path = 'test/path';
+      const mockStream = new Readable();
+      mockStream.push('Test data');
+      mockStream.push(null);
+      storage1.read = jest.fn().mockResolvedValue(mockStream);
+      storage1.exists = jest.fn().mockResolvedValue(true);
+
+      const result = await oneOfStorageStrategy.read(path);
+
+      expect(storage1.exists).toHaveBeenCalledWith(path, undefined);
+      expect(storage1.read).toHaveBeenCalledWith(path, undefined);
+      expect(result).toBe(mockStream);
+    });
+
+    it('should fallback to the next storage if the first fails', async () => {
+      const path = 'test/path';
+      storage1.read = jest.fn().mockRejectedValue(new Error('Read failed'));
+      storage1.exists = jest.fn().mockResolvedValue(true);
+      const mockStream = new Readable();
+      mockStream.push('Test data');
+      mockStream.push(null);
+      storage2.read = jest.fn().mockResolvedValue(mockStream);
+      storage2.exists = jest.fn().mockResolvedValue(true);
+
+      const result = await oneOfStorageStrategy.read(path);
+
+      expect(storage1.exists).toHaveBeenCalled();
+      expect(storage1.read).toHaveBeenCalled();
+      expect(storage2.exists).toHaveBeenCalled();
+      expect(storage2.read).toHaveBeenCalled();
+      expect(result).toBe(mockStream);
+    });
+  });
+
+  describe('exists', () => {
+    it('should return true if the file exists in any storage', async () => {
+      storage1.exists = jest.fn().mockResolvedValue(false);
+      storage2.exists = jest.fn().mockResolvedValue(true);
+
+      const result = await oneOfStorageStrategy.exists('test/path');
+
+      expect(result).toBe(true);
+      expect(storage1.exists).toHaveBeenCalled();
+      expect(storage2.exists).toHaveBeenCalled();
+    });
+
+    it('should return false if the file does not exist in any storage', async () => {
+      storage1.exists = jest.fn().mockResolvedValue(false);
+      storage2.exists = jest.fn().mockResolvedValue(false);
+
+      const result = await oneOfStorageStrategy.exists('test/path');
+
+      expect(result).toBe(false);
+      expect(storage1.exists).toHaveBeenCalled();
+      expect(storage2.exists).toHaveBeenCalled();
+    });
+  });
+
+  describe('remove', () => {
+    it('should remove the file from all storages', async () => {
+      const result = await oneOfStorageStrategy.remove('test/path');
+
+      expect(storage1.remove).toHaveBeenCalledWith('test/path', undefined);
+      expect(storage2.remove).toHaveBeenCalledWith('test/path', undefined);
+      expect(result).toBe(true);
+    });
+
+    it('should return false if any storage fails to remove the file', async () => {
+      storage1.remove = jest.fn().mockResolvedValue(true);
+      storage2.remove = jest.fn().mockResolvedValue(false);
+
+      const result = await oneOfStorageStrategy.remove('test/path');
+
+      expect(result).toBe(false);
+    });
+  });
+});