From 81ee6c167369e3998449794e8aa61cf389469aea Mon Sep 17 00:00:00 2001 From: Montassar Ghanmy Date: Wed, 4 Sep 2024 13:28:31 +0100 Subject: [PATCH 1/5] =?UTF-8?q?=F0=9F=90=9B=20Fix=20OpenSearch=20e2e=20tes?= =?UTF-8?q?t?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tdrive/docker-compose.dev.tests.opensearch.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tdrive/docker-compose.dev.tests.opensearch.yml b/tdrive/docker-compose.dev.tests.opensearch.yml index 62b67948..442d64a2 100644 --- a/tdrive/docker-compose.dev.tests.opensearch.yml +++ b/tdrive/docker-compose.dev.tests.opensearch.yml @@ -41,7 +41,6 @@ services: - "5432:5432" node: - image: tdrive/tdrive-node:test build: context: . dockerfile: docker/tdrive-node/Dockerfile From 8b9fd36c8ca1a551784f918a195009dab5b1212a Mon Sep 17 00:00:00 2001 From: Anton Shepilov Date: Thu, 5 Sep 2024 15:29:25 +0200 Subject: [PATCH 2/5] =?UTF-8?q?=E2=9C=A8=20Ignore=20errors=20during=20next?= =?UTF-8?q?cloud=20migration=20to=20be=20able=20to=20import=20files=20part?= =?UTF-8?q?ially=20(#579)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ Ignore errors during nextcloud migration to be able to import files partially for big migrations * 👷 Fixed Nextcloud migration publishing script * ✏️ Removed wrong comment * 🐛 Fixed nextcloud migration, when there is a folder and a file with the same name --- .../utils/nextcloud-migration/src/executor.ts | 81 +++++++++++++++++++ .../nextcloud-migration/src/express_server.ts | 4 +- .../src/nextcloud_migration.ts | 26 ++++-- .../nextcloud-migration/test/executor.test.ts | 43 ++++++++++ 4 files changed, 145 insertions(+), 9 deletions(-) create mode 100644 tdrive/backend/utils/nextcloud-migration/src/executor.ts create mode 100644 tdrive/backend/utils/nextcloud-migration/test/executor.test.ts diff --git a/tdrive/backend/utils/nextcloud-migration/src/executor.ts b/tdrive/backend/utils/nextcloud-migration/src/executor.ts new file mode 100644 index 00000000..b3b81c24 --- /dev/null +++ b/tdrive/backend/utils/nextcloud-migration/src/executor.ts @@ -0,0 +1,81 @@ +import { logger } from "./logger"; + +type FunctionArgs = any[]; + +type FunctionStats = { + functionName: string; + successfulExecutions: number; + failedExecutions: number; + failedExecutionsArgs: FunctionArgs[]; +}; + +export class FunctionExecutor { + private stats: Map = new Map(); + + async executeWithRetries( + fn: (...args: any[]) => Promise, + args: FunctionArgs, + retries: number = 3, + throwError: boolean = false, + ): Promise { + const functionName = fn.name; + let success = false; + let executionResult = null; + + for (let attempt = 0; attempt < retries; attempt++) { + try { + executionResult = await fn(...args); + success = true; + break; + } catch (error) { + executionResult = error; + logger.info(`Attempt ${attempt + 1} failed for ${functionName}`, error); + } + } + + if (!success && throwError) throw executionResult; + + let stats = this.stats.get(functionName); + if (!stats) { + stats = { + functionName, + successfulExecutions: 0, + failedExecutions: 0, + failedExecutionsArgs: [] + }; + this.stats.set(functionName, stats); + } + + if (success) { + stats.successfulExecutions++; + } else { + stats.failedExecutions++; + stats.failedExecutionsArgs.push( args ); + } + return executionResult; + } + + async getStats() { + return this.stats; + } + + printStatistics(): void { + logger.info("Execution Statistics:"); + this.stats.forEach((stat) => { + logger.info( + `Function: ${stat.functionName}, Successful Executions: ${stat.successfulExecutions}, Failed Executions: ${stat.failedExecutions}` + ); + }); + } + + printFailedExecutions(): void { + logger.info("Failed Executions:"); + this.stats.forEach((execution) => { + if (execution.failedExecutionsArgs.length > 0) { + logger.info(` Function: ${execution.functionName}`); + execution.failedExecutionsArgs.forEach(args => + logger.info(` args: ${JSON.stringify(args)}`)); + } + }); + } +} \ No newline at end of file diff --git a/tdrive/backend/utils/nextcloud-migration/src/express_server.ts b/tdrive/backend/utils/nextcloud-migration/src/express_server.ts index 51bda38f..acd590b4 100644 --- a/tdrive/backend/utils/nextcloud-migration/src/express_server.ts +++ b/tdrive/backend/utils/nextcloud-migration/src/express_server.ts @@ -60,8 +60,8 @@ app.post("/", async (req: Request, res: Response) => { res.status(400).send("Username and password for nextcloud are required"); } try { - await nextcloud.migrate(params.username, params.password, params.dir); - res.status(200).send("Sync DONE ✅"); + const stats = await nextcloud.migrate(params.username, params.password, params.dir); + res.status(200).send(JSON.stringify(stats)); } catch (e) { console.error(e) res.status(500).send("Error during synchronization:: " + e.message) diff --git a/tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts b/tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts index e63776a3..01aa9204 100644 --- a/tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts +++ b/tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts @@ -6,6 +6,7 @@ import { TwakeDriveClient, TwakeDriveUser } from './twake_client'; import path from 'path'; import { logger } from "./logger" import { User, UserProvider, UserProviderFactory, UserProviderType } from "./user_privider"; +import { FunctionExecutor } from "./executor"; export interface NextcloudMigrationConfiguration { shell: { @@ -36,6 +37,8 @@ export class NextcloudMigration { driveClient: TwakeDriveClient; + executor = new FunctionExecutor(); + constructor(config: NextcloudMigrationConfiguration) { this.config = config; this.userProvider = (new UserProviderFactory()).get(config.userProvider, config[config.userProvider]); @@ -55,6 +58,9 @@ export class NextcloudMigration { if(!dir) await this.download(username, password, dirTmp); //upload files to the Twake Drive await this.upload(driveUser, dirTmp); + this.executor.printStatistics(); + this.executor.printFailedExecutions(); + return this.executor.getStats(); } catch (e) { console.error('Error downloading files from next cloud', e); throw e; @@ -119,7 +125,9 @@ export class NextcloudMigration { const parent = await this.driveClient.getDocument(parentDirId); const exists = (filename: string) => { - return parent.children.filter(i => i.name.startsWith(path.parse(filename).name)).length > 0; + let parsedPath = path.parse(filename); + let name = `${parsedPath.name}${parsedPath.ext > '' ? parsedPath.ext : ''}`; + return parent.children.filter(i => i.name == name).length > 0; } logger.debug(`Reading content of the directory ${sourceDirPath} ...`) @@ -144,8 +152,9 @@ export class NextcloudMigration { //check existing files for (const fPath of existingFiles) { logger.debug(`Check existing file ${fPath}`) - let name = path.parse(fPath).name; - let candidatesWithTheSameName = parent.children.filter(i => i.name.startsWith(name)); + let parsedPath = path.parse(fPath); + let name = parsedPath.name + (parsedPath.ext > '' ? parsedPath.ext : ''); + let candidatesWithTheSameName = parent.children.filter(i => i.name === name); if (candidatesWithTheSameName.length > 1) { logger.warn("WE HAVE MORE MORE THAN ONE FILE WITH NAME: " + name); } else { @@ -169,19 +178,22 @@ export class NextcloudMigration { } } - //upload all files logger.debug(`UPLOAD FILES FOR ${sourceDirPath}`) for (const file of filesToUpload) { logger.debug(`Upload file ${file}`) - await this.driveClient.createFile(file, parentDirId); + await this.executor.executeWithRetries(this.driveClient.createFile.bind(this.driveClient), [file, parentDirId], 3) + // await this.driveClient.createFile(file, parentDirId); } logger.debug(`UPLOAD DIRS FOR ${sourceDirPath}`) for (const [name, path] of dirsToUpload) { logger.info(`Create directory ${name}`); - const dir = await this.driveClient.createDirectory(name, parentDirId) - await this.upload(user, path, dir.id); + const dir = await this.executor.executeWithRetries(this.driveClient.createDirectory.bind(this.driveClient), [name, parentDirId], 3) + // const dir = await this.driveClient.createDirectory(name, parentDirId) + if (dir) { + await this.upload(user, path, dir.id); + } } } diff --git a/tdrive/backend/utils/nextcloud-migration/test/executor.test.ts b/tdrive/backend/utils/nextcloud-migration/test/executor.test.ts new file mode 100644 index 00000000..6137e47c --- /dev/null +++ b/tdrive/backend/utils/nextcloud-migration/test/executor.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, test } from "@jest/globals"; +import { FunctionExecutor } from "../src/executor"; + +describe('Function Executor tests', () => { + + const subj = new FunctionExecutor(); + + test('Should successfully execute mock function', async () => { + //when + await subj.executeWithRetries(successFunction, [1, 2], 3); + await subj.executeWithRetries(successFunction, [4, 5], 3); + + //then + let stats = await subj.getStats(); + expect(stats?.get("successFunction")?.successfulExecutions).toEqual(2); + }); + + test('Should successfully gather arguments of the failed executions', async () => { + await subj.executeWithRetries(failedFunction, [1, 2], 3); + + //then + let stats = await subj.getStats(); + expect(stats?.get("failedFunction")?.failedExecutions).toBe(1); + expect(stats?.get("failedFunction")?.failedExecutionsArgs[0]).toStrictEqual([1, 2]); + }); + + test('Should successfully gather arguments of the class member', async () => { + await subj.executeWithRetries(subj.getStats.bind(this), [1, 2], 3); + + //then + let stats = await subj.getStats(); + expect(stats?.get("bound getStats")?.successfulExecutions).toEqual(1); + }); + + const successFunction = async (a: number, b: number) => { + return a + b; + }; + + const failedFunction = async () => { + throw new Error(); + }; + +}); From 492b636e93d042e00304c678a1eecabafaf055b8 Mon Sep 17 00:00:00 2001 From: Anton SHEPILOV Date: Sat, 20 Jul 2024 00:46:20 +0200 Subject: [PATCH 3/5] =?UTF-8?q?=E2=99=BB=EF=B8=8FRemoved=20cumulative=20qu?= =?UTF-8?q?eries=20to=20the=20database=20First=20step=20to=20the=20transac?= =?UTF-8?q?tions=20in=20PostgreSQL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/platform/services/database/api.ts | 2 +- .../services/database/services/orm/manager.ts | 55 ++++--------------- .../services/orm/repository/repository.ts | 11 +--- .../database/services/orm/manager.test.ts | 54 +++++++----------- 4 files changed, 35 insertions(+), 87 deletions(-) diff --git a/tdrive/backend/node/src/core/platform/services/database/api.ts b/tdrive/backend/node/src/core/platform/services/database/api.ts index 105ce97d..e9a3ab96 100644 --- a/tdrive/backend/node/src/core/platform/services/database/api.ts +++ b/tdrive/backend/node/src/core/platform/services/database/api.ts @@ -11,7 +11,7 @@ export interface DatabaseServiceAPI extends TdriveServiceProvider { getConnector(): Connector; /** - * Get entities manager (TODO: Find a better name...) + * Get entities manager */ getManager(): Manager; diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/manager.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/manager.ts index 343720e9..59e769d4 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/manager.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/manager.ts @@ -1,5 +1,4 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -/* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import _ from "lodash"; import { Connector } from "./connectors"; import { getEntityDefinition, unwrapPrimarykey } from "./utils"; @@ -9,13 +8,9 @@ import { DatabaseEntitiesRemovedEvent, DatabaseEntitiesSavedEvent } from "./type import { localEventBus } from "../../../../framework/event-bus"; export default class EntityManager> { - private toInsert: EntityType[] = []; - private toUpdate: EntityType[] = []; - private toRemove: EntityType[] = []; - constructor(readonly connector: Connector) {} - public persist(entity: any): this { + public async persist(entity: any): Promise { logger.trace( `services.database.orm.entity-manager.persist - entity: ${JSON.stringify(entity)}`, ); @@ -62,18 +57,20 @@ export default class EntityManager> { } }); + entity = _.cloneDeep(entity); if (emptyPkFields.length > 0) { - this.toInsert = this.toInsert.filter(e => e !== entity); - this.toInsert.push(_.cloneDeep(entity)); + await this.connector.upsert([entity], { action: "INSERT" }); } else { - this.toUpdate = this.toUpdate.filter(e => e !== entity); - this.toUpdate.push(_.cloneDeep(entity)); + await this.connector.upsert([entity], { action: "UPDATE" }); } + localEventBus.publish("database:entities:saved", { + entities: [entity], + } as DatabaseEntitiesSavedEvent); return this; } - public remove(entity: EntityType, entityType?: EntityType): this { + public async remove(entity: EntityType, entityType?: EntityType): Promise { if (entityType) { // eslint-disable-next-line @typescript-eslint/no-explicit-any entity = _.merge(new (entityType as any)(), entity); @@ -81,45 +78,13 @@ export default class EntityManager> { if (!entity.constructor.prototype._entity || !entity.constructor.prototype._columns) { throw Error("Cannot remove this object: it is not an entity."); } - this.toRemove = this.toRemove.filter(e => e !== entity); - this.toRemove.push(_.cloneDeep(entity)); - return this; - } - - public async flush(): Promise { - this.toInsert = _.uniqWith(this.toInsert, _.isEqual); - this.toUpdate = _.uniqWith(this.toUpdate, _.isEqual); - this.toRemove = _.uniqWith(this.toRemove, _.isEqual); + await this.connector.remove([entity]); localEventBus.publish("database:entities:saved", { - entities: this.toInsert.map(e => _.cloneDeep(e)), - } as DatabaseEntitiesSavedEvent); - - localEventBus.publish("database:entities:saved", { - entities: this.toUpdate.map(e => _.cloneDeep(e)), - } as DatabaseEntitiesSavedEvent); - - localEventBus.publish("database:entities:saved", { - entities: this.toRemove.map(e => _.cloneDeep(e)), + entities: [entity], } as DatabaseEntitiesRemovedEvent); - if (this.toInsert.length > 0) { - await this.connector.upsert(this.toInsert, { action: "INSERT" }); - } - if (this.toUpdate.length > 0) { - await this.connector.upsert(this.toUpdate, { action: "UPDATE" }); - } - if (this.toRemove.length > 0) { - await this.connector.remove(this.toRemove); - } - return this; } - - public reset(): void { - this.toInsert = []; - this.toUpdate = []; - this.toRemove = []; - } } diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/repository/repository.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/repository/repository.ts index 38854a09..25038583 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/repository/repository.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/repository/repository.ts @@ -135,25 +135,20 @@ export default class Repository { }> { if (previousValue === newValue) throw new Error(`Previous and new values are identical: ${JSON.stringify(previousValue)}`); - await this.manager.flush().then(manager => manager.reset()); return this.connector.atomicCompareAndSet(entity, fieldName, previousValue, newValue); } async save(entity: EntityType, _context?: ExecutionContext): Promise { - this.manager.persist(entity); - return this.manager.flush().then(manager => manager.reset()); + await this.manager.persist(entity); } async saveAll(entities: EntityType[] = [], _context?: ExecutionContext): Promise { logger.debug("services.database.repository - Saving entities"); - - entities.forEach(entity => this.manager.persist(entity)); - return this.manager.flush().then(manager => manager.reset()); + await Promise.all(entities.map(entity => this.manager.persist(entity))); } async remove(entity: EntityType, _context?: ExecutionContext): Promise { - this.manager.remove(entity); - return this.manager.flush().then(manager => manager.reset()); + await this.manager.remove(entity); } //Avoid using this except when no choice diff --git a/tdrive/backend/node/test/unit/core/services/database/services/orm/manager.test.ts b/tdrive/backend/node/test/unit/core/services/database/services/orm/manager.test.ts index 3b1cfaca..1dbfec92 100644 --- a/tdrive/backend/node/test/unit/core/services/database/services/orm/manager.test.ts +++ b/tdrive/backend/node/test/unit/core/services/database/services/orm/manager.test.ts @@ -9,50 +9,46 @@ import WorkspaceUser, { getInstance } from "../../../../../../../src/services/wo describe('EntityManager', () => { - const subj: EntityManager = new EntityManager({ } as Connector); + let connector = { upsert: () => void 0}; + const subj: EntityManager = new EntityManager(connector as unknown as Connector); + let upsert; beforeEach(async () => { + upsert = jest.spyOn((subj as any).connector, "upsert"); }); afterEach(() => { jest.clearAllMocks(); - subj.reset(); }); test ("persist should store entity to insert if all fields for pk is empty", () => { //when - subj.persist(new TestDbEntity()); + let entity = new TestDbEntity(); + subj.persist(entity); //then - expect((subj as any).toInsert.length).toEqual(1); - expect((subj as any).toUpdate.length).toEqual(0); - expect((subj as any).toRemove.length).toEqual(0); - expect((subj as any).toInsert[0].id).toBeDefined(); - expect((subj as any).toInsert[0].company_id).toBeDefined(); + expect(upsert).toBeCalledTimes(1); + expect(upsert).toBeCalledWith([entity], {"action": "INSERT"}) }); test ("persist should store entity to insert if id is set", () => { //when - subj.persist(new TestDbEntity({id: randomUUID()})); + let entity = new TestDbEntity({id: randomUUID()}); + subj.persist(entity); //then - expect((subj as any).toInsert.length).toEqual(1); - expect((subj as any).toUpdate.length).toEqual(0); - expect((subj as any).toRemove.length).toEqual(0); - expect((subj as any).toInsert[0].id).toBeDefined(); - expect((subj as any).toInsert[0].company_id).toBeDefined(); + expect(upsert).toBeCalledTimes(1); + expect(upsert).toBeCalledWith([entity], {"action": "INSERT"}) }); test ("persist should store entity to insert if company_id is set", () => { //when - subj.persist(new TestDbEntity({company_id: randomUUID()})); + let entity = new TestDbEntity({company_id: randomUUID()}); + subj.persist(entity); //then - expect((subj as any).toInsert.length).toEqual(1); - expect((subj as any).toUpdate.length).toEqual(0); - expect((subj as any).toRemove.length).toEqual(0); - expect((subj as any).toInsert[0].id).toBeDefined(); - expect((subj as any).toInsert[0].company_id).toBeDefined(); + expect(upsert).toBeCalledTimes(1); + expect(upsert).toBeCalledWith([entity], {"action": "INSERT"}) }); test ("persist should store entity to update if all pk fields are set", () => { @@ -61,10 +57,8 @@ describe('EntityManager', () => { subj.persist(entity); //then - expect((subj as any).toUpdate.length).toEqual(1); - expect((subj as any).toInsert.length).toEqual(0); - expect((subj as any).toRemove.length).toEqual(0); - expect((subj as any).toUpdate[0]).toEqual(entity) + expect(upsert).toBeCalledTimes(1); + expect(upsert).toBeCalledWith([entity], {"action": "UPDATE"}) }); test ("persist should store entity to update if all pk fields are set and column name is different from field name", () => { @@ -73,10 +67,8 @@ describe('EntityManager', () => { subj.persist(entity); //then - expect((subj as any).toUpdate.length).toEqual(1); - expect((subj as any).toInsert.length).toEqual(0); - expect((subj as any).toRemove.length).toEqual(0); - expect((subj as any).toUpdate[0]).toEqual(entity) + expect(upsert).toBeCalledTimes(1) + expect(upsert).toBeCalledWith([entity], {"action": "UPDATE"}) }); test ("persist should store entity to insert if not all pk fields are set and column name is different from field name", () => { @@ -85,11 +77,7 @@ describe('EntityManager', () => { subj.persist(entity); //then - expect((subj as any).toUpdate.length).toEqual(0); - expect((subj as any).toInsert.length).toEqual(1); - expect((subj as any).toRemove.length).toEqual(0); - expect((subj as any).toInsert[0]).toEqual(entity); - expect(entity.userId).toBeDefined(); + expect(upsert).toBeCalledWith([entity], {"action": "INSERT"}) }); }); From f14ec8bd2d3545c6a6284cfa943f02b3a7a6806d Mon Sep 17 00:00:00 2001 From: Montassar Ghanmy Date: Thu, 5 Sep 2024 14:30:27 +0100 Subject: [PATCH 4/5] =?UTF-8?q?=F0=9F=90=9B=20Fix=20infinite=20scroll=20fo?= =?UTF-8?q?r=20shared=20with=20me=20(#641)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../search/adapters/mongosearch/index.ts | 7 +- .../documents/entities/drive-file.search.ts | 1 + .../src/services/documents/services/index.ts | 69 ++++--- .../node/src/services/documents/types.ts | 6 +- .../documents/web/controllers/documents.ts | 43 ++--- .../test/e2e/common/entities/mock_entities.ts | 13 +- .../documents-pagination-sorting.spec.ts | 173 +++++++++++++++++- .../docker-compose.dev.tests.opensearch.yml | 1 + .../features/drive/api-client/api-client.ts | 5 +- .../frontend/src/app/features/drive/types.ts | 5 +- 10 files changed, 253 insertions(+), 70 deletions(-) diff --git a/tdrive/backend/node/src/core/platform/services/search/adapters/mongosearch/index.ts b/tdrive/backend/node/src/core/platform/services/search/adapters/mongosearch/index.ts index ac3884c5..f48b2a63 100644 --- a/tdrive/backend/node/src/core/platform/services/search/adapters/mongosearch/index.ts +++ b/tdrive/backend/node/src/core/platform/services/search/adapters/mongosearch/index.ts @@ -211,7 +211,12 @@ export default class MongoSearch extends SearchAdapter implements SearchAdapterI logger.info(`Search query: ${JSON.stringify(query)}`); console.log(query); - let cursor = collection.find(query).sort(sort); + const sortMapped: any = sort + ? Object.fromEntries( + Object.entries(sort).map(([field, direction]) => [field, direction === "asc" ? 1 : -1]), + ) + : {}; + let cursor = collection.find(query).sort(sortMapped); if (project) { cursor = cursor.project(project); } diff --git a/tdrive/backend/node/src/services/documents/entities/drive-file.search.ts b/tdrive/backend/node/src/services/documents/entities/drive-file.search.ts index d1e6a449..02bda386 100644 --- a/tdrive/backend/node/src/services/documents/entities/drive-file.search.ts +++ b/tdrive/backend/node/src/services/documents/entities/drive-file.search.ts @@ -14,6 +14,7 @@ export default { access_entities: entity.access_info?.entities?.filter(e => e.level != "none").map(e => e.id), last_modified: entity.last_modified, mime_type: entity.last_version_cache?.file_metadata?.mime, + size: entity.last_version_cache?.file_metadata?.size, }), mongoMapping: { text: { diff --git a/tdrive/backend/node/src/services/documents/services/index.ts b/tdrive/backend/node/src/services/documents/services/index.ts index 25d1a8c4..28f32018 100644 --- a/tdrive/backend/node/src/services/documents/services/index.ts +++ b/tdrive/backend/node/src/services/documents/services/index.ts @@ -29,10 +29,8 @@ import { DriveFileAccessLevel, DriveItemDetails, DriveTdriveTab, - PaginateDocumentBody, RootType, SearchDocumentsOptions, - SortDocumentsBody, TrashType, } from "../types"; import { @@ -61,6 +59,7 @@ import archiver from "archiver"; import internal from "stream"; import config from "config"; import { randomUUID } from "crypto"; +import { SortType } from "src/core/platform/services/search/api"; export class DocumentsService { version: "1"; @@ -107,8 +106,6 @@ export class DocumentsService { browse = async ( id: string, options: SearchDocumentsOptions, - sort: SortDocumentsBody, - paginate: PaginateDocumentBody, context: DriveExecutionContext & { public_token?: string }, ): Promise => { if (isSharedWithMeFolder(id)) { @@ -116,7 +113,7 @@ export class DocumentsService { } else { return { nextPage: null, - ...(await this.get(id, context, false, sort, paginate)), + ...(await this.get(id, options, context, false)), }; } }; @@ -125,17 +122,23 @@ export class DocumentsService { options: SearchDocumentsOptions, context: DriveExecutionContext & { public_token?: string }, ): Promise => { - const result = []; - let fileList: ListResult; - do { - fileList = await this.search(options, context); - result.push(...fileList.getEntities()); - options.pagination = fileList.nextPage; - } while (fileList.nextPage?.page_token); + if (options.pagination) { + if (options.pagination.page_token == "1") { + delete options.pagination.page_token; + } + } + + if (options.sort) { + options.sort = this.getSortFieldMapping(options.sort); + } + + const fileList: ListResult = await this.search(options, context); + const result = fileList.getEntities(); + return { access: "read", children: result, - nextPage: null, + nextPage: fileList.nextPage, path: [] as Array, }; }; @@ -157,10 +160,9 @@ export class DocumentsService { */ get = async ( id: string, + options: SearchDocumentsOptions, context: DriveExecutionContext & { public_token?: string }, all?: boolean, - sort?: SortDocumentsBody, - paginate?: PaginateDocumentBody, ): Promise => { if (!context) { this.logger.error("invalid context"); @@ -213,24 +215,21 @@ export class DocumentsService { ) ).getEntities(); - const sortFieldMapping = { - name: "name", - date: "last_modified", - size: "size", - }; - const sortField = {}; - sortField[sortFieldMapping[sort?.by] || "last_modified"] = sort?.order || "desc"; - + let sortField = {}; + if (options?.sort) { + sortField = this.getSortFieldMapping(options.sort); + } const dbType = await globalResolver.database.getConnector().getType(); // Initialize pagination let pagination; - if (paginate) { - const { page, limit } = paginate; - const pageNumber = dbType === "mongodb" ? page : page / limit + 1; + if (options?.pagination) { + const { page_token, limitStr } = options.pagination; + const pageNumber = + dbType === "mongodb" ? parseInt(page_token) : parseInt(page_token) / parseInt(limitStr) + 1; - pagination = new Pagination(`${pageNumber}`, `${limit}`, false); + pagination = new Pagination(`${pageNumber}`, `${limitStr}`, false); } let children = isDirectory @@ -1031,7 +1030,7 @@ export class DocumentsService { context: DriveExecutionContext, ): Promise => { for (const id of ids) { - const item = await this.get(id, context); + const item = await this.get(id, null, context); if (!item) { throw new CrudException("Drive item not found", 404); } @@ -1085,7 +1084,7 @@ export class DocumentsService { size: number; }; }> => { - const item = await this.get(id, context); + const item = await this.get(id, null, context); if (item.item.is_directory) { return { archive: await this.createZip([id], context) }; @@ -1317,4 +1316,16 @@ export class DocumentsService { throw new CrudException(`Not enough space: ${size}, ${leftQuota}.`, 403); } }; + + getSortFieldMapping = (sort: SortType) => { + const sortFieldMapping = { + name: "name", + date: "last_modified", + size: "size", + }; + + const sortField = {}; + sortField[sortFieldMapping[sort?.by] || "last_modified"] = sort?.order || "desc"; + return sortField; + }; } diff --git a/tdrive/backend/node/src/services/documents/types.ts b/tdrive/backend/node/src/services/documents/types.ts index 99c6abec..fc38bbfa 100644 --- a/tdrive/backend/node/src/services/documents/types.ts +++ b/tdrive/backend/node/src/services/documents/types.ts @@ -61,8 +61,8 @@ export type SearchDocumentsOptions = { export type BrowseDocumentsOptions = { filter?: SearchDocumentsBody; - sort?: SortDocumentsBody; - paginate?: PaginateDocumentBody; + sort?: SortType; + paginate?: Paginable; }; export type SearchDocumentsBody = { @@ -85,7 +85,7 @@ export type SortDocumentsBody = { }; export type PaginateDocumentBody = { - page: number; + page?: string; limit: number; }; diff --git a/tdrive/backend/node/src/services/documents/web/controllers/documents.ts b/tdrive/backend/node/src/services/documents/web/controllers/documents.ts index 973c0765..1697d80c 100644 --- a/tdrive/backend/node/src/services/documents/web/controllers/documents.ts +++ b/tdrive/backend/node/src/services/documents/web/controllers/documents.ts @@ -16,12 +16,10 @@ import { DriveItemDetails, DriveTdriveTab, ItemRequestParams, - PaginateDocumentBody, ItemRequestByEditingSessionKeyParams, RequestParams, SearchDocumentsBody, SearchDocumentsOptions, - SortDocumentsBody, } from "../../types"; import { DriveFileDTO } from "../dto/drive-file-dto"; import { DriveFileDTOBuilder } from "../../services/drive-file-dto-builder"; @@ -146,7 +144,7 @@ export class DocumentsController { ): Promise => { const context = getDriveExecutionContext(request); - return await globalResolver.services.documents.documents.get(null, context); + return await globalResolver.services.documents.documents.get(null, null, context); }; /** @@ -165,7 +163,7 @@ export class DocumentsController { const { id } = request.params; return { - ...(await globalResolver.services.documents.documents.get(id, context)), + ...(await globalResolver.services.documents.documents.get(id, null, context)), }; }; @@ -212,19 +210,12 @@ export class DocumentsController { view: DriveFileDTOBuilder.VIEW_SHARED_WITH_ME, onlyDirectlyShared: true, onlyUploadedNotByMe: true, + sort: request.body.sort, + pagination: request.body.paginate, }; - const sortOptions: SortDocumentsBody = request.body.sort; - const paginateOptions: PaginateDocumentBody = request.body.paginate; - return { - ...(await globalResolver.services.documents.documents.browse( - id, - options, - sortOptions, - paginateOptions, - context, - )), + ...(await globalResolver.services.documents.documents.browse(id, options, context)), }; }; @@ -475,12 +466,17 @@ export class DocumentsController { ); if (ids[0] === "root") { - const items = await globalResolver.services.documents.documents.get(ids[0], context); + const items = await globalResolver.services.documents.documents.get(ids[0], null, context); ids = items.children.map(item => item.id); } if (isDirectory === true) { - const items = await globalResolver.services.documents.documents.get(ids[0], context, true); + const items = await globalResolver.services.documents.documents.get( + ids[0], + null, + context, + true, + ); ids = items.children.map(item => item.id); } @@ -594,11 +590,16 @@ export class DocumentsController { type: string; }; }> { - const document = await globalResolver.services.documents.documents.get(req.body.document_id, { - public_token: req.body.token + (req.body.token_password ? "+" + req.body.token_password : ""), - user: null, - company: { id: req.body.company_id }, - }); + const document = await globalResolver.services.documents.documents.get( + req.body.document_id, + null, + { + public_token: + req.body.token + (req.body.token_password ? "+" + req.body.token_password : ""), + user: null, + company: { id: req.body.company_id }, + }, + ); if (!document || !document.access || document.access === "none") throw new CrudException("You don't have access to this document", 401); diff --git a/tdrive/backend/node/test/e2e/common/entities/mock_entities.ts b/tdrive/backend/node/test/e2e/common/entities/mock_entities.ts index 6106028c..9bcf5365 100644 --- a/tdrive/backend/node/test/e2e/common/entities/mock_entities.ts +++ b/tdrive/backend/node/test/e2e/common/entities/mock_entities.ts @@ -37,10 +37,15 @@ export class DriveFileMockClass { } export class DriveItemDetailsMockClass { - path: string[]; - item: DriveFileMockClass; - children: DriveFileMockClass[]; - versions: Record[]; + path: string[]; + item: DriveFileMockClass; + children: DriveFileMockClass[]; + versions: Record[]; + nextPage?: { + page_token: string; + limitStr: string; + reversed: boolean; + }; } export class SearchResultMockClass { diff --git a/tdrive/backend/node/test/e2e/documents/documents-pagination-sorting.spec.ts b/tdrive/backend/node/test/e2e/documents/documents-pagination-sorting.spec.ts index 42c05a48..0ce3eddb 100644 --- a/tdrive/backend/node/test/e2e/documents/documents-pagination-sorting.spec.ts +++ b/tdrive/backend/node/test/e2e/documents/documents-pagination-sorting.spec.ts @@ -36,27 +36,27 @@ describe("The Documents Browser Window and API", () => { const myDriveId = "user_" + currentUser.user.id; await currentUser.uploadAllFilesOneByOne(myDriveId); - let page = 1; - const limit = 2; + let page_token = "1"; + const limitStr = "2"; let docs = await currentUser.browseDocuments(myDriveId, { - paginate: { page, limit }, + paginate: { page_token, limitStr }, }); expect(docs).toBeDefined(); - expect(docs.children).toHaveLength(limit); + expect(docs.children).toHaveLength(parseInt(limitStr)); - page = 2; + page_token = "2"; docs = await currentUser.browseDocuments(myDriveId, { - paginate: { page, limit }, + paginate: { page_token, limitStr }, }); expect(docs).toBeDefined(); - expect(docs.children).toHaveLength(limit); + expect(docs.children).toHaveLength(parseInt(limitStr)); - page = 3; + page_token = "3"; docs = await currentUser.browseDocuments(myDriveId, { - paginate: { page, limit }, + paginate: { page_token, limitStr }, }); expect(docs).toBeDefined(); - expect(docs.children.length).toBeLessThanOrEqual(limit); + expect(docs.children.length).toBeLessThanOrEqual(parseInt(limitStr)); }); it("Should sort documents by name in ascending order", async () => { @@ -152,5 +152,158 @@ describe("The Documents Browser Window and API", () => { const isSorted = docs.children.every((item, i, arr) => !i || arr[i - 1].size >= item.size); expect(isSorted).toBe(true); }); + + it("Should paginate shared with me ", async () => { + const sharedWIthMeFolder = "shared_with_me"; + const oneUser = await UserApi.getInstance(platform, true, { companyRole: "admin" }); + const anotherUser = await UserApi.getInstance(platform, true, { companyRole: "admin" }); + let files = await oneUser.uploadAllFilesOneByOne(); + for (const file of files) { + await oneUser.shareWithPermissions(file, anotherUser.user.id, "read"); + } + + // wait for files to be indexed + await new Promise(r => setTimeout(r, 5000)); + + let page_token: any = "1"; + const limitStr = "2"; + let docs = await anotherUser.browseDocuments(sharedWIthMeFolder, { + paginate: { page_token, limitStr }, + }); + expect(docs).toBeDefined(); + expect(docs.children).toHaveLength(parseInt(limitStr)); + + page_token = docs.nextPage?.page_token || "2"; + docs = await anotherUser.browseDocuments(sharedWIthMeFolder, { + paginate: { page_token, limitStr }, + }); + expect(docs).toBeDefined(); + expect(docs.children).toHaveLength(parseInt(limitStr)); + + page_token = docs.nextPage?.page_token || "3"; + docs = await anotherUser.browseDocuments(sharedWIthMeFolder, { + paginate: { page_token, limitStr }, + }); + expect(docs).toBeDefined(); + expect(docs.children.length).toBeLessThanOrEqual(parseInt(limitStr)); + }); + + it("Should sort shared with me by name in ascending order", async () => { + const sharedWIthMeFolder = "shared_with_me"; + const oneUser = await UserApi.getInstance(platform, true, { companyRole: "admin" }); + const anotherUser = await UserApi.getInstance(platform, true, { companyRole: "admin" }); + let files = await oneUser.uploadAllFilesOneByOne(); + for (const file of files) { + await oneUser.shareWithPermissions(file, anotherUser.user.id, "read"); + } + const sortBy = "name"; + const sortOrder = "asc"; + const docs = await anotherUser.browseDocuments(sharedWIthMeFolder, { + sort: { by: sortBy, order: sortOrder }, + }); + expect(docs).toBeDefined(); + + const isSorted = docs.children.every((item, i, arr) => !i || arr[i - 1].name <= item.name); + expect(isSorted).toBe(true); + }); + + it("Should sort shared with me by name in descending order", async () => { + const sharedWIthMeFolder = "shared_with_me"; + const oneUser = await UserApi.getInstance(platform, true, { companyRole: "admin" }); + const anotherUser = await UserApi.getInstance(platform, true, { companyRole: "admin" }); + let files = await oneUser.uploadAllFilesOneByOne(); + for (const file of files) { + await oneUser.shareWithPermissions(file, anotherUser.user.id, "read"); + } + const sortBy = "name"; + const sortOrder = "desc"; + const docs = await anotherUser.browseDocuments(sharedWIthMeFolder, { + sort: { by: sortBy, order: sortOrder }, + }); + expect(docs).toBeDefined(); + + const isSorted = docs.children.every((item, i, arr) => !i || arr[i - 1].name >= item.name); + expect(isSorted).toBe(true); + }); + + it("Should sort shared with me by size in ascending order", async () => { + const sharedWIthMeFolder = "shared_with_me"; + const oneUser = await UserApi.getInstance(platform, true, { companyRole: "admin" }); + const anotherUser = await UserApi.getInstance(platform, true, { companyRole: "admin" }); + let files = await oneUser.uploadAllFilesOneByOne(); + for (const file of files) { + await oneUser.shareWithPermissions(file, anotherUser.user.id, "read"); + } + const sortBy = "size"; + const sortOrder = "asc"; + const docs = await anotherUser.browseDocuments(sharedWIthMeFolder, { + sort: { by: sortBy, order: sortOrder }, + }); + expect(docs).toBeDefined(); + + const isSorted = docs.children.every((item, i, arr) => !i || arr[i - 1].size <= item.size); + expect(isSorted).toBe(true); + }); + + it("Should sort shared with me by size in descending order", async () => { + const sharedWIthMeFolder = "shared_with_me"; + const oneUser = await UserApi.getInstance(platform, true, { companyRole: "admin" }); + const anotherUser = await UserApi.getInstance(platform, true, { companyRole: "admin" }); + let files = await oneUser.uploadAllFilesOneByOne(); + for (const file of files) { + await oneUser.shareWithPermissions(file, anotherUser.user.id, "read"); + } + const sortBy = "size"; + const sortOrder = "desc"; + const docs = await anotherUser.browseDocuments(sharedWIthMeFolder, { + sort: { by: sortBy, order: sortOrder }, + }); + expect(docs).toBeDefined(); + + const isSorted = docs.children.every((item, i, arr) => !i || arr[i - 1].size >= item.size); + expect(isSorted).toBe(true); + }); + + it("Should sort shared with me by date in ascending order", async () => { + const sharedWIthMeFolder = "shared_with_me"; + const oneUser = await UserApi.getInstance(platform, true, { companyRole: "admin" }); + const anotherUser = await UserApi.getInstance(platform, true, { companyRole: "admin" }); + let files = await oneUser.uploadAllFilesOneByOne(); + for (const file of files) { + await oneUser.shareWithPermissions(file, anotherUser.user.id, "read"); + } + const sortBy = "date"; + const sortOrder = "asc"; + const docs = await anotherUser.browseDocuments(sharedWIthMeFolder, { + sort: { by: sortBy, order: sortOrder }, + }); + expect(docs).toBeDefined(); + + const isSorted = docs.children.every( + (item, i, arr) => !i || new Date(arr[i - 1].added) <= new Date(item.added), + ); + expect(isSorted).toBe(true); + }); + + it("Should sort shared with me by date in descending order", async () => { + const sharedWIthMeFolder = "shared_with_me"; + const oneUser = await UserApi.getInstance(platform, true, { companyRole: "admin" }); + const anotherUser = await UserApi.getInstance(platform, true, { companyRole: "admin" }); + let files = await oneUser.uploadAllFilesOneByOne(); + for (const file of files) { + await oneUser.shareWithPermissions(file, anotherUser.user.id, "read"); + } + const sortBy = "date"; + const sortOrder = "desc"; + const docs = await anotherUser.browseDocuments(sharedWIthMeFolder, { + sort: { by: sortBy, order: sortOrder }, + }); + expect(docs).toBeDefined(); + + const isSorted = docs.children.every( + (item, i, arr) => !i || new Date(arr[i - 1].added) >= new Date(item.added), + ); + expect(isSorted).toBe; + }); }); }); diff --git a/tdrive/docker-compose.dev.tests.opensearch.yml b/tdrive/docker-compose.dev.tests.opensearch.yml index 442d64a2..3cac70f1 100644 --- a/tdrive/docker-compose.dev.tests.opensearch.yml +++ b/tdrive/docker-compose.dev.tests.opensearch.yml @@ -41,6 +41,7 @@ services: - "5432:5432" node: + # Use the build context in the current directory build: context: . dockerfile: docker/tdrive-node/Dockerfile diff --git a/tdrive/frontend/src/app/features/drive/api-client/api-client.ts b/tdrive/frontend/src/app/features/drive/api-client/api-client.ts index 3d2981bf..b82244c2 100644 --- a/tdrive/frontend/src/app/features/drive/api-client/api-client.ts +++ b/tdrive/frontend/src/app/features/drive/api-client/api-client.ts @@ -76,7 +76,10 @@ export class DriveApiClient { { filter, sort, - paginate + paginate: { + page_token: paginate.page.toString(), + limitStr: paginate.limit.toString(), + } }, ); } diff --git a/tdrive/frontend/src/app/features/drive/types.ts b/tdrive/frontend/src/app/features/drive/types.ts index c50743d0..fd605799 100644 --- a/tdrive/frontend/src/app/features/drive/types.ts +++ b/tdrive/frontend/src/app/features/drive/types.ts @@ -1,7 +1,10 @@ export type BrowseQuery = { filter: BrowseFilter; sort: BrowseSort; - paginate: BrowsePaginate; + paginate: { + page_token: string; + limitStr: string; + }; } export type BrowseFilter = { From 66525d7e482542f49af712020f2cb021589e16ca Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Sat, 14 Sep 2024 22:29:50 +0200 Subject: [PATCH 5/5] =?UTF-8?q?=F0=9F=90=9B=20backend:=20fix=20bug=20with?= =?UTF-8?q?=20postgres=20support=20when=20uploading=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tdrive/backend/node/src/services/documents/services/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrive/backend/node/src/services/documents/services/index.ts b/tdrive/backend/node/src/services/documents/services/index.ts index 28f32018..e6779ab4 100644 --- a/tdrive/backend/node/src/services/documents/services/index.ts +++ b/tdrive/backend/node/src/services/documents/services/index.ts @@ -227,7 +227,7 @@ export class DocumentsService { if (options?.pagination) { const { page_token, limitStr } = options.pagination; const pageNumber = - dbType === "mongodb" ? parseInt(page_token) : parseInt(page_token) / parseInt(limitStr) + 1; + dbType === "mongodb" ? parseInt(page_token) : parseInt(page_token) / parseInt(limitStr); pagination = new Pagination(`${pageNumber}`, `${limitStr}`, false); } @@ -441,7 +441,7 @@ export class DocumentsService { ); // TODO: notify the user a document has been added to the directory shared with them try { - if (driveItem.parent_id !== "root" && driveItem.parent_id !== "trash") { + if (!isVirtualFolder(driveItem.parent_id)) { const parentItem = await this.repository.findOne( { id: driveItem.parent_id,