🔀 Merge branch 'postgres-support-fix' into 525-515-548-523-onlyoffice-rework

tdrive/backend/node/src/cli/cmds/editing_session.ts

@@ -0,0 +1,16 @@
+import { CommandModule } from "yargs";
+
+const command: CommandModule = {
+  describe: "Editing sessions tools",
+  command: "editing_session",
+  builder: yargs =>
+    yargs.commandDir("editing_session_cmds", {
+      visit: commandModule => commandModule.default,
+    }),
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  handler: () => {
+    throw new Error("Missing sub-command");
+  },
+};
+
+export default command;

tdrive/backend/node/src/core/platform/services/database/api.ts

@@ -11,7 +11,7 @@ export interface DatabaseServiceAPI extends TdriveServiceProvider {
   getConnector(): Connector;
 
   /**
-   * Get entities manager (TODO: Find a better name...)
+   * Get entities manager
    */
   getManager(): Manager<unknown>;
 

tdrive/backend/node/src/core/platform/services/database/services/orm/manager.ts

@@ -1,5 +1,4 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
-/* eslint-disable @typescript-eslint/explicit-module-boundary-types */
 import _ from "lodash";
 import { Connector } from "./connectors";
 import { getEntityDefinition, unwrapPrimarykey } from "./utils";
@@ -9,13 +8,9 @@ import { DatabaseEntitiesRemovedEvent, DatabaseEntitiesSavedEvent } from "./type
 import { localEventBus } from "../../../../framework/event-bus";
 
 export default class EntityManager<EntityType extends Record<string, any>> {
-  private toInsert: EntityType[] = [];
-  private toUpdate: EntityType[] = [];
-  private toRemove: EntityType[] = [];
-
   constructor(readonly connector: Connector) {}
 
-  public persist(entity: any): this {
+  public async persist(entity: any): Promise<this> {
     logger.trace(
       `services.database.orm.entity-manager.persist - entity: ${JSON.stringify(entity)}`,
     );
@@ -62,18 +57,20 @@ export default class EntityManager<EntityType extends Record<string, any>> {
       }
     });
 
+    entity = _.cloneDeep(entity);
     if (emptyPkFields.length > 0) {
-      this.toInsert = this.toInsert.filter(e => e !== entity);
+      await this.connector.upsert([entity], { action: "INSERT" });
-      this.toInsert.push(_.cloneDeep(entity));
     } else {
-      this.toUpdate = this.toUpdate.filter(e => e !== entity);
+      await this.connector.upsert([entity], { action: "UPDATE" });
-      this.toUpdate.push(_.cloneDeep(entity));
     }
+    localEventBus.publish("database:entities:saved", {
+      entities: [entity],
+    } as DatabaseEntitiesSavedEvent);
 
     return this;
   }
 
-  public remove(entity: EntityType, entityType?: EntityType): this {
+  public async remove(entity: EntityType, entityType?: EntityType): Promise<this> {
     if (entityType) {
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
       entity = _.merge(new (entityType as any)(), entity);
@@ -81,45 +78,13 @@ export default class EntityManager<EntityType extends Record<string, any>> {
     if (!entity.constructor.prototype._entity || !entity.constructor.prototype._columns) {
       throw Error("Cannot remove this object: it is not an entity.");
     }
-    this.toRemove = this.toRemove.filter(e => e !== entity);
-    this.toRemove.push(_.cloneDeep(entity));
 
-    return this;
+    await this.connector.remove([entity]);
-  }
-
-  public async flush(): Promise<this> {
-    this.toInsert = _.uniqWith(this.toInsert, _.isEqual);
-    this.toUpdate = _.uniqWith(this.toUpdate, _.isEqual);
-    this.toRemove = _.uniqWith(this.toRemove, _.isEqual);
 
     localEventBus.publish("database:entities:saved", {
-      entities: this.toInsert.map(e => _.cloneDeep(e)),
+      entities: [entity],
-    } as DatabaseEntitiesSavedEvent);
-
-    localEventBus.publish("database:entities:saved", {
-      entities: this.toUpdate.map(e => _.cloneDeep(e)),
-    } as DatabaseEntitiesSavedEvent);
-
-    localEventBus.publish("database:entities:saved", {
-      entities: this.toRemove.map(e => _.cloneDeep(e)),
     } as DatabaseEntitiesRemovedEvent);
 
-    if (this.toInsert.length > 0) {
-      await this.connector.upsert(this.toInsert, { action: "INSERT" });
-    }
-    if (this.toUpdate.length > 0) {
-      await this.connector.upsert(this.toUpdate, { action: "UPDATE" });
-    }
-    if (this.toRemove.length > 0) {
-      await this.connector.remove(this.toRemove);
-    }
-
     return this;
   }
-
-  public reset(): void {
-    this.toInsert = [];
-    this.toUpdate = [];
-    this.toRemove = [];
-  }
 }

tdrive/backend/node/src/core/platform/services/database/services/orm/repository/repository.ts

@@ -135,25 +135,20 @@ export default class Repository<EntityType> {
   }> {
     if (previousValue === newValue)
       throw new Error(`Previous and new values are identical: ${JSON.stringify(previousValue)}`);
-    await this.manager.flush().then(manager => manager.reset());
     return this.connector.atomicCompareAndSet(entity, fieldName, previousValue, newValue);
   }
 
   async save(entity: EntityType, _context?: ExecutionContext): Promise<void> {
-    this.manager.persist(entity);
+    await this.manager.persist(entity);
-    return this.manager.flush().then(manager => manager.reset());
   }
 
   async saveAll(entities: EntityType[] = [], _context?: ExecutionContext): Promise<void> {
     logger.debug("services.database.repository - Saving entities");
-
+    await Promise.all(entities.map(entity => this.manager.persist(entity)));
-    entities.forEach(entity => this.manager.persist(entity));
-    return this.manager.flush().then(manager => manager.reset());
   }
 
   async remove(entity: EntityType, _context?: ExecutionContext): Promise<void> {
-    this.manager.remove(entity);
+    await this.manager.remove(entity);
-    return this.manager.flush().then(manager => manager.reset());
   }
 
   //Avoid using this except when no choice

tdrive/backend/node/src/core/platform/services/search/adapters/mongosearch/index.ts

@@ -211,7 +211,12 @@ export default class MongoSearch extends SearchAdapter implements SearchAdapterI
     logger.info(`Search query: ${JSON.stringify(query)}`);
     console.log(query);
 
-    let cursor = collection.find(query).sort(sort);
+    const sortMapped: any = sort
+      ? Object.fromEntries(
+          Object.entries(sort).map(([field, direction]) => [field, direction === "asc" ? 1 : -1]),
+        )
+      : {};
+    let cursor = collection.find(query).sort(sortMapped);
     if (project) {
       cursor = cursor.project(project);
     }

tdrive/backend/node/src/services/documents/entities/drive-file.search.ts

@@ -14,6 +14,7 @@ export default {
     access_entities: entity.access_info?.entities?.filter(e => e.level != "none").map(e => e.id),
     last_modified: entity.last_modified,
     mime_type: entity.last_version_cache?.file_metadata?.mime,
+    size: entity.last_version_cache?.file_metadata?.size,
   }),
   mongoMapping: {
     text: {

tdrive/backend/node/src/services/documents/services/index.ts

@@ -29,10 +29,8 @@ import {
   DriveFileAccessLevel,
   DriveItemDetails,
   DriveTdriveTab,
-  PaginateDocumentBody,
   RootType,
   SearchDocumentsOptions,
-  SortDocumentsBody,
   TrashType,
 } from "../types";
 import {
@@ -62,6 +60,7 @@ import internal from "stream";
 import config from "config";
 import { MultipartFile } from "@fastify/multipart";
 import { UploadOptions } from "src/services/files/types";
+import { SortType } from "src/core/platform/services/search/api";
 
 export class DocumentsService {
   version: "1";
@@ -108,8 +107,6 @@ export class DocumentsService {
   browse = async (
     id: string,
     options: SearchDocumentsOptions,
-    sort: SortDocumentsBody,
-    paginate: PaginateDocumentBody,
     context: DriveExecutionContext & { public_token?: string },
   ): Promise<BrowseDetails> => {
     if (isSharedWithMeFolder(id)) {
@@ -117,7 +114,7 @@ export class DocumentsService {
     } else {
       return {
         nextPage: null,
-        ...(await this.get(id, context, false, sort, paginate)),
+        ...(await this.get(id, options, context, false)),
       };
     }
   };
@@ -126,17 +123,23 @@ export class DocumentsService {
     options: SearchDocumentsOptions,
     context: DriveExecutionContext & { public_token?: string },
   ): Promise<BrowseDetails> => {
-    const result = [];
+    if (options.pagination) {
-    let fileList: ListResult<DriveFile>;
+      if (options.pagination.page_token == "1") {
-    do {
+        delete options.pagination.page_token;
-      fileList = await this.search(options, context);
+      }
-      result.push(...fileList.getEntities());
+    }
-      options.pagination = fileList.nextPage;
+
-    } while (fileList.nextPage?.page_token);
+    if (options.sort) {
+      options.sort = this.getSortFieldMapping(options.sort);
+    }
+
+    const fileList: ListResult<DriveFile> = await this.search(options, context);
+    const result = fileList.getEntities();
+
     return {
       access: "read",
       children: result,
-      nextPage: null,
+      nextPage: fileList.nextPage,
       path: [] as Array<DriveFile>,
     };
   };
@@ -158,10 +161,9 @@ export class DocumentsService {
    */
   get = async (
     id: string,
+    options: SearchDocumentsOptions,
     context: DriveExecutionContext & { public_token?: string },
     all?: boolean,
-    sort?: SortDocumentsBody,
-    paginate?: PaginateDocumentBody,
   ): Promise<DriveItemDetails> => {
     if (!context) {
       this.logger.error("invalid context");
@@ -214,24 +216,21 @@ export class DocumentsService {
           )
         ).getEntities();
 
-    const sortFieldMapping = {
+    let sortField = {};
-      name: "name",
+    if (options?.sort) {
-      date: "last_modified",
+      sortField = this.getSortFieldMapping(options.sort);
-      size: "size",
+    }
-    };
-    const sortField = {};
-    sortField[sortFieldMapping[sort?.by] || "last_modified"] = sort?.order || "desc";
-
     const dbType = await globalResolver.database.getConnector().getType();
 
     // Initialize pagination
     let pagination;
 
-    if (paginate) {
+    if (options?.pagination) {
-      const { page, limit } = paginate;
+      const { page_token, limitStr } = options.pagination;
-      const pageNumber = dbType === "mongodb" ? page : page / limit + 1;
+      const pageNumber =
+        dbType === "mongodb" ? parseInt(page_token) : parseInt(page_token) / parseInt(limitStr);
 
-      pagination = new Pagination(`${pageNumber}`, `${limit}`, false);
+      pagination = new Pagination(`${pageNumber}`, `${limitStr}`, false);
     }
 
     let children = isDirectory
@@ -443,7 +442,7 @@ export class DocumentsService {
       );
       // TODO: notify the user a document has been added to the directory shared with them
       try {
-        if (driveItem.parent_id !== "root" && driveItem.parent_id !== "trash") {
+        if (!isVirtualFolder(driveItem.parent_id)) {
           const parentItem = await this.repository.findOne(
             {
               id: driveItem.parent_id,
@@ -1100,7 +1099,7 @@ export class DocumentsService {
     context: DriveExecutionContext,
   ): Promise<string> => {
     for (const id of ids) {
-      const item = await this.get(id, context);
+      const item = await this.get(id, null, context);
       if (!item) {
         throw new CrudException("Drive item not found", 404);
       }
@@ -1154,7 +1153,7 @@ export class DocumentsService {
       size: number;
     };
   }> => {
-    const item = await this.get(id, context);
+    const item = await this.get(id, null, context);
 
     if (item.item.is_directory) {
       return { archive: await this.createZip([id], context) };
@@ -1386,4 +1385,16 @@ export class DocumentsService {
       throw new CrudException(`Not enough space: ${size}, ${leftQuota}.`, 403);
     }
   };
+
+  getSortFieldMapping = (sort: SortType) => {
+    const sortFieldMapping = {
+      name: "name",
+      date: "last_modified",
+      size: "size",
+    };
+
+    const sortField = {};
+    sortField[sortFieldMapping[sort?.by] || "last_modified"] = sort?.order || "desc";
+    return sortField;
+  };
 }

tdrive/backend/node/src/services/documents/types.ts

@@ -61,8 +61,8 @@ export type SearchDocumentsOptions = {
 
 export type BrowseDocumentsOptions = {
   filter?: SearchDocumentsBody;
-  sort?: SortDocumentsBody;
+  sort?: SortType;
-  paginate?: PaginateDocumentBody;
+  paginate?: Paginable;
 };
 
 export type SearchDocumentsBody = {
@@ -85,7 +85,7 @@ export type SortDocumentsBody = {
 };
 
 export type PaginateDocumentBody = {
-  page: number;
+  page?: string;
   limit: number;
 };
 

tdrive/backend/node/src/services/documents/web/controllers/documents.ts

@@ -16,12 +16,10 @@ import {
   DriveItemDetails,
   DriveTdriveTab,
   ItemRequestParams,
-  PaginateDocumentBody,
   ItemRequestByEditingSessionKeyParams,
   RequestParams,
   SearchDocumentsBody,
   SearchDocumentsOptions,
-  SortDocumentsBody,
 } from "../../types";
 import { DriveFileDTO } from "../dto/drive-file-dto";
 import { DriveFileDTOBuilder } from "../../services/drive-file-dto-builder";
@@ -146,7 +144,7 @@ export class DocumentsController {
   ): Promise<DriveItemDetails> => {
     const context = getDriveExecutionContext(request);
 
-    return await globalResolver.services.documents.documents.get(null, context);
+    return await globalResolver.services.documents.documents.get(null, null, context);
   };
 
   /**
@@ -165,7 +163,7 @@ export class DocumentsController {
     const { id } = request.params;
 
     return {
-      ...(await globalResolver.services.documents.documents.get(id, context)),
+      ...(await globalResolver.services.documents.documents.get(id, null, context)),
     };
   };
 
@@ -212,19 +210,12 @@ export class DocumentsController {
       view: DriveFileDTOBuilder.VIEW_SHARED_WITH_ME,
       onlyDirectlyShared: true,
       onlyUploadedNotByMe: true,
+      sort: request.body.sort,
+      pagination: request.body.paginate,
     };
 
-    const sortOptions: SortDocumentsBody = request.body.sort;
-    const paginateOptions: PaginateDocumentBody = request.body.paginate;
-
     return {
-      ...(await globalResolver.services.documents.documents.browse(
+      ...(await globalResolver.services.documents.documents.browse(id, options, context)),
-        id,
-        options,
-        sortOptions,
-        paginateOptions,
-        context,
-      )),
     };
   };
 
@@ -551,12 +542,17 @@ export class DocumentsController {
     );
 
     if (ids[0] === "root") {
-      const items = await globalResolver.services.documents.documents.get(ids[0], context);
+      const items = await globalResolver.services.documents.documents.get(ids[0], null, context);
       ids = items.children.map(item => item.id);
     }
 
     if (isDirectory === true) {
-      const items = await globalResolver.services.documents.documents.get(ids[0], context, true);
+      const items = await globalResolver.services.documents.documents.get(
+        ids[0],
+        null,
+        context,
+        true,
+      );
       ids = items.children.map(item => item.id);
     }
 
@@ -670,11 +666,16 @@ export class DocumentsController {
       type: string;
     };
   }> {
-    const document = await globalResolver.services.documents.documents.get(req.body.document_id, {
+    const document = await globalResolver.services.documents.documents.get(
-      public_token: req.body.token + (req.body.token_password ? "+" + req.body.token_password : ""),
+      req.body.document_id,
-      user: null,
+      null,
-      company: { id: req.body.company_id },
+      {
-    });
+        public_token:
+          req.body.token + (req.body.token_password ? "+" + req.body.token_password : ""),
+        user: null,
+        company: { id: req.body.company_id },
+      },
+    );
 
     if (!document || !document.access || document.access === "none")
       throw new CrudException("You don't have access to this document", 401);

tdrive/backend/node/test/e2e/common/entities/mock_entities.ts

@@ -37,10 +37,15 @@ export class DriveFileMockClass {
 }
 
 export class DriveItemDetailsMockClass {
-    path: string[];
+  path: string[];
-    item: DriveFileMockClass;
+  item: DriveFileMockClass;
-    children: DriveFileMockClass[];
+  children: DriveFileMockClass[];
-    versions: Record<string, unknown>[];
+  versions: Record<string, unknown>[];
+  nextPage?: {
+    page_token: string;
+    limitStr: string;
+    reversed: boolean;
+  };
 }
 
 export class SearchResultMockClass {

tdrive/backend/node/test/e2e/documents/documents-pagination-sorting.spec.ts

@@ -36,27 +36,27 @@ describe("The Documents Browser Window and API", () => {
       const myDriveId = "user_" + currentUser.user.id;
       await currentUser.uploadAllFilesOneByOne(myDriveId);
 
-      let page = 1;
+      let page_token = "1";
-      const limit = 2;
+      const limitStr = "2";
       let docs = await currentUser.browseDocuments(myDriveId, {
-        paginate: { page, limit },
+        paginate: { page_token, limitStr },
       });
       expect(docs).toBeDefined();
-      expect(docs.children).toHaveLength(limit);
+      expect(docs.children).toHaveLength(parseInt(limitStr));
 
-      page = 2;
+      page_token = "2";
       docs = await currentUser.browseDocuments(myDriveId, {
-        paginate: { page, limit },
+        paginate: { page_token, limitStr },
       });
       expect(docs).toBeDefined();
-      expect(docs.children).toHaveLength(limit);
+      expect(docs.children).toHaveLength(parseInt(limitStr));
 
-      page = 3;
+      page_token = "3";
       docs = await currentUser.browseDocuments(myDriveId, {
-        paginate: { page, limit },
+        paginate: { page_token, limitStr },
       });
       expect(docs).toBeDefined();
-      expect(docs.children.length).toBeLessThanOrEqual(limit);
+      expect(docs.children.length).toBeLessThanOrEqual(parseInt(limitStr));
     });
 
     it("Should sort documents by name in ascending order", async () => {
@@ -152,5 +152,158 @@ describe("The Documents Browser Window and API", () => {
       const isSorted = docs.children.every((item, i, arr) => !i || arr[i - 1].size >= item.size);
       expect(isSorted).toBe(true);
     });
+
+    it("Should paginate shared with me ", async () => {
+      const sharedWIthMeFolder = "shared_with_me";
+      const oneUser = await UserApi.getInstance(platform, true, { companyRole: "admin" });
+      const anotherUser = await UserApi.getInstance(platform, true, { companyRole: "admin" });
+      let files = await oneUser.uploadAllFilesOneByOne();
+      for (const file of files) {
+        await oneUser.shareWithPermissions(file, anotherUser.user.id, "read");
+      }
+
+      // wait for files to be indexed
+      await new Promise(r => setTimeout(r, 5000));
+
+      let page_token: any = "1";
+      const limitStr = "2";
+      let docs = await anotherUser.browseDocuments(sharedWIthMeFolder, {
+        paginate: { page_token, limitStr },
+      });
+      expect(docs).toBeDefined();
+      expect(docs.children).toHaveLength(parseInt(limitStr));
+
+      page_token = docs.nextPage?.page_token || "2";
+      docs = await anotherUser.browseDocuments(sharedWIthMeFolder, {
+        paginate: { page_token, limitStr },
+      });
+      expect(docs).toBeDefined();
+      expect(docs.children).toHaveLength(parseInt(limitStr));
+
+      page_token = docs.nextPage?.page_token || "3";
+      docs = await anotherUser.browseDocuments(sharedWIthMeFolder, {
+        paginate: { page_token, limitStr },
+      });
+      expect(docs).toBeDefined();
+      expect(docs.children.length).toBeLessThanOrEqual(parseInt(limitStr));
+    });
+
+    it("Should sort shared with me by name in ascending order", async () => {
+      const sharedWIthMeFolder = "shared_with_me";
+      const oneUser = await UserApi.getInstance(platform, true, { companyRole: "admin" });
+      const anotherUser = await UserApi.getInstance(platform, true, { companyRole: "admin" });
+      let files = await oneUser.uploadAllFilesOneByOne();
+      for (const file of files) {
+        await oneUser.shareWithPermissions(file, anotherUser.user.id, "read");
+      }
+      const sortBy = "name";
+      const sortOrder = "asc";
+      const docs = await anotherUser.browseDocuments(sharedWIthMeFolder, {
+        sort: { by: sortBy, order: sortOrder },
+      });
+      expect(docs).toBeDefined();
+
+      const isSorted = docs.children.every((item, i, arr) => !i || arr[i - 1].name <= item.name);
+      expect(isSorted).toBe(true);
+    });
+
+    it("Should sort shared with me by name in descending order", async () => {
+      const sharedWIthMeFolder = "shared_with_me";
+      const oneUser = await UserApi.getInstance(platform, true, { companyRole: "admin" });
+      const anotherUser = await UserApi.getInstance(platform, true, { companyRole: "admin" });
+      let files = await oneUser.uploadAllFilesOneByOne();
+      for (const file of files) {
+        await oneUser.shareWithPermissions(file, anotherUser.user.id, "read");
+      }
+      const sortBy = "name";
+      const sortOrder = "desc";
+      const docs = await anotherUser.browseDocuments(sharedWIthMeFolder, {
+        sort: { by: sortBy, order: sortOrder },
+      });
+      expect(docs).toBeDefined();
+
+      const isSorted = docs.children.every((item, i, arr) => !i || arr[i - 1].name >= item.name);
+      expect(isSorted).toBe(true);
+    });
+
+    it("Should sort shared with me by size in ascending order", async () => {
+      const sharedWIthMeFolder = "shared_with_me";
+      const oneUser = await UserApi.getInstance(platform, true, { companyRole: "admin" });
+      const anotherUser = await UserApi.getInstance(platform, true, { companyRole: "admin" });
+      let files = await oneUser.uploadAllFilesOneByOne();
+      for (const file of files) {
+        await oneUser.shareWithPermissions(file, anotherUser.user.id, "read");
+      }
+      const sortBy = "size";
+      const sortOrder = "asc";
+      const docs = await anotherUser.browseDocuments(sharedWIthMeFolder, {
+        sort: { by: sortBy, order: sortOrder },
+      });
+      expect(docs).toBeDefined();
+
+      const isSorted = docs.children.every((item, i, arr) => !i || arr[i - 1].size <= item.size);
+      expect(isSorted).toBe(true);
+    });
+
+    it("Should sort shared with me by size in descending order", async () => {
+      const sharedWIthMeFolder = "shared_with_me";
+      const oneUser = await UserApi.getInstance(platform, true, { companyRole: "admin" });
+      const anotherUser = await UserApi.getInstance(platform, true, { companyRole: "admin" });
+      let files = await oneUser.uploadAllFilesOneByOne();
+      for (const file of files) {
+        await oneUser.shareWithPermissions(file, anotherUser.user.id, "read");
+      }
+      const sortBy = "size";
+      const sortOrder = "desc";
+      const docs = await anotherUser.browseDocuments(sharedWIthMeFolder, {
+        sort: { by: sortBy, order: sortOrder },
+      });
+      expect(docs).toBeDefined();
+
+      const isSorted = docs.children.every((item, i, arr) => !i || arr[i - 1].size >= item.size);
+      expect(isSorted).toBe(true);
+    });
+
+    it("Should sort shared with me by date in ascending order", async () => {
+      const sharedWIthMeFolder = "shared_with_me";
+      const oneUser = await UserApi.getInstance(platform, true, { companyRole: "admin" });
+      const anotherUser = await UserApi.getInstance(platform, true, { companyRole: "admin" });
+      let files = await oneUser.uploadAllFilesOneByOne();
+      for (const file of files) {
+        await oneUser.shareWithPermissions(file, anotherUser.user.id, "read");
+      }
+      const sortBy = "date";
+      const sortOrder = "asc";
+      const docs = await anotherUser.browseDocuments(sharedWIthMeFolder, {
+        sort: { by: sortBy, order: sortOrder },
+      });
+      expect(docs).toBeDefined();
+
+      const isSorted = docs.children.every(
+        (item, i, arr) => !i || new Date(arr[i - 1].added) <= new Date(item.added),
+      );
+      expect(isSorted).toBe(true);
+    });
+
+    it("Should sort shared with me by date in descending order", async () => {
+      const sharedWIthMeFolder = "shared_with_me";
+      const oneUser = await UserApi.getInstance(platform, true, { companyRole: "admin" });
+      const anotherUser = await UserApi.getInstance(platform, true, { companyRole: "admin" });
+      let files = await oneUser.uploadAllFilesOneByOne();
+      for (const file of files) {
+        await oneUser.shareWithPermissions(file, anotherUser.user.id, "read");
+      }
+      const sortBy = "date";
+      const sortOrder = "desc";
+      const docs = await anotherUser.browseDocuments(sharedWIthMeFolder, {
+        sort: { by: sortBy, order: sortOrder },
+      });
+      expect(docs).toBeDefined();
+
+      const isSorted = docs.children.every(
+        (item, i, arr) => !i || new Date(arr[i - 1].added) >= new Date(item.added),
+      );
+      expect(isSorted).toBe;
+    });
   });
 });

tdrive/backend/node/test/unit/core/services/database/services/orm/manager.test.ts

@@ -9,50 +9,46 @@ import WorkspaceUser, { getInstance } from "../../../../../../../src/services/wo
 
 describe('EntityManager', () => {
 
-  const subj: EntityManager<TestDbEntity> = new EntityManager<TestDbEntity>({ } as Connector);
+  let connector = { upsert: () => void 0};
+  const subj: EntityManager<TestDbEntity> = new EntityManager<TestDbEntity>(connector as unknown as Connector);
+  let upsert;
 
   beforeEach(async () => {
+    upsert = jest.spyOn((subj as any).connector, "upsert");
   });
 
   afterEach(() => {
     jest.clearAllMocks();
-    subj.reset();
   });
 
   test ("persist should store entity to insert if all fields for pk is empty", () => {
     //when
-    subj.persist(new TestDbEntity());
+    let entity = new TestDbEntity();
+    subj.persist(entity);
 
     //then
-    expect((subj as any).toInsert.length).toEqual(1);
+    expect(upsert).toBeCalledTimes(1);
-    expect((subj as any).toUpdate.length).toEqual(0);
+    expect(upsert).toBeCalledWith([entity], {"action": "INSERT"})
-    expect((subj as any).toRemove.length).toEqual(0);
-    expect((subj as any).toInsert[0].id).toBeDefined();
-    expect((subj as any).toInsert[0].company_id).toBeDefined();
   });
 
   test ("persist should store entity to insert if id is set", () => {
     //when
-    subj.persist(new TestDbEntity({id: randomUUID()}));
+    let entity = new TestDbEntity({id: randomUUID()});
+    subj.persist(entity);
 
     //then
-    expect((subj as any).toInsert.length).toEqual(1);
+    expect(upsert).toBeCalledTimes(1);
-    expect((subj as any).toUpdate.length).toEqual(0);
+    expect(upsert).toBeCalledWith([entity], {"action": "INSERT"})
-    expect((subj as any).toRemove.length).toEqual(0);
-    expect((subj as any).toInsert[0].id).toBeDefined();
-    expect((subj as any).toInsert[0].company_id).toBeDefined();
   });
 
   test ("persist should store entity to insert if company_id is set", () => {
     //when
-    subj.persist(new TestDbEntity({company_id: randomUUID()}));
+    let entity = new TestDbEntity({company_id: randomUUID()});
+    subj.persist(entity);
 
     //then
-    expect((subj as any).toInsert.length).toEqual(1);
+    expect(upsert).toBeCalledTimes(1);
-    expect((subj as any).toUpdate.length).toEqual(0);
+    expect(upsert).toBeCalledWith([entity], {"action": "INSERT"})
-    expect((subj as any).toRemove.length).toEqual(0);
-    expect((subj as any).toInsert[0].id).toBeDefined();
-    expect((subj as any).toInsert[0].company_id).toBeDefined();
   });
 
   test ("persist should store entity to update if all pk fields are set", () => {
@@ -61,10 +57,8 @@ describe('EntityManager', () => {
     subj.persist(entity);
 
     //then
-    expect((subj as any).toUpdate.length).toEqual(1);
+    expect(upsert).toBeCalledTimes(1);
-    expect((subj as any).toInsert.length).toEqual(0);
+    expect(upsert).toBeCalledWith([entity], {"action": "UPDATE"})
-    expect((subj as any).toRemove.length).toEqual(0);
-    expect((subj as any).toUpdate[0]).toEqual(entity)
   });
 
   test ("persist should store entity to update if all pk fields are set and column name is different from field name", () => {
@@ -73,10 +67,8 @@ describe('EntityManager', () => {
     subj.persist(entity);
 
     //then
-    expect((subj as any).toUpdate.length).toEqual(1);
+    expect(upsert).toBeCalledTimes(1)
-    expect((subj as any).toInsert.length).toEqual(0);
+    expect(upsert).toBeCalledWith([entity], {"action": "UPDATE"})
-    expect((subj as any).toRemove.length).toEqual(0);
-    expect((subj as any).toUpdate[0]).toEqual(entity)
   });
 
   test ("persist should store entity to insert if not all pk fields are set and column name is different from field name", () => {
@@ -85,11 +77,7 @@ describe('EntityManager', () => {
     subj.persist(entity);
 
     //then
-    expect((subj as any).toUpdate.length).toEqual(0);
+    expect(upsert).toBeCalledWith([entity], {"action": "INSERT"})
-    expect((subj as any).toInsert.length).toEqual(1);
-    expect((subj as any).toRemove.length).toEqual(0);
-    expect((subj as any).toInsert[0]).toEqual(entity);
-    expect(entity.userId).toBeDefined();
   });
 
 });

tdrive/backend/utils/nextcloud-migration/src/executor.ts

@@ -0,0 +1,81 @@
+import { logger } from "./logger";
+
+type FunctionArgs = any[];
+
+type FunctionStats = {
+  functionName: string;
+  successfulExecutions: number;
+  failedExecutions: number;
+  failedExecutionsArgs: FunctionArgs[];
+};
+
+export class FunctionExecutor {
+  private stats: Map<string, FunctionStats> = new Map();
+
+  async executeWithRetries(
+    fn: (...args: any[]) => Promise<any>,
+    args: FunctionArgs,
+    retries: number = 3,
+    throwError: boolean = false,
+  ): Promise<any> {
+    const functionName = fn.name;
+    let success = false;
+    let executionResult = null;
+
+    for (let attempt = 0; attempt < retries; attempt++) {
+      try {
+        executionResult = await fn(...args);
+        success = true;
+        break;
+      } catch (error) {
+        executionResult = error;
+        logger.info(`Attempt ${attempt + 1} failed for ${functionName}`, error);
+      }
+    }
+
+    if (!success && throwError) throw executionResult;
+
+    let stats = this.stats.get(functionName);
+    if (!stats) {
+      stats = {
+        functionName,
+        successfulExecutions: 0,
+        failedExecutions: 0,
+        failedExecutionsArgs: []
+      };
+      this.stats.set(functionName, stats);
+    }
+
+    if (success) {
+      stats.successfulExecutions++;
+    } else {
+      stats.failedExecutions++;
+      stats.failedExecutionsArgs.push( args );
+    }
+    return executionResult;
+  }
+
+  async getStats() {
+    return this.stats;
+  }
+
+  printStatistics(): void {
+    logger.info("Execution Statistics:");
+    this.stats.forEach((stat) => {
+      logger.info(
+        `Function: ${stat.functionName}, Successful Executions: ${stat.successfulExecutions}, Failed Executions: ${stat.failedExecutions}`
+      );
+    });
+  }
+
+  printFailedExecutions(): void {
+    logger.info("Failed Executions:");
+    this.stats.forEach((execution) => {
+      if (execution.failedExecutionsArgs.length > 0) {
+        logger.info(`   Function: ${execution.functionName}`);
+        execution.failedExecutionsArgs.forEach(args =>
+          logger.info(`       args: ${JSON.stringify(args)}`));
+      }
+    });
+  }
+}

tdrive/backend/utils/nextcloud-migration/src/express_server.ts

@@ -60,8 +60,8 @@ app.post("/", async (req: Request, res: Response) => {
     res.status(400).send("Username and password for nextcloud are required");
   }
   try {
-    await nextcloud.migrate(params.username, params.password, params.dir);
+    const stats = await nextcloud.migrate(params.username, params.password, params.dir);
-    res.status(200).send("Sync DONE ✅");
+    res.status(200).send(JSON.stringify(stats));
   } catch (e) {
     console.error(e)
     res.status(500).send("Error during synchronization:: " + e.message)

tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts

@@ -6,6 +6,7 @@ import { TwakeDriveClient, TwakeDriveUser } from './twake_client';
 import path from 'path';
 import { logger } from "./logger"
 import { User, UserProvider, UserProviderFactory, UserProviderType } from "./user_privider";
+import { FunctionExecutor } from "./executor";
 
 export interface NextcloudMigrationConfiguration {
   shell: {
@@ -36,6 +37,8 @@ export class NextcloudMigration {
 
   driveClient: TwakeDriveClient;
 
+  executor = new FunctionExecutor();
+
   constructor(config: NextcloudMigrationConfiguration) {
     this.config = config;
     this.userProvider = (new UserProviderFactory()).get(config.userProvider, config[config.userProvider]);
@@ -55,6 +58,9 @@ export class NextcloudMigration {
       if(!dir) await this.download(username, password, dirTmp);
       //upload files to the Twake Drive
       await this.upload(driveUser, dirTmp);
+      this.executor.printStatistics();
+      this.executor.printFailedExecutions();
+      return this.executor.getStats();
     } catch (e) {
       console.error('Error downloading files from next cloud', e);
       throw e;
@@ -119,7 +125,9 @@ export class NextcloudMigration {
     const parent = await this.driveClient.getDocument(parentDirId);
 
     const exists = (filename: string) => {
-      return parent.children.filter(i => i.name.startsWith(path.parse(filename).name)).length > 0;
+      let parsedPath = path.parse(filename);
+      let name = `${parsedPath.name}${parsedPath.ext > '' ? parsedPath.ext : ''}`;
+      return parent.children.filter(i => i.name == name).length > 0;
     }
 
     logger.debug(`Reading content of the directory ${sourceDirPath} ...`)
@@ -144,8 +152,9 @@ export class NextcloudMigration {
     //check existing files
     for (const fPath of existingFiles) {
       logger.debug(`Check existing file ${fPath}`)
-      let name = path.parse(fPath).name;
+      let parsedPath = path.parse(fPath);
-      let candidatesWithTheSameName = parent.children.filter(i => i.name.startsWith(name));
+      let name = parsedPath.name + (parsedPath.ext > '' ? parsedPath.ext : '');
+      let candidatesWithTheSameName = parent.children.filter(i => i.name === name);
       if (candidatesWithTheSameName.length > 1) {
         logger.warn("WE HAVE MORE MORE THAN ONE FILE WITH NAME: " + name);
       } else {
@@ -169,19 +178,22 @@ export class NextcloudMigration {
       }
     }
 
-
     //upload all files
     logger.debug(`UPLOAD FILES FOR  ${sourceDirPath}`)
     for (const file of filesToUpload) {
       logger.debug(`Upload file ${file}`)
-      await this.driveClient.createFile(file, parentDirId);
+      await this.executor.executeWithRetries(this.driveClient.createFile.bind(this.driveClient), [file, parentDirId], 3)
+      // await this.driveClient.createFile(file, parentDirId);
     }
 
     logger.debug(`UPLOAD DIRS FOR  ${sourceDirPath}`)
     for (const [name, path] of dirsToUpload) {
       logger.info(`Create directory ${name}`);
-      const dir = await this.driveClient.createDirectory(name, parentDirId)
+      const dir = await this.executor.executeWithRetries(this.driveClient.createDirectory.bind(this.driveClient), [name, parentDirId], 3)
-      await this.upload(user, path, dir.id);
+      // const dir = await this.driveClient.createDirectory(name, parentDirId)
+      if (dir) {
+        await this.upload(user, path, dir.id);
+      }
     }
   }
 

tdrive/backend/utils/nextcloud-migration/test/executor.test.ts

@@ -0,0 +1,43 @@
+import { describe, expect, test } from "@jest/globals";
+import { FunctionExecutor } from "../src/executor";
+
+describe('Function Executor tests', () => {
+
+  const subj = new FunctionExecutor();
+
+  test('Should successfully execute mock function', async () => {
+    //when
+    await subj.executeWithRetries(successFunction, [1, 2], 3);
+    await subj.executeWithRetries(successFunction, [4, 5], 3);
+
+    //then
+    let stats = await subj.getStats();
+    expect(stats?.get("successFunction")?.successfulExecutions).toEqual(2);
+  });
+
+  test('Should successfully gather arguments of the failed executions', async () => {
+    await subj.executeWithRetries(failedFunction, [1, 2], 3);
+
+    //then
+    let stats = await subj.getStats();
+    expect(stats?.get("failedFunction")?.failedExecutions).toBe(1);
+    expect(stats?.get("failedFunction")?.failedExecutionsArgs[0]).toStrictEqual([1, 2]);
+  });
+
+  test('Should successfully gather arguments of the class member', async () => {
+    await subj.executeWithRetries(subj.getStats.bind(this), [1, 2], 3);
+
+    //then
+    let stats = await subj.getStats();
+    expect(stats?.get("bound getStats")?.successfulExecutions).toEqual(1);
+  });
+
+  const successFunction = async (a: number, b: number) => {
+    return a + b;
+  };
+
+  const failedFunction = async () => {
+    throw new Error();
+  };
+
+});

tdrive/docker-compose.dev.tests.opensearch.yml

@@ -41,7 +41,7 @@ services:
       - "5432:5432"
 
   node:
-    image: tdrive/tdrive-node:test
+    # Use the build context in the current directory
     build:
       context: .
       dockerfile: docker/tdrive-node/Dockerfile

tdrive/frontend/src/app/features/drive/api-client/api-client.ts

@@ -76,7 +76,10 @@ export class DriveApiClient {
       {
         filter,
         sort,
-        paginate
+        paginate: {
+          page_token: paginate.page.toString(),
+          limitStr: paginate.limit.toString(),
+        }
       },
     );
   }

tdrive/frontend/src/app/features/drive/types.ts

@@ -1,7 +1,10 @@
 export type BrowseQuery = {
   filter: BrowseFilter;
   sort: BrowseSort;
-  paginate: BrowsePaginate;
+  paginate: {
+    page_token: string;
+    limitStr: string;
+  };
 }
 
 export type BrowseFilter = {