✨ Sort files chronologically + infinite scroll (#535)

tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts

@@ -264,12 +264,22 @@ export class MongoConnector extends AbstractConnector<MongoConnectionOptions> {
       options,
     );
 
-    const sort: any = {};
+    let sort: any = {};
     for (const key of entityDefinition.options.primaryKey.slice(1)) {
       const defaultOrder =
         (columnsDefinition[key as string].options.order || "ASC") === "ASC" ? 1 : -1;
       sort[key as string] = (options?.pagination?.reversed ? -1 : 1) * defaultOrder;
     }
+    if (options?.sort) {
+      sort = options.sort
+        ? Object.fromEntries(
+            Object.entries(options.sort).map(([field, direction]) => [
+              field,
+              direction === "asc" ? 1 : -1,
+            ]),
+          )
+        : {};
+    }
 
     logger.debug(`services.database.orm.mongodb.find - Query: ${JSON.stringify(query)}`);
 
@@ -277,7 +287,11 @@ export class MongoConnector extends AbstractConnector<MongoConnectionOptions> {
       .find(query)
       .sort(sort)
       .skip(Math.max(0, parseInt(options.pagination.page_token || "0")))
-      .limit(Math.max(0, parseInt(options.pagination.limitStr || "100")));
+      .limit(Math.max(0, parseInt(options.pagination.limitStr || "100")))
+      .collation({
+        locale: "en_US",
+        numericOrdering: true,
+      });
 
     const entities: Table[] = [];
     while (await cursor.hasNext()) {

tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres-query-builder.ts

@@ -88,13 +88,20 @@ export class PostgresQueryBuilder {
     if (whereClause && whereClause.endsWith("AND ")) whereClause = whereClause.slice(0, -4);
 
     // ==== ORDER BY =====
-    const orderByClause = `${entityDefinition.options.primaryKey
+    let orderByClause = `${entityDefinition.options.primaryKey
       .slice(1)
       .map(
         (key: string) =>
           `${key} ${(columnsDefinition[key].options.order || "ASC") === "ASC" ? "DESC" : "ASC"}`,
       )}`;
 
+    // ==== ORDER BY CUSTOM COLUMN =====
+    if (options?.sort) {
+      orderByClause = Object.keys(options.sort)
+        .map(key => `${key} ${options.sort[key].toUpperCase()}`)
+        .join(", ");
+    }
+
     // ==== PAGING =====
     let limit = 100;
     let offset = 0;

tdrive/backend/node/src/core/platform/services/database/services/orm/repository/repository.ts

@@ -22,6 +22,12 @@ export type inType = [string, Array<any>];
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 type likeType = [string, any];
 
+type SortDirection = "asc" | "desc";
+
+type SortOption = {
+  [field: string]: SortDirection;
+};
+
 export type FindOptions = {
   pagination?: Pagination;
   $lt?: comparisonType[];
@@ -34,6 +40,7 @@ export type FindOptions = {
   $in?: inType[];
   $nin?: inType[];
   $like?: likeType[];
+  sort?: SortOption;
 };
 
 /**

tdrive/backend/node/src/services/console/service.ts

@@ -35,14 +35,14 @@ export class ConsoleServiceImpl implements TdriveServiceProvider {
 
     this.consoleOptions = {
       type: type,
-      authority: s.authority,
-      client_id: s.client_id,
-      client_secret: s.client_secret,
-      audience: s.audience,
-      issuer: s.issuer,
-      jwks_uri: s.jwks_uri,
-      redirect_uris: s.redirect_uris,
-      disable_account_creation: s.disable_account_creation,
+      authority: s?.authority,
+      client_id: s?.client_id,
+      client_secret: s?.client_secret,
+      audience: s?.audience,
+      issuer: s?.issuer,
+      jwks_uri: s?.jwks_uri,
+      redirect_uris: s?.redirect_uris,
+      disable_account_creation: s?.disable_account_creation,
     };
 
     this.consoleOptions.type = type;

tdrive/backend/node/src/services/documents/services/index.ts

@@ -29,8 +29,10 @@ import {
   DriveFileAccessLevel,
   DriveItemDetails,
   DriveTdriveTab,
+  PaginateDocumentBody,
   RootType,
   SearchDocumentsOptions,
+  SortDocumentsBody,
   TrashType,
 } from "../types";
 import {
@@ -105,6 +107,8 @@ export class DocumentsService {
   browse = async (
     id: string,
     options: SearchDocumentsOptions,
+    sort: SortDocumentsBody,
+    paginate: PaginateDocumentBody,
     context: DriveExecutionContext & { public_token?: string },
   ): Promise<BrowseDetails> => {
     if (isSharedWithMeFolder(id)) {
@@ -112,7 +116,7 @@ export class DocumentsService {
     } else {
       return {
         nextPage: null,
-        ...(await this.get(id, context)),
+        ...(await this.get(id, context, false, sort, paginate)),
       };
     }
   };
@@ -154,6 +158,9 @@ export class DocumentsService {
   get = async (
     id: string,
     context: DriveExecutionContext & { public_token?: string },
+    all?: boolean,
+    sort?: SortDocumentsBody,
+    paginate?: PaginateDocumentBody,
   ): Promise<DriveItemDetails> => {
     if (!context) {
       this.logger.error("invalid context");
@@ -206,7 +213,26 @@ export class DocumentsService {
           )
         ).getEntities();
 
-    //Get children if it is a directory
+    const sortFieldMapping = {
+      name: "name",
+      date: "last_modified",
+      size: "size",
+    };
+    let sortField = {};
+    sortField[sortFieldMapping[sort?.by] || "last_modified"] = sort?.order || "desc";
+
+    const dbType = await globalResolver.database.getConnector().getType();
+
+    // Initialize pagination
+    let pagination;
+
+    if (paginate) {
+      const { page, limit } = paginate;
+      const pageNumber = dbType === "mongodb" ? page : page / limit + 1;
+
+      pagination = new Pagination(`${pageNumber}`, `${limit}`, false);
+    }
+
     let children = isDirectory
       ? (
           await this.repository.find(
@@ -229,7 +255,15 @@ export class DocumentsService {
                     is_in_trash: false,
                   }),
             },
-            {},
+            all
+              ? {}
+              : {
+                  sort: {
+                    is_directory: "desc",
+                    ...sortField,
+                  },
+                  pagination,
+                },
             context,
           )
         ).getEntities()

tdrive/backend/node/src/services/documents/types.ts

@@ -59,6 +59,12 @@ export type SearchDocumentsOptions = {
   pagination?: Paginable;
 };
 
+export type BrowseDocumentsOptions = {
+  filter?: SearchDocumentsBody;
+  sort?: SortDocumentsBody;
+  paginate?: PaginateDocumentBody;
+};
+
 export type SearchDocumentsBody = {
   search?: string;
   company_id?: string;
@@ -73,6 +79,16 @@ export type SearchDocumentsBody = {
   fields?: string[];
 };
 
+export type SortDocumentsBody = {
+  by: string;
+  order: string;
+};
+
+export type PaginateDocumentBody = {
+  page: number;
+  limit: number;
+};
+
 export type DocumentsMessageQueueRequest = {
   item: DriveFile;
   version: FileVersion;

tdrive/backend/node/src/services/documents/web/controllers/documents.ts

@@ -9,16 +9,19 @@ import { CompanyUserRole, PaginationQueryParameters } from "../../../../utils/ty
 import { DriveFile } from "../../entities/drive-file";
 import { FileVersion } from "../../entities/file-version";
 import {
+  BrowseDocumentsOptions,
   CompanyExecutionContext,
   DriveExecutionContext,
   DriveFileAccessLevel,
   DriveItemDetails,
   DriveTdriveTab,
   ItemRequestParams,
+  PaginateDocumentBody,
   ItemRequestByEditingSessionKeyParams,
   RequestParams,
   SearchDocumentsBody,
   SearchDocumentsOptions,
+  SortDocumentsBody,
 } from "../../types";
 import { DriveFileDTO } from "../dto/drive-file-dto";
 import { DriveFileDTOBuilder } from "../../services/drive-file-dto-builder";
@@ -196,7 +199,7 @@ export class DocumentsController {
   browse = async (
     request: FastifyRequest<{
       Params: ItemRequestParams;
-      Body: SearchDocumentsBody;
+      Body: BrowseDocumentsOptions;
       Querystring: PaginationQueryParameters & { public_token?: string };
     }>,
   ): Promise<DriveItemDetails> => {
@@ -204,15 +207,24 @@ export class DocumentsController {
     const { id } = request.params;
 
     const options: SearchDocumentsOptions = {
-      ...request.body,
-      company_id: request.body.company_id || context.company.id,
+      ...request.body.filter,
+      company_id: request.body.filter?.company_id || context.company.id,
       view: DriveFileDTOBuilder.VIEW_SHARED_WITH_ME,
       onlyDirectlyShared: true,
       onlyUploadedNotByMe: true,
     };
 
+    const sortOptions: SortDocumentsBody = request.body.sort;
+    const paginateOptions: PaginateDocumentBody = request.body.paginate;
+
     return {
-      ...(await globalResolver.services.documents.documents.browse(id, options, context)),
+      ...(await globalResolver.services.documents.documents.browse(
+        id,
+        options,
+        sortOptions,
+        paginateOptions,
+        context,
+      )),
     };
   };
 
@@ -446,11 +458,12 @@ export class DocumentsController {
   downloadZip = async (
     request: FastifyRequest<{
       Params: RequestParams;
-      Querystring: { token?: string; items: string; public_token?: string };
+      Querystring: { token?: string; items: string; public_token?: string; is_directory?: boolean };
     }>,
     reply: FastifyReply,
   ): Promise<void> => {
     const context = getDriveExecutionContext(request);
+    const isDirectory = request.query.is_directory || false;
     let ids = (request.query.items || "").split(",");
     const token = request.query.token;
 
@@ -466,6 +479,11 @@ export class DocumentsController {
       ids = items.children.map(item => item.id);
     }
 
+    if (isDirectory) {
+      const items = await globalResolver.services.documents.documents.get(ids[0], context, true);
+      ids = items.children.map(item => item.id);
+    }
+
     try {
       const archive = await globalResolver.services.documents.documents.createZip(ids, context);
 

tdrive/backend/node/test/e2e/documents/documents-pagination-sorting.spec.ts

@@ -0,0 +1,156 @@
+import { describe, beforeAll, afterAll, it, expect } from "@jest/globals";
+import { init, TestPlatform } from "../setup";
+import UserApi from "../common/user-api";
+
+describe("The Documents Browser Window and API", () => {
+  let platform: TestPlatform;
+  let currentUser: UserApi;
+
+  beforeAll(async () => {
+    platform = await init({
+      services: [
+        "webserver",
+        "database",
+        "applications",
+        "search",
+        "storage",
+        "message-queue",
+        "user",
+        "files",
+        "auth",
+        "statistics",
+        "platform-services",
+        "documents",
+      ],
+    });
+    currentUser = await UserApi.getInstance(platform);
+  });
+
+  afterAll(async () => {
+    await platform?.tearDown();
+    platform = null;
+  });
+
+  describe("Pagination and Sorting", () => {
+    it("Should paginate documents correctly", async () => {
+      const myDriveId = "user_" + currentUser.user.id;
+      await currentUser.uploadAllFilesOneByOne(myDriveId);
+
+      let page = 1;
+      const limit = 2;
+      let docs = await currentUser.browseDocuments(myDriveId, {
+        paginate: { page, limit },
+      });
+      expect(docs).toBeDefined();
+      expect(docs.children).toHaveLength(limit);
+
+      page = 2;
+      docs = await currentUser.browseDocuments(myDriveId, {
+        paginate: { page, limit },
+      });
+      expect(docs).toBeDefined();
+      expect(docs.children).toHaveLength(limit);
+
+      page = 3;
+      docs = await currentUser.browseDocuments(myDriveId, {
+        paginate: { page, limit },
+      });
+      expect(docs).toBeDefined();
+      expect(docs.children.length).toBeLessThanOrEqual(limit);
+    });
+
+    it("Should sort documents by name in ascending order", async () => {
+      const myDriveId = "user_" + currentUser.user.id;
+      await currentUser.uploadAllFilesOneByOne(myDriveId);
+
+      const sortBy = "name";
+      const sortOrder = "asc";
+      const docs = await currentUser.browseDocuments(myDriveId, {
+        sort: { by: sortBy, order: sortOrder },
+      });
+      expect(docs).toBeDefined();
+
+      const isSorted = docs.children.every((item, i, arr) => !i || arr[i - 1].name <= item.name);
+      expect(isSorted).toBe(true);
+    });
+
+    it("Should sort documents by name in descending order", async () => {
+      const myDriveId = "user_" + currentUser.user.id;
+      await currentUser.uploadAllFilesOneByOne(myDriveId);
+
+      const sortBy = "name";
+      const sortOrder = "desc";
+      const docs = await currentUser.browseDocuments(myDriveId, {
+        sort: { by: sortBy, order: sortOrder },
+      });
+      expect(docs).toBeDefined();
+
+      const isSorted = docs.children.every((item, i, arr) => !i || arr[i - 1].name >= item.name);
+      expect(isSorted).toBe(true);
+    });
+
+    it("Should sort documents by date in ascending order", async () => {
+      const myDriveId = "user_" + currentUser.user.id;
+      await currentUser.uploadAllFilesOneByOne(myDriveId);
+
+      const sortBy = "date";
+      const sortOrder = "asc";
+      const docs = await currentUser.browseDocuments(myDriveId, {
+        sort: { by: sortBy, order: sortOrder },
+      });
+      expect(docs).toBeDefined();
+
+      const isSorted = docs.children.every(
+        (item, i, arr) => !i || new Date(arr[i - 1].added) <= new Date(item.added),
+      );
+      expect(isSorted).toBe(true);
+    });
+
+    it("Should sort documents by date in descending order", async () => {
+      const myDriveId = "user_" + currentUser.user.id;
+      await currentUser.uploadAllFilesOneByOne(myDriveId);
+
+      const sortBy = "date";
+      const sortOrder = "desc";
+      const docs = await currentUser.browseDocuments(myDriveId, {
+        sort: { by: sortBy, order: sortOrder },
+      });
+      expect(docs).toBeDefined();
+
+      const isSorted = docs.children.every(
+        (item, i, arr) => !i || new Date(arr[i - 1].added) >= new Date(item.added),
+      );
+      expect(isSorted).toBe(true);
+    });
+
+    it("Should sort documents by size in ascending order", async () => {
+      const myDriveId = "user_" + currentUser.user.id;
+      await currentUser.uploadAllFilesOneByOne(myDriveId);
+
+      const sortBy = "size";
+      const sortOrder = "asc";
+      const docs = await currentUser.browseDocuments(myDriveId, {
+        sort: { by: sortBy, order: sortOrder },
+      });
+      expect(docs).toBeDefined();
+
+      const isSorted = docs.children.every((item, i, arr) => !i || arr[i - 1].size <= item.size);
+      expect(isSorted).toBe(true);
+    });
+
+    it("Should sort documents by size in descending order", async () => {
+      const myDriveId = "user_" + currentUser.user.id;
+      await currentUser.uploadAllFilesOneByOne(myDriveId);
+
+      const sortBy = "size";
+      const sortOrder = "desc";
+      const docs = await currentUser.browseDocuments(myDriveId, {
+        sort: { by: sortBy, order: sortOrder },
+      });
+      expect(docs).toBeDefined();
+
+      const isSorted = docs.children.every((item, i, arr) => !i || arr[i - 1].size >= item.size);
+      expect(isSorted).toBe(true);
+    });
+  });
+});

tdrive/backend/node/test/unit/core/services/database/services/orm/connectors/postgres/postgres-query-builder.test.ts

@@ -8,6 +8,20 @@ import {
   comparisonType
 } from "../../../../../../../../../src/core/platform/services/database/services/orm/repository/repository";
 
+type SortDirection = "asc" | "desc";
+
+type SortOption = {
+  [field: string]: SortDirection;
+};
+
+interface FindOptions {
+  sort?: SortOption;
+  pagination?: {
+    limitStr: string;
+    page_token: string;
+  };
+}
+
 describe('The PostgresQueryBuilder', () => {
 
   const subj: PostgresQueryBuilder = new PostgresQueryBuilder("");
@@ -103,6 +117,43 @@ describe('The PostgresQueryBuilder', () => {
     expect(query[1]).toEqual([entity.company_id, entity.id]);
   });
 
+  test('buildSelect query with sort option', async () => {
+    //given
+    const options = { sort: { added: "asc" as SortDirection, id: "desc" as SortDirection } };
+
+    //when
+    const query = subj.buildSelect(TestDbEntity, null, options);
+
+    //then
+    expect(normalizeWhitespace(query[0] as string)).toBe(`SELECT * FROM "test_table" ORDER BY added ASC, id DESC LIMIT 100 OFFSET 0`);
+    expect(query[1]).toEqual([]);
+  });
+
+  test('buildSelect query with pagination', async () => {
+    //given
+    const options = { pagination: { limitStr: "50", page_token: "2" } };
+
+    //when
+    const query = subj.buildSelect(TestDbEntity, null, options);
+
+    //then
+    expect(normalizeWhitespace(query[0] as string)).toBe(`SELECT * FROM "test_table" ORDER BY id DESC LIMIT 50 OFFSET 100`);
+    expect(query[1]).toEqual([]);
+  });
+
+  test('buildSelect query with filter, sort, and pagination', async () => {
+    //given
+    const filter = { id: randomUUID(), company_id: randomUUID() };
+    const options = { sort: { added: "asc" as SortDirection, id: "desc" as SortDirection }, pagination: { limitStr: "10", page_token: "3" } };
+
+    //when
+    const query = subj.buildSelect(TestDbEntity, filter as { [key: string]: any }, options);
+
+    //then
+    expect(normalizeWhitespace(query[0] as string)).toBe(`SELECT * FROM "test_table" WHERE id = $1 AND company_id = $2 ORDER BY added ASC, id DESC LIMIT 10 OFFSET 30`);
+    expect(query[1]).toEqual([filter.id, filter.company_id]);
+  });
+
   test('buildInsert', async () => {
     //given
     const entity = newTestDbEntity();