🐛 Not all the data is shown in the shared with me section (#376)

* 🐛Not all the data is shown in the shared with me section

tdrive/backend/node/src/core/platform/services/search/adapters/elasticsearch/elastic-open-search-adapter.ts

@@ -283,11 +283,12 @@ export default class ESAndOpenSearch extends SearchAdapter implements SearchAdap
     const esParamsWithScroll = {
       ...esParams,
       size: parseInt(options.pagination.limitStr || "100"),
-      scroll: "1m",
+      scroll: "10m",
     };
 
     let esResponse: any;
 
+    let nextToken;
     if (options.pagination.page_token) {
       // eslint-disable-next-line @typescript-eslint/ban-ts-comment
       // @ts-ignore
@@ -297,18 +298,23 @@ export default class ESAndOpenSearch extends SearchAdapter implements SearchAdap
         },
         esOptions,
       );
+      //the scroll token is also the same, and we do not get it from response
+      nextToken = options.pagination.page_token;
     } else {
       // eslint-disable-next-line @typescript-eslint/ban-ts-comment
       // @ts-ignore
       esResponse = await this.client.search(esParamsWithScroll, esOptions);
+      nextToken = esResponse.body?._scroll_id || "";
     }
 
     if (esResponse.statusCode !== 200) {
       logger.error(`${this.name} -  ${JSON.stringify(esResponse.body)}`);
     }
 
-    const nextToken = esResponse.body?._scroll_id || "";
     const hits = esResponse.body?.hits?.hits || [];
+    if (hits.length == 0) {
+      nextToken = false;
+    }
 
     logger.debug(`${this.name} got response: ${JSON.stringify(esResponse)}`);
 

tdrive/backend/node/src/core/platform/services/search/adapters/mongosearch/index.ts

@@ -200,9 +200,10 @@ export default class MongoSearch extends SearchAdapter implements SearchAdapterI
 
     const { query, sort, project } = buildSearchQuery<EntityType>(entityType, filters, options);
 
+    logger.info(`Search query: ${JSON.stringify(query)}`);
     console.log(query);
 
-    let cursor = collection.find({ ...query }).sort(sort);
+    let cursor = collection.find(query).sort(sort);
     if (project) {
       cursor = cursor.project(project);
     }
@@ -230,7 +231,7 @@ export default class MongoSearch extends SearchAdapter implements SearchAdapterI
 
     const nextToken =
       entities.length === parseInt(options.pagination.limitStr) &&
-      (parseInt(options.pagination.page_token) + 1).toString(10);
+      (parseInt(options.pagination.page_token || "0") + 1).toString(10);
     const nextPage: Paginable = new Pagination(nextToken, options.pagination.limitStr || "100");
 
     logger.info(`Found ${entities.length} results on entity ${searchPrefix}${index}`);

tdrive/backend/node/src/core/platform/services/search/repository.ts

@@ -40,7 +40,7 @@ export default class SearchRepository<EntityType> {
         options,
       );
 
-      //2. Get database original objects from theses primary keys
+      //2. Get database original objects from these primary keys
       for (const searchEntity of searchResults.getEntities().sort((a, b) => b.score - a.score)) {
         const sourceEntity = await repository.findOne(searchEntity.primaryKey, {}, context);
         if (sourceEntity) {

tdrive/backend/node/src/services/documents/services/index.ts

@@ -1,6 +1,10 @@
 import SearchRepository from "../../../core/platform/services/search/repository";
 import { getLogger, logger, TdriveLogger } from "../../../core/platform/framework";
-import { CrudException, ListResult } from "../../../core/platform/framework/api/crud-service";
+import {
+  CrudException,
+  ListResult,
+  Pagination,
+} from "../../../core/platform/framework/api/crud-service";
 import Repository, {
   comparisonType,
   inType,
@@ -96,13 +100,7 @@ export class DocumentsService {
     context: DriveExecutionContext & { public_token?: string },
   ): Promise<BrowseDetails> => {
     if (isSharedWithMeFolder(id)) {
-      const children = await this.search(options, context);
+      return this.sharedWithMe(options, context);
-      return {
-        access: "read",
-        children: children.getEntities(),
-        nextPage: children.nextPage,
-        path: [] as Array<DriveFile>,
-      };
     } else {
       return {
         nextPage: null,
@@ -111,6 +109,25 @@ export class DocumentsService {
     }
   };
 
+  sharedWithMe = async (
+    options: SearchDocumentsOptions,
+    context: DriveExecutionContext & { public_token?: string },
+  ): Promise<BrowseDetails> => {
+    const result = [];
+    let fileList: ListResult<DriveFile>;
+    do {
+      fileList = await this.search(options, context);
+      result.push(...fileList.getEntities());
+      options.pagination = fileList.nextPage;
+    } while (fileList.nextPage?.page_token);
+    return {
+      access: "read",
+      children: result,
+      nextPage: null,
+      path: [] as Array<DriveFile>,
+    };
+  };
+
   userQuota = async (context: CompanyExecutionContext): Promise<number> => {
     const children = await this.repository.find({
       parent_id: "user_" + context.user.id,
@@ -955,9 +972,9 @@ export class DocumentsService {
     const result = await this.searchRepository.search(
       {},
       {
-        pagination: {
+        pagination: options.pagination
-          limitStr: "100",
+          ? Pagination.fromPaginable(options.pagination)
-        },
+          : new Pagination(),
         $in: [
           ...(options.onlyDirectlyShared ? [["access_entities", [context.user.id]] as inType] : []),
           ...(options.company_id ? [["company_id", [options.company_id]] as inType] : []),
@@ -999,7 +1016,7 @@ export class DocumentsService {
     if (!options.onlyDirectlyShared) {
       const filteredResult = await this.filter(result.getEntities(), async item => {
         try {
-          // Check access for each item
+          //skip all the fiels
           return await checkAccess(item.id, null, "read", this.repository, context);
         } catch (error) {
           this.logger.warn("failed to check item access", error);
@@ -1017,7 +1034,6 @@ export class DocumentsService {
 
       return new ListResult(result.type, filteredResult, result.nextPage);
     }
-
     return result;
   };
 

tdrive/backend/node/src/services/documents/types.ts

@@ -52,6 +52,7 @@ export type SearchDocumentsOptions = {
   fields?: string[];
   onlyDirectlyShared?: boolean;
   onlyUploadedNotByMe?: boolean;
+  pagination?: Paginable;
 };
 
 export type SearchDocumentsBody = {

tdrive/backend/node/src/services/documents/web/controllers/documents.ts

@@ -6,9 +6,9 @@ import { File } from "../../../../services/files/entities/file";
 import { UploadOptions } from "../../../../services/files/types";
 import globalResolver from "../../../../services/global-resolver";
 import {
+  CompanyUserRole,
   PaginationQueryParameters,
   ResourceWebsocket,
-  CompanyUserRole,
 } from "../../../../utils/types";
 import { DriveFile } from "../../entities/drive-file";
 import { FileVersion } from "../../entities/file-version";
@@ -199,48 +199,10 @@ export class DocumentsController {
       onlyUploadedNotByMe: true,
     };
 
-    const data = {
+    return {
       ...(await globalResolver.services.documents.documents.browse(id, options, context)),
-      websockets: request.currentUser?.id
+      websockets: [],
-        ? globalResolver.platformServices.realtime.sign(
-            [{ room: `/companies/${context.company.id}/documents/item/${id}` }],
-            request.currentUser?.id,
-          )
-        : [],
     };
-
-    return data;
-  };
-
-  sharedWithMe = async (
-    request: FastifyRequest<{
-      Params: RequestParams;
-      Body: SearchDocumentsBody;
-      Querystring: { public_token?: string };
-    }>,
-  ): Promise<ListResult<DriveFileDTO>> => {
-    try {
-      const context = getDriveExecutionContext(request);
-
-      const options: SearchDocumentsOptions = {
-        ...request.body,
-        company_id: request.body.company_id || context.company.id,
-        view: DriveFileDTOBuilder.VIEW_SHARED_WITH_ME,
-        onlyDirectlyShared: true,
-        onlyUploadedNotByMe: true,
-      };
-
-      if (!Object.keys(options).length) {
-        this.throw500Search();
-      }
-
-      const fileList = await globalResolver.services.documents.documents.search(options, context);
-
-      return this.driveFileDTOBuilder.build(fileList, context, options.fields, options.view);
-    } catch (error) {
-      logger.error({ error: `${error}` }, "error while searching for document");
-      this.throw500Search();
-    }
   };
 
   /**

tdrive/backend/node/src/services/documents/web/routes.ts

@@ -107,13 +107,6 @@ const routes: FastifyPluginCallback = (fastify: FastifyInstance, _options, next)
     handler: documentsController.search.bind(documentsController),
   });
 
-  fastify.route({
-    method: "POST",
-    url: `${baseUrl}/shared-with-me`,
-    preValidation: [fastify.authenticate],
-    handler: documentsController.sharedWithMe.bind(documentsController),
-  });
-
   fastify.route({
     method: "POST",
     url: `${baseUrl}/browse/:id`,

tdrive/backend/node/test/e2e/common/user-api.ts

@@ -343,15 +343,15 @@ export default class UserApi {
   ) {
     const response = await this.platform.app.inject({
       method: "POST",
-      url: `${UserApi.DOC_URL}/companies/${this.platform.workspace.company_id}/shared-with-me`,
+      url: `${UserApi.DOC_URL}/companies/${this.platform.workspace.company_id}/browse/shared_with_me`,
       headers: {
         authorization: `Bearer ${this.jwt}`
       },
       payload
     });
 
-    return deserialize<SearchResultMockClass>(
+    return deserialize<DriveItemDetailsMockClass>(
-      SearchResultMockClass,
+      DriveItemDetailsMockClass,
       response.body);
   };
 

tdrive/backend/node/test/e2e/documents/documents-browser.spec.ts

@@ -126,10 +126,32 @@ describe("The Documents Browser Window and API", () => {
       await new Promise(r => setTimeout(r, 3000));
 
       //then file become searchable
-      expect((await anotherUser.browseDocuments("shared_with_me", {})).children).toHaveLength(1);
+      expect((await anotherUser.browseDocuments("shared_with_me", {pageSize: 1})).children).toHaveLength(1);
+    });
+
+    it("Share With Me should return all the files that was share by user at one", async () => {
+      const sharedWIthMeFolder = "shared_with_me";
+      const oneUser = await UserApi.getInstance(platform, true, {companyRole: "admin"});
+      const anotherUser = await UserApi.getInstance(platform, true, {companyRole: "admin"});
+
+      let files = await oneUser.uploadAllFilesOneByOne();
+
+      await anotherUser.uploadAllFilesOneByOne();
+      await new Promise(r => setTimeout(r, 5000));
+
+      //give permissions to the file
+      files[2].access_info.entities.push({
+        type: "user",
+        id: anotherUser.user.id,
+        level: "read",
+        grantor: null,
+      });
+      await oneUser.updateDocument(files[2].id, files[2]);
+      await new Promise(r => setTimeout(r, 3000));
+
+      //then file become searchable
+      expect((await anotherUser.browseDocuments("shared_with_me", {pagination: {limitStr: 100}})).children).toHaveLength(1);
     });
   });
-
-
 });
 

tdrive/backend/node/test/e2e/documents/documents-search.spec.ts

@@ -106,7 +106,7 @@ describe("the Drive Search feature", () => {
     const searchResponse = await userTwo.sharedWithMeDocuments({});
 
     //then::
-    expect(searchResponse.entities?.length).toEqual(1);
+    expect(searchResponse.children?.length).toEqual(1);
   })
 
   it("did search for an item that doesn't exist", async () => {
@@ -146,7 +146,7 @@ describe("the Drive Search feature", () => {
 
   it("did search a file shared by another user", async () => {
     //then file become searchable
-    expect((await userTwo.sharedWithMeDocuments({})).entities).toHaveLength(1);
+    expect((await userTwo.sharedWithMeDocuments({})).children).toHaveLength(1);
   });
 
   it("did search a file by file owner", async () => {

tdrive/backend/node/test/http/elasticsearch.http

@@ -20,13 +20,61 @@ Content-Type: application/json
   }
 }
 
+### Query data
+GET https://localhost:9200/drive_files/_search
+Authorization: Basic admin admin
+Content-Type: application/json
+
+{
+  "query": {
+    "bool": {
+      "boost": 1,
+      "must": [
+        {
+          "bool": {
+            "should": [
+              {
+                "match": {
+                  "access_entities": {
+                    "query": "7f2e4e80-d4be-11ee-8a11-0544b51cf428",
+                    "operator": "AND"
+                  }
+                }
+              }
+            ],
+            "minimum_should_match": 1
+          }
+        },
+        {
+          "bool": {
+            "should": [
+              {
+                "match": {
+                  "company_id": {
+                    "query": "7efb7eb1-d4be-11ee-8a11-0544b51cf428",
+                    "operator": "AND"
+                  }
+                }
+              }
+            ],
+            "minimum_should_match": 1
+          }
+        }
+      ]
+    }
+  }
+}
 
 
 ### Check that it is created
 GET https://localhost:9200/_cat/indices
+Authorization: Basic admin admin
+Content-Type: application/json
 
 ### GET index info
-GET https://localhost:9200/drive_files
+GET https://localhost:9200/user
+Authorization: Basic admin admin
+Content-Type: application/json
 
 ### GET data to the index
 GET https://localhost:9200/drive_files/_doc/8c03c5a1-0146-11ee-82d9-f503f8a58e9d