From 77e485f9dffaa02a942997413bf3320ee157e4a1 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Wed, 27 Mar 2024 13:24:30 +0100 Subject: [PATCH] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20=20CLI=20refactoring:=20re?= =?UTF-8?q?named=20files=20to=20kebab=20case,=20extracted=20iterateOverRep?= =?UTF-8?q?oPages=20and=20waitTimeout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../{index_all.ts => index-all.ts} | 62 +++++++++++-------- .../utils/iterate-over-repository-pages.ts | 27 ++++++++ .../node/src/cli/utils/wait-timeout.ts | 2 + ...gs_comma_array.ts => yargs-comma-array.ts} | 0 4 files changed, 66 insertions(+), 25 deletions(-) rename tdrive/backend/node/src/cli/cmds/search_cmds/{index_all.ts => index-all.ts} (91%) create mode 100644 tdrive/backend/node/src/cli/utils/iterate-over-repository-pages.ts create mode 100644 tdrive/backend/node/src/cli/utils/wait-timeout.ts rename tdrive/backend/node/src/cli/utils/{yargs_comma_array.ts => yargs-comma-array.ts} (100%) diff --git a/tdrive/backend/node/src/cli/cmds/search_cmds/index_all.ts b/tdrive/backend/node/src/cli/cmds/search_cmds/index-all.ts similarity index 91% rename from tdrive/backend/node/src/cli/cmds/search_cmds/index_all.ts rename to tdrive/backend/node/src/cli/cmds/search_cmds/index-all.ts index 42975677..1f789ed3 100644 --- a/tdrive/backend/node/src/cli/cmds/search_cmds/index_all.ts +++ b/tdrive/backend/node/src/cli/cmds/search_cmds/index-all.ts @@ -3,14 +3,11 @@ import ora from "ora"; import { TdrivePlatform } from "../../../core/platform/platform"; import { DatabaseServiceAPI } from "../../../core/platform/services/database/api"; -import { Pagination } from "../../../core/platform/framework/api/crud-service"; import User, { TYPE as UserTYPE } from "../../../services/user/entities/user"; import { DriveFile, TYPE as DriveFileTYPE } from "../../../services/documents/entities/drive-file"; -import Repository, { - FindFilter, -} from "../../../core/platform/services/database/services/orm/repository/repository"; +import Repository from "../../../core/platform/services/database/services/orm/repository/repository"; import { EntityTarget, FindOptions, @@ -19,7 +16,9 @@ import { import CompanyUser, { TYPE as CompanyUserTYPE } from "../../../services/user/entities/company_user"; import runWithPlatform from "../../lib/run_with_platform"; import SearchRepository from "src/core/platform/services/search/repository"; -import parseYargsCommaSeparatedStringArray from "../../utils/yargs_comma_array"; +import parseYargsCommaSeparatedStringArray from "../../utils/yargs-comma-array"; +import waitTimeoutMS from "../../utils/wait-timeout"; +import iterateOverRepoPages from "../../utils/iterate-over-repository-pages"; import globalResolver from "../../../services/global-resolver"; import { couldGetKeywordsOfFile, getKeywordsOfFile } from "../../../services/documents/utils"; @@ -30,26 +29,6 @@ type Options = { }; const repairEntitiesArgumentDetails = []; -const waitTimeoutMS = (ms: number) => ms > 0 && new Promise(r => setTimeout(r, ms)); -const defaultPageSize = "100"; - -async function iterateOverRepoPages( - repository: Repository, - forEachPage: (entities: Entity[]) => Promise, - findOptions: FindOptions = {}, - filter: FindFilter = {}, - pageSizeAsStringForReasons: string = defaultPageSize, - delayPerPageMS: number = 200, -) { - let page: Pagination = { limitStr: pageSizeAsStringForReasons }; - do { - const options = { ...findOptions, pagination: page }; - const list = await repository.find(filter, options, undefined); - await forEachPage(list.getEntities()); - page = list.nextPage as Pagination; - await waitTimeoutMS(delayPerPageMS); - } while (page.page_token); -} /** This is an abstract base class for re-index cli commands; it stores runtime options * into fields; runs repair first if request and re-indexes generically from the database @@ -231,6 +210,21 @@ class DocumentsReindexerCLICommand extends ReindexerCLICommand { return { $in: [["creator", [...rawUserIds, ...emailsToIds.values()]]] }; } protected override async repairEntities(findOptions: FindOptions): Promise { + /* + //Todo: + - [ ] Batch saves + - [ ] Refactor repair into mapping + - [ ] Check save does upsert + - [ ] Remove logs + - [ ] Document this: + For each item in DB + Download + - [ ] Decide if download fails ? Flag file ? Delete entity ? + If keywords changed + Save to DB + - [ ] Delete file + */ + const repository = await this.dbRepository(); await iterateOverRepoPages( repository, @@ -286,6 +280,24 @@ RepositoryNameToCTOR.set( repairEntitiesArgumentDetails.push( "documents: Download and re-extract keywords before re-indexing", ); +class _BufferedAction { + private buffer: Entity[]; + constructor( + private readonly batchSize: number, + private readonly action: (entities: Entity[]) => Promise, + ) {} + public async flush() { + const buffer = this.buffer; + if (!buffer.length) return; + this.buffer = new Array(this.batchSize); + return this.action(buffer); + } + public async save(entity: Entity) { + this.buffer.push(entity); + if (this.buffer.length >= this.batchSize) await this.flush(); + } +} +import { logger } from "../../../core/platform/framework/logger"; const reindexingArgumentGroupTitle = "Re-indexing options"; const repositoryArgumentName = "repository"; diff --git a/tdrive/backend/node/src/cli/utils/iterate-over-repository-pages.ts b/tdrive/backend/node/src/cli/utils/iterate-over-repository-pages.ts new file mode 100644 index 00000000..b7140dd3 --- /dev/null +++ b/tdrive/backend/node/src/cli/utils/iterate-over-repository-pages.ts @@ -0,0 +1,27 @@ +import { Pagination } from "../../core/platform/framework/api/crud-service"; +import Repository, { + FindFilter, +} from "../../core/platform/services/database/services/orm/repository/repository"; +import { FindOptions } from "../../core/platform/services/search/api"; + +import waitTimeoutMS from "./wait-timeout"; + +const defaultPageSize = "100"; + +export default async function iterateOverRepositoryPages( + repository: Repository, + forEachPage: (entities: Entity[]) => Promise, + findOptions: FindOptions = {}, + filter: FindFilter = {}, + pageSizeAsStringForReasons: string = defaultPageSize, + delayPerPageMS: number = 200, +) { + let page: Pagination = { limitStr: pageSizeAsStringForReasons }; + do { + const options = { ...findOptions, pagination: page }; + const list = await repository.find(filter, options, undefined); + await forEachPage(list.getEntities()); + page = list.nextPage as Pagination; + await waitTimeoutMS(delayPerPageMS); + } while (page.page_token); +} diff --git a/tdrive/backend/node/src/cli/utils/wait-timeout.ts b/tdrive/backend/node/src/cli/utils/wait-timeout.ts new file mode 100644 index 00000000..c71cee1f --- /dev/null +++ b/tdrive/backend/node/src/cli/utils/wait-timeout.ts @@ -0,0 +1,2 @@ +/** Create a promise that resolves after `ms` milliseconds */ +export default (ms: number) => ms > 0 && new Promise(r => setTimeout(r, ms)); diff --git a/tdrive/backend/node/src/cli/utils/yargs_comma_array.ts b/tdrive/backend/node/src/cli/utils/yargs-comma-array.ts similarity index 100% rename from tdrive/backend/node/src/cli/utils/yargs_comma_array.ts rename to tdrive/backend/node/src/cli/utils/yargs-comma-array.ts