From 0aabc2e26e7dd589f583982ca6019a3bfdcf9bd1 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Tue, 26 Mar 2024 12:06:22 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20cli:=20add=20document=20re-indexing?= =?UTF-8?q?=20and=20repair=20(#438)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/cli/cmds/search_cmds/index_all.ts | 100 +++++++++++++++++- 1 file changed, 99 insertions(+), 1 deletion(-) diff --git a/tdrive/backend/node/src/cli/cmds/search_cmds/index_all.ts b/tdrive/backend/node/src/cli/cmds/search_cmds/index_all.ts index f03ac7e7..d0b89a79 100644 --- a/tdrive/backend/node/src/cli/cmds/search_cmds/index_all.ts +++ b/tdrive/backend/node/src/cli/cmds/search_cmds/index_all.ts @@ -20,10 +20,13 @@ import CompanyUser, { TYPE as CompanyUserTYPE } from "../../../services/user/ent import runWithPlatform from "../../lib/run_with_platform"; import SearchRepository from "src/core/platform/services/search/repository"; import parseYargsCommaSeparatedStringArray from "../../utils/yargs_comma_array"; +import globalResolver from "../../../services/global-resolver"; +import { couldGetKeywordsOfFile, getKeywordsOfFile } from "../../../services/documents/utils"; type Options = { spinner: ora.Ora; repairEntities: boolean; + filterDocumentsByUserEMail: string[]; }; const repairEntitiesArgumentDetails = []; @@ -56,7 +59,7 @@ abstract class ReindexerCLICommand { protected readonly search: SearchServiceAPI; constructor( protected readonly platform: TdrivePlatform, - private readonly options: Options, + protected readonly options: Options, private readonly table: string, private readonly entity: EntityTarget, ) { @@ -191,6 +194,90 @@ class DocumentsReindexerCLICommand extends ReindexerCLICommand { constructor(platform: TdrivePlatform, options: Options) { super(platform, options, DriveFileTYPE, DriveFile); } + protected override async prepareFindOptionsForItemsToReIndex(): Promise { + if (this.options.filterDocumentsByUserEMail.length === 0) + throw new Error( + "No creator e-mail specified. This would affect all documents. To do this on purpose, add '--filterDocumentsByUserEMail all'", + ); + if (this.options.filterDocumentsByUserEMail.join("-") === "all") { + this.statusWarn("All users included in document re-index"); + return {}; + } + if (this.options.filterDocumentsByUserEMail.indexOf("all") > -1) + throw new Error( + "If specifying 'all' to include all users, don't include another email argument.", + ); + const userRepo = await this.database.getRepository(UserTYPE, User); + const emailsToIds = new Map(); + const rawUserIds = []; + this.options.filterDocumentsByUserEMail.forEach(email => { + if (email.match(/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i)) rawUserIds.push(email); + else emailsToIds.set(email, false); + }); + await iterateOverRepoPages( + userRepo, + async entities => { + entities.forEach(({ id, email_canonical }) => emailsToIds.set(email_canonical, id)); + }, + { $in: [["email_canonical", this.options.filterDocumentsByUserEMail]] }, + ); + const failedToFindEmails = []; + for (const [email, id] of emailsToIds) if (id === false) failedToFindEmails.push(email); + if (failedToFindEmails.length > 0) + throw new Error("No user id found for email(s): " + JSON.stringify(failedToFindEmails)); + this.statusInfo("Documents of user(s):"); + for (const [email, id] of emailsToIds) this.statusInfo(`\t- ${email}: ${id}`); + for (const id of rawUserIds) this.statusInfo(`\t- ${id}`); + return { $in: [["creator", [...rawUserIds, ...emailsToIds.values()]]] }; + } + protected override async repairEntities(findOptions: FindOptions): Promise { + const repository = await this.dbRepository(); + await iterateOverRepoPages( + repository, + async entities => { + for (const entity of entities) { + if (entity.is_directory) + // || entity.is_in_trash) // why not index trash too after all + continue; + this.statusInfo( + `-> Downloading ${entity.name} (${entity.size}b id: ${entity.id} creator: ${entity.creator}`, + ); + try { + if ( + entity.size === 0 || + !couldGetKeywordsOfFile( + entity.last_version_cache.file_metadata.mime, + entity.last_version_cache.file_metadata.name || entity.name, + ) + ) { + this.statusInfo("\tSkipping, 0 size or couldGetKeywordsOfFile returned nope"); + continue; + } + const storedFile = await globalResolver.services.files.download( + entity.last_version_cache.file_metadata.external_id, + { company: { id: entity.company_id }, user: null }, + ); + const content_keywords = await getKeywordsOfFile( + storedFile.mime, + storedFile.name, + storedFile.file, + ); + if (storedFile.size != entity.size) + this.statusWarn( + `Warning, file ${entity.id} (${entity.name}) has size ${entity.size} in DB but ${storedFile.size} when downloaded`, + ); + if (content_keywords !== entity.content_keywords) { + entity.content_keywords = content_keywords; + repository.save(entity); + } + } catch (err) { + this.statusFail(err.stack); + } + } + }, + findOptions, + ); + } } RepositoryNameToCTOR.set( "documents", @@ -221,17 +308,28 @@ const command: yargs.CommandModule = { ), group: reindexingArgumentGroupTitle, }, + filterDocumentsByUserEMail: { + type: "string", + alias: "e", + description: + "When processing documents, limit to those owned by the users with the provided comma separated e-mails", + group: "Filtering for documents repository", + }, }, handler: async argv => { const repositories = parseYargsCommaSeparatedStringArray( argv[repositoryArgumentName] as string /* ignore typechecker */, ); + const filterDocumentsByUserEMail = parseYargsCommaSeparatedStringArray( + argv.filterDocumentsByUserEMail as string /* ignore typechecker */, + ); runWithPlatform("Re-index", async ({ spinner, platform }) => { try { for (const repositoryName of repositories) await RepositoryNameToCTOR.get(repositoryName)(platform, { spinner, repairEntities: !!argv.repairEntities, + filterDocumentsByUserEMail, }).run(); } catch (err) { spinner.fail(err.stack || err);