diff --git a/tdrive/backend/node/src/cli/cmds/search_cmds/index-all.ts b/tdrive/backend/node/src/cli/cmds/search_cmds/index-all.ts index 5474270a..d87383e2 100644 --- a/tdrive/backend/node/src/cli/cmds/search_cmds/index-all.ts +++ b/tdrive/backend/node/src/cli/cmds/search_cmds/index-all.ts @@ -3,6 +3,7 @@ import yargs from "yargs"; import type { TdrivePlatform } from "../../../core/platform/platform"; import runWithPlatform from "../../lib/run_with_platform"; import parseYargsCommaSeparatedStringArray from "../../utils/yargs-comma-array"; +import runWithLoggerLevel from "../../utils/run-with-logger-level"; import type BaseReindexer from "./index-all/base-reindexer"; import type ReindexerOptions from "./index-all/reindexer-options"; @@ -58,6 +59,14 @@ const command: yargs.CommandModule = { "When processing documents, limit to those owned by the users with the provided comma separated e-mails", group: "Filtering for documents repository", }, + verboseDuringRun: { + type: "boolean", + alias: "V", + description: + "Set to verbose logging except for startup and shutdown of Platform. Use flag once for 'info' level, twice for 'debug'.", + group: "Output", + count: true, + }, }, handler: async argv => { const repositories = parseYargsCommaSeparatedStringArray( @@ -67,19 +76,28 @@ const command: yargs.CommandModule = { argv.filterDocumentsByUserEMail as string /* ignore typechecker */, ); await runWithPlatform("Re-index", async ({ spinner, platform }) => { - try { - for (const repositoryName of repositories) - await RepositoryNameToCTOR.get(repositoryName) - .ctor(platform, { - spinner, - repairEntities: !!argv.repairEntities, - filterDocumentsByUserEMail, - }) - .run(); - } catch (err) { - spinner.fail(err.stack || err); - return 1; - } + return await runWithLoggerLevel( + argv.verboseDuringRun + ? (argv.verboseDuringRun as number) > 1 + ? "debug" + : "info" + : undefined, + async () => { + try { + for (const repositoryName of repositories) + await RepositoryNameToCTOR.get(repositoryName) + .ctor(platform, { + spinner, + repairEntities: !!argv.repairEntities, + filterDocumentsByUserEMail, + }) + .run(); + } catch (err) { + spinner.fail(err.stack || err); + return 1; + } + }, + ); }); }, }; diff --git a/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/base-reindexer.ts b/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/base-reindexer.ts index 0ac1aff4..e8fb8cac 100644 --- a/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/base-reindexer.ts +++ b/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/base-reindexer.ts @@ -12,12 +12,16 @@ import waitTimeoutMS from "../../../utils/wait-timeout"; import iterateOverRepoPages from "../../../utils/iterate-over-repository-pages"; import type ReindexerOptions from "./reindexer-options"; +const pluralificationate = (noun: string) => noun.replace(/s$/i, "").replace(/y$/i, "ie") + "s"; + /** This is an abstract base class for re-index cli commands; it stores runtime options * into fields; runs repair first if requested and re-indexes generically from the database * to search services. */ export default abstract class BaseReindexer { protected readonly database: DatabaseServiceAPI; protected readonly search: SearchServiceAPI; + private readonly tablePlural: string; + constructor( protected readonly platform: TdrivePlatform, protected readonly options: ReindexerOptions, @@ -26,12 +30,15 @@ export default abstract class BaseReindexer { ) { this.database = this.platform.getProvider("database"); this.search = this.platform.getProvider("search"); + this.tablePlural = pluralificationate(this.table); } + private _dbRepo: Repository; protected async dbRepository() { return (this._dbRepo = this._dbRepo || (await this.database.getRepository(this.table, this.entity))); } + private _searchRepo: SearchRepository; protected async searchRepository() { return (this._searchRepo = @@ -39,19 +46,19 @@ export default abstract class BaseReindexer { } protected statusStart(info: string) { - this.options.spinner.start(`${this.table} > ${info}`); + this.options.spinner.start(`${this.tablePlural} > ${info}`); } protected statusSucceed(info: string) { - this.options.spinner.succeed(`${this.table} > ${info}`); + this.options.spinner.succeed(`${this.tablePlural} > ${info}`); } protected statusWarn(info: string) { - this.options.spinner.warn(`${this.table} > ${info}`); + this.options.spinner.warn(`${this.tablePlural} > ${info}`); } protected statusFail(info: string) { - this.options.spinner.fail(`${this.table} > ${info}`); + this.options.spinner.fail(`${this.tablePlural} > ${info}`); } protected statusInfo(info: string) { - this.options.spinner.info(`${this.table} > ${info}`); + this.options.spinner.info(`${this.tablePlural} > ${info}`); } /** Override in sub-classes to translate a page from database to search entities. @@ -71,7 +78,7 @@ export default abstract class BaseReindexer { * on the entity type. */ protected async repairEntities(_findOptions: FindOptions): Promise { - this.statusWarn(`repairEntities > No repair action for ${this.table}`); + this.statusWarn(`repairEntities > No repair action for ${this.tablePlural}`); } /** Override in a sub-class to completely replace the re-indexing logic. @@ -89,12 +96,12 @@ export default abstract class BaseReindexer { entities = await this.mapEntitiesToReIndexFromDBToSearch(entities); await this.search.upsert(entities); count += entities.length; - this.statusStart(`Indexed ${count} ${this.table}...`); + this.statusStart(`Indexed ${count} ${this.tablePlural}...`); }, findOptions, ); - if (count === 0) this.statusWarn(`Index ${this.table} finished; but 0 items included`); - else this.statusSucceed(`${count} ${this.table} indexed`); + if (count === 0) this.statusWarn(`Index ${this.tablePlural} finished; but 0 items included`); + else this.statusSucceed(`${count} ${this.tablePlural} indexed`); const giveFlushAChanceDurationMS = 10000; this.statusStart(`Emptying flush (${giveFlushAChanceDurationMS / 1000}s)...`); await waitTimeoutMS(giveFlushAChanceDurationMS); diff --git a/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/documents-reindexer.ts b/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/documents-reindexer.ts index d071fe0f..035379b5 100644 --- a/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/documents-reindexer.ts +++ b/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/documents-reindexer.ts @@ -60,6 +60,8 @@ export default class DocumentsReindexer extends BaseReindexer { "Download and re-extract keywords before re-indexing"; protected override async repairEntities(findOptions: FindOptions): Promise { const repository = await this.dbRepository(); + let count = 0, + countChanged = 0; await iterateOverRepoPages( repository, async entities => { @@ -70,6 +72,7 @@ export default class DocumentsReindexer extends BaseReindexer { this.statusInfo( `-> Downloading ${entity.name} (${entity.size}b id: ${entity.id} creator: ${entity.creator}`, ); + count++; try { if ( entity.size === 0 || @@ -97,11 +100,13 @@ export default class DocumentsReindexer extends BaseReindexer { if (content_keywords !== entity.content_keywords) { entity.content_keywords = content_keywords; repository.save(entity); + countChanged++; } } catch (err) { this.statusFail(err.stack); } } + this.statusStart(`Repaired ${count} documents (${countChanged} changed)`); }, findOptions, ); diff --git a/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/users-reindexer.ts b/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/users-reindexer.ts index c6b07aa9..0ce70dab 100644 --- a/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/users-reindexer.ts +++ b/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/users-reindexer.ts @@ -24,7 +24,8 @@ export default class UsersReindexer extends BaseReindexer { CompanyUserTYPE, CompanyUser, ); - let count = 0; + let count = 0, + countChanged = 0; const repository = await this.dbRepository(); await iterateOverRepoPages(repository, async entities => { for (const user of entities) { @@ -33,11 +34,18 @@ export default class UsersReindexer extends BaseReindexer { user.cache ||= { companies: [] }; user.cache.companies = companies.getEntities().map(company => company.group_id); const newCache = JSON.stringify(user.cache); - if (prevCache != newCache) await repository.save(user, undefined); + if (prevCache != newCache) { + await repository.save(user, undefined); + countChanged++; + } } count += entities.length; - this.statusStart(`repairEntities > Adding companies to cache of ${count} users...`); + this.statusStart( + `repairEntities > Adding companies to cache of ${count} users (${countChanged} changed)...`, + ); }); - this.statusSucceed(`repairEntities > Added companies to cache of ${count} users`); + this.statusSucceed( + `repairEntities > Added companies to cache of ${count} users (${countChanged} changed)`, + ); } } diff --git a/tdrive/backend/node/src/cli/lib/run_with_platform.ts b/tdrive/backend/node/src/cli/lib/run_with_platform.ts index 35059f30..ddc1b3ba 100644 --- a/tdrive/backend/node/src/cli/lib/run_with_platform.ts +++ b/tdrive/backend/node/src/cli/lib/run_with_platform.ts @@ -5,6 +5,8 @@ import tdrive from "../../tdrive"; import gr from "../../services/global-resolver"; import type { TdrivePlatform } from "../../core/platform/platform"; +//TODO: When this gets used for all commands; update verboseDuringRun from search/index-all and move to root index + /** * Start the platform and its services, run the command (passed as * the `handler` callback), then cleanly shut down the platform. diff --git a/tdrive/backend/node/src/cli/utils/run-with-logger-level.ts b/tdrive/backend/node/src/cli/utils/run-with-logger-level.ts new file mode 100644 index 00000000..0a679bbe --- /dev/null +++ b/tdrive/backend/node/src/cli/utils/run-with-logger-level.ts @@ -0,0 +1,20 @@ +import { logger } from "../../core/platform/framework/logger"; + +type LogLevel = "debug" | "info" | "warn" | "error" | "fatal"; + +/** Run the callback with the provided log level set (if undefined it's left identical). + * Restores the previous level before returning. + */ +export default async function runWithLoggerLevel( + level: LogLevel | undefined, + callback: () => R, +) { + if (level === undefined) return await callback(); + const previousLogLevel = logger.level; + try { + logger.level = level; + return await callback(); + } finally { + logger.level = previousLogLevel; + } +}