eduvia/workavia-drive
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

🔊 CLI better logging: new option for finer level, changed count tracking, bit of grammar

Browse Source
This commit is contained in:
Eric Doughty-Papassideris
2024-03-28 00:33:56 +01:00
committed by ericlinagora
parent b72a958817
commit bde006c63e
6 changed files with 86 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tdrive/backend/node/src/cli/cmds/search_cmds/index-all.ts
+31 -13
View File
@@ -3,6 +3,7 @@ import yargs from "yargs";
import type { TdrivePlatform } from "../../../core/platform/platform";
import runWithPlatform from "../../lib/run_with_platform";
import parseYargsCommaSeparatedStringArray from "../../utils/yargs-comma-array";
import runWithLoggerLevel from "../../utils/run-with-logger-level";
import type BaseReindexer from "./index-all/base-reindexer";
import type ReindexerOptions from "./index-all/reindexer-options";
@@ -58,6 +59,14 @@ const command: yargs.CommandModule<unknown, unknown> = {
"When processing documents, limit to those owned by the users with the provided comma separated e-mails",
group: "Filtering for documents repository",
},
verboseDuringRun: {
type: "boolean",
alias: "V",
description:
"Set to verbose logging except for startup and shutdown of Platform. Use flag once for 'info' level, twice for 'debug'.",
group: "Output",
count: true,
},
},
handler: async argv => {
const repositories = parseYargsCommaSeparatedStringArray(
@@ -67,19 +76,28 @@ const command: yargs.CommandModule<unknown, unknown> = {
argv.filterDocumentsByUserEMail as string /* ignore typechecker */,
);
await runWithPlatform("Re-index", async ({ spinner, platform }) => {
try {
for (const repositoryName of repositories)
await RepositoryNameToCTOR.get(repositoryName)
.ctor(platform, {
spinner,
repairEntities: !!argv.repairEntities,
filterDocumentsByUserEMail,
})
.run();
} catch (err) {
spinner.fail(err.stack || err);
return 1;
}
return await runWithLoggerLevel(
argv.verboseDuringRun
? (argv.verboseDuringRun as number) > 1
? "debug"
: "info"
: undefined,
async () => {
try {
for (const repositoryName of repositories)
await RepositoryNameToCTOR.get(repositoryName)
.ctor(platform, {
spinner,
repairEntities: !!argv.repairEntities,
filterDocumentsByUserEMail,
})
.run();
} catch (err) {
spinner.fail(err.stack || err);
return 1;
}
},
);
});
},
};
tdrive/backend/node/src/cli/cmds/search_cmds/index-all/base-reindexer.ts
+16 -9
View File
@@ -12,12 +12,16 @@ import waitTimeoutMS from "../../../utils/wait-timeout";
import iterateOverRepoPages from "../../../utils/iterate-over-repository-pages";
import type ReindexerOptions from "./reindexer-options";
const pluralificationate = (noun: string) => noun.replace(/s$/i, "").replace(/y$/i, "ie") + "s";
/** This is an abstract base class for re-index cli commands; it stores runtime options
* into fields; runs repair first if requested and re-indexes generically from the database
* to search services. */
export default abstract class BaseReindexer<Entity> {
protected readonly database: DatabaseServiceAPI;
protected readonly search: SearchServiceAPI;
private readonly tablePlural: string;
constructor(
protected readonly platform: TdrivePlatform,
protected readonly options: ReindexerOptions,
@@ -26,12 +30,15 @@ export default abstract class BaseReindexer<Entity> {
) {
this.database = this.platform.getProvider<DatabaseServiceAPI>("database");
this.search = this.platform.getProvider<SearchServiceAPI>("search");
this.tablePlural = pluralificationate(this.table);
}
private _dbRepo: Repository<Entity>;
protected async dbRepository() {
return (this._dbRepo =
this._dbRepo || (await this.database.getRepository<Entity>(this.table, this.entity)));
}
private _searchRepo: SearchRepository<Entity>;
protected async searchRepository() {
return (this._searchRepo =
@@ -39,19 +46,19 @@ export default abstract class BaseReindexer<Entity> {
}
protected statusStart(info: string) {
this.options.spinner.start(`${this.table} > ${info}`);
this.options.spinner.start(`${this.tablePlural} > ${info}`);
}
protected statusSucceed(info: string) {
this.options.spinner.succeed(`${this.table} > ${info}`);
this.options.spinner.succeed(`${this.tablePlural} > ${info}`);
}
protected statusWarn(info: string) {
this.options.spinner.warn(`${this.table} > ${info}`);
this.options.spinner.warn(`${this.tablePlural} > ${info}`);
}
protected statusFail(info: string) {
this.options.spinner.fail(`${this.table} > ${info}`);
this.options.spinner.fail(`${this.tablePlural} > ${info}`);
}
protected statusInfo(info: string) {
this.options.spinner.info(`${this.table} > ${info}`);
this.options.spinner.info(`${this.tablePlural} > ${info}`);
}
/** Override in sub-classes to translate a page from database to search entities.
@@ -71,7 +78,7 @@ export default abstract class BaseReindexer<Entity> {
* on the entity type.
*/
protected async repairEntities(_findOptions: FindOptions): Promise<void> {
this.statusWarn(`repairEntities > No repair action for ${this.table}`);
this.statusWarn(`repairEntities > No repair action for ${this.tablePlural}`);
}
/** Override in a sub-class to completely replace the re-indexing logic.
@@ -89,12 +96,12 @@ export default abstract class BaseReindexer<Entity> {
entities = await this.mapEntitiesToReIndexFromDBToSearch(entities);
await this.search.upsert(entities);
count += entities.length;
this.statusStart(`Indexed ${count} ${this.table}...`);
this.statusStart(`Indexed ${count} ${this.tablePlural}...`);
},
findOptions,
);
if (count === 0) this.statusWarn(`Index ${this.table} finished; but 0 items included`);
else this.statusSucceed(`${count} ${this.table} indexed`);
if (count === 0) this.statusWarn(`Index ${this.tablePlural} finished; but 0 items included`);
else this.statusSucceed(`${count} ${this.tablePlural} indexed`);
const giveFlushAChanceDurationMS = 10000;
this.statusStart(`Emptying flush (${giveFlushAChanceDurationMS / 1000}s)...`);
await waitTimeoutMS(giveFlushAChanceDurationMS);
tdrive/backend/node/src/cli/cmds/search_cmds/index-all/documents-reindexer.ts
+5
View File
@@ -60,6 +60,8 @@ export default class DocumentsReindexer extends BaseReindexer<DriveFile> {
"Download and re-extract keywords before re-indexing";
protected override async repairEntities(findOptions: FindOptions): Promise<void> {
const repository = await this.dbRepository();
let count = 0,
countChanged = 0;
await iterateOverRepoPages(
repository,
async entities => {
@@ -70,6 +72,7 @@ export default class DocumentsReindexer extends BaseReindexer<DriveFile> {
this.statusInfo(
`-> Downloading ${entity.name} (${entity.size}b id: ${entity.id} creator: ${entity.creator}`,
);
count++;
try {
if (
entity.size === 0 ||
@@ -97,11 +100,13 @@ export default class DocumentsReindexer extends BaseReindexer<DriveFile> {
if (content_keywords !== entity.content_keywords) {
entity.content_keywords = content_keywords;
repository.save(entity);
countChanged++;
}
} catch (err) {
this.statusFail(err.stack);
}
}
this.statusStart(`Repaired ${count} documents (${countChanged} changed)`);
},
findOptions,
);
tdrive/backend/node/src/cli/cmds/search_cmds/index-all/users-reindexer.ts
+12 -4
View File
@@ -24,7 +24,8 @@ export default class UsersReindexer extends BaseReindexer<User> {
CompanyUserTYPE,
CompanyUser,
);
let count = 0;
let count = 0,
countChanged = 0;
const repository = await this.dbRepository();
await iterateOverRepoPages(repository, async entities => {
for (const user of entities) {
@@ -33,11 +34,18 @@ export default class UsersReindexer extends BaseReindexer<User> {
user.cache ||= { companies: [] };
user.cache.companies = companies.getEntities().map(company => company.group_id);
const newCache = JSON.stringify(user.cache);
if (prevCache != newCache) await repository.save(user, undefined);
if (prevCache != newCache) {
await repository.save(user, undefined);
countChanged++;
}
}
count += entities.length;
this.statusStart(`repairEntities > Adding companies to cache of ${count} users...`);
this.statusStart(
`repairEntities > Adding companies to cache of ${count} users (${countChanged} changed)...`,
);
});
this.statusSucceed(`repairEntities > Added companies to cache of ${count} users`);
this.statusSucceed(
`repairEntities > Added companies to cache of ${count} users (${countChanged} changed)`,
);
}
}
tdrive/backend/node/src/cli/lib/run_with_platform.ts
+2
View File
@@ -5,6 +5,8 @@ import tdrive from "../../tdrive";
import gr from "../../services/global-resolver";
import type { TdrivePlatform } from "../../core/platform/platform";
//TODO: When this gets used for all commands; update verboseDuringRun from search/index-all and move to root index
/**
* Start the platform and its services, run the command (passed as
* the `handler` callback), then cleanly shut down the platform.
tdrive/backend/node/src/cli/utils/run-with-logger-level.ts
+20
View File
@@ -0,0 +1,20 @@
import { logger } from "../../core/platform/framework/logger";
type LogLevel = "debug" | "info" | "warn" | "error" | "fatal";
/** Run the callback with the provided log level set (if undefined it's left identical).
* Restores the previous level before returning.
*/
export default async function runWithLoggerLevel<R = void>(
level: LogLevel | undefined,
callback: () => R,
) {
if (level === undefined) return await callback();
const previousLogLevel = logger.level;
try {
logger.level = level;
return await callback();
} finally {
logger.level = previousLogLevel;
}
}