🔊 CLI better logging: new option for finer level, changed count tracking, bit of grammar

tdrive/backend/node/src/cli/cmds/search_cmds/index-all.ts

@@ -3,6 +3,7 @@ import yargs from "yargs";
 import type { TdrivePlatform } from "../../../core/platform/platform";
 import runWithPlatform from "../../lib/run_with_platform";
 import parseYargsCommaSeparatedStringArray from "../../utils/yargs-comma-array";
+import runWithLoggerLevel from "../../utils/run-with-logger-level";
 
 import type BaseReindexer from "./index-all/base-reindexer";
 import type ReindexerOptions from "./index-all/reindexer-options";
@@ -58,6 +59,14 @@ const command: yargs.CommandModule<unknown, unknown> = {
         "When processing documents, limit to those owned by the users with the provided comma separated e-mails",
       group: "Filtering for documents repository",
     },
+    verboseDuringRun: {
+      type: "boolean",
+      alias: "V",
+      description:
+        "Set to verbose logging except for startup and shutdown of Platform. Use flag once for 'info' level, twice for 'debug'.",
+      group: "Output",
+      count: true,
+    },
   },
   handler: async argv => {
     const repositories = parseYargsCommaSeparatedStringArray(
@@ -67,19 +76,28 @@ const command: yargs.CommandModule<unknown, unknown> = {
       argv.filterDocumentsByUserEMail as string /* ignore typechecker */,
     );
     await runWithPlatform("Re-index", async ({ spinner, platform }) => {
-      try {
+      return await runWithLoggerLevel(
-        for (const repositoryName of repositories)
+        argv.verboseDuringRun
-          await RepositoryNameToCTOR.get(repositoryName)
+          ? (argv.verboseDuringRun as number) > 1
-            .ctor(platform, {
+            ? "debug"
-              spinner,
+            : "info"
-              repairEntities: !!argv.repairEntities,
+          : undefined,
-              filterDocumentsByUserEMail,
+        async () => {
-            })
+          try {
-            .run();
+            for (const repositoryName of repositories)
-      } catch (err) {
+              await RepositoryNameToCTOR.get(repositoryName)
-        spinner.fail(err.stack || err);
+                .ctor(platform, {
-        return 1;
+                  spinner,
-      }
+                  repairEntities: !!argv.repairEntities,
+                  filterDocumentsByUserEMail,
+                })
+                .run();
+          } catch (err) {
+            spinner.fail(err.stack || err);
+            return 1;
+          }
+        },
+      );
     });
   },
 };

tdrive/backend/node/src/cli/cmds/search_cmds/index-all/base-reindexer.ts

@@ -12,12 +12,16 @@ import waitTimeoutMS from "../../../utils/wait-timeout";
 import iterateOverRepoPages from "../../../utils/iterate-over-repository-pages";
 import type ReindexerOptions from "./reindexer-options";
 
+const pluralificationate = (noun: string) => noun.replace(/s$/i, "").replace(/y$/i, "ie") + "s";
+
 /** This is an abstract base class for re-index cli commands; it stores runtime options
  *  into fields; runs repair first if requested and re-indexes generically from the database
  *  to search services. */
 export default abstract class BaseReindexer<Entity> {
   protected readonly database: DatabaseServiceAPI;
   protected readonly search: SearchServiceAPI;
+  private readonly tablePlural: string;
+
   constructor(
     protected readonly platform: TdrivePlatform,
     protected readonly options: ReindexerOptions,
@@ -26,12 +30,15 @@ export default abstract class BaseReindexer<Entity> {
   ) {
     this.database = this.platform.getProvider<DatabaseServiceAPI>("database");
     this.search = this.platform.getProvider<SearchServiceAPI>("search");
+    this.tablePlural = pluralificationate(this.table);
   }
+
   private _dbRepo: Repository<Entity>;
   protected async dbRepository() {
     return (this._dbRepo =
       this._dbRepo || (await this.database.getRepository<Entity>(this.table, this.entity)));
   }
+
   private _searchRepo: SearchRepository<Entity>;
   protected async searchRepository() {
     return (this._searchRepo =
@@ -39,19 +46,19 @@ export default abstract class BaseReindexer<Entity> {
   }
 
   protected statusStart(info: string) {
-    this.options.spinner.start(`${this.table} > ${info}`);
+    this.options.spinner.start(`${this.tablePlural} > ${info}`);
   }
   protected statusSucceed(info: string) {
-    this.options.spinner.succeed(`${this.table} > ${info}`);
+    this.options.spinner.succeed(`${this.tablePlural} > ${info}`);
   }
   protected statusWarn(info: string) {
-    this.options.spinner.warn(`${this.table} > ${info}`);
+    this.options.spinner.warn(`${this.tablePlural} > ${info}`);
   }
   protected statusFail(info: string) {
-    this.options.spinner.fail(`${this.table} > ${info}`);
+    this.options.spinner.fail(`${this.tablePlural} > ${info}`);
   }
   protected statusInfo(info: string) {
-    this.options.spinner.info(`${this.table} > ${info}`);
+    this.options.spinner.info(`${this.tablePlural} > ${info}`);
   }
 
   /** Override in sub-classes to translate a page from database to search entities.
@@ -71,7 +78,7 @@ export default abstract class BaseReindexer<Entity> {
    * on the entity type.
    */
   protected async repairEntities(_findOptions: FindOptions): Promise<void> {
-    this.statusWarn(`repairEntities > No repair action for ${this.table}`);
+    this.statusWarn(`repairEntities > No repair action for ${this.tablePlural}`);
   }
 
   /** Override in a sub-class to completely replace the re-indexing logic.
@@ -89,12 +96,12 @@ export default abstract class BaseReindexer<Entity> {
         entities = await this.mapEntitiesToReIndexFromDBToSearch(entities);
         await this.search.upsert(entities);
         count += entities.length;
-        this.statusStart(`Indexed ${count} ${this.table}...`);
+        this.statusStart(`Indexed ${count} ${this.tablePlural}...`);
       },
       findOptions,
     );
-    if (count === 0) this.statusWarn(`Index ${this.table} finished; but 0 items included`);
+    if (count === 0) this.statusWarn(`Index ${this.tablePlural} finished; but 0 items included`);
-    else this.statusSucceed(`${count} ${this.table} indexed`);
+    else this.statusSucceed(`${count} ${this.tablePlural} indexed`);
     const giveFlushAChanceDurationMS = 10000;
     this.statusStart(`Emptying flush (${giveFlushAChanceDurationMS / 1000}s)...`);
     await waitTimeoutMS(giveFlushAChanceDurationMS);

tdrive/backend/node/src/cli/cmds/search_cmds/index-all/documents-reindexer.ts

@@ -60,6 +60,8 @@ export default class DocumentsReindexer extends BaseReindexer<DriveFile> {
     "Download and re-extract keywords before re-indexing";
   protected override async repairEntities(findOptions: FindOptions): Promise<void> {
     const repository = await this.dbRepository();
+    let count = 0,
+      countChanged = 0;
     await iterateOverRepoPages(
       repository,
       async entities => {
@@ -70,6 +72,7 @@ export default class DocumentsReindexer extends BaseReindexer<DriveFile> {
           this.statusInfo(
             `-> Downloading ${entity.name} (${entity.size}b id: ${entity.id} creator: ${entity.creator}`,
           );
+          count++;
           try {
             if (
               entity.size === 0 ||
@@ -97,11 +100,13 @@ export default class DocumentsReindexer extends BaseReindexer<DriveFile> {
             if (content_keywords !== entity.content_keywords) {
               entity.content_keywords = content_keywords;
               repository.save(entity);
+              countChanged++;
             }
           } catch (err) {
             this.statusFail(err.stack);
           }
         }
+        this.statusStart(`Repaired ${count} documents (${countChanged} changed)`);
       },
       findOptions,
     );

tdrive/backend/node/src/cli/cmds/search_cmds/index-all/users-reindexer.ts

@@ -24,7 +24,8 @@ export default class UsersReindexer extends BaseReindexer<User> {
       CompanyUserTYPE,
       CompanyUser,
     );
-    let count = 0;
+    let count = 0,
+      countChanged = 0;
     const repository = await this.dbRepository();
     await iterateOverRepoPages(repository, async entities => {
       for (const user of entities) {
@@ -33,11 +34,18 @@ export default class UsersReindexer extends BaseReindexer<User> {
         user.cache ||= { companies: [] };
         user.cache.companies = companies.getEntities().map(company => company.group_id);
         const newCache = JSON.stringify(user.cache);
-        if (prevCache != newCache) await repository.save(user, undefined);
+        if (prevCache != newCache) {
+          await repository.save(user, undefined);
+          countChanged++;
+        }
       }
       count += entities.length;
-      this.statusStart(`repairEntities > Adding companies to cache of ${count} users...`);
+      this.statusStart(
+        `repairEntities > Adding companies to cache of ${count} users (${countChanged} changed)...`,
+      );
     });
-    this.statusSucceed(`repairEntities > Added companies to cache of ${count} users`);
+    this.statusSucceed(
+      `repairEntities > Added companies to cache of ${count} users (${countChanged} changed)`,
+    );
   }
 }

tdrive/backend/node/src/cli/lib/run_with_platform.ts

@@ -5,6 +5,8 @@ import tdrive from "../../tdrive";
 import gr from "../../services/global-resolver";
 import type { TdrivePlatform } from "../../core/platform/platform";
 
+//TODO: When this gets used for all commands; update verboseDuringRun from search/index-all and move to root index
+
 /**
  * Start the platform and its services, run the command (passed as
  * the `handler` callback), then cleanly shut down the platform.

tdrive/backend/node/src/cli/utils/run-with-logger-level.ts

@@ -0,0 +1,20 @@
+import { logger } from "../../core/platform/framework/logger";
+
+type LogLevel = "debug" | "info" | "warn" | "error" | "fatal";
+
+/** Run the callback with the provided log level set (if undefined it's left identical).
+ * Restores the previous level before returning.
+ */
+export default async function runWithLoggerLevel<R = void>(
+  level: LogLevel | undefined,
+  callback: () => R,
+) {
+  if (level === undefined) return await callback();
+  const previousLogLevel = logger.level;
+  try {
+    logger.level = level;
+    return await callback();
+  } finally {
+    logger.level = previousLogLevel;
+  }
+}