♻️ back: separate and generic service level diagnostics (#762)

tdrive/backend/node/src/core/platform/framework/api/diagnostics.ts

@@ -13,18 +13,28 @@ export type TDiagnosticTag =
   | "startup" // Tests that are absolutely required (and light) to even begin the other tests
   | "ready" // Tests required before traffic can be sent our way
   | "live" // Tests required to prevent from being restarted
-  | "stats"; // Expensive diagnostics that should not often be ran
+  | "stats" // Expensive diagnostics that should not often be ran, but can be system wide
+  | "stats-full"; // Expensive diagnostics that should not often be ran, and then probably for a single key
 
 /** Detail requested from platform service self-diagnostics */
 export enum TServiceDiagnosticDepth {
   /** Minimal cost information that tests functioning service */
-  critical = 0,
-  /** Statistics that have a little impact enough for regular tracking into a time series */
-  tracked_statistics = 1,
+  alive = "alive",
+  /** Statistics that have a little impact enough for periodic tracking into a time series */
+  stats_track = "stats_track",
+  /** Statistics that should be included when looking specifically at general statistics */
+  stats_basic = "stats_basic",
   /** Statistics possibly expensive and large to calculate, for occasional debug operations */
-  deep_statistics = 2,
+  stats_deep = "stats_deep",
 }
 
+const serviceDiagnosticDepthToTags: { [depth in TServiceDiagnosticDepth]: TDiagnosticTag[] } = {
+  [TServiceDiagnosticDepth.alive]: ["startup", "live", "ready"],
+  [TServiceDiagnosticDepth.stats_track]: ["ready", "stats"],
+  [TServiceDiagnosticDepth.stats_basic]: ["stats", "stats-full"],
+  [TServiceDiagnosticDepth.stats_deep]: ["stats-full"],
+};
+
 interface IDiagnosticsConfig {
   // Diagnostic keys that should be considered ok without evaluation
   skipKeys?: string[];
@@ -183,6 +193,31 @@ export default {
     });
   },
 
+  /** Create providers to match from {@link IServiceDiagnosticProvider} to multiple {@link IDiagnosticProvider}s */
+  registerServiceProviders(
+    name: string,
+    getService: () => IServiceDiagnosticProvider,
+    overrideTags: Partial<
+      typeof serviceDiagnosticDepthToTags | { [key in TServiceDiagnosticDepth]: false }
+    > = {},
+  ) {
+    this.registerProviders(
+      ...Object.values(TServiceDiagnosticDepth)
+        .map(depth => {
+          const defaultTags = serviceDiagnosticDepthToTags[depth];
+          if (!defaultTags) throw new Error(`Unknown depth ${JSON.stringify(depth)}`);
+          const tags = overrideTags[depth] ?? defaultTags;
+          if (tags === false) return null;
+          return {
+            key: `${name}-${depth}`,
+            tags,
+            get: () => getService().getDiagnostics(depth),
+          };
+        })
+        .filter(x => !!x),
+    );
+  },
+
   /** Cancel all pending diagnostic updates */
   shutdown() {
     ensureHasntShutdown();

tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts

@@ -52,18 +52,45 @@ export class MongoConnector extends AbstractConnector<MongoConnectionOptions> {
     return !wasConnected;
   }
 
+  private async dbStats(): Promise<object> {
+    return (await this.getDatabase()).stats();
+  }
+
+  private async collectionsStats(deep: boolean): Promise<object> {
+    const db = await this.getDatabase();
+    const result = { collections: {} };
+    for (const collection of await db.collections()) {
+      const stats = await collection.aggregate([
+        {
+          $collStats: {
+            latencyStats: { histograms: true },
+            storageStats: deep ? {} : undefined, // Really a lot of keys with 0 occurances
+            count: {},
+            queryExecStats: {},
+          },
+        },
+      ]);
+      result.collections[collection.collectionName] = await stats.toArray();
+    }
+    return result;
+  }
+
   async getDiagnostics(depth: TServiceDiagnosticDepth): Promise<TDiagnosticResult> {
     switch (depth) {
-      case TServiceDiagnosticDepth.critical:
+      case TServiceDiagnosticDepth.alive:
         return { ok: true, didConnect: await this.ping() };
-      case TServiceDiagnosticDepth.deep_statistics:
-      case TServiceDiagnosticDepth.tracked_statistics:
-        return { ok: true, warn: "unsupported_depth" };
+      case TServiceDiagnosticDepth.stats_track:
+        return { ok: true, ...(await this.dbStats()) };
+      case TServiceDiagnosticDepth.stats_basic:
+        return { ok: true, ...(await this.collectionsStats(false)) };
+      case TServiceDiagnosticDepth.stats_deep:
+        return { ok: true, ...(await this.collectionsStats(true)) };
 
       default:
         throw new Error(`Unexpected TServiceDiagnosticDepth: ${JSON.stringify(depth)}`);
     }
   }
+
   getClient(): mongo.MongoClient {
     return this.client;
   }

tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres.ts

@@ -72,12 +72,13 @@ export class PostgresConnector extends AbstractConnector<PostgresConnectionOptio
 
   async getDiagnostics(depth: TServiceDiagnosticDepth): Promise<TDiagnosticResult> {
     switch (depth) {
-      case TServiceDiagnosticDepth.critical:
+      case TServiceDiagnosticDepth.alive:
         return { ok: true, didConnect: await this.ping() };
       case TServiceDiagnosticDepth.stats_basic:
       case TServiceDiagnosticDepth.stats_track:
       case TServiceDiagnosticDepth.stats_deep:
         return { ok: true, warn: "unsupported_depth" };
+
       default:
         throw new Error(`Unexpected TServiceDiagnosticDepth: ${JSON.stringify(depth)}`);
     }

tdrive/backend/node/src/core/platform/services/diagnostics/providers/database-service.ts

@@ -0,0 +1,4 @@
+import diagnostics from "../../../framework/api/diagnostics";
+import globalResolver from "../../../../../services/global-resolver";
+
+export default () => diagnostics.registerServiceProviders("db", () => globalResolver.database, {});

tdrive/backend/node/src/core/platform/services/diagnostics/providers/db-ping.ts

@@ -1,9 +0,0 @@
-import diagnostics, { TServiceDiagnosticDepth } from "../../../framework/api/diagnostics";
-import globalResolver from "../../../../../services/global-resolver";
-
-export default () =>
-  diagnostics.registerProviders({
-    key: "db-ping",
-    tags: ["startup", "live", "ready"],
-    get: () => globalResolver.database.getDiagnostics(TServiceDiagnosticDepth.critical),
-  });

tdrive/backend/node/src/core/platform/services/diagnostics/providers/index.ts

@@ -1,4 +1,4 @@
-import registerDBPingProvider from "./db-ping";
+import registerDBPingProvider from "./database-service";
 import registerPlatformProvider from "./platform-started";
 import registerProcessProvider from "./process";
 

tdrive/backend/node/src/core/platform/services/diagnostics/providers/process.ts

@@ -8,8 +8,8 @@ export default () =>
       return {
         ok: true,
         gc: !!global.gc,
-        mem: process.memoryUsage(),
         pid: process.pid,
+        mem: process.memoryUsage(),
         res: process.resourceUsage(),
       };
     },