From 6f8ff34051a168477dfe768032e564823b942862 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Sun, 8 Dec 2024 22:51:58 +0100 Subject: [PATCH] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20back:=20separate=20and=20g?= =?UTF-8?q?eneric=20service=20level=20diagnostics=20(#762)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/framework/api/diagnostics.ts | 45 ++++++++++++++++--- .../orm/connectors/mongodb/mongodb.ts | 35 +++++++++++++-- .../orm/connectors/postgres/postgres.ts | 3 +- .../diagnostics/providers/database-service.ts | 4 ++ .../services/diagnostics/providers/db-ping.ts | 9 ---- .../services/diagnostics/providers/index.ts | 2 +- .../services/diagnostics/providers/process.ts | 2 +- 7 files changed, 79 insertions(+), 21 deletions(-) create mode 100644 tdrive/backend/node/src/core/platform/services/diagnostics/providers/database-service.ts delete mode 100644 tdrive/backend/node/src/core/platform/services/diagnostics/providers/db-ping.ts diff --git a/tdrive/backend/node/src/core/platform/framework/api/diagnostics.ts b/tdrive/backend/node/src/core/platform/framework/api/diagnostics.ts index d26010e1..a4e31d80 100644 --- a/tdrive/backend/node/src/core/platform/framework/api/diagnostics.ts +++ b/tdrive/backend/node/src/core/platform/framework/api/diagnostics.ts @@ -13,18 +13,28 @@ export type TDiagnosticTag = | "startup" // Tests that are absolutely required (and light) to even begin the other tests | "ready" // Tests required before traffic can be sent our way | "live" // Tests required to prevent from being restarted - | "stats"; // Expensive diagnostics that should not often be ran + | "stats" // Expensive diagnostics that should not often be ran, but can be system wide + | "stats-full"; // Expensive diagnostics that should not often be ran, and then probably for a single key /** Detail requested from platform service self-diagnostics */ export enum TServiceDiagnosticDepth { /** Minimal cost information that tests functioning service */ - critical = 0, - /** Statistics that have a little impact enough for regular tracking into a time series */ - tracked_statistics = 1, + alive = "alive", + /** Statistics that have a little impact enough for periodic tracking into a time series */ + stats_track = "stats_track", + /** Statistics that should be included when looking specifically at general statistics */ + stats_basic = "stats_basic", /** Statistics possibly expensive and large to calculate, for occasional debug operations */ - deep_statistics = 2, + stats_deep = "stats_deep", } +const serviceDiagnosticDepthToTags: { [depth in TServiceDiagnosticDepth]: TDiagnosticTag[] } = { + [TServiceDiagnosticDepth.alive]: ["startup", "live", "ready"], + [TServiceDiagnosticDepth.stats_track]: ["ready", "stats"], + [TServiceDiagnosticDepth.stats_basic]: ["stats", "stats-full"], + [TServiceDiagnosticDepth.stats_deep]: ["stats-full"], +}; + interface IDiagnosticsConfig { // Diagnostic keys that should be considered ok without evaluation skipKeys?: string[]; @@ -183,6 +193,31 @@ export default { }); }, + /** Create providers to match from {@link IServiceDiagnosticProvider} to multiple {@link IDiagnosticProvider}s */ + registerServiceProviders( + name: string, + getService: () => IServiceDiagnosticProvider, + overrideTags: Partial< + typeof serviceDiagnosticDepthToTags | { [key in TServiceDiagnosticDepth]: false } + > = {}, + ) { + this.registerProviders( + ...Object.values(TServiceDiagnosticDepth) + .map(depth => { + const defaultTags = serviceDiagnosticDepthToTags[depth]; + if (!defaultTags) throw new Error(`Unknown depth ${JSON.stringify(depth)}`); + const tags = overrideTags[depth] ?? defaultTags; + if (tags === false) return null; + return { + key: `${name}-${depth}`, + tags, + get: () => getService().getDiagnostics(depth), + }; + }) + .filter(x => !!x), + ); + }, + /** Cancel all pending diagnostic updates */ shutdown() { ensureHasntShutdown(); diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts index 395fffc4..3890361e 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts @@ -52,18 +52,45 @@ export class MongoConnector extends AbstractConnector { return !wasConnected; } + private async dbStats(): Promise { + return (await this.getDatabase()).stats(); + } + + private async collectionsStats(deep: boolean): Promise { + const db = await this.getDatabase(); + const result = { collections: {} }; + for (const collection of await db.collections()) { + const stats = await collection.aggregate([ + { + $collStats: { + latencyStats: { histograms: true }, + storageStats: deep ? {} : undefined, // Really a lot of keys with 0 occurances + count: {}, + queryExecStats: {}, + }, + }, + ]); + result.collections[collection.collectionName] = await stats.toArray(); + } + return result; + } + async getDiagnostics(depth: TServiceDiagnosticDepth): Promise { switch (depth) { - case TServiceDiagnosticDepth.critical: + case TServiceDiagnosticDepth.alive: return { ok: true, didConnect: await this.ping() }; - case TServiceDiagnosticDepth.deep_statistics: - case TServiceDiagnosticDepth.tracked_statistics: - return { ok: true, warn: "unsupported_depth" }; + case TServiceDiagnosticDepth.stats_track: + return { ok: true, ...(await this.dbStats()) }; + case TServiceDiagnosticDepth.stats_basic: + return { ok: true, ...(await this.collectionsStats(false)) }; + case TServiceDiagnosticDepth.stats_deep: + return { ok: true, ...(await this.collectionsStats(true)) }; default: throw new Error(`Unexpected TServiceDiagnosticDepth: ${JSON.stringify(depth)}`); } } + getClient(): mongo.MongoClient { return this.client; } diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres.ts index 0fb00784..f2a06f95 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres.ts @@ -72,12 +72,13 @@ export class PostgresConnector extends AbstractConnector { switch (depth) { - case TServiceDiagnosticDepth.critical: + case TServiceDiagnosticDepth.alive: return { ok: true, didConnect: await this.ping() }; case TServiceDiagnosticDepth.stats_basic: case TServiceDiagnosticDepth.stats_track: case TServiceDiagnosticDepth.stats_deep: return { ok: true, warn: "unsupported_depth" }; + default: throw new Error(`Unexpected TServiceDiagnosticDepth: ${JSON.stringify(depth)}`); } diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/providers/database-service.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/database-service.ts new file mode 100644 index 00000000..ed403b49 --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/database-service.ts @@ -0,0 +1,4 @@ +import diagnostics from "../../../framework/api/diagnostics"; +import globalResolver from "../../../../../services/global-resolver"; + +export default () => diagnostics.registerServiceProviders("db", () => globalResolver.database, {}); diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/providers/db-ping.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/db-ping.ts deleted file mode 100644 index eef99c7c..00000000 --- a/tdrive/backend/node/src/core/platform/services/diagnostics/providers/db-ping.ts +++ /dev/null @@ -1,9 +0,0 @@ -import diagnostics, { TServiceDiagnosticDepth } from "../../../framework/api/diagnostics"; -import globalResolver from "../../../../../services/global-resolver"; - -export default () => - diagnostics.registerProviders({ - key: "db-ping", - tags: ["startup", "live", "ready"], - get: () => globalResolver.database.getDiagnostics(TServiceDiagnosticDepth.critical), - }); diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/providers/index.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/index.ts index 10e24bd6..d9350f7f 100644 --- a/tdrive/backend/node/src/core/platform/services/diagnostics/providers/index.ts +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/index.ts @@ -1,4 +1,4 @@ -import registerDBPingProvider from "./db-ping"; +import registerDBPingProvider from "./database-service"; import registerPlatformProvider from "./platform-started"; import registerProcessProvider from "./process"; diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/providers/process.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/process.ts index 4abe1b18..35d085d4 100644 --- a/tdrive/backend/node/src/core/platform/services/diagnostics/providers/process.ts +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/process.ts @@ -8,8 +8,8 @@ export default () => return { ok: true, gc: !!global.gc, - mem: process.memoryUsage(), pid: process.pid, + mem: process.memoryUsage(), res: process.resourceUsage(), }; },