diff --git a/tdrive/backend/node/config/custom-environment-variables.json b/tdrive/backend/node/config/custom-environment-variables.json index 3e337b86..5a0d030e 100644 --- a/tdrive/backend/node/config/custom-environment-variables.json +++ b/tdrive/backend/node/config/custom-environment-variables.json @@ -16,6 +16,15 @@ "logger": { "level": "LOG_LEVEL" }, + "diagnostics": { + "skipKeys": { + "__name": "DIAG_SKIP_KEYS", + "__format": "json" + }, + "probeSecret": "DIAG_PROBE_SECRET", + "statsLogPeriodMs": "DIAG_STATS_PRINT_PERIOD_MS", + "statsFullStatsLogPeriodMs": "DIAG_FULL_STATS_PRINT_PERIOD_MS" + }, "webserver": { "host": "TWAKE_DRIVE_HOST", "logger": { diff --git a/tdrive/backend/node/config/default.json b/tdrive/backend/node/config/default.json index 30da3a6f..22c107d5 100644 --- a/tdrive/backend/node/config/default.json +++ b/tdrive/backend/node/config/default.json @@ -35,6 +35,12 @@ "logger": { "level": "debug" }, + "diagnostics": { + "skipKeys": [], + "probeSecret": "", + "statsLogPeriodMs": 120000, + "statsFullStatsLogPeriodMs": 600000 + }, "tracker": { "type": "segment", "segment": { @@ -226,6 +232,7 @@ }, "services": [ "auth", + "diagnostics", "push", "storage", "webserver", diff --git a/tdrive/backend/node/src/core/platform/framework/api/diagnostics.ts b/tdrive/backend/node/src/core/platform/framework/api/diagnostics.ts new file mode 100644 index 00000000..09b2aaa8 --- /dev/null +++ b/tdrive/backend/node/src/core/platform/framework/api/diagnostics.ts @@ -0,0 +1,266 @@ +import assert from "node:assert"; +import config from "../../../config"; +import { getLogger } from "../logger"; + +const logger = getLogger("Diagnostics"); + +/** + * Values that can match a set of diagnostic providers. + * + * `startup`, `ready` and `live` are meant to match the meanings of the corresponding + * kubernetes probes: + * https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ + */ +export type TDiagnosticTag = + | "*" // Special value that includes everything, and is always included + | "startup" // Tests that are absolutely required (and light) to even begin the other tests + | "ready" // Tests required before traffic can be sent our way + | "live" // Tests required to prevent from being restarted + | "stats" // Expensive diagnostics that should not often be ran, but can be system wide + | "stats-full"; // Expensive diagnostics that should not often be ran, and then probably for a single key + +/** Detail requested from platform service self-diagnostics */ +export enum TServiceDiagnosticDepth { + /** Minimal cost information that tests functioning service */ + alive = "alive", + /** Statistics that have a little impact enough for periodic tracking into a time series */ + stats_track = "stats_track", + /** Statistics that should be included when looking specifically at general statistics */ + stats_basic = "stats_basic", + /** Statistics possibly expensive and large to calculate, for occasional debug operations */ + stats_deep = "stats_deep", +} + +const serviceDiagnosticDepthToTags: { [depth in TServiceDiagnosticDepth]: TDiagnosticTag[] } = { + [TServiceDiagnosticDepth.alive]: ["startup", "live", "ready"], + [TServiceDiagnosticDepth.stats_track]: ["ready", "stats"], + [TServiceDiagnosticDepth.stats_basic]: ["stats", "stats-full"], + [TServiceDiagnosticDepth.stats_deep]: ["stats-full"], +}; + +interface IDiagnosticsConfig { + // Diagnostic keys that should be considered ok without evaluation + skipKeys?: string[]; + // This secret must be provided to the diagnostic endpoints as a query param + // therefor it's likely to leak, through logs etc, for ex. and should not + // relied on for security because disabling diagnostics. At worst this + // provides access to the DB statistics. + probeSecret?: string; + // Period at which to log TDiagnosticTag `stats`. 0 to disable. + statsLogPeriodMs: number; + // Period at which to log TDiagnosticTag `stats-full`. 0 to disable. + statsFullStatsLogPeriodMs: number; +} + +export const getConfig = (): IDiagnosticsConfig => { + let configSection = config.get("diagnostics") as IDiagnosticsConfig; + if (typeof configSection.skipKeys === "string") + configSection = { + ...configSection, + skipKeys: (configSection.skipKeys as string) + .trim() + .split(/[,\s]+/g) + .filter(x => !!x), + }; + + const getNumberFromConfig = (value): number => { + if (typeof value == "number") return value; + if (typeof value == "string") return parseInt(value, 10) ?? 0; + return 0; + }; + return { + ...configSection, + statsLogPeriodMs: getNumberFromConfig(configSection.statsLogPeriodMs), + statsFullStatsLogPeriodMs: getNumberFromConfig(configSection.statsFullStatsLogPeriodMs), + }; +}; + +/** Code-wide unique key for each provider */ +export type TDiagnosticKey = string; + +/** Each provider should return an object of this format. The key of the provider defines the schema. */ +export type TDiagnosticResult = { ok: boolean; warn?: string; empty?: boolean } & { + [key: string]: unknown; +}; + +/** Implemented by objects that want to provide data to the diagnostic check */ +export interface IDiagnosticProvider { + /** Code-wide unique key underwhich the result of `get` will be included */ + key: TDiagnosticKey; + + /** This result is present in any included request tag */ + tags: TDiagnosticTag[] | "*"; + + /** + * If set, this provider will be polled at that interval. + * If `undefined`, this provider will be ran at each request. + */ + pollPeriodMs?: number; + + /** + * Returns an object as presented to a diagnostic requester. + * Warning: this could be public and readable to the internet. + */ + get(): Promise; +} + +/** + * Platform services that can provide generic diagnostic implementations may use this interface. + * + * Matching from {@link TDiagnosticTag} to {@link TServiceDiagnosticDepth} is expected to + * be done by intermediary providers. This is because not all services are equally critical, + * or have the same tolerable down times. + */ +export interface IServiceDiagnosticProvider { + /** The return format is specific to each service, but should include a `{ok: boolean}` field. */ + getDiagnostics(depth: TServiceDiagnosticDepth): Promise; +} + +const isProviderIncludedInTag = ( + tag: TDiagnosticTag, + provider: IDiagnosticProvider, + config: IDiagnosticsConfig, +) => + (provider.tags === "*" || provider.tags.indexOf(tag) >= 0 || provider.tags.indexOf("*") >= 0) && + (!config.skipKeys?.length || !config.skipKeys.includes(provider.key)); + +// registered providers with `pollPeriodMs === undefined` +const immediateDiagnosticProviders: IDiagnosticProvider[] = []; +// registered providers with `pollPeriodMs !== undefined` +const periodicDiagnosticProviders: IDiagnosticProvider[] = []; + +const now = () => Math.round(process.uptime() * 1000); + +const isKeyAlreadyRegistered = (key: TDiagnosticKey) => + immediateDiagnosticProviders.some(provider => key == provider.key) || + periodicDiagnosticProviders.some(provider => key == provider.key); + +// stores results of all the `pollPeriodMs` truthy providers +const latestPeriodicDiagnostics: { [key: TDiagnosticKey]: object } = {}; +const recordDiagnostic = (startMs: number, key: TDiagnosticKey, data?: object, error?: object) => + (latestPeriodicDiagnostics[key] = { + durationMs: Math.round(now() - startMs), + ...(error ? { ok: false, error } : { ...data }), + }); + +const runProvider = async (provider, log) => { + const startMs = now(); + try { + const result = await provider.get(); + if (!result.ok) + logger.error( + { diagnostic: provider.key, ...result }, + "Got diagnostic provider result with ok=false", + ); + else if (result.warn) + logger.warn( + { diagnostic: provider.key, ...result }, + "Got diagnostic provider result with ok=true but a warning", + ); + else if (log && !result.empty) + logger.info({ diagnostic: provider.key, ...result }, "Diagnostic provider result"); + return recordDiagnostic(startMs, provider.key, result); + } catch (err) { + logger.error({ err, provider: provider.key }, "Failed to read diagnostic provider"); + return recordDiagnostic(startMs, provider.key, undefined, err); + } +}; + +const pendingTimeouts: number[] = []; // Pending return values from `setTimeout` calls +const forgetPendingTimeout = (timeoutId: number) => { + const index = pendingTimeouts.indexOf(timeoutId); + assert(index >= 0); + pendingTimeouts.splice(index, 1); +}; + +let hasShutdown = false; +const ensureHasntShutdown = () => { + if (hasShutdown) throw new Error("Diagnostics service already shutdown"); +}; + +export default { + /** Add a provider to be included in diagnostics output */ + registerProviders(...providers: IDiagnosticProvider[]) { + ensureHasntShutdown(); + providers.forEach(provider => { + if (isKeyAlreadyRegistered(provider.key)) throw new Error("Provider with duplicate key"); + if (provider.pollPeriodMs) { + periodicDiagnosticProviders.push(provider); + } else { + immediateDiagnosticProviders.push(provider); + return; + } + let triggerUpdate: () => void = () => undefined; // The empty function is for the linter. I love you linter <3 + const updateProvider = (timeoutId: number) => async () => { + forgetPendingTimeout(timeoutId); + await runProvider(provider, false); + triggerUpdate(); + }; + triggerUpdate = () => pendingTimeouts.push(setTimeout(updateProvider, provider.pollPeriodMs)); + triggerUpdate(); + }); + }, + + /** Create providers to match from {@link IServiceDiagnosticProvider} to multiple {@link IDiagnosticProvider}s */ + registerServiceProviders( + name: string, + getService: () => IServiceDiagnosticProvider, + overrideTags: Partial< + typeof serviceDiagnosticDepthToTags | { [key in TServiceDiagnosticDepth]: false } + > = {}, + ) { + this.registerProviders( + ...Object.values(TServiceDiagnosticDepth) + .map(depth => { + const defaultTags = serviceDiagnosticDepthToTags[depth]; + if (!defaultTags) throw new Error(`Unknown depth ${JSON.stringify(depth)}`); + const tags = overrideTags[depth] ?? defaultTags; + if (tags === false) return null; + return { + key: `${name}-${depth}`, + tags, + get: () => getService().getDiagnostics(depth), + }; + }) + .filter(x => !!x), + ); + }, + + /** Cancel all pending diagnostic updates */ + shutdown() { + ensureHasntShutdown(); + pendingTimeouts.forEach(timeout => clearTimeout(timeout)); + hasShutdown = true; + }, + + /** + * Return the values of all providers which include the provided tag. + * + * @param log if `true`, print each individual log output even if succesful + */ + async get( + tag: TDiagnosticTag, + log: boolean, + ): Promise<{ ok: boolean } | { [key: TDiagnosticKey]: TDiagnosticResult }> { + const config = getConfig(); + const result = { ok: true }; + let atLeastOneCheck = false; + periodicDiagnosticProviders.forEach(provider => { + if (!isProviderIncludedInTag(tag, provider, config)) return; + atLeastOneCheck = true; + result[provider.key] = latestPeriodicDiagnostics[provider.key]; + if (!result[provider.key].ok) result.ok = false; + }); + await Promise.all( + immediateDiagnosticProviders.map(async provider => { + if (!isProviderIncludedInTag(tag, provider, config)) return; + atLeastOneCheck = true; + const providerResult = await runProvider(provider, log); + if (!providerResult.ok) result.ok = false; + return (result[provider.key] = providerResult); + }), + ); + if (!atLeastOneCheck) result.ok = false; + return result; + }, +}; diff --git a/tdrive/backend/node/src/core/platform/services/database/api.ts b/tdrive/backend/node/src/core/platform/services/database/api.ts index e9a3ab96..259fc38f 100644 --- a/tdrive/backend/node/src/core/platform/services/database/api.ts +++ b/tdrive/backend/node/src/core/platform/services/database/api.ts @@ -1,10 +1,11 @@ import { TdriveServiceProvider } from "../../framework"; +import type { IServiceDiagnosticProvider } from "../../framework/api/diagnostics"; import { Connector } from "./services/orm/connectors"; import Manager from "./services/orm/manager"; import Repository from "./services/orm/repository/repository"; import { EntityTarget } from "./services/orm/types"; -export interface DatabaseServiceAPI extends TdriveServiceProvider { +export interface DatabaseServiceAPI extends TdriveServiceProvider, IServiceDiagnosticProvider { /** * Get the database connector */ diff --git a/tdrive/backend/node/src/core/platform/services/database/services/index.ts b/tdrive/backend/node/src/core/platform/services/database/services/index.ts index d389d621..5d0df015 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/index.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/index.ts @@ -7,6 +7,10 @@ import { MongoConnectionOptions } from "./orm/connectors/mongodb/mongodb"; import { EntityTarget } from "./orm/types"; import { RepositoryManager } from "./orm/repository/manager"; import { PostgresConnectionOptions } from "./orm/connectors/postgres/postgres"; +import type { + TDiagnosticResult, + TServiceDiagnosticDepth, +} from "../../../framework/api/diagnostics"; export default class DatabaseService implements DatabaseServiceAPI { version = "1"; @@ -38,6 +42,15 @@ export default class DatabaseService implements DatabaseServiceAPI { } } + async getDiagnostics(depth: TServiceDiagnosticDepth): Promise { + const connector = this.getConnector(); + const result = await connector.getDiagnostics(depth); + return { + type: connector.getType(), + ...result, + }; + } + getManager(): Manager { return new Manager(this.connector); } diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/abstract-connector.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/abstract-connector.ts index daaf5810..50db7c98 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/abstract-connector.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/abstract-connector.ts @@ -4,12 +4,17 @@ import { ConnectionOptions, DatabaseType } from "../.."; import { FindOptions } from "../repository/repository"; import { ColumnDefinition, EntityDefinition } from "../types"; import { ListResult, Paginable, Pagination } from "../../../../../framework/api/crud-service"; +import type { + TDiagnosticResult, + TServiceDiagnosticDepth, +} from "../../../../../framework/api/diagnostics"; export abstract class AbstractConnector implements Connector { constructor(protected type: DatabaseType, protected options: T, protected secret: string) {} abstract connect(): Promise; abstract disconnect(): Promise; + abstract getDiagnostics(depth: TServiceDiagnosticDepth): Promise; abstract drop(): Promise; diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts index 6e1f50d6..55e9538d 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts @@ -5,6 +5,7 @@ import { ColumnDefinition, EntityDefinition } from "../types"; import { FindOptions } from "../repository/repository"; import { ListResult, Paginable, Pagination } from "../../../../../framework/api/crud-service"; import { PostgresConnectionOptions } from "./postgres/postgres"; +import type { IServiceDiagnosticProvider } from "../../../../../framework/api/diagnostics"; export * from "./mongodb/mongodb"; @@ -14,7 +15,7 @@ export type UpsertOptions = { export type RemoveOptions = any; -export interface Connector extends Initializable { +export interface Connector extends Initializable, IServiceDiagnosticProvider { /** * Connect to the database */ diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts index 55c7c2bf..3890361e 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts @@ -8,6 +8,10 @@ import { AbstractConnector } from "../abstract-connector"; import { buildSelectQuery } from "./query-builder"; import { transformValueFromDbString, transformValueToDbString } from "./typeTransforms"; import { logger } from "../../../../../../framework"; +import { + TDiagnosticResult, + TServiceDiagnosticDepth, +} from "../../../../../../framework/api/diagnostics"; export interface MongoConnectionOptions { // TODO: More options @@ -42,6 +46,51 @@ export class MongoConnector extends AbstractConnector { return this; } + private async ping(): Promise { + const wasConnected = !!this.client; + await (await this.getDatabase()).admin().ping(); + return !wasConnected; + } + + private async dbStats(): Promise { + return (await this.getDatabase()).stats(); + } + + private async collectionsStats(deep: boolean): Promise { + const db = await this.getDatabase(); + const result = { collections: {} }; + for (const collection of await db.collections()) { + const stats = await collection.aggregate([ + { + $collStats: { + latencyStats: { histograms: true }, + storageStats: deep ? {} : undefined, // Really a lot of keys with 0 occurances + count: {}, + queryExecStats: {}, + }, + }, + ]); + result.collections[collection.collectionName] = await stats.toArray(); + } + return result; + } + + async getDiagnostics(depth: TServiceDiagnosticDepth): Promise { + switch (depth) { + case TServiceDiagnosticDepth.alive: + return { ok: true, didConnect: await this.ping() }; + case TServiceDiagnosticDepth.stats_track: + return { ok: true, ...(await this.dbStats()) }; + case TServiceDiagnosticDepth.stats_basic: + return { ok: true, ...(await this.collectionsStats(false)) }; + case TServiceDiagnosticDepth.stats_deep: + return { ok: true, ...(await this.collectionsStats(true)) }; + + default: + throw new Error(`Unexpected TServiceDiagnosticDepth: ${JSON.stringify(depth)}`); + } + } + getClient(): mongo.MongoClient { return this.client; } diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres.ts index 696e8899..663f06fd 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres.ts @@ -8,6 +8,10 @@ import { getEntityDefinition } from "../../../orm/utils"; import { UpsertOptions } from "src/core/platform/services/database/services/orm/connectors"; import { PostgresDataTransformer, TypeMappings } from "./postgres-data-transform"; import { PostgresQueryBuilder, Query } from "./postgres-query-builder"; +import { + TDiagnosticResult, + TServiceDiagnosticDepth, +} from "../../../../../../framework/api/diagnostics"; export interface PostgresConnectionOptions { database: string; @@ -25,6 +29,7 @@ export interface PostgresConnectionOptions { export class PostgresConnector extends AbstractConnector { private logger = getLogger("PostgresConnector"); private client: Client = new Client(this.options); + private connected = false; private dataTransformer = new PostgresDataTransformer({ secret: this.secret }); private queryBuilder = new PostgresQueryBuilder(this.secret); @@ -40,10 +45,12 @@ export class PostgresConnector extends AbstractConnector { - this.logger.warn(err, "PostgreSQL connection error"); + this.logger.error(err, "PostgreSQL connection error"); }); + this.client.on("end", () => (this.connected = false)); await this.client.connect(); + this.connected = true; this.logger.info("Connection pool created"); await this.healthcheck(); } @@ -56,6 +63,59 @@ export class PostgresConnector extends AbstractConnector { + const wasConnected = this.connected; + if (wasConnected) await this.healthcheck(); + else await this.connect(); + return !wasConnected; + } + + async getDiagnostics(depth: TServiceDiagnosticDepth): Promise { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const safeRequest = async (query: string, values?: any[], singleRow = false) => { + try { + const rows = (await this.client.query(query, values)).rows; + return singleRow && rows?.length === 1 ? rows[0] : rows; + } catch (err) { + const logId = "pg-diags-error-" + Math.floor(process.uptime() * 1000); + logger.error( + { err, query, values, logId, errCode: err.code }, + `Error running postgresql statistics at ${depth} ( ${logId} ) `, + ); + return { error: true, logId }; + } + }; + switch (depth) { + // This is the only required `ok` + case TServiceDiagnosticDepth.alive: + return { ok: true, didConnect: await this.ping() }; + + // Statistics can silently fail, and do it granularly if there is + // a permission issue only on some of the stats + case TServiceDiagnosticDepth.stats_track: + return { + ok: true, + db: await safeRequest( + "select * from pg_stat_database where datname = $1", + [this.options.database], + true, + ), + }; + case TServiceDiagnosticDepth.stats_basic: + return { ok: true, empty: true }; + case TServiceDiagnosticDepth.stats_deep: + return { + ok: true, + databases: await safeRequest("select * from pg_stat_database"), + tables: await safeRequest("select * from pg_stat_user_tables"), + indexes: await safeRequest("select * from pg_stat_user_indexes"), + }; + + default: + throw new Error(`Unexpected TServiceDiagnosticDepth: ${JSON.stringify(depth)}`); + } + } + async init(): Promise { if (!this.client) { await this.connect(); @@ -111,7 +171,7 @@ export class PostgresConnector extends AbstractConnector { const query = ` - DO $$ - DECLARE + DO $$ + DECLARE tablename text; - BEGIN - FOR tablename IN (SELECT table_name FROM information_schema.tables WHERE table_schema = 'public') - LOOP - EXECUTE 'DELETE FROM "' || tablename || '" CASCADE'; - END LOOP; + BEGIN + FOR tablename IN (SELECT table_name FROM information_schema.tables WHERE table_schema = 'public') + LOOP + EXECUTE 'DELETE FROM "' || tablename || '" CASCADE'; + END LOOP; END $$;`; logger.debug(`service.database.orm.postgres.drop - Query: "${query}"`); await this.client.query(query); @@ -188,14 +248,14 @@ export class PostgresConnector extends AbstractConnector { const query = ` - DO $$ - DECLARE + DO $$ + DECLARE tablename text; - BEGIN - FOR tablename IN (SELECT table_name FROM information_schema.tables WHERE table_schema = 'public') - LOOP - EXECUTE 'DROP TABLE IF EXISTS "' || tablename || '" CASCADE'; - END LOOP; + BEGIN + FOR tablename IN (SELECT table_name FROM information_schema.tables WHERE table_schema = 'public') + LOOP + EXECUTE 'DROP TABLE IF EXISTS "' || tablename || '" CASCADE'; + END LOOP; END $$;`; logger.debug(`service.database.orm.postgres.dropTables - Query: "${query}"`); await this.client.query(query); @@ -343,13 +403,13 @@ export class PostgresConnector extends AbstractConnector { try { - const query = `SELECT - table_name, - column_name, - data_type - FROM + const query = `SELECT + table_name, + column_name, + data_type + FROM information_schema.columns - WHERE + WHERE table_name = $1`; const dbResult: QueryResult = await this.client.query(query, [name]); return dbResult.rows.map(row => row.column_name); diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/index.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/index.ts new file mode 100644 index 00000000..35217965 --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/index.ts @@ -0,0 +1,66 @@ +import { TdriveService, Consumes, Prefix, ServiceName } from "../../framework"; +import web from "./web"; +import DiagnosticsServiceAPI from "./service-provider"; +import DiagnosticsServiceImpl from "./service"; +import WebServerAPI from "../webserver/provider"; +import registerBasicProviders from "./providers"; +import diagnostics, { + getConfig as getDiagnosticsGetConfig, + TDiagnosticTag, +} from "../../framework/api/diagnostics"; +import registerFastifyRoutesDiagnosticsProvider from "./web/provider"; + +/** + * The diagnostics service exposes endpoint that are of use for operational reasons. + * + */ +@Prefix("/diagnostics") +@Consumes(["webserver"]) +@ServiceName("diagnostics") +export default class DiagnosticsService extends TdriveService { + name = "diagnostics"; + service: DiagnosticsServiceAPI; + private runningIntervalStatsLog?: ReturnType; + private runningIntervalStatsFullLog?: ReturnType; + + api(): DiagnosticsServiceAPI { + return this.service; + } + + public async doInit(): Promise { + this.service = new DiagnosticsServiceImpl(); + const fastify = this.context.getProvider("webserver").getServer(); + + registerBasicProviders(); + registerFastifyRoutesDiagnosticsProvider(fastify); + + fastify.register((instance, _opts, next) => { + web(instance, { prefix: this.prefix }); + next(); + }); + + const config = getDiagnosticsGetConfig(); + const printStatsToLog = (tag: TDiagnosticTag) => () => diagnostics.get(tag, true); + const startLoggingStats = (tag, periodMs) => + periodMs && periodMs > 0 ? setInterval(printStatsToLog(tag), periodMs) : undefined; + this.runningIntervalStatsLog = startLoggingStats("stats", config.statsLogPeriodMs); + this.runningIntervalStatsFullLog = startLoggingStats( + "stats-full", + config.statsFullStatsLogPeriodMs, + ); + + return this; + } + + public async doStop(): Promise { + if (this.runningIntervalStatsLog) { + clearInterval(this.runningIntervalStatsLog); + this.runningIntervalStatsLog = null; + } + if (this.runningIntervalStatsFullLog) { + clearInterval(this.runningIntervalStatsFullLog); + this.runningIntervalStatsFullLog = null; + } + return this; + } +} diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/providers/database-service.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/database-service.ts new file mode 100644 index 00000000..0df00d3c --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/database-service.ts @@ -0,0 +1,4 @@ +import diagnostics from "../../../framework/api/diagnostics"; +import globalResolver from "../../../../../services/global-resolver"; + +export default () => diagnostics.registerServiceProviders("db", () => globalResolver.database); diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/providers/index.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/index.ts new file mode 100644 index 00000000..08f2b5d7 --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/index.ts @@ -0,0 +1,11 @@ +import registerDBServiceProvider from "./database-service"; +import registerStorageServiceProvider from "./storage-service"; +import registerPlatformProvider from "./platform-started"; +import registerProcessProvider from "./process"; + +export default () => { + registerDBServiceProvider(); + registerStorageServiceProvider(); + registerPlatformProvider(); + registerProcessProvider(); +}; diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/providers/platform-started.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/platform-started.ts new file mode 100644 index 00000000..d3eff017 --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/platform-started.ts @@ -0,0 +1,14 @@ +import diagnostics from "../../../framework/api/diagnostics"; +import globalResolver from "../../../../../services/global-resolver"; + +export default () => + diagnostics.registerProviders({ + key: "platform", + tags: "*", + get: async () => { + return { + ok: globalResolver.isPlatformStarted(), + uptimeMs: Math.floor((process.uptime() + Math.random()) * 1000), + }; + }, + }); diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/providers/process.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/process.ts new file mode 100644 index 00000000..eca74dfc --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/process.ts @@ -0,0 +1,21 @@ +import diagnostics from "../../../framework/api/diagnostics"; + +export const e2eTestOverride = { + // Set to true to force fail to test diagnostic aggregation + forceFail: false, +}; + +export default () => + diagnostics.registerProviders({ + key: "process", + tags: ["live", "ready"], + async get() { + return { + ok: !e2eTestOverride.forceFail, + gc: !!global.gc, + pid: process.pid, + mem: process.memoryUsage(), + res: process.resourceUsage(), + }; + }, + }); diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/providers/storage-service.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/storage-service.ts new file mode 100644 index 00000000..5ad43456 --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/storage-service.ts @@ -0,0 +1,9 @@ +import diagnostics from "../../../framework/api/diagnostics"; +import globalResolver from "../../../../../services/global-resolver"; + +export default () => + diagnostics.registerServiceProviders("storage", () => globalResolver.platformServices.storage, { + stats_basic: false, + stats_track: false, + stats_deep: false, + }); diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/service-provider.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/service-provider.ts new file mode 100644 index 00000000..00b72786 --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/service-provider.ts @@ -0,0 +1,5 @@ +import type { TdriveServiceProvider } from "../../framework"; + +type DiagnosticsServiceAPI = TdriveServiceProvider; + +export default DiagnosticsServiceAPI; diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/service.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/service.ts new file mode 100644 index 00000000..614e10c6 --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/service.ts @@ -0,0 +1,5 @@ +import DiagnosticsServiceAPI from "./service-provider"; + +export default class DiagnosticsServiceImpl implements DiagnosticsServiceAPI { + version: "1"; +} diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/utils.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/utils.ts new file mode 100644 index 00000000..375a1ebe --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/utils.ts @@ -0,0 +1,40 @@ +import { logger } from "../../framework"; +import { Readable, Transform } from "node:stream"; +import { Session } from "node:inspector"; + +/** Time the execution of the callback, and return its duration in ms with the result */ +export async function timeCallback( + cb: () => Promise, +): Promise<{ durationMs: number; result: T }> { + const startMs = new Date().getTime(); + const result = await cb(); + return { durationMs: new Date().getTime() - startMs, result }; +} + +/** Use `node:inspector` to create a snapshot of the heap, and + * synchroneously (because of session.post) pipe it to a stream + * readable from the first argument of the callback. + */ +export function getHeapSnapshotSync(cb: (readable: Readable) => void) { + logger.info({ gcExposed: !!global.gc }, "Beginning heap snapshot"); + if (global.gc) global.gc(); + const session = new Session(); + session.connect(); + try { + const transform = new Transform(); + try { + let size = 0; + session.on("HeapProfiler.addHeapSnapshotChunk", message => { + size += message.params.chunk.length; + transform.push(message.params.chunk); + }); + cb(transform); + session.post("HeapProfiler.takeHeapSnapshot", null); + logger.info({ size }, "Heap snapshot sent"); + } finally { + transform.end(); + } + } finally { + session.disconnect(); + } +} diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/web/heap-routes.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/web/heap-routes.ts new file mode 100644 index 00000000..eb294cf8 --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/web/heap-routes.ts @@ -0,0 +1,27 @@ +import type { FastifyInstance, FastifyPluginCallback } from "fastify"; +import { getConfig as getDiagnosticsGetConfig } from "../../../framework/api/diagnostics"; +import { getHeapSnapshotSync } from "../utils"; + +const routes: FastifyPluginCallback = (fastify: FastifyInstance, _opts, next) => { + const diagnosticsConfig = getDiagnosticsGetConfig(); + if (diagnosticsConfig?.probeSecret?.length) { + fastify.post("/heap", async (request, reply) => { + if ((request.query as { secret: string }).secret !== diagnosticsConfig.probeSecret) + return reply.status(403).send(); + const filenameTimestamp = new Date() + .toISOString() + .replace(/(\.\d\d\d)?Z$/, "") + .replace(/\D/g, "-"); + reply.header( + "Content-Disposition", + `attachment; filename="twake-drive-snap-${filenameTimestamp}.heapsnapshot"`, + ); + let replyResult; + getHeapSnapshotSync(readable => (replyResult = reply.send(readable))); + return replyResult; + }); + } + next(); +}; + +export default routes; diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/web/index.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/web/index.ts new file mode 100644 index 00000000..3091541c --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/web/index.ts @@ -0,0 +1,11 @@ +import type { FastifyInstance, FastifyRegisterOptions } from "fastify"; +import probeRoutes from "./probe-routes"; +import heapRoutes from "./heap-routes"; + +export default ( + fastify: FastifyInstance, + opts: FastifyRegisterOptions<{ prefix: string }>, +): void => { + fastify.register(probeRoutes, opts); + fastify.register(heapRoutes, opts); +}; diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/web/probe-routes.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/web/probe-routes.ts new file mode 100644 index 00000000..2ffc2130 --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/web/probe-routes.ts @@ -0,0 +1,25 @@ +import type { FastifyInstance, FastifyPluginCallback } from "fastify"; +import diagnostics, { + getConfig as getDiagnosticsGetConfig, +} from "../../../framework/api/diagnostics"; + +const routes: FastifyPluginCallback = (fastify: FastifyInstance, _opts, next) => { + const diagnosticsConfig = getDiagnosticsGetConfig(); + if (diagnosticsConfig?.probeSecret?.length) { + const tagParamName = "diagnosticTags"; + fastify.get(`/t/:${tagParamName}`, async (request, reply) => { + const tag = request.params[tagParamName]; + if ( + tag === "*" || + (request.query as { secret: string }).secret !== diagnosticsConfig.probeSecret + ) + return reply.status(403).send(); + const results = await diagnostics.get(tag, false); + if (!results.ok) reply.status(503); + return reply.send(results); + }); + } + next(); +}; + +export default routes; diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/web/provider.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/web/provider.ts new file mode 100644 index 00000000..87a543a9 --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/web/provider.ts @@ -0,0 +1,23 @@ +import diagnostics from "../../../framework/api/diagnostics"; + +export default fastify => { + // TODO: registering @fastify/routes-stats creates errors (performance mark not found) + // on the request performance marks, the hook doesn't seem to be called for every route + // and some of the more important ones like browse start failing... + // Version tested had to be 3.4.0 because 4+ needs Fastify 5. + return; + // Don't require it at all until it's fixed + // eslint-disable-next-line @typescript-eslint/no-var-requires + fastify.register(require("@fastify/routes-stats"), { + printInterval: 4000, // milliseconds + decoratorName: "performanceMarked", // decorator is set to true if a performace.mark was called for the request + }); + diagnostics.registerProviders({ + key: "fastify-routes", + tags: ["stats", "stats-full"], + async get() { + fastify.measurements(); + return { ok: true, ...fastify.stats() }; + }, + }); +}; diff --git a/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts b/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts index e70cfc4a..e5c1d1f3 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts @@ -4,6 +4,7 @@ import { Readable } from "stream"; import { StorageConnectorAPI, WriteMetadata } from "../../provider"; import { randomUUID } from "crypto"; import _ from "lodash"; +import { TDiagnosticResult, TServiceDiagnosticDepth } from "../../../../framework/api/diagnostics"; export type S3Configuration = { id: string; @@ -43,6 +44,21 @@ export default class S3ConnectorService implements StorageConnectorAPI { return this.id; } + async getDiagnostics(depth: TServiceDiagnosticDepth): Promise { + switch (depth) { + case TServiceDiagnosticDepth.alive: + return { ok: await this.client.bucketExists(this.minioConfiguration.bucket) }; + case TServiceDiagnosticDepth.stats_basic: + case TServiceDiagnosticDepth.stats_track: + case TServiceDiagnosticDepth.stats_deep: + // Local store is always ok... should never be used outside dev environments + return { ok: true, empty: true }; + + default: + throw new Error(`Unexpected TServiceDiagnosticDepth: ${JSON.stringify(depth)}`); + } + } + write(path: string, stream: Readable): Promise { return new Promise((resolve, reject) => { let totalSize = 0; diff --git a/tdrive/backend/node/src/core/platform/services/storage/connectors/local/service.ts b/tdrive/backend/node/src/core/platform/services/storage/connectors/local/service.ts index 16cf356f..682411c3 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/connectors/local/service.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/connectors/local/service.ts @@ -6,6 +6,7 @@ import { StorageConnectorAPI, WriteMetadata } from "../../provider"; import fs from "fs"; import { logger } from "../../../../framework/logger"; import { randomUUID } from "crypto"; +import { TDiagnosticResult, TServiceDiagnosticDepth } from "../../../../framework/api/diagnostics"; export type LocalConfiguration = { id: string; @@ -29,6 +30,19 @@ export default class LocalConnectorService implements StorageConnectorAPI { return this.id; } + async getDiagnostics(depth: TServiceDiagnosticDepth): Promise { + switch (depth) { + case TServiceDiagnosticDepth.alive: + case TServiceDiagnosticDepth.stats_basic: + case TServiceDiagnosticDepth.stats_track: + case TServiceDiagnosticDepth.stats_deep: + return { ok: true, empty: true }; + + default: + throw new Error(`Unexpected TServiceDiagnosticDepth: ${JSON.stringify(depth)}`); + } + } + write(relativePath: string, stream: Readable): Promise { const path = this.getFullPath(relativePath); logger.trace(`Writing file ${path}`); diff --git a/tdrive/backend/node/src/core/platform/services/storage/default-storage-strategy.ts b/tdrive/backend/node/src/core/platform/services/storage/default-storage-strategy.ts index c14034ca..8e14112d 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/default-storage-strategy.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/default-storage-strategy.ts @@ -19,6 +19,10 @@ export class DefaultStorageStrategy implements StorageConnectorAPI { return this.connector.getId(); } + getDiagnostics(depth) { + return this.connector.getDiagnostics(depth); + } + write = ( path: string, stream: Stream, diff --git a/tdrive/backend/node/src/core/platform/services/storage/oneof-storage-strategy.ts b/tdrive/backend/node/src/core/platform/services/storage/oneof-storage-strategy.ts index b71c7026..0223b8a6 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/oneof-storage-strategy.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/oneof-storage-strategy.ts @@ -8,6 +8,7 @@ import { } from "../storage/provider"; import { logger } from "../../../platform/framework"; import { FileNotFountException, WriteFileException } from "./exceptions"; +import type { TDiagnosticResult, TServiceDiagnosticDepth } from "../../framework/api/diagnostics"; /** * OneOfStorageStrategy is responsible for managing multiple storage backends. @@ -34,6 +35,17 @@ export class OneOfStorageStrategy implements StorageConnectorAPI { return this.id; } + async getDiagnostics(depth: TServiceDiagnosticDepth): Promise { + const states = await Promise.all( + this.storages.map(async s => ({ id: s.getId(), ...(await s.getDiagnostics(depth)) })), + ); + return { + ...(states.every(s => s.ok) ? {} : { warn: "not_all_storages_ok" }), + ok: states.some(s => s.ok), + states: states, + }; + } + /** * Writes a file to all configured storages. * The write operation is considered successful if one of the storage succeed. diff --git a/tdrive/backend/node/src/core/platform/services/storage/provider.ts b/tdrive/backend/node/src/core/platform/services/storage/provider.ts index d46a554c..fb258876 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/provider.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/provider.ts @@ -1,6 +1,7 @@ import { Stream, Readable } from "stream"; import { TdriveServiceProvider } from "../../framework"; import { ExecutionContext } from "../../framework/api/crud-service"; +import { IServiceDiagnosticProvider } from "../../framework/api/diagnostics"; export type WriteMetadata = { size: number; @@ -22,7 +23,7 @@ export type DeleteOptions = { totalChunks?: number; }; -export interface StorageConnectorAPI { +export interface StorageConnectorAPI extends IServiceDiagnosticProvider { /** * Returns identifier of a storage that should've been set in configuration. * diff --git a/tdrive/backend/node/src/core/platform/services/storage/storage-service.ts b/tdrive/backend/node/src/core/platform/services/storage/storage-service.ts index 4eaea4ba..b2763250 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/storage-service.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/storage-service.ts @@ -64,6 +64,10 @@ export default class StorageService extends TdriveService implements return this.connector; } + getDiagnostics(depth) { + return this.getConnector().getDiagnostics(depth); + } + getHomeDir(): string { return this.homeDir; } diff --git a/tdrive/backend/node/src/services/av/service/index.ts b/tdrive/backend/node/src/services/av/service/index.ts index 8bcf44fb..867f54f5 100644 --- a/tdrive/backend/node/src/services/av/service/index.ts +++ b/tdrive/backend/node/src/services/av/service/index.ts @@ -32,8 +32,8 @@ export class AVServiceImpl implements TdriveServiceProvider, Initializable { }, }); } - } catch (error) { - logger.error({ error: `${error}` }, "Error while initializing Antivirus Service"); + } catch (err) { + logger.error({ err }, "Error while initializing Antivirus Service"); throw AVException.initializationFailed("Failed to initialize Antivirus service"); } return this; diff --git a/tdrive/backend/node/src/services/console/web/controller.ts b/tdrive/backend/node/src/services/console/web/controller.ts index 35a3217f..703149f2 100644 --- a/tdrive/backend/node/src/services/console/web/controller.ts +++ b/tdrive/backend/node/src/services/console/web/controller.ts @@ -37,6 +37,20 @@ export class ConsoleController { } async auth(request: FastifyRequest<{ Body: AuthRequest }>): Promise { + const censoredHeadersLC = { + authorization: true, + cookie: true, + }; + logger.info( + { + http_headers: Object.fromEntries( + Object.entries(request.headers) + .map(([k, v]) => (censoredHeadersLC[k.toLowerCase()] ? null : [k, v])) + .filter(x => !!x), + ), + }, + "auth_attempt", + ); if (request.body.oidc_id_token) { return { access_token: await this.authByToken(request.body.oidc_id_token) }; } else if (request.body.email && request.body.password) { diff --git a/tdrive/backend/node/src/services/global-resolver.ts b/tdrive/backend/node/src/services/global-resolver.ts index 3e68328b..42088f6e 100644 --- a/tdrive/backend/node/src/services/global-resolver.ts +++ b/tdrive/backend/node/src/services/global-resolver.ts @@ -15,7 +15,7 @@ import TrackerAPI from "../core/platform/services/tracker/provider"; import WebServerAPI from "../core/platform/services/webserver/provider"; import assert from "assert"; -import { logger } from "../core/platform/framework"; +import { logger, TdriveServiceState } from "../core/platform/framework"; import { ApplicationServiceImpl } from "./applications/services/applications"; import { CompanyApplicationServiceImpl } from "./applications/services/company-applications"; import { ApplicationHooksService } from "./applications/services/hooks"; @@ -78,6 +78,7 @@ class GlobalResolver { public services: TdriveServices; public platformServices: PlatformServices; public database: DatabaseServiceAPI; + private platform?: TdrivePlatform; public fastify: FastifyInstance; @@ -87,6 +88,7 @@ class GlobalResolver { if (this.alreadyInitialized) { return; } + this.platform = platform; this.database = platform.getProvider("database"); this.platformServices = { @@ -152,6 +154,11 @@ class GlobalResolver { logger.info("Global resolver finished initializing services"); this.alreadyInitialized = true; } + + /** `true` if all components are in the started state. This means we should respond to http at least (for kubernetes Startup probe). */ + isPlatformStarted(): boolean { + return this.alreadyInitialized && this.platform?.state.getValue() == TdriveServiceState.Started; + } } export default new GlobalResolver(); diff --git a/tdrive/backend/node/test/e2e/diagnostics/diagnostics.spec.ts b/tdrive/backend/node/test/e2e/diagnostics/diagnostics.spec.ts new file mode 100644 index 00000000..276bd689 --- /dev/null +++ b/tdrive/backend/node/test/e2e/diagnostics/diagnostics.spec.ts @@ -0,0 +1,63 @@ +import "reflect-metadata"; +import { afterAll, beforeEach, describe, expect, it, jest } from "@jest/globals"; +import { init, TestPlatform } from "../setup"; +import { getConfig as getDiagnosticsConfig } from "../../../src/core/platform/framework/api/diagnostics"; +import { e2eTestOverride as processDiagnosticProviderE2EOverride } from "../../../src/core/platform/services/diagnostics/providers/process"; + +describe("The diagnostics infrastucture", () => { + let platform: TestPlatform; + const diagnosticConfig = getDiagnosticsConfig(); + beforeEach(async () => { + platform = await init({ + services: [ + "webserver", + "database", + "applications", + "search", + "storage", + "diagnostics", + "message-queue", + "user", + "files", + "auth", + "statistics", + "platform-services", + "documents", + ], + }); + }); + + afterAll(async () => { + await platform?.tearDown(); + // @ts-ignore + platform = null; + }); + + const getDiagnosticTags = (tag: string, secret: string) => + platform.app.inject({ + method: "GET", + url: `/diagnostics/t/${encodeURIComponent(tag)}?secret=${encodeURIComponent(secret)}`, + }); + + it("should refuse invalid probe secrets", async () => { + const result = await getDiagnosticTags("ready", "ooooh look at me ! I'm like totally such an invalid probeSecret value"); + expect(result.statusCode).toBe(403); + }); + + it("should at least report alive", async () => { + const result = await getDiagnosticTags("ready", diagnosticConfig.probeSecret!); + expect(result.statusCode).toBe(200); + expect(result.json().ok).toBe(true); + }); + + it("should aggregate failure", async () => { + try { + processDiagnosticProviderE2EOverride.forceFail = true; + const result = await getDiagnosticTags("ready", diagnosticConfig.probeSecret!); + expect(result.statusCode).toBe(503); + expect(result.json().ok).toBe(false); + } finally { + processDiagnosticProviderE2EOverride.forceFail = false; + } + }); +}); diff --git a/tdrive/docker-compose.dev.tests.opensearch.yml b/tdrive/docker-compose.dev.tests.opensearch.yml index 7c025bd0..69cb853d 100644 --- a/tdrive/docker-compose.dev.tests.opensearch.yml +++ b/tdrive/docker-compose.dev.tests.opensearch.yml @@ -68,6 +68,7 @@ services: - SEARCH_OS_PASSWORD=admin - SEARCH_OS_USE_AUTH=true - SEARCH_OS_USERNAME=admin + - DIAG_PROBE_SECRET=super_diagnostic_probe_secret depends_on: - postgres - opensearch-node1 diff --git a/tdrive/docker-compose.tests.yml b/tdrive/docker-compose.tests.yml index 515b6cc4..2b887a95 100644 --- a/tdrive/docker-compose.tests.yml +++ b/tdrive/docker-compose.tests.yml @@ -84,6 +84,7 @@ services: - STORAGE_S3_ENDPOINT=minio - STORAGE_S3_ACCESS_KEY=admin - STORAGE_S3_SECRET_KEY=adminminio + - DIAG_PROBE_SECRET=super_diagnostic_probe_secret depends_on: minio: condition: service_healthy