diff --git a/tdrive/backend/node/config/custom-environment-variables.json b/tdrive/backend/node/config/custom-environment-variables.json index 3e337b86..97880a2b 100644 --- a/tdrive/backend/node/config/custom-environment-variables.json +++ b/tdrive/backend/node/config/custom-environment-variables.json @@ -16,6 +16,14 @@ "logger": { "level": "LOG_LEVEL" }, + "diagnostics": { + "skipKeys": { + "__name": "DIAG_SKIP_KEYS", + "__format": "json" + }, + "probeSecret": "DIAG_PROBE_SECRET", + "secret": "DIAG_SECRET" + }, "webserver": { "host": "TWAKE_DRIVE_HOST", "logger": { diff --git a/tdrive/backend/node/config/default.json b/tdrive/backend/node/config/default.json index 30da3a6f..d59e7664 100644 --- a/tdrive/backend/node/config/default.json +++ b/tdrive/backend/node/config/default.json @@ -35,6 +35,12 @@ "logger": { "level": "debug" }, + "diagnostics": { + "skipKeys": [], + "probeSecret": "", + "secret": "", + "secretChallengeRefreshS": 15 + }, "tracker": { "type": "segment", "segment": { @@ -226,6 +232,7 @@ }, "services": [ "auth", + "diagnostics", "push", "storage", "webserver", diff --git a/tdrive/backend/node/src/core/platform/framework/api/diagnostics.ts b/tdrive/backend/node/src/core/platform/framework/api/diagnostics.ts new file mode 100644 index 00000000..d26010e1 --- /dev/null +++ b/tdrive/backend/node/src/core/platform/framework/api/diagnostics.ts @@ -0,0 +1,218 @@ +import assert from "node:assert"; +import config from "../../../config"; + +/** + * Values that can match a set of diagnostic providers. + * + * `startup`, `ready` and `live` are meant to match the meanings of the corresponding + * kubernetes probes: + * https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ + */ +export type TDiagnosticTag = + | "*" // Special value that includes everything, and is always included + | "startup" // Tests that are absolutely required (and light) to even begin the other tests + | "ready" // Tests required before traffic can be sent our way + | "live" // Tests required to prevent from being restarted + | "stats"; // Expensive diagnostics that should not often be ran + +/** Detail requested from platform service self-diagnostics */ +export enum TServiceDiagnosticDepth { + /** Minimal cost information that tests functioning service */ + critical = 0, + /** Statistics that have a little impact enough for regular tracking into a time series */ + tracked_statistics = 1, + /** Statistics possibly expensive and large to calculate, for occasional debug operations */ + deep_statistics = 2, +} + +interface IDiagnosticsConfig { + // Diagnostic keys that should be considered ok without evaluation + skipKeys?: string[]; + // This secret must be provided to the diagnostic endpoints as a query param + // therefor it's likely to leak, through logs etc, for ex. and should not + // relied on for security because disabling diagnostics. At worst this + // provides access to the DB statistics. + probeSecret?: string; + // This secret is required to sign more dangerous diagnostic endpoints, such + // the heap snapshot. It should never be sent over the network. + secret?: string; + // Maximum time to keep the same challenge token for diagnostic endpoints, in seconds. + // Must be large enough to have a reasonable chance of running the token request + // then the action on the same backend instance. + secretChallengeRefreshS: number; +} + +export const getConfig = (): IDiagnosticsConfig => { + let configSection = config.get("diagnostics") as IDiagnosticsConfig; + if (typeof configSection.skipKeys === "string") + configSection = { + ...configSection, + skipKeys: (configSection.skipKeys as string) + .trim() + .split(/[,\s]+/g) + .filter(x => !!x), + }; + if (typeof configSection.secretChallengeRefreshS === "string") + configSection = { + ...configSection, + secretChallengeRefreshS: parseInt(configSection.secretChallengeRefreshS, 10), + }; + return configSection; +}; + +/** Code-wide unique key for each provider */ +export type TDiagnosticKey = string; + +/** Each provider should return an object of this format. The key of the provider defines the schema. */ +export type TDiagnosticResult = { ok: boolean; warn?: string } & { [key: string]: unknown }; + +/** Implemented by objects that want to provide data to the diagnostic check */ +export interface IDiagnosticProvider { + /** Code-wide unique key underwhich the result of `get` will be included */ + key: TDiagnosticKey; + + /** This result is present in any included request tag */ + tags: TDiagnosticTag[] | "*"; + + /** + * If set, this provider will be polled at that interval. + * If `undefined`, this provider will be ran at each request. + */ + pollPeriodMs?: number; + + /** + * Returns an object as presented to a diagnostic requester. + * Warning: this could be public and readable to the internet. + * @param completeButSlow If `true`, perform additional operations for a + * more informative + */ + get(): Promise; +} + +/** + * Platform services that can provide generic diagnostic implementations may use this interface. + * + * Matching from {@link TDiagnosticTag} to {@link TServiceDiagnosticDepth} is expected to + * be done by intermediary providers. This is because not all services are equally critical, + * or have the same tolerable down times. + */ +export interface IServiceDiagnosticProvider { + /** The return format is specific to each service, but should include a `{ok: boolean}` field. */ + getDiagnostics(depth: TServiceDiagnosticDepth): Promise; +} + +const isProviderIncludedInTag = ( + tag: TDiagnosticTag, + provider: IDiagnosticProvider, + config: IDiagnosticsConfig, +) => + (provider.tags === "*" || provider.tags.indexOf(tag) >= 0 || provider.tags.indexOf("*") >= 0) && + (!config.skipKeys?.length || !config.skipKeys.includes(provider.key)); + +// registered providers with `pollPeriodMs === undefined` +const immediateDiagnosticProviders: IDiagnosticProvider[] = []; +// registered providers with `pollPeriodMs !== undefined` +const periodicDiagnosticProviders: IDiagnosticProvider[] = []; + +const now = () => Math.round(process.uptime() * 1000); + +const isKeyAlreadyRegistered = (key: TDiagnosticKey) => + immediateDiagnosticProviders.some(provider => key == provider.key) || + periodicDiagnosticProviders.some(provider => key == provider.key); + +// stores results of all the `pollPeriodMs` truthy providers +const latestPeriodicDiagnostics: { [key: TDiagnosticKey]: object } = {}; +const recordDiagnostic = (startMs: number, key: TDiagnosticKey, data?: object, error?: object) => + (latestPeriodicDiagnostics[key] = { + durationMs: Math.round(now() - startMs), + ...(error ? { ok: false, error } : { ...data }), + }); + +const runProvider = async provider => { + const startMs = now(); + try { + const result = await provider.get(); + if (!result.ok || result.warn) + logger.error( + { provider: provider.key, result }, + "Got diagnostic provider result with ok=false", + ); + else if (result.warn) + logger.warn( + { provider: provider.key, result }, + "Got diagnostic provider result with ok=true but a warning", + ); + return recordDiagnostic(startMs, provider.key, result); + } catch (error) { + return recordDiagnostic(startMs, provider.key, undefined, error); + } +}; + +const pendingTimeouts: number[] = []; // Pending return values from `setTimeout` calls +const forgetPendingTimeout = (timeoutId: number) => { + const index = pendingTimeouts.indexOf(timeoutId); + assert(index >= 0); + pendingTimeouts.splice(index, 1); +}; + +let hasShutdown = false; +const ensureHasntShutdown = () => { + if (hasShutdown) throw new Error("Diagnostics service already shutdown"); +}; + +export default { + /** Add a provider to be included in diagnostics output */ + registerProviders(...providers: IDiagnosticProvider[]) { + ensureHasntShutdown(); + providers.forEach(provider => { + if (isKeyAlreadyRegistered(provider.key)) throw new Error("Provider with duplicate key"); + if (provider.pollPeriodMs) { + periodicDiagnosticProviders.push(provider); + } else { + immediateDiagnosticProviders.push(provider); + return; + } + let triggerUpdate: () => void = () => undefined; // The empty function is for the linter. I love you linter <3 + const updateProvider = (timeoutId: number) => async () => { + forgetPendingTimeout(timeoutId); + await runProvider(provider); + triggerUpdate(); + }; + triggerUpdate = () => pendingTimeouts.push(setTimeout(updateProvider, provider.pollPeriodMs)); + triggerUpdate(); + }); + }, + + /** Cancel all pending diagnostic updates */ + shutdown() { + ensureHasntShutdown(); + pendingTimeouts.forEach(timeout => clearTimeout(timeout)); + hasShutdown = true; + }, + + /** Return the values of all providers which include the provided tag */ + async get( + tag: TDiagnosticTag, + ): Promise<{ ok: boolean } | { [key: TDiagnosticKey]: TDiagnosticResult }> { + const config = getConfig(); + const result = { ok: true }; + let atLeastOneCheck = false; + periodicDiagnosticProviders.forEach(provider => { + if (!isProviderIncludedInTag(tag, provider, config)) return; + atLeastOneCheck = true; + result[provider.key] = latestPeriodicDiagnostics[provider.key]; + if (!result[provider.key].ok) result.ok = false; + }); + await Promise.all( + immediateDiagnosticProviders.map(async provider => { + if (!isProviderIncludedInTag(tag, provider, config)) return; + atLeastOneCheck = true; + const providerResult = await runProvider(provider); + if (!providerResult.ok) result.ok = false; + return (result[provider.key] = providerResult); + }), + ); + if (!atLeastOneCheck) result.ok = false; + return result; + }, +}; diff --git a/tdrive/backend/node/src/core/platform/services/database/api.ts b/tdrive/backend/node/src/core/platform/services/database/api.ts index e9a3ab96..259fc38f 100644 --- a/tdrive/backend/node/src/core/platform/services/database/api.ts +++ b/tdrive/backend/node/src/core/platform/services/database/api.ts @@ -1,10 +1,11 @@ import { TdriveServiceProvider } from "../../framework"; +import type { IServiceDiagnosticProvider } from "../../framework/api/diagnostics"; import { Connector } from "./services/orm/connectors"; import Manager from "./services/orm/manager"; import Repository from "./services/orm/repository/repository"; import { EntityTarget } from "./services/orm/types"; -export interface DatabaseServiceAPI extends TdriveServiceProvider { +export interface DatabaseServiceAPI extends TdriveServiceProvider, IServiceDiagnosticProvider { /** * Get the database connector */ diff --git a/tdrive/backend/node/src/core/platform/services/database/services/index.ts b/tdrive/backend/node/src/core/platform/services/database/services/index.ts index d389d621..5d0df015 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/index.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/index.ts @@ -7,6 +7,10 @@ import { MongoConnectionOptions } from "./orm/connectors/mongodb/mongodb"; import { EntityTarget } from "./orm/types"; import { RepositoryManager } from "./orm/repository/manager"; import { PostgresConnectionOptions } from "./orm/connectors/postgres/postgres"; +import type { + TDiagnosticResult, + TServiceDiagnosticDepth, +} from "../../../framework/api/diagnostics"; export default class DatabaseService implements DatabaseServiceAPI { version = "1"; @@ -38,6 +42,15 @@ export default class DatabaseService implements DatabaseServiceAPI { } } + async getDiagnostics(depth: TServiceDiagnosticDepth): Promise { + const connector = this.getConnector(); + const result = await connector.getDiagnostics(depth); + return { + type: connector.getType(), + ...result, + }; + } + getManager(): Manager { return new Manager(this.connector); } diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/abstract-connector.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/abstract-connector.ts index daaf5810..50db7c98 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/abstract-connector.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/abstract-connector.ts @@ -4,12 +4,17 @@ import { ConnectionOptions, DatabaseType } from "../.."; import { FindOptions } from "../repository/repository"; import { ColumnDefinition, EntityDefinition } from "../types"; import { ListResult, Paginable, Pagination } from "../../../../../framework/api/crud-service"; +import type { + TDiagnosticResult, + TServiceDiagnosticDepth, +} from "../../../../../framework/api/diagnostics"; export abstract class AbstractConnector implements Connector { constructor(protected type: DatabaseType, protected options: T, protected secret: string) {} abstract connect(): Promise; abstract disconnect(): Promise; + abstract getDiagnostics(depth: TServiceDiagnosticDepth): Promise; abstract drop(): Promise; diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts index 6e1f50d6..55e9538d 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts @@ -5,6 +5,7 @@ import { ColumnDefinition, EntityDefinition } from "../types"; import { FindOptions } from "../repository/repository"; import { ListResult, Paginable, Pagination } from "../../../../../framework/api/crud-service"; import { PostgresConnectionOptions } from "./postgres/postgres"; +import type { IServiceDiagnosticProvider } from "../../../../../framework/api/diagnostics"; export * from "./mongodb/mongodb"; @@ -14,7 +15,7 @@ export type UpsertOptions = { export type RemoveOptions = any; -export interface Connector extends Initializable { +export interface Connector extends Initializable, IServiceDiagnosticProvider { /** * Connect to the database */ diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts index 55c7c2bf..395fffc4 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts @@ -8,6 +8,10 @@ import { AbstractConnector } from "../abstract-connector"; import { buildSelectQuery } from "./query-builder"; import { transformValueFromDbString, transformValueToDbString } from "./typeTransforms"; import { logger } from "../../../../../../framework"; +import { + TDiagnosticResult, + TServiceDiagnosticDepth, +} from "../../../../../../framework/api/diagnostics"; export interface MongoConnectionOptions { // TODO: More options @@ -42,6 +46,24 @@ export class MongoConnector extends AbstractConnector { return this; } + private async ping(): Promise { + const wasConnected = !!this.client; + await (await this.getDatabase()).admin().ping(); + return !wasConnected; + } + + async getDiagnostics(depth: TServiceDiagnosticDepth): Promise { + switch (depth) { + case TServiceDiagnosticDepth.critical: + return { ok: true, didConnect: await this.ping() }; + case TServiceDiagnosticDepth.deep_statistics: + case TServiceDiagnosticDepth.tracked_statistics: + return { ok: true, warn: "unsupported_depth" }; + + default: + throw new Error(`Unexpected TServiceDiagnosticDepth: ${JSON.stringify(depth)}`); + } + } getClient(): mongo.MongoClient { return this.client; } diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres.ts index 696e8899..0fb00784 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres.ts @@ -8,6 +8,10 @@ import { getEntityDefinition } from "../../../orm/utils"; import { UpsertOptions } from "src/core/platform/services/database/services/orm/connectors"; import { PostgresDataTransformer, TypeMappings } from "./postgres-data-transform"; import { PostgresQueryBuilder, Query } from "./postgres-query-builder"; +import { + TDiagnosticResult, + TServiceDiagnosticDepth, +} from "../../../../../../framework/api/diagnostics"; export interface PostgresConnectionOptions { database: string; @@ -25,6 +29,7 @@ export interface PostgresConnectionOptions { export class PostgresConnector extends AbstractConnector { private logger = getLogger("PostgresConnector"); private client: Client = new Client(this.options); + private connected = false; private dataTransformer = new PostgresDataTransformer({ secret: this.secret }); private queryBuilder = new PostgresQueryBuilder(this.secret); @@ -40,10 +45,12 @@ export class PostgresConnector extends AbstractConnector { - this.logger.warn(err, "PostgreSQL connection error"); + this.logger.error(err, "PostgreSQL connection error"); }); + this.client.on("end", () => (this.connected = false)); await this.client.connect(); + this.connected = true; this.logger.info("Connection pool created"); await this.healthcheck(); } @@ -56,6 +63,26 @@ export class PostgresConnector extends AbstractConnector { + const wasConnected = this.connected; + if (wasConnected) await this.healthcheck(); + else await this.connect(); + return !wasConnected; + } + + async getDiagnostics(depth: TServiceDiagnosticDepth): Promise { + switch (depth) { + case TServiceDiagnosticDepth.critical: + return { ok: true, didConnect: await this.ping() }; + case TServiceDiagnosticDepth.stats_basic: + case TServiceDiagnosticDepth.stats_track: + case TServiceDiagnosticDepth.stats_deep: + return { ok: true, warn: "unsupported_depth" }; + default: + throw new Error(`Unexpected TServiceDiagnosticDepth: ${JSON.stringify(depth)}`); + } + } + async init(): Promise { if (!this.client) { await this.connect(); @@ -111,7 +138,7 @@ export class PostgresConnector extends AbstractConnector { const query = ` - DO $$ - DECLARE + DO $$ + DECLARE tablename text; - BEGIN - FOR tablename IN (SELECT table_name FROM information_schema.tables WHERE table_schema = 'public') - LOOP - EXECUTE 'DELETE FROM "' || tablename || '" CASCADE'; - END LOOP; + BEGIN + FOR tablename IN (SELECT table_name FROM information_schema.tables WHERE table_schema = 'public') + LOOP + EXECUTE 'DELETE FROM "' || tablename || '" CASCADE'; + END LOOP; END $$;`; logger.debug(`service.database.orm.postgres.drop - Query: "${query}"`); await this.client.query(query); @@ -188,14 +215,14 @@ export class PostgresConnector extends AbstractConnector { const query = ` - DO $$ - DECLARE + DO $$ + DECLARE tablename text; - BEGIN - FOR tablename IN (SELECT table_name FROM information_schema.tables WHERE table_schema = 'public') - LOOP - EXECUTE 'DROP TABLE IF EXISTS "' || tablename || '" CASCADE'; - END LOOP; + BEGIN + FOR tablename IN (SELECT table_name FROM information_schema.tables WHERE table_schema = 'public') + LOOP + EXECUTE 'DROP TABLE IF EXISTS "' || tablename || '" CASCADE'; + END LOOP; END $$;`; logger.debug(`service.database.orm.postgres.dropTables - Query: "${query}"`); await this.client.query(query); @@ -343,13 +370,13 @@ export class PostgresConnector extends AbstractConnector { try { - const query = `SELECT - table_name, - column_name, - data_type - FROM + const query = `SELECT + table_name, + column_name, + data_type + FROM information_schema.columns - WHERE + WHERE table_name = $1`; const dbResult: QueryResult = await this.client.query(query, [name]); return dbResult.rows.map(row => row.column_name); diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/index.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/index.ts new file mode 100644 index 00000000..6d71d25e --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/index.ts @@ -0,0 +1,35 @@ +import { TdriveService, Consumes, Prefix, ServiceName } from "../../framework"; +import web from "./web"; +import DiagnosticsServiceAPI from "./service-provider"; +import DiagnosticsServiceImpl from "./service"; +import WebServerAPI from "../webserver/provider"; +import registerBasicProviders from "./providers"; + +/** + * The diagnostics service exposes endpoint that are of use for operational reasons. + * + */ +@Prefix("/api/diagnostics") +@Consumes(["webserver"]) +@ServiceName("diagnostics") +export default class DiagnosticsService extends TdriveService { + name = "diagnostics"; + service: DiagnosticsServiceAPI; + + api(): DiagnosticsServiceAPI { + return this.service; + } + + public async doInit(): Promise { + this.service = new DiagnosticsServiceImpl(); + const fastify = this.context.getProvider("webserver").getServer(); + registerBasicProviders(); + + fastify.register((instance, _opts, next) => { + web(instance, { prefix: this.prefix }); + next(); + }); + + return this; + } +} diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/providers/db-ping.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/db-ping.ts new file mode 100644 index 00000000..eef99c7c --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/db-ping.ts @@ -0,0 +1,9 @@ +import diagnostics, { TServiceDiagnosticDepth } from "../../../framework/api/diagnostics"; +import globalResolver from "../../../../../services/global-resolver"; + +export default () => + diagnostics.registerProviders({ + key: "db-ping", + tags: ["startup", "live", "ready"], + get: () => globalResolver.database.getDiagnostics(TServiceDiagnosticDepth.critical), + }); diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/providers/index.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/index.ts new file mode 100644 index 00000000..10e24bd6 --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/index.ts @@ -0,0 +1,9 @@ +import registerDBPingProvider from "./db-ping"; +import registerPlatformProvider from "./platform-started"; +import registerProcessProvider from "./process"; + +export default () => { + registerDBPingProvider(); + registerPlatformProvider(); + registerProcessProvider(); +}; diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/providers/platform-started.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/platform-started.ts new file mode 100644 index 00000000..d3eff017 --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/platform-started.ts @@ -0,0 +1,14 @@ +import diagnostics from "../../../framework/api/diagnostics"; +import globalResolver from "../../../../../services/global-resolver"; + +export default () => + diagnostics.registerProviders({ + key: "platform", + tags: "*", + get: async () => { + return { + ok: globalResolver.isPlatformStarted(), + uptimeMs: Math.floor((process.uptime() + Math.random()) * 1000), + }; + }, + }); diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/providers/process.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/process.ts new file mode 100644 index 00000000..4abe1b18 --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/process.ts @@ -0,0 +1,16 @@ +import diagnostics from "../../../framework/api/diagnostics"; + +export default () => + diagnostics.registerProviders({ + key: "process", + tags: ["live", "ready"], + async get() { + return { + ok: true, + gc: !!global.gc, + mem: process.memoryUsage(), + pid: process.pid, + res: process.resourceUsage(), + }; + }, + }); diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/service-provider.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/service-provider.ts new file mode 100644 index 00000000..00b72786 --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/service-provider.ts @@ -0,0 +1,5 @@ +import type { TdriveServiceProvider } from "../../framework"; + +type DiagnosticsServiceAPI = TdriveServiceProvider; + +export default DiagnosticsServiceAPI; diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/service.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/service.ts new file mode 100644 index 00000000..614e10c6 --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/service.ts @@ -0,0 +1,5 @@ +import DiagnosticsServiceAPI from "./service-provider"; + +export default class DiagnosticsServiceImpl implements DiagnosticsServiceAPI { + version: "1"; +} diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/utils.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/utils.ts new file mode 100644 index 00000000..375a1ebe --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/utils.ts @@ -0,0 +1,40 @@ +import { logger } from "../../framework"; +import { Readable, Transform } from "node:stream"; +import { Session } from "node:inspector"; + +/** Time the execution of the callback, and return its duration in ms with the result */ +export async function timeCallback( + cb: () => Promise, +): Promise<{ durationMs: number; result: T }> { + const startMs = new Date().getTime(); + const result = await cb(); + return { durationMs: new Date().getTime() - startMs, result }; +} + +/** Use `node:inspector` to create a snapshot of the heap, and + * synchroneously (because of session.post) pipe it to a stream + * readable from the first argument of the callback. + */ +export function getHeapSnapshotSync(cb: (readable: Readable) => void) { + logger.info({ gcExposed: !!global.gc }, "Beginning heap snapshot"); + if (global.gc) global.gc(); + const session = new Session(); + session.connect(); + try { + const transform = new Transform(); + try { + let size = 0; + session.on("HeapProfiler.addHeapSnapshotChunk", message => { + size += message.params.chunk.length; + transform.push(message.params.chunk); + }); + cb(transform); + session.post("HeapProfiler.takeHeapSnapshot", null); + logger.info({ size }, "Heap snapshot sent"); + } finally { + transform.end(); + } + } finally { + session.disconnect(); + } +} diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/web/heap-routes.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/web/heap-routes.ts new file mode 100644 index 00000000..72f6385d --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/web/heap-routes.ts @@ -0,0 +1,86 @@ +import type { FastifyInstance, FastifyPluginCallback } from "fastify"; +import { getConfig as getDiagnosticsGetConfig } from "../../../framework/api/diagnostics"; +import { getHeapSnapshotSync } from "../utils"; +import { createHash, randomUUID } from "node:crypto"; + +const routes: FastifyPluginCallback = (fastify: FastifyInstance, _opts, next) => { + const diagnosticsConfig = getDiagnosticsGetConfig(); + + if (diagnosticsConfig?.probeSecret?.length && diagnosticsConfig?.secret?.length) { + const getRunningToken = (() => { + const newToken = () => [randomUUID(), randomUUID()].join("*"); + let token = newToken(); + let updatedS = process.uptime(); + return () => { + const nowS = process.uptime(); + if (nowS - updatedS > diagnosticsConfig.secretChallengeRefreshS) { + token = newToken(); + updatedS = nowS; + } + return token; + }; + })(); + const hashToken = (token: string = getRunningToken()) => + createHash("sha512").update(`+${token}+${diagnosticsConfig.secret}+`).digest("hex"); + + /* + Example flow: + Configuration: + { secret: 'secretValue', probeSecret: 'probeSecretValue' } + GET current token using the probeSecret: + $ curl "$SERVER/api/diagnostics/heap?secret=probeSecretValue" + {"token":"d5888f91-d929-4775-b9dc-de6e2fb4d7cd*b2a57b32-7029-4957-82e9-c7c25794727f"} + Hash `+${token}+${secret}+` in SHA-512: + Shell example: + $ echo -n '+d5888f91-d929-4775-b9dc-de6e2fb4d7cd*b2a57b32-7029-4957-82e9-c7c25794727f+secretValue+' | sha512sum | cut -f1 -d' ' + 320070a508da218baf0f3363e837080fdb902bd0d986bd0f33b806b1230608c0868accdd10bd25261ca91b57c0459edf76218deb26571c72f6b93b077846abe3 + JS example: + require('crypto').createHash('sha512').update(`+${token}+${secret}+`).digest('hex') + POST to download snapshot: + $ curl "$SERVER/api/diagnostics/heap" + + Shell script example to download heap snapshot (requires curl, node and jq): + set -euo pipefail + SERVER="http://localhost:4000" + DIAG_PROBE_SECRET="probeSecretValue" + DIAG_SECRET="secretValue" + + urlencode() { node -e 'console.log(encodeURIComponent(process.argv[1]))' "$@" ; } + DIAG_PROBE_SECRET_URLENCODED="$(urlencode "$DIAG_PROBE_SECRET")" + TOKEN="$(curl --fail-with-body "$SERVER/api/diagnostics/heap?secret=$DIAG_PROBE_SECRET_URLENCODED" | jq -r .token)" + HASHED="$(echo -n "+$TOKEN+$DIAG_SECRET+" | sha512sum | cut -f1 -d' ')" + curl --fail-with-body --remote-name --remote-header-name \ + --header "Content-Type: application/json" \ + --data '{"hash":"'"$HASHED"'"}' \ + "$SERVER/api/diagnostics/heap" + # The file should be downloaded as `twake-drive-snap-$date.heapsnapshot` + ls twake-drive-snap-*.heapsnapshot + */ + fastify.get("/heap", async (request, reply) => { + if ((request.query as { secret: string }).secret !== diagnosticsConfig.probeSecret) + return reply.status(403).send(); + return reply.send({ token: getRunningToken() }); + }); + + fastify.post("/heap", async (request, reply) => { + const hashExpected = hashToken(); + const hashProvided = (request.body as { hash: string })?.hash; + if (hashProvided !== hashExpected) return reply.status(403).send(); + const filenameTimestamp = new Date() + .toISOString() + .replace(/(\.\d\d\d)?Z$/, "") + .replace(/\D/g, "-"); + reply.header( + "Content-Disposition", + `attachment; filename="twake-drive-snap-${filenameTimestamp}.heapsnapshot"`, + ); + let replyResult; + getHeapSnapshotSync(readable => (replyResult = reply.send(readable))); + return replyResult; + }); + } + + next(); +}; + +export default routes; diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/web/index.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/web/index.ts new file mode 100644 index 00000000..3091541c --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/web/index.ts @@ -0,0 +1,11 @@ +import type { FastifyInstance, FastifyRegisterOptions } from "fastify"; +import probeRoutes from "./probe-routes"; +import heapRoutes from "./heap-routes"; + +export default ( + fastify: FastifyInstance, + opts: FastifyRegisterOptions<{ prefix: string }>, +): void => { + fastify.register(probeRoutes, opts); + fastify.register(heapRoutes, opts); +}; diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/web/probe-routes.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/web/probe-routes.ts new file mode 100644 index 00000000..054fcabc --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/web/probe-routes.ts @@ -0,0 +1,25 @@ +import type { FastifyInstance, FastifyPluginCallback } from "fastify"; +import diagnostics, { + getConfig as getDiagnosticsGetConfig, +} from "../../../framework/api/diagnostics"; + +const routes: FastifyPluginCallback = (fastify: FastifyInstance, _opts, next) => { + const diagnosticsConfig = getDiagnosticsGetConfig(); + if (diagnosticsConfig?.probeSecret?.length) { + const tagParamName = "diagnosticTags"; + fastify.get(`/t/:${tagParamName}`, async (request, reply) => { + const tag = request.params[tagParamName]; + if ( + tag === "*" || + (request.query as { secret: string }).secret !== diagnosticsConfig.probeSecret + ) + return reply.status(403).send(); + const results = await diagnostics.get(tag); + if (!results.ok) reply.status(503); + return reply.send(results); + }); + } + next(); +}; + +export default routes; diff --git a/tdrive/backend/node/src/services/av/service/index.ts b/tdrive/backend/node/src/services/av/service/index.ts index 8bcf44fb..867f54f5 100644 --- a/tdrive/backend/node/src/services/av/service/index.ts +++ b/tdrive/backend/node/src/services/av/service/index.ts @@ -32,8 +32,8 @@ export class AVServiceImpl implements TdriveServiceProvider, Initializable { }, }); } - } catch (error) { - logger.error({ error: `${error}` }, "Error while initializing Antivirus Service"); + } catch (err) { + logger.error({ err }, "Error while initializing Antivirus Service"); throw AVException.initializationFailed("Failed to initialize Antivirus service"); } return this; diff --git a/tdrive/backend/node/src/services/global-resolver.ts b/tdrive/backend/node/src/services/global-resolver.ts index 3e68328b..42088f6e 100644 --- a/tdrive/backend/node/src/services/global-resolver.ts +++ b/tdrive/backend/node/src/services/global-resolver.ts @@ -15,7 +15,7 @@ import TrackerAPI from "../core/platform/services/tracker/provider"; import WebServerAPI from "../core/platform/services/webserver/provider"; import assert from "assert"; -import { logger } from "../core/platform/framework"; +import { logger, TdriveServiceState } from "../core/platform/framework"; import { ApplicationServiceImpl } from "./applications/services/applications"; import { CompanyApplicationServiceImpl } from "./applications/services/company-applications"; import { ApplicationHooksService } from "./applications/services/hooks"; @@ -78,6 +78,7 @@ class GlobalResolver { public services: TdriveServices; public platformServices: PlatformServices; public database: DatabaseServiceAPI; + private platform?: TdrivePlatform; public fastify: FastifyInstance; @@ -87,6 +88,7 @@ class GlobalResolver { if (this.alreadyInitialized) { return; } + this.platform = platform; this.database = platform.getProvider("database"); this.platformServices = { @@ -152,6 +154,11 @@ class GlobalResolver { logger.info("Global resolver finished initializing services"); this.alreadyInitialized = true; } + + /** `true` if all components are in the started state. This means we should respond to http at least (for kubernetes Startup probe). */ + isPlatformStarted(): boolean { + return this.alreadyInitialized && this.platform?.state.getValue() == TdriveServiceState.Started; + } } export default new GlobalResolver();