🛠️ probe and diagnostic endpoints (#763)

basic lifecycle probes for kubes and an endpoint for diagnosing mem
leaks

missing tests for av, message queue, possibly sso

tdrive/backend/node/config/custom-environment-variables.json

@@ -16,6 +16,15 @@
   "logger": {
     "level": "LOG_LEVEL"
   },
+  "diagnostics": {
+    "skipKeys": {
+      "__name": "DIAG_SKIP_KEYS",
+      "__format": "json"
+    },
+    "probeSecret": "DIAG_PROBE_SECRET",
+    "statsLogPeriodMs": "DIAG_STATS_PRINT_PERIOD_MS",
+    "statsFullStatsLogPeriodMs": "DIAG_FULL_STATS_PRINT_PERIOD_MS"
+  },
   "webserver": {
     "host": "TWAKE_DRIVE_HOST",
     "logger": {

tdrive/backend/node/config/default.json

@@ -35,6 +35,12 @@
   "logger": {
     "level": "debug"
   },
+  "diagnostics": {
+    "skipKeys": [],
+    "probeSecret": "",
+    "statsLogPeriodMs": 120000,
+    "statsFullStatsLogPeriodMs": 600000
+  },
   "tracker": {
     "type": "segment",
     "segment": {
@@ -226,6 +232,7 @@
   },
   "services": [
     "auth",
+    "diagnostics",
     "push",
     "storage",
     "webserver",

tdrive/backend/node/src/core/platform/framework/api/diagnostics.ts

@@ -0,0 +1,266 @@
+import assert from "node:assert";
+import config from "../../../config";
+import { getLogger } from "../logger";
+
+const logger = getLogger("Diagnostics");
+
+/**
+ * Values that can match a set of diagnostic providers.
+ *
+ * `startup`, `ready` and `live` are meant to match the meanings of the corresponding
+ * kubernetes probes:
+ * https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
+ */
+export type TDiagnosticTag =
+  | "*" // Special value that includes everything, and is always included
+  | "startup" // Tests that are absolutely required (and light) to even begin the other tests
+  | "ready" // Tests required before traffic can be sent our way
+  | "live" // Tests required to prevent from being restarted
+  | "stats" // Expensive diagnostics that should not often be ran, but can be system wide
+  | "stats-full"; // Expensive diagnostics that should not often be ran, and then probably for a single key
+
+/** Detail requested from platform service self-diagnostics */
+export enum TServiceDiagnosticDepth {
+  /** Minimal cost information that tests functioning service */
+  alive = "alive",
+  /** Statistics that have a little impact enough for periodic tracking into a time series */
+  stats_track = "stats_track",
+  /** Statistics that should be included when looking specifically at general statistics */
+  stats_basic = "stats_basic",
+  /** Statistics possibly expensive and large to calculate, for occasional debug operations */
+  stats_deep = "stats_deep",
+}
+
+const serviceDiagnosticDepthToTags: { [depth in TServiceDiagnosticDepth]: TDiagnosticTag[] } = {
+  [TServiceDiagnosticDepth.alive]: ["startup", "live", "ready"],
+  [TServiceDiagnosticDepth.stats_track]: ["ready", "stats"],
+  [TServiceDiagnosticDepth.stats_basic]: ["stats", "stats-full"],
+  [TServiceDiagnosticDepth.stats_deep]: ["stats-full"],
+};
+
+interface IDiagnosticsConfig {
+  // Diagnostic keys that should be considered ok without evaluation
+  skipKeys?: string[];
+  // This secret must be provided to the diagnostic endpoints as a query param
+  // therefor it's likely to leak, through logs etc, for ex. and should not
+  // relied on for security because disabling diagnostics. At worst this
+  // provides access to the DB statistics.
+  probeSecret?: string;
+  // Period at which to log TDiagnosticTag `stats`. 0 to disable.
+  statsLogPeriodMs: number;
+  // Period at which to log TDiagnosticTag `stats-full`. 0 to disable.
+  statsFullStatsLogPeriodMs: number;
+}
+
+export const getConfig = (): IDiagnosticsConfig => {
+  let configSection = config.get("diagnostics") as IDiagnosticsConfig;
+  if (typeof configSection.skipKeys === "string")
+    configSection = {
+      ...configSection,
+      skipKeys: (configSection.skipKeys as string)
+        .trim()
+        .split(/[,\s]+/g)
+        .filter(x => !!x),
+    };
+
+  const getNumberFromConfig = (value): number => {
+    if (typeof value == "number") return value;
+    if (typeof value == "string") return parseInt(value, 10) ?? 0;
+    return 0;
+  };
+  return {
+    ...configSection,
+    statsLogPeriodMs: getNumberFromConfig(configSection.statsLogPeriodMs),
+    statsFullStatsLogPeriodMs: getNumberFromConfig(configSection.statsFullStatsLogPeriodMs),
+  };
+};
+
+/** Code-wide unique key for each provider */
+export type TDiagnosticKey = string;
+
+/** Each provider should return an object of this format. The key of the provider defines the schema. */
+export type TDiagnosticResult = { ok: boolean; warn?: string; empty?: boolean } & {
+  [key: string]: unknown;
+};
+
+/** Implemented by objects that want to provide data to the diagnostic check */
+export interface IDiagnosticProvider {
+  /** Code-wide unique key underwhich the result of `get` will be included */
+  key: TDiagnosticKey;
+
+  /** This result is present in any included request tag */
+  tags: TDiagnosticTag[] | "*";
+
+  /**
+   * If set, this provider will be polled at that interval.
+   * If `undefined`, this provider will be ran at each request.
+   */
+  pollPeriodMs?: number;
+
+  /**
+   * Returns an object as presented to a diagnostic requester.
+   * Warning: this could be public and readable to the internet.
+   */
+  get(): Promise<TDiagnosticResult>;
+}
+
+/**
+ * Platform services that can provide generic diagnostic implementations may use this interface.
+ *
+ * Matching from {@link TDiagnosticTag} to {@link TServiceDiagnosticDepth} is expected to
+ * be done by intermediary providers. This is because not all services are equally critical,
+ * or have the same tolerable down times.
+ */
+export interface IServiceDiagnosticProvider {
+  /** The return format is specific to each service, but should include a `{ok: boolean}` field. */
+  getDiagnostics(depth: TServiceDiagnosticDepth): Promise<TDiagnosticResult>;
+}
+
+const isProviderIncludedInTag = (
+  tag: TDiagnosticTag,
+  provider: IDiagnosticProvider,
+  config: IDiagnosticsConfig,
+) =>
+  (provider.tags === "*" || provider.tags.indexOf(tag) >= 0 || provider.tags.indexOf("*") >= 0) &&
+  (!config.skipKeys?.length || !config.skipKeys.includes(provider.key));
+
+// registered providers with `pollPeriodMs === undefined`
+const immediateDiagnosticProviders: IDiagnosticProvider[] = [];
+// registered providers with `pollPeriodMs !== undefined`
+const periodicDiagnosticProviders: IDiagnosticProvider[] = [];
+
+const now = () => Math.round(process.uptime() * 1000);
+
+const isKeyAlreadyRegistered = (key: TDiagnosticKey) =>
+  immediateDiagnosticProviders.some(provider => key == provider.key) ||
+  periodicDiagnosticProviders.some(provider => key == provider.key);
+
+// stores results of all the `pollPeriodMs` truthy providers
+const latestPeriodicDiagnostics: { [key: TDiagnosticKey]: object } = {};
+const recordDiagnostic = (startMs: number, key: TDiagnosticKey, data?: object, error?: object) =>
+  (latestPeriodicDiagnostics[key] = {
+    durationMs: Math.round(now() - startMs),
+    ...(error ? { ok: false, error } : { ...data }),
+  });
+
+const runProvider = async (provider, log) => {
+  const startMs = now();
+  try {
+    const result = await provider.get();
+    if (!result.ok)
+      logger.error(
+        { diagnostic: provider.key, ...result },
+        "Got diagnostic provider result with ok=false",
+      );
+    else if (result.warn)
+      logger.warn(
+        { diagnostic: provider.key, ...result },
+        "Got diagnostic provider result with ok=true but a warning",
+      );
+    else if (log && !result.empty)
+      logger.info({ diagnostic: provider.key, ...result }, "Diagnostic provider result");
+    return recordDiagnostic(startMs, provider.key, result);
+  } catch (err) {
+    logger.error({ err, provider: provider.key }, "Failed to read diagnostic provider");
+    return recordDiagnostic(startMs, provider.key, undefined, err);
+  }
+};
+
+const pendingTimeouts: number[] = []; // Pending return values from `setTimeout` calls
+const forgetPendingTimeout = (timeoutId: number) => {
+  const index = pendingTimeouts.indexOf(timeoutId);
+  assert(index >= 0);
+  pendingTimeouts.splice(index, 1);
+};
+
+let hasShutdown = false;
+const ensureHasntShutdown = () => {
+  if (hasShutdown) throw new Error("Diagnostics service already shutdown");
+};
+
+export default {
+  /** Add a provider to be included in diagnostics output */
+  registerProviders(...providers: IDiagnosticProvider[]) {
+    ensureHasntShutdown();
+    providers.forEach(provider => {
+      if (isKeyAlreadyRegistered(provider.key)) throw new Error("Provider with duplicate key");
+      if (provider.pollPeriodMs) {
+        periodicDiagnosticProviders.push(provider);
+      } else {
+        immediateDiagnosticProviders.push(provider);
+        return;
+      }
+      let triggerUpdate: () => void = () => undefined; // The empty function is for the linter. I love you linter <3
+      const updateProvider = (timeoutId: number) => async () => {
+        forgetPendingTimeout(timeoutId);
+        await runProvider(provider, false);
+        triggerUpdate();
+      };
+      triggerUpdate = () => pendingTimeouts.push(setTimeout(updateProvider, provider.pollPeriodMs));
+      triggerUpdate();
+    });
+  },
+
+  /** Create providers to match from {@link IServiceDiagnosticProvider} to multiple {@link IDiagnosticProvider}s */
+  registerServiceProviders(
+    name: string,
+    getService: () => IServiceDiagnosticProvider,
+    overrideTags: Partial<
+      typeof serviceDiagnosticDepthToTags | { [key in TServiceDiagnosticDepth]: false }
+    > = {},
+  ) {
+    this.registerProviders(
+      ...Object.values(TServiceDiagnosticDepth)
+        .map(depth => {
+          const defaultTags = serviceDiagnosticDepthToTags[depth];
+          if (!defaultTags) throw new Error(`Unknown depth ${JSON.stringify(depth)}`);
+          const tags = overrideTags[depth] ?? defaultTags;
+          if (tags === false) return null;
+          return {
+            key: `${name}-${depth}`,
+            tags,
+            get: () => getService().getDiagnostics(depth),
+          };
+        })
+        .filter(x => !!x),
+    );
+  },
+
+  /** Cancel all pending diagnostic updates */
+  shutdown() {
+    ensureHasntShutdown();
+    pendingTimeouts.forEach(timeout => clearTimeout(timeout));
+    hasShutdown = true;
+  },
+
+  /**
+   * Return the values of all providers which include the provided tag.
+   *
+   * @param log if `true`, print each individual log output even if succesful
+   */
+  async get(
+    tag: TDiagnosticTag,
+    log: boolean,
+  ): Promise<{ ok: boolean } | { [key: TDiagnosticKey]: TDiagnosticResult }> {
+    const config = getConfig();
+    const result = { ok: true };
+    let atLeastOneCheck = false;
+    periodicDiagnosticProviders.forEach(provider => {
+      if (!isProviderIncludedInTag(tag, provider, config)) return;
+      atLeastOneCheck = true;
+      result[provider.key] = latestPeriodicDiagnostics[provider.key];
+      if (!result[provider.key].ok) result.ok = false;
+    });
+    await Promise.all(
+      immediateDiagnosticProviders.map(async provider => {
+        if (!isProviderIncludedInTag(tag, provider, config)) return;
+        atLeastOneCheck = true;
+        const providerResult = await runProvider(provider, log);
+        if (!providerResult.ok) result.ok = false;
+        return (result[provider.key] = providerResult);
+      }),
+    );
+    if (!atLeastOneCheck) result.ok = false;
+    return result;
+  },
+};

tdrive/backend/node/src/core/platform/services/database/api.ts

@@ -1,10 +1,11 @@
 import { TdriveServiceProvider } from "../../framework";
+import type { IServiceDiagnosticProvider } from "../../framework/api/diagnostics";
 import { Connector } from "./services/orm/connectors";
 import Manager from "./services/orm/manager";
 import Repository from "./services/orm/repository/repository";
 import { EntityTarget } from "./services/orm/types";
 
-export interface DatabaseServiceAPI extends TdriveServiceProvider {
+export interface DatabaseServiceAPI extends TdriveServiceProvider, IServiceDiagnosticProvider {
   /**
    * Get the database connector
    */

tdrive/backend/node/src/core/platform/services/database/services/index.ts

@@ -7,6 +7,10 @@ import { MongoConnectionOptions } from "./orm/connectors/mongodb/mongodb";
 import { EntityTarget } from "./orm/types";
 import { RepositoryManager } from "./orm/repository/manager";
 import { PostgresConnectionOptions } from "./orm/connectors/postgres/postgres";
+import type {
+  TDiagnosticResult,
+  TServiceDiagnosticDepth,
+} from "../../../framework/api/diagnostics";
 
 export default class DatabaseService implements DatabaseServiceAPI {
   version = "1";
@@ -38,6 +42,15 @@ export default class DatabaseService implements DatabaseServiceAPI {
     }
   }
 
+  async getDiagnostics(depth: TServiceDiagnosticDepth): Promise<TDiagnosticResult> {
+    const connector = this.getConnector();
+    const result = await connector.getDiagnostics(depth);
+    return {
+      type: connector.getType(),
+      ...result,
+    };
+  }
+
   getManager(): Manager<unknown> {
     return new Manager<unknown>(this.connector);
   }

tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/abstract-connector.ts

@@ -4,12 +4,17 @@ import { ConnectionOptions, DatabaseType } from "../..";
 import { FindOptions } from "../repository/repository";
 import { ColumnDefinition, EntityDefinition } from "../types";
 import { ListResult, Paginable, Pagination } from "../../../../../framework/api/crud-service";
+import type {
+  TDiagnosticResult,
+  TServiceDiagnosticDepth,
+} from "../../../../../framework/api/diagnostics";
 
 export abstract class AbstractConnector<T extends ConnectionOptions> implements Connector {
   constructor(protected type: DatabaseType, protected options: T, protected secret: string) {}
 
   abstract connect(): Promise<this>;
   abstract disconnect(): Promise<this>;
+  abstract getDiagnostics(depth: TServiceDiagnosticDepth): Promise<TDiagnosticResult>;
 
   abstract drop(): Promise<this>;
 

tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts

@@ -5,6 +5,7 @@ import { ColumnDefinition, EntityDefinition } from "../types";
 import { FindOptions } from "../repository/repository";
 import { ListResult, Paginable, Pagination } from "../../../../../framework/api/crud-service";
 import { PostgresConnectionOptions } from "./postgres/postgres";
+import type { IServiceDiagnosticProvider } from "../../../../../framework/api/diagnostics";
 
 export * from "./mongodb/mongodb";
 
@@ -14,7 +15,7 @@ export type UpsertOptions = {
 
 export type RemoveOptions = any;
 
-export interface Connector extends Initializable {
+export interface Connector extends Initializable, IServiceDiagnosticProvider {
   /**
    * Connect to the database
    */

tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts

@@ -8,6 +8,10 @@ import { AbstractConnector } from "../abstract-connector";
 import { buildSelectQuery } from "./query-builder";
 import { transformValueFromDbString, transformValueToDbString } from "./typeTransforms";
 import { logger } from "../../../../../../framework";
+import {
+  TDiagnosticResult,
+  TServiceDiagnosticDepth,
+} from "../../../../../../framework/api/diagnostics";
 
 export interface MongoConnectionOptions {
   // TODO: More options
@@ -42,6 +46,51 @@ export class MongoConnector extends AbstractConnector<MongoConnectionOptions> {
     return this;
   }
 
+  private async ping(): Promise<boolean> {
+    const wasConnected = !!this.client;
+    await (await this.getDatabase()).admin().ping();
+    return !wasConnected;
+  }
+
+  private async dbStats(): Promise<object> {
+    return (await this.getDatabase()).stats();
+  }
+
+  private async collectionsStats(deep: boolean): Promise<object> {
+    const db = await this.getDatabase();
+    const result = { collections: {} };
+    for (const collection of await db.collections()) {
+      const stats = await collection.aggregate([
+        {
+          $collStats: {
+            latencyStats: { histograms: true },
+            storageStats: deep ? {} : undefined, // Really a lot of keys with 0 occurances
+            count: {},
+            queryExecStats: {},
+          },
+        },
+      ]);
+      result.collections[collection.collectionName] = await stats.toArray();
+    }
+    return result;
+  }
+
+  async getDiagnostics(depth: TServiceDiagnosticDepth): Promise<TDiagnosticResult> {
+    switch (depth) {
+      case TServiceDiagnosticDepth.alive:
+        return { ok: true, didConnect: await this.ping() };
+      case TServiceDiagnosticDepth.stats_track:
+        return { ok: true, ...(await this.dbStats()) };
+      case TServiceDiagnosticDepth.stats_basic:
+        return { ok: true, ...(await this.collectionsStats(false)) };
+      case TServiceDiagnosticDepth.stats_deep:
+        return { ok: true, ...(await this.collectionsStats(true)) };
+
+      default:
+        throw new Error(`Unexpected TServiceDiagnosticDepth: ${JSON.stringify(depth)}`);
+    }
+  }
+
   getClient(): mongo.MongoClient {
     return this.client;
   }

tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres.ts

@@ -8,6 +8,10 @@ import { getEntityDefinition } from "../../../orm/utils";
 import { UpsertOptions } from "src/core/platform/services/database/services/orm/connectors";
 import { PostgresDataTransformer, TypeMappings } from "./postgres-data-transform";
 import { PostgresQueryBuilder, Query } from "./postgres-query-builder";
+import {
+  TDiagnosticResult,
+  TServiceDiagnosticDepth,
+} from "../../../../../../framework/api/diagnostics";
 
 export interface PostgresConnectionOptions {
   database: string;
@@ -25,6 +29,7 @@ export interface PostgresConnectionOptions {
 export class PostgresConnector extends AbstractConnector<PostgresConnectionOptions> {
   private logger = getLogger("PostgresConnector");
   private client: Client = new Client(this.options);
+  private connected = false;
   private dataTransformer = new PostgresDataTransformer({ secret: this.secret });
   private queryBuilder = new PostgresQueryBuilder(this.secret);
 
@@ -40,10 +45,12 @@ export class PostgresConnector extends AbstractConnector<PostgresConnectionOptio
     if (this.client) {
       this.logger.info("Connecting to DB");
       this.client.on("error", err => {
-        this.logger.warn(err, "PostgreSQL connection error");
+        this.logger.error(err, "PostgreSQL connection error");
       });
+      this.client.on("end", () => (this.connected = false));
 
       await this.client.connect();
+      this.connected = true;
       this.logger.info("Connection pool created");
       await this.healthcheck();
     }
@@ -56,6 +63,59 @@ export class PostgresConnector extends AbstractConnector<PostgresConnectionOptio
     return this;
   }
 
+  private async ping(): Promise<boolean> {
+    const wasConnected = this.connected;
+    if (wasConnected) await this.healthcheck();
+    else await this.connect();
+    return !wasConnected;
+  }
+
+  async getDiagnostics(depth: TServiceDiagnosticDepth): Promise<TDiagnosticResult> {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const safeRequest = async (query: string, values?: any[], singleRow = false) => {
+      try {
+        const rows = (await this.client.query(query, values)).rows;
+        return singleRow && rows?.length === 1 ? rows[0] : rows;
+      } catch (err) {
+        const logId = "pg-diags-error-" + Math.floor(process.uptime() * 1000);
+        logger.error(
+          { err, query, values, logId, errCode: err.code },
+          `Error running postgresql statistics at ${depth} ( ${logId} ) `,
+        );
+        return { error: true, logId };
+      }
+    };
+    switch (depth) {
+      // This is the only required `ok`
+      case TServiceDiagnosticDepth.alive:
+        return { ok: true, didConnect: await this.ping() };
+
+      // Statistics can silently fail, and do it granularly if there is
+      // a permission issue only on some of the stats
+      case TServiceDiagnosticDepth.stats_track:
+        return {
+          ok: true,
+          db: await safeRequest(
+            "select * from pg_stat_database where datname = $1",
+            [this.options.database],
+            true,
+          ),
+        };
+      case TServiceDiagnosticDepth.stats_basic:
+        return { ok: true, empty: true };
+      case TServiceDiagnosticDepth.stats_deep:
+        return {
+          ok: true,
+          databases: await safeRequest("select * from pg_stat_database"),
+          tables: await safeRequest("select * from pg_stat_user_tables"),
+          indexes: await safeRequest("select * from pg_stat_user_indexes"),
+        };
+
+      default:
+        throw new Error(`Unexpected TServiceDiagnosticDepth: ${JSON.stringify(depth)}`);
+    }
+  }
+
   async init(): Promise<this> {
     if (!this.client) {
       await this.connect();

tdrive/backend/node/src/core/platform/services/diagnostics/index.ts

@@ -0,0 +1,66 @@
+import { TdriveService, Consumes, Prefix, ServiceName } from "../../framework";
+import web from "./web";
+import DiagnosticsServiceAPI from "./service-provider";
+import DiagnosticsServiceImpl from "./service";
+import WebServerAPI from "../webserver/provider";
+import registerBasicProviders from "./providers";
+import diagnostics, {
+  getConfig as getDiagnosticsGetConfig,
+  TDiagnosticTag,
+} from "../../framework/api/diagnostics";
+import registerFastifyRoutesDiagnosticsProvider from "./web/provider";
+
+/**
+ * The diagnostics service exposes endpoint that are of use for operational reasons.
+ *
+ */
+@Prefix("/diagnostics")
+@Consumes(["webserver"])
+@ServiceName("diagnostics")
+export default class DiagnosticsService extends TdriveService<DiagnosticsServiceAPI> {
+  name = "diagnostics";
+  service: DiagnosticsServiceAPI;
+  private runningIntervalStatsLog?: ReturnType<typeof setInterval>;
+  private runningIntervalStatsFullLog?: ReturnType<typeof setInterval>;
+
+  api(): DiagnosticsServiceAPI {
+    return this.service;
+  }
+
+  public async doInit(): Promise<this> {
+    this.service = new DiagnosticsServiceImpl();
+    const fastify = this.context.getProvider<WebServerAPI>("webserver").getServer();
+
+    registerBasicProviders();
+    registerFastifyRoutesDiagnosticsProvider(fastify);
+
+    fastify.register((instance, _opts, next) => {
+      web(instance, { prefix: this.prefix });
+      next();
+    });
+
+    const config = getDiagnosticsGetConfig();
+    const printStatsToLog = (tag: TDiagnosticTag) => () => diagnostics.get(tag, true);
+    const startLoggingStats = (tag, periodMs) =>
+      periodMs && periodMs > 0 ? setInterval(printStatsToLog(tag), periodMs) : undefined;
+    this.runningIntervalStatsLog = startLoggingStats("stats", config.statsLogPeriodMs);
+    this.runningIntervalStatsFullLog = startLoggingStats(
+      "stats-full",
+      config.statsFullStatsLogPeriodMs,
+    );
+
+    return this;
+  }
+
+  public async doStop(): Promise<this> {
+    if (this.runningIntervalStatsLog) {
+      clearInterval(this.runningIntervalStatsLog);
+      this.runningIntervalStatsLog = null;
+    }
+    if (this.runningIntervalStatsFullLog) {
+      clearInterval(this.runningIntervalStatsFullLog);
+      this.runningIntervalStatsFullLog = null;
+    }
+    return this;
+  }
+}

tdrive/backend/node/src/core/platform/services/diagnostics/providers/database-service.ts

@@ -0,0 +1,4 @@
+import diagnostics from "../../../framework/api/diagnostics";
+import globalResolver from "../../../../../services/global-resolver";
+
+export default () => diagnostics.registerServiceProviders("db", () => globalResolver.database);

tdrive/backend/node/src/core/platform/services/diagnostics/providers/index.ts

@@ -0,0 +1,11 @@
+import registerDBServiceProvider from "./database-service";
+import registerStorageServiceProvider from "./storage-service";
+import registerPlatformProvider from "./platform-started";
+import registerProcessProvider from "./process";
+
+export default () => {
+  registerDBServiceProvider();
+  registerStorageServiceProvider();
+  registerPlatformProvider();
+  registerProcessProvider();
+};

tdrive/backend/node/src/core/platform/services/diagnostics/providers/platform-started.ts

@@ -0,0 +1,14 @@
+import diagnostics from "../../../framework/api/diagnostics";
+import globalResolver from "../../../../../services/global-resolver";
+
+export default () =>
+  diagnostics.registerProviders({
+    key: "platform",
+    tags: "*",
+    get: async () => {
+      return {
+        ok: globalResolver.isPlatformStarted(),
+        uptimeMs: Math.floor((process.uptime() + Math.random()) * 1000),
+      };
+    },
+  });

tdrive/backend/node/src/core/platform/services/diagnostics/providers/process.ts

@@ -0,0 +1,21 @@
+import diagnostics from "../../../framework/api/diagnostics";
+
+export const e2eTestOverride = {
+  // Set to true to force fail to test diagnostic aggregation
+  forceFail: false,
+};
+
+export default () =>
+  diagnostics.registerProviders({
+    key: "process",
+    tags: ["live", "ready"],
+    async get() {
+      return {
+        ok: !e2eTestOverride.forceFail,
+        gc: !!global.gc,
+        pid: process.pid,
+        mem: process.memoryUsage(),
+        res: process.resourceUsage(),
+      };
+    },
+  });

tdrive/backend/node/src/core/platform/services/diagnostics/providers/storage-service.ts

@@ -0,0 +1,9 @@
+import diagnostics from "../../../framework/api/diagnostics";
+import globalResolver from "../../../../../services/global-resolver";
+
+export default () =>
+  diagnostics.registerServiceProviders("storage", () => globalResolver.platformServices.storage, {
+    stats_basic: false,
+    stats_track: false,
+    stats_deep: false,
+  });

tdrive/backend/node/src/core/platform/services/diagnostics/service-provider.ts

@@ -0,0 +1,5 @@
+import type { TdriveServiceProvider } from "../../framework";
+
+type DiagnosticsServiceAPI = TdriveServiceProvider;
+
+export default DiagnosticsServiceAPI;

tdrive/backend/node/src/core/platform/services/diagnostics/service.ts

@@ -0,0 +1,5 @@
+import DiagnosticsServiceAPI from "./service-provider";
+
+export default class DiagnosticsServiceImpl implements DiagnosticsServiceAPI {
+  version: "1";
+}

tdrive/backend/node/src/core/platform/services/diagnostics/utils.ts

@@ -0,0 +1,40 @@
+import { logger } from "../../framework";
+import { Readable, Transform } from "node:stream";
+import { Session } from "node:inspector";
+
+/** Time the execution of the callback, and return its duration in ms with the result */
+export async function timeCallback<T>(
+  cb: () => Promise<T>,
+): Promise<{ durationMs: number; result: T }> {
+  const startMs = new Date().getTime();
+  const result = await cb();
+  return { durationMs: new Date().getTime() - startMs, result };
+}
+
+/** Use `node:inspector` to create a snapshot of the heap, and
+ * synchroneously (because of session.post) pipe it to a stream
+ * readable from the first argument of the callback.
+ */
+export function getHeapSnapshotSync(cb: (readable: Readable) => void) {
+  logger.info({ gcExposed: !!global.gc }, "Beginning heap snapshot");
+  if (global.gc) global.gc();
+  const session = new Session();
+  session.connect();
+  try {
+    const transform = new Transform();
+    try {
+      let size = 0;
+      session.on("HeapProfiler.addHeapSnapshotChunk", message => {
+        size += message.params.chunk.length;
+        transform.push(message.params.chunk);
+      });
+      cb(transform);
+      session.post("HeapProfiler.takeHeapSnapshot", null);
+      logger.info({ size }, "Heap snapshot sent");
+    } finally {
+      transform.end();
+    }
+  } finally {
+    session.disconnect();
+  }
+}

tdrive/backend/node/src/core/platform/services/diagnostics/web/heap-routes.ts

@@ -0,0 +1,27 @@
+import type { FastifyInstance, FastifyPluginCallback } from "fastify";
+import { getConfig as getDiagnosticsGetConfig } from "../../../framework/api/diagnostics";
+import { getHeapSnapshotSync } from "../utils";
+
+const routes: FastifyPluginCallback = (fastify: FastifyInstance, _opts, next) => {
+  const diagnosticsConfig = getDiagnosticsGetConfig();
+  if (diagnosticsConfig?.probeSecret?.length) {
+    fastify.post("/heap", async (request, reply) => {
+      if ((request.query as { secret: string }).secret !== diagnosticsConfig.probeSecret)
+        return reply.status(403).send();
+      const filenameTimestamp = new Date()
+        .toISOString()
+        .replace(/(\.\d\d\d)?Z$/, "")
+        .replace(/\D/g, "-");
+      reply.header(
+        "Content-Disposition",
+        `attachment; filename="twake-drive-snap-${filenameTimestamp}.heapsnapshot"`,
+      );
+      let replyResult;
+      getHeapSnapshotSync(readable => (replyResult = reply.send(readable)));
+      return replyResult;
+    });
+  }
+  next();
+};
+
+export default routes;

tdrive/backend/node/src/core/platform/services/diagnostics/web/index.ts

@@ -0,0 +1,11 @@
+import type { FastifyInstance, FastifyRegisterOptions } from "fastify";
+import probeRoutes from "./probe-routes";
+import heapRoutes from "./heap-routes";
+
+export default (
+  fastify: FastifyInstance,
+  opts: FastifyRegisterOptions<{ prefix: string }>,
+): void => {
+  fastify.register(probeRoutes, opts);
+  fastify.register(heapRoutes, opts);
+};

tdrive/backend/node/src/core/platform/services/diagnostics/web/probe-routes.ts

@@ -0,0 +1,25 @@
+import type { FastifyInstance, FastifyPluginCallback } from "fastify";
+import diagnostics, {
+  getConfig as getDiagnosticsGetConfig,
+} from "../../../framework/api/diagnostics";
+
+const routes: FastifyPluginCallback = (fastify: FastifyInstance, _opts, next) => {
+  const diagnosticsConfig = getDiagnosticsGetConfig();
+  if (diagnosticsConfig?.probeSecret?.length) {
+    const tagParamName = "diagnosticTags";
+    fastify.get(`/t/:${tagParamName}`, async (request, reply) => {
+      const tag = request.params[tagParamName];
+      if (
+        tag === "*" ||
+        (request.query as { secret: string }).secret !== diagnosticsConfig.probeSecret
+      )
+        return reply.status(403).send();
+      const results = await diagnostics.get(tag, false);
+      if (!results.ok) reply.status(503);
+      return reply.send(results);
+    });
+  }
+  next();
+};
+
+export default routes;

tdrive/backend/node/src/core/platform/services/diagnostics/web/provider.ts

@@ -0,0 +1,23 @@
+import diagnostics from "../../../framework/api/diagnostics";
+
+export default fastify => {
+  // TODO: registering @fastify/routes-stats creates errors (performance mark not found)
+  // on the request performance marks, the hook doesn't seem to be called for every route
+  // and some of the more important ones like browse start failing...
+  // Version tested had to be 3.4.0 because 4+ needs Fastify 5.
+  return;
+  // Don't require it at all until it's fixed
+  // eslint-disable-next-line @typescript-eslint/no-var-requires
+  fastify.register(require("@fastify/routes-stats"), {
+    printInterval: 4000, // milliseconds
+    decoratorName: "performanceMarked", // decorator is set to true if a performace.mark was called for the request
+  });
+  diagnostics.registerProviders({
+    key: "fastify-routes",
+    tags: ["stats", "stats-full"],
+    async get() {
+      fastify.measurements();
+      return { ok: true, ...fastify.stats() };
+    },
+  });
+};

tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts

@@ -4,6 +4,7 @@ import { Readable } from "stream";
 import { StorageConnectorAPI, WriteMetadata } from "../../provider";
 import { randomUUID } from "crypto";
 import _ from "lodash";
+import { TDiagnosticResult, TServiceDiagnosticDepth } from "../../../../framework/api/diagnostics";
 
 export type S3Configuration = {
   id: string;
@@ -43,6 +44,21 @@ export default class S3ConnectorService implements StorageConnectorAPI {
     return this.id;
   }
 
+  async getDiagnostics(depth: TServiceDiagnosticDepth): Promise<TDiagnosticResult> {
+    switch (depth) {
+      case TServiceDiagnosticDepth.alive:
+        return { ok: await this.client.bucketExists(this.minioConfiguration.bucket) };
+      case TServiceDiagnosticDepth.stats_basic:
+      case TServiceDiagnosticDepth.stats_track:
+      case TServiceDiagnosticDepth.stats_deep:
+        // Local store is always ok... should never be used outside dev environments
+        return { ok: true, empty: true };
+
+      default:
+        throw new Error(`Unexpected TServiceDiagnosticDepth: ${JSON.stringify(depth)}`);
+    }
+  }
+
   write(path: string, stream: Readable): Promise<WriteMetadata> {
     return new Promise((resolve, reject) => {
       let totalSize = 0;

tdrive/backend/node/src/core/platform/services/storage/connectors/local/service.ts

@@ -6,6 +6,7 @@ import { StorageConnectorAPI, WriteMetadata } from "../../provider";
 import fs from "fs";
 import { logger } from "../../../../framework/logger";
 import { randomUUID } from "crypto";
+import { TDiagnosticResult, TServiceDiagnosticDepth } from "../../../../framework/api/diagnostics";
 
 export type LocalConfiguration = {
   id: string;
@@ -29,6 +30,19 @@ export default class LocalConnectorService implements StorageConnectorAPI {
     return this.id;
   }
 
+  async getDiagnostics(depth: TServiceDiagnosticDepth): Promise<TDiagnosticResult> {
+    switch (depth) {
+      case TServiceDiagnosticDepth.alive:
+      case TServiceDiagnosticDepth.stats_basic:
+      case TServiceDiagnosticDepth.stats_track:
+      case TServiceDiagnosticDepth.stats_deep:
+        return { ok: true, empty: true };
+
+      default:
+        throw new Error(`Unexpected TServiceDiagnosticDepth: ${JSON.stringify(depth)}`);
+    }
+  }
+
   write(relativePath: string, stream: Readable): Promise<WriteMetadata> {
     const path = this.getFullPath(relativePath);
     logger.trace(`Writing file ${path}`);

tdrive/backend/node/src/core/platform/services/storage/default-storage-strategy.ts

@@ -19,6 +19,10 @@ export class DefaultStorageStrategy implements StorageConnectorAPI {
     return this.connector.getId();
   }
 
+  getDiagnostics(depth) {
+    return this.connector.getDiagnostics(depth);
+  }
+
   write = (
     path: string,
     stream: Stream,

tdrive/backend/node/src/core/platform/services/storage/oneof-storage-strategy.ts

@@ -8,6 +8,7 @@ import {
 } from "../storage/provider";
 import { logger } from "../../../platform/framework";
 import { FileNotFountException, WriteFileException } from "./exceptions";
+import type { TDiagnosticResult, TServiceDiagnosticDepth } from "../../framework/api/diagnostics";
 
 /**
  * OneOfStorageStrategy is responsible for managing multiple storage backends.
@@ -34,6 +35,17 @@ export class OneOfStorageStrategy implements StorageConnectorAPI {
     return this.id;
   }
 
+  async getDiagnostics(depth: TServiceDiagnosticDepth): Promise<TDiagnosticResult> {
+    const states = await Promise.all(
+      this.storages.map(async s => ({ id: s.getId(), ...(await s.getDiagnostics(depth)) })),
+    );
+    return {
+      ...(states.every(s => s.ok) ? {} : { warn: "not_all_storages_ok" }),
+      ok: states.some(s => s.ok),
+      states: states,
+    };
+  }
+
   /**
    * Writes a file to all configured storages.
    * The write operation is considered successful if one of the storage succeed.

tdrive/backend/node/src/core/platform/services/storage/provider.ts

@@ -1,6 +1,7 @@
 import { Stream, Readable } from "stream";
 import { TdriveServiceProvider } from "../../framework";
 import { ExecutionContext } from "../../framework/api/crud-service";
+import { IServiceDiagnosticProvider } from "../../framework/api/diagnostics";
 
 export type WriteMetadata = {
   size: number;
@@ -22,7 +23,7 @@ export type DeleteOptions = {
   totalChunks?: number;
 };
 
-export interface StorageConnectorAPI {
+export interface StorageConnectorAPI extends IServiceDiagnosticProvider {
   /**
    * Returns identifier of a storage that should've been set in configuration.
    *

tdrive/backend/node/src/core/platform/services/storage/storage-service.ts

@@ -64,6 +64,10 @@ export default class StorageService extends TdriveService<StorageAPI> implements
     return this.connector;
   }
 
+  getDiagnostics(depth) {
+    return this.getConnector().getDiagnostics(depth);
+  }
+
   getHomeDir(): string {
     return this.homeDir;
   }

tdrive/backend/node/src/services/av/service/index.ts

@@ -32,8 +32,8 @@ export class AVServiceImpl implements TdriveServiceProvider, Initializable {
           },
         });
       }
-    } catch (error) {
+    } catch (err) {
-      logger.error({ error: `${error}` }, "Error while initializing Antivirus Service");
+      logger.error({ err }, "Error while initializing Antivirus Service");
       throw AVException.initializationFailed("Failed to initialize Antivirus service");
     }
     return this;

tdrive/backend/node/src/services/console/web/controller.ts

@@ -37,6 +37,20 @@ export class ConsoleController {
   }
 
   async auth(request: FastifyRequest<{ Body: AuthRequest }>): Promise<AuthResponse> {
+    const censoredHeadersLC = {
+      authorization: true,
+      cookie: true,
+    };
+    logger.info(
+      {
+        http_headers: Object.fromEntries(
+          Object.entries(request.headers)
+            .map(([k, v]) => (censoredHeadersLC[k.toLowerCase()] ? null : [k, v]))
+            .filter(x => !!x),
+        ),
+      },
+      "auth_attempt",
+    );
     if (request.body.oidc_id_token) {
       return { access_token: await this.authByToken(request.body.oidc_id_token) };
     } else if (request.body.email && request.body.password) {

tdrive/backend/node/src/services/global-resolver.ts

@@ -15,7 +15,7 @@ import TrackerAPI from "../core/platform/services/tracker/provider";
 import WebServerAPI from "../core/platform/services/webserver/provider";
 
 import assert from "assert";
-import { logger } from "../core/platform/framework";
+import { logger, TdriveServiceState } from "../core/platform/framework";
 import { ApplicationServiceImpl } from "./applications/services/applications";
 import { CompanyApplicationServiceImpl } from "./applications/services/company-applications";
 import { ApplicationHooksService } from "./applications/services/hooks";
@@ -78,6 +78,7 @@ class GlobalResolver {
   public services: TdriveServices;
   public platformServices: PlatformServices;
   public database: DatabaseServiceAPI;
+  private platform?: TdrivePlatform;
 
   public fastify: FastifyInstance<Server, IncomingMessage, ServerResponse>;
 
@@ -87,6 +88,7 @@ class GlobalResolver {
     if (this.alreadyInitialized) {
       return;
     }
+    this.platform = platform;
     this.database = platform.getProvider<DatabaseServiceAPI>("database");
 
     this.platformServices = {
@@ -152,6 +154,11 @@ class GlobalResolver {
     logger.info("Global resolver finished initializing services");
     this.alreadyInitialized = true;
   }
+
+  /** `true` if all components are in the started state. This means we should respond to http at least (for kubernetes Startup probe). */
+  isPlatformStarted(): boolean {
+    return this.alreadyInitialized && this.platform?.state.getValue() == TdriveServiceState.Started;
+  }
 }
 
 export default new GlobalResolver();

tdrive/backend/node/test/e2e/diagnostics/diagnostics.spec.ts

@@ -0,0 +1,63 @@
+import "reflect-metadata";
+import { afterAll, beforeEach, describe, expect, it, jest } from "@jest/globals";
+import { init, TestPlatform } from "../setup";
+import { getConfig as getDiagnosticsConfig } from "../../../src/core/platform/framework/api/diagnostics";
+import { e2eTestOverride as processDiagnosticProviderE2EOverride } from "../../../src/core/platform/services/diagnostics/providers/process";
+
+describe("The diagnostics infrastucture", () => {
+  let platform: TestPlatform;
+  const diagnosticConfig = getDiagnosticsConfig();
+  beforeEach(async () => {
+    platform = await init({
+      services: [
+        "webserver",
+        "database",
+        "applications",
+        "search",
+        "storage",
+        "diagnostics",
+        "message-queue",
+        "user",
+        "files",
+        "auth",
+        "statistics",
+        "platform-services",
+        "documents",
+      ],
+    });
+  });
+
+  afterAll(async () => {
+    await platform?.tearDown();
+    // @ts-ignore
+    platform = null;
+  });
+
+  const getDiagnosticTags = (tag: string, secret: string) =>
+    platform.app.inject({
+      method: "GET",
+      url: `/diagnostics/t/${encodeURIComponent(tag)}?secret=${encodeURIComponent(secret)}`,
+    });
+
+  it("should refuse invalid probe secrets", async () => {
+    const result = await getDiagnosticTags("ready", "ooooh look at me ! I'm like totally such an invalid probeSecret value");
+    expect(result.statusCode).toBe(403);
+  });
+
+  it("should at least report alive", async () => {
+    const result = await getDiagnosticTags("ready", diagnosticConfig.probeSecret!);
+    expect(result.statusCode).toBe(200);
+    expect(result.json().ok).toBe(true);
+  });
+
+  it("should aggregate failure", async () => {
+    try {
+      processDiagnosticProviderE2EOverride.forceFail = true;
+      const result = await getDiagnosticTags("ready", diagnosticConfig.probeSecret!);
+      expect(result.statusCode).toBe(503);
+      expect(result.json().ok).toBe(false);
+    } finally {
+      processDiagnosticProviderE2EOverride.forceFail = false;
+    }
+  });
+});

tdrive/docker-compose.dev.tests.opensearch.yml

@@ -68,6 +68,7 @@ services:
       - SEARCH_OS_PASSWORD=admin
       - SEARCH_OS_USE_AUTH=true
       - SEARCH_OS_USERNAME=admin
+      - DIAG_PROBE_SECRET=super_diagnostic_probe_secret
     depends_on:
       - postgres
       - opensearch-node1

tdrive/docker-compose.tests.yml

@@ -84,6 +84,7 @@ services:
       - STORAGE_S3_ENDPOINT=minio
       - STORAGE_S3_ACCESS_KEY=admin
       - STORAGE_S3_SECRET_KEY=adminminio
+      - DIAG_PROBE_SECRET=super_diagnostic_probe_secret
     depends_on:
       minio:
         condition: service_healthy