♻️🐛 back: removing heap roundtrip and minor cleanup (#762)

tdrive/backend/node/config/custom-environment-variables.json

@@ -21,8 +21,7 @@
       "__name": "DIAG_SKIP_KEYS",
       "__format": "json"
     },
-    "probeSecret": "DIAG_PROBE_SECRET",
+    "probeSecret": "DIAG_PROBE_SECRET"
-    "secret": "DIAG_SECRET"
   },
   "webserver": {
     "host": "TWAKE_DRIVE_HOST",

tdrive/backend/node/config/default.json

@@ -37,9 +37,7 @@
   },
   "diagnostics": {
     "skipKeys": [],
-    "probeSecret": "",
+    "probeSecret": ""
-    "secret": "",
-    "secretChallengeRefreshS": 15
   },
   "tracker": {
     "type": "segment",

tdrive/backend/node/src/core/platform/framework/api/diagnostics.ts

@@ -44,13 +44,6 @@ interface IDiagnosticsConfig {
   // relied on for security because disabling diagnostics. At worst this
   // provides access to the DB statistics.
   probeSecret?: string;
-  // This secret is required to sign more dangerous diagnostic endpoints, such
-  // the heap snapshot. It should never be sent over the network.
-  secret?: string;
-  // Maximum time to keep the same challenge token for diagnostic endpoints, in seconds.
-  // Must be large enough to have a reasonable chance of running the token request
-  // then the action on the same backend instance.
-  secretChallengeRefreshS: number;
 }
 
 export const getConfig = (): IDiagnosticsConfig => {
@@ -63,11 +56,6 @@ export const getConfig = (): IDiagnosticsConfig => {
         .split(/[,\s]+/g)
         .filter(x => !!x),
     };
-  if (typeof configSection.secretChallengeRefreshS === "string")
-    configSection = {
-      ...configSection,
-      secretChallengeRefreshS: parseInt(configSection.secretChallengeRefreshS, 10),
-    };
   return configSection;
 };
 
@@ -94,8 +82,6 @@ export interface IDiagnosticProvider {
   /**
    * Returns an object as presented to a diagnostic requester.
    * Warning: this could be public and readable to the internet.
-   * @param completeButSlow If `true`, perform additional operations for a
-   * more informative
    */
   get(): Promise<TDiagnosticResult>;
 }

tdrive/backend/node/src/core/platform/services/diagnostics/web/heap-routes.ts

@@ -1,71 +1,13 @@
 import type { FastifyInstance, FastifyPluginCallback } from "fastify";
 import { getConfig as getDiagnosticsGetConfig } from "../../../framework/api/diagnostics";
 import { getHeapSnapshotSync } from "../utils";
-import { createHash, randomUUID } from "node:crypto";
 
 const routes: FastifyPluginCallback = (fastify: FastifyInstance, _opts, next) => {
   const diagnosticsConfig = getDiagnosticsGetConfig();
-
+  if (diagnosticsConfig?.probeSecret?.length) {
-  if (diagnosticsConfig?.probeSecret?.length && diagnosticsConfig?.secret?.length) {
+    fastify.post("/heap", async (request, reply) => {
-    const getRunningToken = (() => {
-      const newToken = () => [randomUUID(), randomUUID()].join("*");
-      let token = newToken();
-      let updatedS = process.uptime();
-      return () => {
-        const nowS = process.uptime();
-        if (nowS - updatedS > diagnosticsConfig.secretChallengeRefreshS) {
-          token = newToken();
-          updatedS = nowS;
-        }
-        return token;
-      };
-    })();
-    const hashToken = (token: string = getRunningToken()) =>
-      createHash("sha512").update(`+${token}+${diagnosticsConfig.secret}+`).digest("hex");
-
-    /*
-      Example flow:
-        Configuration:
-          { secret: 'secretValue', probeSecret: 'probeSecretValue' }
-        GET current token using the probeSecret:
-          $ curl "$SERVER/api/diagnostics/heap?secret=probeSecretValue"
-          {"token":"d5888f91-d929-4775-b9dc-de6e2fb4d7cd*b2a57b32-7029-4957-82e9-c7c25794727f"}
-        Hash `+${token}+${secret}+` in SHA-512:
-          Shell example:
-            $ echo -n '+d5888f91-d929-4775-b9dc-de6e2fb4d7cd*b2a57b32-7029-4957-82e9-c7c25794727f+secretValue+' | sha512sum | cut -f1 -d' '
-            320070a508da218baf0f3363e837080fdb902bd0d986bd0f33b806b1230608c0868accdd10bd25261ca91b57c0459edf76218deb26571c72f6b93b077846abe3
-          JS example:
-            require('crypto').createHash('sha512').update(`+${token}+${secret}+`).digest('hex')
-        POST to download snapshot:
-          $ curl "$SERVER/api/diagnostics/heap"
-
-      Shell script example to download heap snapshot (requires curl, node and jq):
-          set -euo pipefail
-          SERVER="http://localhost:4000"
-          DIAG_PROBE_SECRET="probeSecretValue"
-          DIAG_SECRET="secretValue"
-
-          urlencode() { node -e 'console.log(encodeURIComponent(process.argv[1]))' "$@" ; }
-          DIAG_PROBE_SECRET_URLENCODED="$(urlencode "$DIAG_PROBE_SECRET")"
-          TOKEN="$(curl --fail-with-body "$SERVER/api/diagnostics/heap?secret=$DIAG_PROBE_SECRET_URLENCODED" | jq -r .token)"
-          HASHED="$(echo -n "+$TOKEN+$DIAG_SECRET+" | sha512sum | cut -f1 -d' ')"
-          curl --fail-with-body --remote-name --remote-header-name \
-              --header "Content-Type: application/json" \
-              --data '{"hash":"'"$HASHED"'"}' \
-              "$SERVER/api/diagnostics/heap"
-          # The file should be downloaded as `twake-drive-snap-$date.heapsnapshot`
-          ls twake-drive-snap-*.heapsnapshot
-    */
-    fastify.get("/heap", async (request, reply) => {
       if ((request.query as { secret: string }).secret !== diagnosticsConfig.probeSecret)
         return reply.status(403).send();
-      return reply.send({ token: getRunningToken() });
-    });
-
-    fastify.post("/heap", async (request, reply) => {
-      const hashExpected = hashToken();
-      const hashProvided = (request.body as { hash: string })?.hash;
-      if (hashProvided !== hashExpected) return reply.status(403).send();
       const filenameTimestamp = new Date()
         .toISOString()
         .replace(/(\.\d\d\d)?Z$/, "")
@@ -79,7 +21,6 @@ const routes: FastifyPluginCallback = (fastify: FastifyInstance, _opts, next) =>
       return replyResult;
     });
   }
-
   next();
 };
 

tdrive/backend/node/src/core/platform/services/storage/connectors/local/service.ts

@@ -36,7 +36,7 @@ export default class LocalConnectorService implements StorageConnectorAPI {
       case TServiceDiagnosticDepth.stats_basic:
       case TServiceDiagnosticDepth.stats_track:
       case TServiceDiagnosticDepth.stats_deep:
-        return { ok: true, warn: "local_s3_alway_ok" };
+        return { ok: true, warn: "local_storage_always_ok" };
 
       default:
         throw new Error(`Unexpected TServiceDiagnosticDepth: ${JSON.stringify(depth)}`);

tdrive/docker-compose.dev.tests.opensearch.yml

@@ -69,7 +69,6 @@ services:
       - SEARCH_OS_USE_AUTH=true
       - SEARCH_OS_USERNAME=admin
       - DIAG_PROBE_SECRET=super_diagnostic_probe_secret
-      - DIAG_SECRET=super_diagnostic_secret
     depends_on:
       - postgres
       - opensearch-node1

tdrive/docker-compose.tests.yml

@@ -85,7 +85,6 @@ services:
       - STORAGE_S3_ACCESS_KEY=admin
       - STORAGE_S3_SECRET_KEY=adminminio
       - DIAG_PROBE_SECRET=super_diagnostic_probe_secret
-      - DIAG_SECRET=super_diagnostic_secret
     depends_on:
       minio:
         condition: service_healthy