From 7c0f3597599a098cae09bdc0e98179b5854d5a2c Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Sun, 8 Dec 2024 22:53:43 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20back:=20adding=20storage=20service?= =?UTF-8?q?=20diagnostics=20(#762)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../services/diagnostics/providers/index.ts | 6 ++++-- .../diagnostics/providers/storage-service.ts | 9 +++++++++ .../services/storage/connectors/S3/s3-service.ts | 15 +++++++++++++++ .../services/storage/connectors/local/service.ts | 14 ++++++++++++++ .../services/storage/default-storage-strategy.ts | 4 ++++ .../services/storage/oneof-storage-strategy.ts | 12 ++++++++++++ .../core/platform/services/storage/provider.ts | 3 ++- .../platform/services/storage/storage-service.ts | 4 ++++ 8 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 tdrive/backend/node/src/core/platform/services/diagnostics/providers/storage-service.ts diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/providers/index.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/index.ts index d9350f7f..08f2b5d7 100644 --- a/tdrive/backend/node/src/core/platform/services/diagnostics/providers/index.ts +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/index.ts @@ -1,9 +1,11 @@ -import registerDBPingProvider from "./database-service"; +import registerDBServiceProvider from "./database-service"; +import registerStorageServiceProvider from "./storage-service"; import registerPlatformProvider from "./platform-started"; import registerProcessProvider from "./process"; export default () => { - registerDBPingProvider(); + registerDBServiceProvider(); + registerStorageServiceProvider(); registerPlatformProvider(); registerProcessProvider(); }; diff --git a/tdrive/backend/node/src/core/platform/services/diagnostics/providers/storage-service.ts b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/storage-service.ts new file mode 100644 index 00000000..5ad43456 --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/diagnostics/providers/storage-service.ts @@ -0,0 +1,9 @@ +import diagnostics from "../../../framework/api/diagnostics"; +import globalResolver from "../../../../../services/global-resolver"; + +export default () => + diagnostics.registerServiceProviders("storage", () => globalResolver.platformServices.storage, { + stats_basic: false, + stats_track: false, + stats_deep: false, + }); diff --git a/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts b/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts index e70cfc4a..a2c05b08 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts @@ -4,6 +4,7 @@ import { Readable } from "stream"; import { StorageConnectorAPI, WriteMetadata } from "../../provider"; import { randomUUID } from "crypto"; import _ from "lodash"; +import { TDiagnosticResult, TServiceDiagnosticDepth } from "../../../../framework/api/diagnostics"; export type S3Configuration = { id: string; @@ -43,6 +44,20 @@ export default class S3ConnectorService implements StorageConnectorAPI { return this.id; } + async getDiagnostics(depth: TServiceDiagnosticDepth): Promise { + switch (depth) { + case TServiceDiagnosticDepth.alive: + return { ok: await this.client.bucketExists(this.minioConfiguration.bucket) }; + case TServiceDiagnosticDepth.stats_basic: + case TServiceDiagnosticDepth.stats_track: + case TServiceDiagnosticDepth.stats_deep: + return { ok: true, warn: "s3_statistics_not_implemented" }; + + default: + throw new Error(`Unexpected TServiceDiagnosticDepth: ${JSON.stringify(depth)}`); + } + } + write(path: string, stream: Readable): Promise { return new Promise((resolve, reject) => { let totalSize = 0; diff --git a/tdrive/backend/node/src/core/platform/services/storage/connectors/local/service.ts b/tdrive/backend/node/src/core/platform/services/storage/connectors/local/service.ts index 16cf356f..ed7fa9af 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/connectors/local/service.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/connectors/local/service.ts @@ -6,6 +6,7 @@ import { StorageConnectorAPI, WriteMetadata } from "../../provider"; import fs from "fs"; import { logger } from "../../../../framework/logger"; import { randomUUID } from "crypto"; +import { TDiagnosticResult, TServiceDiagnosticDepth } from "../../../../framework/api/diagnostics"; export type LocalConfiguration = { id: string; @@ -29,6 +30,19 @@ export default class LocalConnectorService implements StorageConnectorAPI { return this.id; } + async getDiagnostics(depth: TServiceDiagnosticDepth): Promise { + switch (depth) { + case TServiceDiagnosticDepth.alive: + case TServiceDiagnosticDepth.stats_basic: + case TServiceDiagnosticDepth.stats_track: + case TServiceDiagnosticDepth.stats_deep: + return { ok: true, warn: "local_s3_alway_ok" }; + + default: + throw new Error(`Unexpected TServiceDiagnosticDepth: ${JSON.stringify(depth)}`); + } + } + write(relativePath: string, stream: Readable): Promise { const path = this.getFullPath(relativePath); logger.trace(`Writing file ${path}`); diff --git a/tdrive/backend/node/src/core/platform/services/storage/default-storage-strategy.ts b/tdrive/backend/node/src/core/platform/services/storage/default-storage-strategy.ts index c14034ca..8e14112d 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/default-storage-strategy.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/default-storage-strategy.ts @@ -19,6 +19,10 @@ export class DefaultStorageStrategy implements StorageConnectorAPI { return this.connector.getId(); } + getDiagnostics(depth) { + return this.connector.getDiagnostics(depth); + } + write = ( path: string, stream: Stream, diff --git a/tdrive/backend/node/src/core/platform/services/storage/oneof-storage-strategy.ts b/tdrive/backend/node/src/core/platform/services/storage/oneof-storage-strategy.ts index b71c7026..0223b8a6 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/oneof-storage-strategy.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/oneof-storage-strategy.ts @@ -8,6 +8,7 @@ import { } from "../storage/provider"; import { logger } from "../../../platform/framework"; import { FileNotFountException, WriteFileException } from "./exceptions"; +import type { TDiagnosticResult, TServiceDiagnosticDepth } from "../../framework/api/diagnostics"; /** * OneOfStorageStrategy is responsible for managing multiple storage backends. @@ -34,6 +35,17 @@ export class OneOfStorageStrategy implements StorageConnectorAPI { return this.id; } + async getDiagnostics(depth: TServiceDiagnosticDepth): Promise { + const states = await Promise.all( + this.storages.map(async s => ({ id: s.getId(), ...(await s.getDiagnostics(depth)) })), + ); + return { + ...(states.every(s => s.ok) ? {} : { warn: "not_all_storages_ok" }), + ok: states.some(s => s.ok), + states: states, + }; + } + /** * Writes a file to all configured storages. * The write operation is considered successful if one of the storage succeed. diff --git a/tdrive/backend/node/src/core/platform/services/storage/provider.ts b/tdrive/backend/node/src/core/platform/services/storage/provider.ts index d46a554c..fb258876 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/provider.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/provider.ts @@ -1,6 +1,7 @@ import { Stream, Readable } from "stream"; import { TdriveServiceProvider } from "../../framework"; import { ExecutionContext } from "../../framework/api/crud-service"; +import { IServiceDiagnosticProvider } from "../../framework/api/diagnostics"; export type WriteMetadata = { size: number; @@ -22,7 +23,7 @@ export type DeleteOptions = { totalChunks?: number; }; -export interface StorageConnectorAPI { +export interface StorageConnectorAPI extends IServiceDiagnosticProvider { /** * Returns identifier of a storage that should've been set in configuration. * diff --git a/tdrive/backend/node/src/core/platform/services/storage/storage-service.ts b/tdrive/backend/node/src/core/platform/services/storage/storage-service.ts index 4eaea4ba..b2763250 100644 --- a/tdrive/backend/node/src/core/platform/services/storage/storage-service.ts +++ b/tdrive/backend/node/src/core/platform/services/storage/storage-service.ts @@ -64,6 +64,10 @@ export default class StorageService extends TdriveService implements return this.connector; } + getDiagnostics(depth) { + return this.getConnector().getDiagnostics(depth); + } + getHomeDir(): string { return this.homeDir; }