eduvia/workavia-drive
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

🌟 Support for PostgreSQL (#306)

Browse Source
This commit is contained in:
Anton Shepilov
2024-01-13 23:49:34 +03:00
committed by GitHub
parent 81777154af
commit 70c033aabe
38 changed files with 2202 additions and 364 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/backend.yml
+1 -1
View File
@@ -24,7 +24,7 @@ jobs:
- name: e2e-mongo-test-clean
run: cd tdrive && docker-compose -f docker-compose.tests.yml stop
- name: e2e-opensearch-test
run: cd tdrive && docker-compose -f docker-compose.dev.tests.opensearch.yml run -e NODE_OPTIONS=--unhandled-rejections=warn -e SEARCH_DRIVER=opensearch -e DB_DRIVER=mongodb -e PUBSUB_TYPE=local node npm run test:all
run: cd tdrive && docker-compose -f docker-compose.dev.tests.opensearch.yml run -e NODE_OPTIONS=--unhandled-rejections=warn -e SEARCH_DRIVER=opensearch -e DB_DRIVER=postgres -e PUBSUB_TYPE=local node npm run test:all
- name: e2e-opensearch-test-clean
run: cd tdrive && docker-compose -f docker-compose.dev.tests.opensearch.yml stop
- name: e2e-cassandra-test
.run/Run all e2e [MongoDB].run.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<node-interpreter value="project" />
<jest-package value="$PROJECT_DIR$/tdrive/backend/node/node_modules/jest" />
<working-dir value="$PROJECT_DIR$/tdrive/backend/node" />
<jest-options value="--forceExit --testTimeout=3000000 --maxWorkers=1" />
<jest-options value="--forceExit --testTimeout=30000 --maxWorkers=1" />
<envs>
<env name="DB_DRIVER" value="mongodb" />
<env name="DB_MONGO_URI" value="mongodb://localhost:27017" />
.run/Run all e2e [OpenSearch].run.xml
+27
View File
@@ -0,0 +1,27 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Run all e2e [OpenSearch]" type="JavaScriptTestRunnerJest">
<node-interpreter value="project" />
<jest-package value="$PROJECT_DIR$/tdrive/backend/node/node_modules/jest" />
<working-dir value="$PROJECT_DIR$/tdrive/backend/node" />
<jest-options value="--forceExit --coverage --maxWorkers=1 --testTimeout=30000" />
<envs>
<env name="DB_DRIVER" value="postgres" />
<env name="DB_MONGO_URI" value="mongodb://localhost:27017" />
<env name="DB_POSTGRES_DBNAME" value="tdrive" />
<env name="DB_POSTGRES_HOST" value="localhost" />
<env name="DB_POSTGRES_PASSWORD" value="tdrive_secret" />
<env name="DB_POSTGRES_PORT" value="5432" />
<env name="DB_POSTGRES_USERNAME" value="tdrive_user" />
<env name="PUBSUB_TYPE" value="local" />
<env name="SEARCH_DRIVER" value="opensearch" />
<env name="SEARCH_OS_ENDPOINT" value="https://localhost:9200" />
<env name="SEARCH_OS_PASSWORD" value="admin" />
<env name="SEARCH_OS_USE_AUTH" value="true" />
<env name="SEARCH_OS_USERNAME" value="admin" />
<env name="STORAGE_LOCAL_PATH" value="/tmp" />
</envs>
<scope-kind value="DIRECTORY" />
<test-directory value="$PROJECT_DIR$/tdrive/backend/node/test/e2e" />
<method v="2" />
</configuration>
</component>
.run/TDrive[Postgres] local.run.xml
+27
View File
@@ -0,0 +1,27 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="TDrive[Postgres] local" type="js.build_tools.npm">
<package-json value="$PROJECT_DIR$/tdrive/backend/node/package.json" />
<command value="run" />
<scripts>
<script value="serve" />
</scripts>
<node-interpreter value="project" />
<envs>
<env name="DB_DRIVER" value="postgres" />
<env name="DB_POSTGRES_DBNAME" value="tdrive" />
<env name="DB_POSTGRES_PASSWORD" value="tdrive_secret" />
<env name="DB_POSTGRES_PORT" value="5432" />
<env name="DB_POSTGRES_URI" value="postgres://localhost:27017" />
<env name="DB_POSTGRES_USERNAME" value="tdrive_user" />
<env name="DRIVE_ROOT_ADMINS" value="ashepilov@gmail.com" />
<env name="PUBSUB_TYPE" value="local" />
<env name="SEARCH_DRIVER" value="opensearch" />
<env name="SEARCH_OS_ENDPOINT" value="https://localhost:9200" />
<env name="SEARCH_OS_PASSWORD" value="admin" />
<env name="SEARCH_OS_USE_AUTH" value="true" />
<env name="SEARCH_OS_USERNAME" value="admin" />
<env name="STORAGE_LOCAL_PATH" value="$USER_HOME$/dev/linagora/TDrive/tdrive/docker-data" />
</envs>
<method v="2" />
</configuration>
</component>
tdrive/backend/node/config/custom-environment-variables.json
+8
View File
@@ -53,6 +53,14 @@
"contactPoints": "DB_CASSANDRA_URI",
"localDataCenter": "DB_CASSANDRA_LOCALDATACENTER",
"keyspace": "DB_CASSANDRA_KEYSPACE"
},
"postgres": {
"database": "DB_POSTGRES_DBNAME",
"host": "DB_POSTGRES_HOST",
"user": "DB_POSTGRES_USERNAME",
"password": "DB_POSTGRES_PASSWORD",
"port": "DB_POSTGRES_PORT",
"ssl": "DB_POSTGRES_USE_SSL"
}
},
"search": {
tdrive/backend/node/package-lock.json
Generated
+758 -238
View File
File diff suppressed because it is too large Load Diff
tdrive/backend/node/package.json
+6 -3
View File
@@ -74,6 +74,7 @@
"@types/node-fetch": "^2.5.12",
"@types/node-uuid": "^0.0.28",
"@types/pdf-image": "^2.0.1",
"@types/pg-pool": "^2.0.6",
"@types/pump": "^1.1.1",
"@types/random-useragent": "^0.3.1",
"@types/socket.io-client": "^1.4.36",
@@ -153,9 +154,9 @@
"jsonwebtoken": "^9.0.0",
"jwks-rsa": "^3.0.1",
"lodash": "^4.17.21",
"minio": "7.0.18",
"minio": "^7.1.3",
"moment": "^2.29.4",
"mongodb": "^5.6.0",
"mongodb": "^5.9.2",
"multistream": "^4.1.0",
"njwt": "^2.0.0",
"node-abort-controller": "^3.1.1",
@@ -167,6 +168,8 @@
"ora": "^5.4.0",
"pdf-parse": "^1.1.1",
"pdf2pic": "^2.1.4",
"pg": "^8.11.3",
"pg-pool": "^3.6.1",
"pino": "^6.8.0",
"pino-std-serializers": "^6.1.0",
"probe-image-size": "^7.2.3",
@@ -175,7 +178,7 @@
"redis": "3",
"reflect-metadata": "^0.1.13",
"rxjs": "^6.6.3",
"sharp": "^0.30.5",
"sharp": "^0.33.1",
"short-uuid": "^4.2.2",
"socket.io": "^4.6.2",
"socket.io-client": "^3.0.0",
tdrive/backend/node/src/core/platform/services/database/services/connector-factory.ts
+3
View File
@@ -5,6 +5,7 @@ import {
CassandraConnector,
} from "./orm/connectors/cassandra/cassandra";
import { MongoConnectionOptions, MongoConnector } from "./orm/connectors/mongodb/mongodb";
import { PostgresConnectionOptions, PostgresConnector } from "./orm/connectors/postgres/postgres";
export class ConnectorFactory {
public create(type: DatabaseType, options: ConnectionOptions, secret: string): Connector {
@@ -13,6 +14,8 @@ export class ConnectorFactory {
return new CassandraConnector(type, options as CassandraConnectionOptions, secret);
case "mongodb":
return new MongoConnector(type, options as MongoConnectionOptions, secret);
case "postgres":
return new PostgresConnector(type, options as PostgresConnectionOptions, secret);
default:
throw new Error(`${type} is not supported`);
}
tdrive/backend/node/src/core/platform/services/database/services/index.ts
+6 -2
View File
@@ -7,6 +7,7 @@ import { CassandraConnectionOptions } from "./orm/connectors/cassandra/cassandra
import { MongoConnectionOptions } from "./orm/connectors/mongodb/mongodb";
import { EntityTarget } from "./orm/types";
import { RepositoryManager } from "./orm/repository/manager";
import { PostgresConnectionOptions } from "./orm/connectors/postgres/postgres";
export default class DatabaseService implements DatabaseServiceAPI {
version = "1";
@@ -40,6 +41,9 @@ export default class DatabaseService implements DatabaseServiceAPI {
}
}
export declare type ConnectionOptions = MongoConnectionOptions | CassandraConnectionOptions;
export declare type ConnectionOptions =
| MongoConnectionOptions
| CassandraConnectionOptions
| PostgresConnectionOptions;
export declare type DatabaseType = "mongodb" | "cassandra";
export declare type DatabaseType = "mongodb" | "cassandra" | "postgres";
tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/abstract-connector.ts
+3 -11
View File
@@ -1,27 +1,23 @@
/* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/explicit-module-boundary-types */
import { Connector } from ".";
import { Connector, UpsertOptions } from ".";
import { ConnectionOptions, DatabaseType } from "../..";
import { FindOptions } from "../repository/repository";
import { ColumnDefinition, EntityDefinition } from "../types";
import { ListResult } from "../../../../../framework/api/crud-service";
export abstract class AbstractConnector<T extends ConnectionOptions, DatabaseClient>
implements Connector
{
export abstract class AbstractConnector<T extends ConnectionOptions> implements Connector {
constructor(protected type: DatabaseType, protected options: T, protected secret: string) {}
abstract connect(): Promise<this>;
abstract drop(): Promise<this>;
abstract getClient(): DatabaseClient;
abstract createTable(
entity: EntityDefinition,
columns: { [name: string]: ColumnDefinition },
): Promise<boolean>;
abstract upsert(entities: any[]): Promise<boolean[]>;
abstract upsert(entities: any[], _options: UpsertOptions): Promise<boolean[]>;
abstract remove(entities: any[]): Promise<boolean[]>;
@@ -38,8 +34,4 @@ export abstract class AbstractConnector<T extends ConnectionOptions, DatabaseCli
getType(): DatabaseType {
return this.type;
}
getSecret(): string {
return this.secret;
}
}
tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/cassandra/cassandra.ts
+2 -5
View File
@@ -1,4 +1,4 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
/* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/explicit-module-boundary-types */
import cassandra, { types } from "cassandra-driver";
import { md5 } from "../../../../../../../crypto";
import { defer, Subject, throwError, timer } from "rxjs";
@@ -48,10 +48,7 @@ export interface CassandraConnectionOptions {
delay?: number;
}
export class CassandraConnector extends AbstractConnector<
CassandraConnectionOptions,
cassandra.Client
> {
export class CassandraConnector extends AbstractConnector<CassandraConnectionOptions> {
private client: cassandra.Client;
private keyspaceExists = false;
tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts
+9 -3
View File
@@ -5,11 +5,14 @@ import { MongoConnectionOptions } from "./mongodb/mongodb";
import { ColumnDefinition, EntityDefinition } from "../types";
import { FindOptions } from "../repository/repository";
import { ListResult } from "../../../../../framework/api/crud-service";
import { PostgresConnectionOptions } from "./postgres/postgres";
export * from "./mongodb/mongodb";
export * from "./cassandra/cassandra";
export type UpsertOptions = any;
export type UpsertOptions = {
action?: "INSERT" | "UPDATE";
};
export type RemoveOptions = any;
@@ -41,7 +44,7 @@ export interface Connector extends Initializable {
* Upsert
* returns true if the object was created/updated, false otherwise
*/
upsert(entities: any[]): Promise<boolean[]>;
upsert(entities: any[], _options: UpsertOptions): Promise<boolean[]>;
/**
* Remove
@@ -60,4 +63,7 @@ export interface Connector extends Initializable {
): Promise<ListResult<EntityType>>;
}
export declare type ConnectionOptions = MongoConnectionOptions | CassandraConnectionOptions;
export declare type ConnectionOptions =
| MongoConnectionOptions
| CassandraConnectionOptions
| PostgresConnectionOptions;
tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts
+7 -3
View File
@@ -9,15 +9,13 @@ import { buildSelectQuery } from "./query-builder";
import { transformValueFromDbString, transformValueToDbString } from "./typeTransforms";
import { logger } from "../../../../../../framework";
export { MongoPagination } from "./pagination";
export interface MongoConnectionOptions {
// TODO: More options
uri: string;
database: string;
}
export class MongoConnector extends AbstractConnector<MongoConnectionOptions, mongo.MongoClient> {
export class MongoConnector extends AbstractConnector<MongoConnectionOptions> {
private client: mongo.MongoClient;
async init(): Promise<this> {
@@ -75,6 +73,7 @@ export class MongoConnector extends AbstractConnector<MongoConnectionOptions, mo
return true;
}
async upsert(entities: any[], _options: UpsertOptions = {}): Promise<boolean[]> {
logger.trace(`services.database.orm.mongodb.upsert - entities count: ${entities.length}`);
return new Promise(async resolve => {
const promises: Promise<mongo.UpdateResult>[] = [];
@@ -83,6 +82,11 @@ export class MongoConnector extends AbstractConnector<MongoConnectionOptions, mo
entities.forEach(entity => {
const { columnsDefinition, entityDefinition } = getEntityDefinition(entity);
const primaryKey = unwrapPrimarykey(entityDefinition);
logger.trace(
`services.database.orm.mongodb.upsert[${_options.action}] - Entity{${
entityDefinition.name
}: ${JSON.stringify(entity)}`,
);
//Set updated content
const set: any = {};
tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres-data-transform.ts
+156
View File
@@ -0,0 +1,156 @@
import { ColumnType } from "../../../orm/types";
import { isBoolean, isInteger, isNull, isString, isUndefined } from "lodash";
import { decrypt, encrypt } from "../../../../../../../crypto";
import { logger } from "../../../../../../framework";
export type TransformOptions = {
secret?: string;
};
export const TypeMappings = {
encoded_string: "TEXT",
encoded_json: "TEXT",
json: "TEXT",
string: "TEXT",
number: "BIGINT",
timeuuid: "UUID",
uuid: "UUID",
boolean: "BOOLEAN",
counter: "BIGINT",
// backward compatibility
tdrive_boolean: "BOOLEAN",
tdrive_datetime: "BIGINT", //Deprecated
};
/**
* Possible data transformations:
* string --> everything
* encoded_string --> everything
* json --> everything
* encoded_json --> everything
*
* number --> number, string
* tdrive_int --> number, string
* tdrive_datetime --> number. string
*
* timeuuid --> string
* uuid --> string
*
* boolean --> boolean, number
* tdrive_boolean --> boolean, number
*
* tdrive_datetime --> number
*
* null and undefined --> null
*/
export class PostgresDataTransformer {
constructor(readonly options: TransformOptions) {}
fromDbString(v: any, type: ColumnType): any {
if (isNull(v) || isUndefined(v)) {
return null;
}
this.checkType(type);
if (v !== null && (type === "encoded_string" || type === "encoded_json")) {
let decryptedValue: string;
if (typeof v === "string" && v.trim() === "") {
return v;
}
try {
decryptedValue = decrypt(v, this.options.secret).data;
} catch (err) {
logger.debug(`Can not decrypt data (${err.message}) %o of type ${type}`, v);
decryptedValue = v;
}
if (type === "encoded_json") {
try {
decryptedValue = JSON.parse(decryptedValue);
} catch (err) {
logger.debug(
{ err },
`Can not parse JSON from decrypted data %o of type ${type}`,
decryptedValue,
);
decryptedValue = null;
}
}
return decryptedValue;
}
if (type === "number" || type === "tdrive_datetime" || type === "counter") {
const number = parseInt(v, 10);
if (isNaN(number)) throw new Error(`Can't parse ${v} to int`);
return number;
}
if (type === "json") {
return JSON.parse(v);
}
return v;
}
toDbString(v: any, type: ColumnType): any {
this.checkType(type);
if (isNull(v) || isUndefined(v)) {
return null;
}
if (type === "number" || type === "tdrive_datetime" || type === "counter") {
const number = parseInt(v, 10);
if (isNaN(number)) throw new Error(`Can't parse ${v} to int`);
return number;
}
if (type === "uuid" || type === "timeuuid") {
if (!isString(v))
throw new Error(`uuid or timeuuid could be only strings, and ${v} is not string`);
return v;
}
if (type === "boolean" || type === "tdrive_boolean") {
//Security to avoid string with "false" in it
if (!isInteger(v) && !isBoolean(v)) {
throw new Error(`'${v}' is not a ${type}`);
}
return !!v;
}
if (type === "string") {
if (isString(v)) return v;
else throw "We can't store in string fields only strings";
}
if (type === "json") {
return JSON.stringify(v);
}
if (type === "encoded_string" || type === "encoded_json") {
if (type === "encoded_string" && !isString(v))
throw "We can't store in string fields only strings";
if (type === "encoded_json") {
v = JSON.stringify(v);
}
const encrypted = encrypt(v, this.options.secret);
return `${(encrypted.data || "").toString().replace(/'/gm, "''")}`;
}
return v.toString();
}
private checkType(type: ColumnType) {
if (!Object.keys(TypeMappings).includes(type)) {
throw new Error(`Support of ${type} not implemented yet for PostgreSQL`);
}
}
}
tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres-query-builder.ts
+185
View File
@@ -0,0 +1,185 @@
import { comparisonType, FindOptions } from "../../repository/repository";
import { ObjectType } from "../../types";
import { getEntityDefinition, unwrapPrimarykey } from "../../utils";
import { PostgresDataTransformer } from "./postgres-data-transform";
import _ from "lodash";
export type Query = [string, any[]];
export class PostgresQueryBuilder {
private dataTransformer: PostgresDataTransformer;
constructor(private secret: string) {
this.dataTransformer = new PostgresDataTransformer({ secret: this.secret });
}
buildSelect<Entity>(
entityType: ObjectType<Entity>,
filters: Record<string, unknown>,
options: FindOptions,
): Query {
const instance = new (entityType as any)();
const { columnsDefinition, entityDefinition } = getEntityDefinition(instance);
const query = (whereClause: string, orderByClause: string, limit: number, offset: number) => {
return `SELECT * FROM "${entityDefinition.name}"
${whereClause?.length ? "WHERE " + whereClause : ""}
${orderByClause?.length ? "ORDER BY " + orderByClause : ""}
LIMIT ${limit} OFFSET ${offset}`;
};
// === EQUAL or IN operator ===
let idx = 1;
const values = [];
let whereClause = "";
if (filters) {
Object.keys(filters)
.filter(key => !_.isUndefined(filters[key]))
.forEach(key => {
const filter = filters[key];
if (Array.isArray(filter)) {
if (filter.length) {
const inClause: string[] = filter.map(
value => `${this.dataTransformer.toDbString(value, columnsDefinition[key].type)}`,
);
whereClause += `${key} IN ($${inClause.map(() => idx++).join(",$")}) AND `;
values.push(...inClause);
}
} else {
const value = `${this.dataTransformer.toDbString(filter, columnsDefinition[key].type)}`;
whereClause += `${key} = $${idx++} AND `;
values.push(value);
}
});
}
if (options) {
// ==== Comparison operators ===
const appendComparison = (predicates: comparisonType[], operator: string) => {
if (predicates) {
predicates.forEach(element => {
whereClause += `${element[0]} ${operator} $${idx++} AND `;
values.push(
this.dataTransformer.toDbString(element[1], columnsDefinition[element[0]].type),
);
});
}
};
appendComparison(options.$lt, "<");
appendComparison(options.$lte, "<=");
appendComparison(options.$gt, ">");
appendComparison(options.$gte, ">=");
// === IN ===
options.$in?.forEach(e => {
whereClause += `${e[0]} IN ($${e[1].map(() => idx++).join(",$")}) AND `;
values.push(...e[1]);
});
// === LIKE ====
options.$like?.forEach(e => {
whereClause += `${e[0]} LIKE $${idx++} AND `;
values.push(`%${e[1]}%`);
});
}
if (whereClause && whereClause.endsWith("AND ")) whereClause = whereClause.slice(0, -4);
// ==== ORDER BY =====
const orderByClause = `${entityDefinition.options.primaryKey
.slice(1)
.map(
(key: string) =>
`${key} ${(columnsDefinition[key].options.order || "ASC") === "ASC" ? "DESC" : "ASC"}`,
)}`;
// ==== PAGING =====
let limit = 100;
let offset = 0;
if (options?.pagination) {
if (options.pagination.limitStr) {
limit = Number.parseInt(options.pagination.limitStr);
}
if (options.pagination.page_token) {
offset = (Number.parseInt(options.pagination.page_token) - 1) * limit;
}
}
return [query(whereClause, orderByClause, limit, offset), values];
}
buildDelete<Entity>(entity: Entity): Query {
const { columnsDefinition, entityDefinition } = getEntityDefinition(entity);
const primaryKey = unwrapPrimarykey(entityDefinition);
const toValueKeyDBStringPair = (key: string) => {
return [
key,
this.dataTransformer.toDbString(
entity[columnsDefinition[key].nodename],
columnsDefinition[key].type,
),
];
};
const where = primaryKey.map(key => toValueKeyDBStringPair(key));
const query = `DELETE FROM "${entityDefinition.name}"
WHERE ${where.map((e, idx) => `${e[0]} = $${idx + 1}`).join(" AND ")}`;
return [query, where.map(f => f[1])];
}
buildInsert<Entity>(entity: Entity): Query {
const { columnsDefinition, entityDefinition } = getEntityDefinition(entity);
const toValueKeyDBStringPair = (key: string) => {
return [
key,
this.dataTransformer.toDbString(
entity[columnsDefinition[key].nodename],
columnsDefinition[key].type,
),
];
};
const fields = Object.keys(columnsDefinition)
.filter(key => entity[columnsDefinition[key].nodename] !== undefined)
.map(key => toValueKeyDBStringPair(key));
const query = `INSERT INTO "${entityDefinition.name}" (${fields.map(e => e[0]).join(", ")})
VALUES (${fields.map((e, idx) => `$${idx + 1}`).join(", ")})`;
return [query, fields.map(f => f[1])];
}
buildUpdate<Entity>(entity: Entity): Query {
const { columnsDefinition, entityDefinition } = getEntityDefinition(entity);
const primaryKey = unwrapPrimarykey(entityDefinition);
const toValueKeyDBStringPair = (key: string) => {
return [
key,
this.dataTransformer.toDbString(
entity[columnsDefinition[key].nodename],
columnsDefinition[key].type,
),
];
};
// Set updated content
const set = Object.keys(columnsDefinition)
.filter(key => primaryKey.indexOf(key) === -1)
.filter(key => entity[columnsDefinition[key].nodename] !== undefined)
.map(key => toValueKeyDBStringPair(key));
//Set primary key
const where = primaryKey.map(key => toValueKeyDBStringPair(key));
//Start index for where clause params
const whereIdx = set.length + 1;
const query = `UPDATE "${entityDefinition.name}"
SET ${set.map((e, idx) => `${e[0]} = $${idx + 1}`).join(", ")}
WHERE ${where.map((e, idx) => `${e[0]} = $${whereIdx + idx}`).join(" AND ")}`;
const values = [];
values.push(...set.map(f => f[1]));
values.push(...where.map(f => f[1]));
return [query, values];
}
}
tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres.ts
+283
View File
@@ -0,0 +1,283 @@
import { AbstractConnector } from "../abstract-connector";
import { ColumnDefinition, EntityDefinition, ObjectType } from "../../types";
import { ListResult, Paginable, Pagination } from "../../../../../../framework/api/crud-service";
import { FindOptions } from "../../repository/repository";
import { Client, QueryResult } from "pg";
import { getLogger, logger } from "../../../../../../framework";
import { getEntityDefinition } from "../../../orm/utils";
import { UpsertOptions } from "src/core/platform/services/database/services/orm/connectors";
import { PostgresDataTransformer, TypeMappings } from "./postgres-data-transform";
import { PostgresQueryBuilder, Query } from "./postgres-query-builder";
export interface PostgresConnectionOptions {
database: string;
user: string;
password: string;
port: number;
host: string;
ssl: false;
idleTimeoutMillis: 1000; // close idle clients after 1 second
connectionTimeoutMillis: 1000; // return an error after 1 second if connection could not be established
statement_timeout: number; // number of milliseconds before a statement in query will time out, default is no timeout
query_timeout: number; // number of milliseconds before a query call will time out, default is no timeout
}
export class PostgresConnector extends AbstractConnector<PostgresConnectionOptions> {
private logger = getLogger("PostgresConnector");
private client: Client = new Client(this.options);
private dataTransformer = new PostgresDataTransformer({ secret: this.secret });
private queryBuilder = new PostgresQueryBuilder(this.secret);
private async healthcheck(): Promise<void> {
const result: QueryResult = await this.client.query("SELECT NOW()");
if (!result || result.rowCount != 1) {
throw new Error("Connection Error");
}
this.logger.info(`DB connection is fine, current time is ${result.rows[0].now}`);
}
async connect(): Promise<this> {
if (this.client) {
this.logger.info("Connecting to DB");
this.client.on("error", err => {
this.logger.warn(err, "PostgreSQL connection error");
});
await this.client.connect();
this.logger.info("Connection pool created");
await this.healthcheck();
}
return this;
}
async init(): Promise<this> {
if (!this.client) {
await this.connect();
}
return this;
}
async createTable(
entity: EntityDefinition,
columns: { [p: string]: ColumnDefinition },
): Promise<boolean> {
const columnsString = Object.keys(columns)
.map(colName => {
const definition = columns[colName];
return `${colName} ${TypeMappings[definition.type]}`;
})
.join(",\n");
// --- Generate final create table query --- //
const query = `
CREATE TABLE IF NOT EXISTS "${entity.name}"
(
${columnsString}
);`;
try {
this.logger.debug(
`service.database.orm.createTable - Creating table ${entity.name} : ${query}`,
);
const result: QueryResult = await this.client.query(query);
this.logger.info(`Table is created with the result ${JSON.stringify(result)}`);
} catch (err) {
this.logger.warn(
{ err },
`service.database.orm.createTable - creation error for table ${entity.name} : ${err.message}`,
);
return false;
}
//--- Alter table if not up to date --- //
await this.alterTable(entity, columns);
// --- Create primary key --- //
await this.alterTablePrimaryKey(entity);
// --- Create indexes --- //
return await this.alterTableIndexes(entity);
}
private async alterTableIndexes(entity: EntityDefinition) {
if (entity.options.globalIndexes) {
for (const globalIndex of entity.options.globalIndexes) {
const indexName = globalIndex.join("_");
const indexDbName = `index_${entity.name}_${indexName}`;
const query = `CREATE INDEX IF NOT EXISTS ${indexDbName} ON "${entity.name}"
(${globalIndex.length === 1 ? globalIndex[0] : `(${globalIndex[0]}), ${globalIndex[1]}`})`;
try {
this.logger.debug(`Creating index ${indexName} (${indexDbName}) : ${query}`);
await this.client.query(query);
} catch (err) {
this.logger.warn(
err,
`Creation error for index ${indexName} (${indexDbName}) : ${err.message}`,
);
return false;
}
}
}
}
private async alterTablePrimaryKey(entity: EntityDefinition) {
if (entity.options.primaryKey) {
const query = `ALTER TABLE "${entity.name}" ADD PRIMARY KEY (
${entity.options.primaryKey.join(", ")});`;
try {
await this.client.query(query);
} catch (err) {
this.logger.warn(err, `Error creating primary key for "${entity.name}"`);
}
}
}
private async alterTable(entity: EntityDefinition, columns: { [p: string]: ColumnDefinition }) {
const existingColumns = await this.getTableDefinition(entity.name);
if (existingColumns.length > 0) {
this.logger.debug(
`Existing columns for table "${entity.name}", generating alter table queries ...`,
);
const alterQueryColumns = Object.keys(columns)
.filter(colName => existingColumns.indexOf(colName) < 0)
.map(colName => `ADD COLUMN ${colName} ${TypeMappings[columns[colName].type]}`)
.join(", ");
if (alterQueryColumns.length > 0) {
const alterQuery = `ALTER TABLE "${entity.name}" ${alterQueryColumns}`;
const queryResult: QueryResult = await this.client.query(alterQuery);
this.logger.info(`Table is altered with the result ${queryResult}`);
}
}
}
async drop(): Promise<this> {
logger.info("Drop database is not implemented for security reasons.");
return this;
}
async dropTables(): Promise<this> {
const query = `
DO $$
DECLARE
tablename text;
BEGIN
FOR tablename IN (SELECT table_name FROM information_schema.tables WHERE table_schema = 'public')
LOOP
EXECUTE 'DROP TABLE IF EXISTS "' || tablename || '" CASCADE';
END LOOP;
END $$;`;
logger.debug(`service.database.orm.postgres.dropTables - Query: "${query}"`);
await this.client.query(query);
return this;
}
async find<EntityType>(
entityType: ObjectType<EntityType>,
filters: Record<string, unknown>,
options: FindOptions,
): Promise<ListResult<EntityType>> {
const query = this.queryBuilder.buildSelect(entityType, filters, options);
logger.debug(`services.database.orm.postgres.find - Query: ${query}`);
const results = await this.client.query(query[0] as string, query[1] as never[]);
const { columnsDefinition, entityDefinition } = getEntityDefinition(
new (entityType as ObjectType<EntityType>)(),
);
const entities: EntityType[] = [];
results.rows.forEach(row => {
const entity = new (entityType as ObjectType<EntityType>)();
Object.keys(row).forEach(key => {
if (columnsDefinition[key]) {
entity[columnsDefinition[key].nodename] = this.dataTransformer.fromDbString(
row[key],
columnsDefinition[key].type,
);
}
});
entities.push(entity);
});
const nextPageToken = options?.pagination?.page_token || "0";
const limit = parseInt(options?.pagination?.limitStr);
const nextToken = entities.length === limit && (parseInt(nextPageToken) + limit).toString(10);
const nextPage: Paginable = new Pagination(nextToken, options?.pagination?.limitStr || "100");
logger.debug(
`services.database.orm.postgres.find - Query Result (items=${entities.length}): ${query}`,
);
return new ListResult<EntityType>(entityDefinition.type, entities, nextPage);
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
remove<Entity>(entities: Entity[]): Promise<boolean[]> {
return Promise.all(entities.map(entity => this.removeOne(entity)));
}
async removeOne<Entity>(entity: Entity): Promise<boolean> {
const query = this.queryBuilder.buildDelete(entity);
logger.debug(`service.database.orm.postgres.remove - Query: "${query}"`);
const result = await this.client.query(query[0] as string, query[1] as never[]);
return result.rowCount > 0;
}
async upsert<Entity>(entities: Entity[], _options: UpsertOptions): Promise<boolean[]> {
if (!_options?.action) {
throw new Error("Can't perform unknown operation");
} else {
return Promise.all(entities.map(entity => this.upsertOne(entity, _options)));
}
}
private async upsertOne<Entity>(entity: Entity, _options: UpsertOptions): Promise<boolean> {
if (_options.action == "INSERT") {
const query = this.queryBuilder.buildInsert(entity);
return this.execute(query);
} else if (_options.action == "UPDATE") {
if (!(await this.execute(this.queryBuilder.buildUpdate(entity)))) {
return this.execute(this.queryBuilder.buildInsert(entity));
}
} else {
return false;
}
}
private async execute(query: Query) {
logger.debug(`service.database.orm.postgres - Query: "${query[0]}"`);
try {
const result: QueryResult = await this.client.query(query[0] as string, query[1] as never[]);
return result.rowCount > 0;
} catch (err) {
logger.error({ err }, `services.database.orm.postgres - Error with SQL query: ${query[0]}`);
return false;
}
}
async getTableDefinition(name: string): Promise<string[]> {
try {
const query = `SELECT
table_name,
column_name,
data_type
FROM
information_schema.columns
WHERE
table_name = $1`;
const dbResult: QueryResult<TableRowInfo> = await this.client.query(query, [name]);
return dbResult.rows.map(row => row.column_name);
} catch (err) {
this.logger.warn("Error querying table information", err);
throw err;
}
}
}
export type TableRowInfo = {
table_name: string;
column_name: string;
data_type: string;
};
tdrive/backend/node/src/core/platform/services/database/services/orm/manager.ts
+56 -33
View File
@@ -9,12 +9,16 @@ import { DatabaseEntitiesRemovedEvent, DatabaseEntitiesSavedEvent } from "./type
import { localEventBus } from "../../../../framework/event-bus";
export default class EntityManager<EntityType extends Record<string, any>> {
private toPersist: EntityType[] = [];
private toInsert: EntityType[] = [];
private toUpdate: EntityType[] = [];
private toRemove: EntityType[] = [];
constructor(readonly connector: Connector) {}
public persist(entity: any): this {
logger.trace(
`services.database.orm.entity-manager.persist - entity: ${JSON.stringify(entity)}`,
);
if (!entity.constructor.prototype._entity || !entity.constructor.prototype._columns) {
logger.error("Can not persist this object %o", entity);
throw Error("Cannot persist this object: it is not an entity.");
@@ -24,32 +28,7 @@ export default class EntityManager<EntityType extends Record<string, any>> {
const { columnsDefinition, entityDefinition } = getEntityDefinition(entity);
const primaryKey: string[] = unwrapPrimarykey(entityDefinition);
primaryKey.forEach(pk => {
if (entity[pk] === undefined) {
const definition = columnsDefinition[pk];
if (!definition) {
throw Error(`There is no definition for primary key ${pk}`);
}
//Create default value
switch (definition.options.generator || definition.type) {
case "uuid":
entity[pk] = uuidv4();
break;
case "timeuuid":
entity[pk] = uuidv1();
break;
case "number":
entity[pk] = 0;
break;
default:
entity[pk] = "";
}
}
});
//Apply the onUpsert
//apply on upsert(generating created_at, updated_at fields)
for (const column in columnsDefinition) {
const definition = columnsDefinition[column];
if (definition.options.onUpsert) {
@@ -57,8 +36,39 @@ export default class EntityManager<EntityType extends Record<string, any>> {
}
}
this.toPersist = this.toPersist.filter(e => e !== entity);
this.toPersist.push(_.cloneDeep(entity));
// generate primary key
const emptyPkFields = primaryKey.filter(
pk => entity[columnsDefinition[pk].nodename] === undefined,
);
emptyPkFields.forEach(pk => {
const definition = columnsDefinition[pk];
if (!definition) {
throw Error(`There is no definition for primary key ${pk}`);
}
//Create default value
switch (definition.options.generator || definition.type) {
case "uuid":
entity[columnsDefinition[pk].nodename] = uuidv4();
break;
case "timeuuid":
entity[columnsDefinition[pk].nodename] = uuidv1();
break;
case "number":
entity[columnsDefinition[pk].nodename] = 0;
break;
default:
entity[columnsDefinition[pk].nodename] = "";
}
});
if (emptyPkFields.length > 0) {
this.toInsert = this.toInsert.filter(e => e !== entity);
this.toInsert.push(_.cloneDeep(entity));
} else {
this.toUpdate = this.toUpdate.filter(e => e !== entity);
this.toUpdate.push(_.cloneDeep(entity));
}
return this;
}
@@ -78,25 +88,38 @@ export default class EntityManager<EntityType extends Record<string, any>> {
}
public async flush(): Promise<this> {
this.toPersist = _.uniqWith(this.toPersist, _.isEqual);
this.toInsert = _.uniqWith(this.toInsert, _.isEqual);
this.toUpdate = _.uniqWith(this.toUpdate, _.isEqual);
this.toRemove = _.uniqWith(this.toRemove, _.isEqual);
localEventBus.publish("database:entities:saved", {
entities: this.toPersist.map(e => _.cloneDeep(e)),
entities: this.toInsert.map(e => _.cloneDeep(e)),
} as DatabaseEntitiesSavedEvent);
localEventBus.publish("database:entities:saved", {
entities: this.toUpdate.map(e => _.cloneDeep(e)),
} as DatabaseEntitiesSavedEvent);
localEventBus.publish("database:entities:saved", {
entities: this.toRemove.map(e => _.cloneDeep(e)),
} as DatabaseEntitiesRemovedEvent);
await this.connector.upsert(this.toPersist);
if (this.toInsert.length > 0) {
await this.connector.upsert(this.toInsert, { action: "INSERT" });
}
if (this.toUpdate.length > 0) {
await this.connector.upsert(this.toUpdate, { action: "UPDATE" });
}
if (this.toRemove.length > 0) {
await this.connector.remove(this.toRemove);
}
return this;
}
public reset(): void {
this.toPersist = [];
this.toInsert = [];
this.toUpdate = [];
this.toRemove = [];
}
}
tdrive/backend/node/src/core/platform/services/database/services/orm/repository/repository.ts
+5 -3
View File
@@ -96,18 +96,20 @@ export default class Repository<EntityType> {
}
async save(entity: EntityType, _context?: ExecutionContext): Promise<void> {
(await this.manager.persist(entity).flush()).reset();
this.manager.persist(entity);
return this.manager.flush().then(manager => manager.reset());
}
async saveAll(entities: EntityType[] = [], _context?: ExecutionContext): Promise<void> {
logger.debug("services.database.repository - Saving entities");
entities.forEach(entity => this.manager.persist(entity));
await (await this.manager.flush()).reset();
return this.manager.flush().then(manager => manager.reset());
}
async remove(entity: EntityType, _context?: ExecutionContext): Promise<void> {
await (await this.manager.remove(entity).flush()).reset();
this.manager.remove(entity);
return this.manager.flush().then(manager => manager.reset());
}
//Avoid using this except when no choice
tdrive/backend/node/src/core/platform/services/database/services/orm/types.ts
+1 -1
View File
@@ -46,7 +46,7 @@ export type ColumnType =
export type EntityTarget<Entity> = ObjectType<Entity>;
// eslint-disable-next-line @typescript-eslint/ban-types
export type ObjectType<T> = { new (): T } | Function;
export type ObjectType<T> = { new (): T };
/** Local Event bus */
tdrive/backend/node/src/core/platform/services/database/services/orm/utils.ts
+5 -8
View File
@@ -6,10 +6,10 @@ export function getEntityDefinition(instance: any): {
entityDefinition: EntityDefinition;
columnsDefinition: { [name: string]: ColumnDefinition };
} {
const entityConfituration = _.cloneDeep(instance.constructor.prototype._entity);
const entityConfiguration = _.cloneDeep(instance.constructor.prototype._entity);
const entityColumns = _.cloneDeep(instance.constructor.prototype._columns);
return {
entityDefinition: entityConfituration,
entityDefinition: entityConfiguration,
columnsDefinition: entityColumns,
};
}
@@ -17,11 +17,10 @@ export function getEntityDefinition(instance: any): {
export function unwrapPrimarykey(entityDefinition: EntityDefinition): string[] {
const initial = [...entityDefinition.options.primaryKey];
const partitionKey = initial.shift();
const primaryKey: string[] = [
return [
...(typeof partitionKey === "string" ? [partitionKey] : partitionKey),
...(initial as string[]),
];
return primaryKey;
}
export function unwrapIndexes(entityDefinition: EntityDefinition): string[] {
@@ -89,8 +88,7 @@ export function fromMongoDbOrderable(orderable: string): string {
return null;
}
const uuid_arr = orderable.split("-");
const timeuuid = [uuid_arr[2], uuid_arr[1], uuid_arr[0], uuid_arr[3], uuid_arr[4]].join("-");
return timeuuid;
return [uuid_arr[2], uuid_arr[1], uuid_arr[0], uuid_arr[3], uuid_arr[4]].join("-");
}
/**
@@ -103,8 +101,7 @@ export function toMongoDbOrderable(timeuuid?: string): string {
return null;
}
const uuid_arr = timeuuid.split("-");
const time_str = [uuid_arr[2], uuid_arr[1], uuid_arr[0], uuid_arr[3], uuid_arr[4]].join("-");
return time_str;
return [uuid_arr[2], uuid_arr[1], uuid_arr[0], uuid_arr[3], uuid_arr[4]].join("-");
}
/**
tdrive/backend/node/src/core/platform/services/storage/connectors/local/service.ts
+5
View File
@@ -4,6 +4,7 @@ import p from "path";
import { rm } from "fs/promises"; // Do not change the import, this is not the same function import { rm } from "fs"
import { StorageConnectorAPI, WriteMetadata } from "../../provider";
import fs from "fs";
import { logger } from "../../../../framework/logger";
export type LocalConfiguration = {
path: string;
@@ -18,6 +19,7 @@ export default class LocalConnectorService implements StorageConnectorAPI {
write(relativePath: string, stream: Readable): Promise<WriteMetadata> {
const path = this.getFullPath(relativePath);
logger.trace(`Writing file ${path}`);
const directory = p.dirname(path);
if (!existsSync(directory)) {
@@ -30,10 +32,12 @@ export default class LocalConnectorService implements StorageConnectorAPI {
const file = createWriteStream(path);
file
.on("error", function (err) {
logger.error(`Error ${err.message} during writing file ${path}`);
reject(err);
})
.on("finish", () => {
const stats = statSync(path);
logger.trace(`File ${path} have been written`);
resolve({
size: stats.size,
});
@@ -43,6 +47,7 @@ export default class LocalConnectorService implements StorageConnectorAPI {
}
async read(path: string): Promise<Readable> {
logger.trace(`Reading file ... ${path}`);
const fullPath = this.getFullPath(path);
if (!fs.existsSync(fullPath)) {
throw new Error(`File doesn't exists ${fullPath}`);
tdrive/backend/node/src/core/platform/services/storage/index.ts
+2 -1
View File
@@ -51,7 +51,8 @@ export default class StorageService extends TdriveService<StorageAPI> implements
} else {
logger.info("Using 'local' connector for storage.");
const defaultHomeDir = this.configuration.get<string>("local.path");
if (defaultHomeDir) this.homeDir = `/${defaultHomeDir}`;
if (defaultHomeDir) this.homeDir = `${defaultHomeDir}`;
logger.trace(`Home directory for the storage: ${this.homeDir}`);
}
logger.info(
`Using 'local' connector for storage${
tdrive/backend/node/src/services/applications-api/index.ts
+6 -5
View File
@@ -5,6 +5,7 @@ import { Prefix, TdriveService } from "../../core/platform/framework";
import WebServerAPI from "../../core/platform/services/webserver/provider";
import Application from "../applications/entities/application";
import web from "./web/index";
import { logger } from "../../core/platform/framework/logger";
@Prefix("/api")
export default class ApplicationsApiService extends TdriveService<undefined> {
@@ -26,7 +27,7 @@ export default class ApplicationsApiService extends TdriveService<undefined> {
if (domain && prefix) {
try {
fastify.all("/" + prefix + "/*", async (req, rep) => {
console.log("Proxying", req.method, req.url, "to", domain);
logger.info("Proxying", req.method, req.url, "to", domain);
try {
const response = await axios.request({
url: domain + req.url,
@@ -54,15 +55,15 @@ export default class ApplicationsApiService extends TdriveService<undefined> {
rep.send(response.data);
} catch (err) {
console.error(`${err}`);
logger.error(`${err}`);
rep.raw.statusCode = 500;
rep.raw.end(JSON.stringify({ error: err.message }));
}
});
console.log("Listening at ", "/" + prefix + "/*");
logger.info("Listening at ", "/" + prefix + "/*");
} catch (e) {
console.log(e);
console.log("Can't listen to ", "/" + prefix + "/*");
logger.error(e);
logger.info("Can't listen to ", "/" + prefix + "/*");
}
}
}
tdrive/backend/node/src/services/documents/web/controllers/documents.ts
-8
View File
@@ -25,8 +25,6 @@ import {
} from "../../types";
import { DriveFileDTO } from "../dto/drive-file-dto";
import { DriveFileDTOBuilder } from "../../services/drive-file-dto-builder";
import { ExecutionContext } from "../../../../core/platform/framework/api/crud-service";
import gr from "../../../global-resolver";
import config from "config";
export class DocumentsController {
@@ -35,12 +33,6 @@ export class DocumentsController {
? config.get("drive.rootAdmins")
: [];
private getCompanyUserRole(companyId: string, userId: string, context?: ExecutionContext) {
return gr.services.companies
.getCompanyUser({ id: companyId }, { id: userId }, context)
.then(a => (a ? a.level : null));
}
/**
* Creates a DriveFile item
*
tdrive/backend/node/src/services/user/entities/company_user.ts
-16
View File
@@ -35,22 +35,6 @@ export default class CompanyUser {
@Column("last_update_day", "number")
lastUpdateDay: number;
/**
* 0: member,
* 1, 2, 3: admin,
*/
@Column("level", "number")
level: number; //Depreciated
@Column("is_externe", "tdrive_boolean")
isExterne: boolean; //Depreciated
@Column("did_connect_today", "tdrive_boolean")
didConnectToday: boolean; //Depreciated
@Column("app_used_today", "json")
appUsedToday: Array<string>; //Depreciated
}
export type CompanyUserPrimaryKey = Partial<Pick<CompanyUser, "group_id" | "user_id">>;
tdrive/backend/node/src/services/user/services/companies.ts
-3
View File
@@ -276,9 +276,6 @@ export class CompanyServiceImpl {
}
entity.role = role;
if (level) {
entity.level = level;
}
entity.applications = applications;
await this.companyUserRepository.save(entity, context);
tdrive/backend/node/src/services/workspaces/entities/workspace_user.ts
-3
View File
@@ -29,9 +29,6 @@ export default class WorkspaceUser {
@Column("last_access", "number")
lastAccess: number;
@Column("is_externe", "tdrive_boolean")
isExternal: boolean; //Depreciated
}
export type WorkspaceUserPrimaryKey = Partial<Pick<WorkspaceUser, "workspaceId" | "userId">>;
tdrive/backend/node/test/e2e/common/entities/mock_entities.ts
+1 -1
View File
@@ -25,7 +25,7 @@ export class DriveFileMockClass {
extension: string;
description: string;
tags: string[];
last_modified: string;
last_modified: number;
access_info: MockAccessInformation;
creator: string;
is_directory: boolean;
tdrive/backend/node/test/e2e/users/users.search.spec.ts
+1 -1
View File
@@ -81,7 +81,7 @@ describe("The /users API", () => {
//Wait for indexation to happen
await new Promise(r => setTimeout(r, 5000));
let resources = await search("ha");
let resources = await search("ha", platform.workspace.company_id);
expect(resources.length).toBe(2);
resources = await search("bob rabiot");
tdrive/backend/node/test/e2e/workspaces/workspace-users.spec.ts
-4
View File
@@ -285,10 +285,6 @@ describe("The /workspace users API", () => {
expect(response.statusCode).toBe(201);
const resource = response.json()["resource"];
checkUserObject(resource);
workspaceUsersCount = await testDbService.getWorkspaceUsersCountFromDb(workspaceId);
expect(workspaceUsersCount).toBe(5);
});
});
tdrive/backend/node/test/unit/core/services/database/services/orm/connectors/postgres/postgres-data-transform.test.ts
+173
View File
@@ -0,0 +1,173 @@
import {
PostgresDataTransformer, TypeMappings
} from "../../../../../../../../../src/core/platform/services/database/services/orm/connectors/postgres/postgres-data-transform";
import { randomInt, randomUUID } from "crypto";
import { ColumnType } from "../../../../../../../../../src/core/platform/services/database/services/orm/types";
import { v4 } from "uuid";
import { DriveScope } from "../../../../../../../../../src/services/documents/entities/drive-file";
describe('The PostgresDataTransformer', () => {
test('toDbString anything to counter, blob and other not implemented types should throw an error', async () => {
//given
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
//then
expect(() => subj.toDbString(randomInt(1000), "blob")).toThrow(Error);
expect(() => subj.toDbString(null, "blob")).toThrow(Error);
});
test.each(Object.keys(TypeMappings))('toDbString for "%s" and null or undefined --> to null', async (type: ColumnType) => {
//given
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
//then
expect(subj.toDbString(null, type)).toBeNull();
expect(subj.toDbString(undefined, type)).toBeNull();
});
test('toDbString for tdrive_int, tdrive_datetime should return the same number', async () => {
//given
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
const number = randomInt(1000);
//then
expect(subj.toDbString(number, "number")).toBe(number);
expect(subj.toDbString(number, "counter")).toBe(number);
expect(subj.toDbString(number, "tdrive_datetime")).toBe(number);
expect(subj.toDbString(String(number), "tdrive_datetime")).toBe(number);
});
test('toDbString for tdrive_int, tdrive_datetime should throw error if its not a number', async () => {
//given
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
//then
expect(() => subj.toDbString("", "number")).toThrow(Error);
expect(() => subj.toDbString("str_" + randomUUID(), "tdrive_datetime")).toThrow(Error);
expect(() => subj.toDbString(true, "tdrive_datetime")).toThrow(Error);
expect(() => subj.toDbString({ idx: 1} , "tdrive_datetime")).toThrow(Error);
});
test('toDbString for uuid and timeuuid should throw exception if value is not string', async () => {
//given
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
//then
expect(() => subj.toDbString({ }, "uuid")).toThrow(Error);
expect(() => subj.toDbString({ }, "timeuuid")).toThrow(Error);
expect(() => subj.toDbString(true, "uuid")).toThrow(Error);
expect(() => subj.toDbString(1 , "timeuuid")).toThrow(Error);
});
test('toDbString for uuid and timeuuid should be transformed to the same string', async () => {
//given
const uuid = v4()
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
//then
expect(subj.toDbString(uuid, "uuid")).toBe(uuid);
expect(subj.toDbString(uuid, "timeuuid")).toBe(uuid);
});
test('toDbString for boolean and to tdrive_boolean should return boolean', async () => {
//given
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
const number = randomInt(1000) + 1;
//then
expect(subj.toDbString(number, "tdrive_boolean")).toBe(true);
expect(subj.toDbString(number, "boolean")).toBe(true);
expect(subj.toDbString(0, "tdrive_boolean")).toBe(false);
expect(subj.toDbString(-number, "boolean")).toBe(true);
expect(subj.toDbString(true, "boolean")).toBe(true);
expect(subj.toDbString(false, "boolean")).toBe(false);
});
test('toDbString for boolean and to tdrive_boolean should throw error for any not boolean', async () => {
//given
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
//then
expect(() => subj.toDbString("true", "tdrive_boolean")).toThrow(Error);
expect(() => subj.toDbString("true", "boolean")).toThrow(Error);
expect(() => subj.toDbString("false", "tdrive_boolean")).toThrow(Error);
expect(() => subj.toDbString("false", "boolean")).toThrow(Error);
expect(() => subj.toDbString({ }, "boolean")).toThrow(Error);
expect(() => subj.toDbString([true], "tdrive_boolean")).toThrow(Error);
});
test('toDbString for string should return boolean the result of toString() method', async () => {
//given
const scope: DriveScope = "personal";
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
//then
expect(subj.toDbString("test_string", "string")).toBe("test_string");
expect(subj.toDbString(scope, "string")).toBe("personal");
});
test('toDbString for json should return stringified object', async () => {
//given
const file = { id: "123", amount: 10};
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
//then
expect(subj.toDbString(file, "json")).toBe('{"id":"123","amount":10}');
expect(subj.toDbString(10, "json")).toBe("10");
});
test('toDbString for encoded_string and empty secret should return plain string', async () => {
//given
const str = randomUUID();
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
//then
expect(subj.toDbString(str, "encoded_string")).toBe(str);
});
test('toDbString for encoded_json and empty secret should return plain string', async () => {
//then
//given
const file = { id: "123", amount: 10};
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
//then
expect(subj.toDbString(file, "encoded_json")).toBe('{"id":"123","amount":10}');
expect(subj.toDbString(10, "encoded_json")).toBe("10");
});
test('toDbString for encoded_json should return decodable string different from the source one', async () => {
//then
//given
const file = { id: "123", amount: 10};
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "hey it's secret" });
//then
const encodedJson = subj.toDbString(file, "encoded_json");
expect(encodedJson).not.toBe('{"id":"123","amount":10}');
const encodedNumber = subj.toDbString(10, "encoded_json");
expect(encodedNumber).not.toBe("10");
expect(subj.fromDbString(encodedJson, "encoded_json")).toEqual(file);
expect(subj.fromDbString(encodedNumber, "encoded_json")).toEqual(10);
});
test('toDbString for encoded_string should return decodable string different from the source one', async () => {
//then
//given
const str = randomUUID();
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "hey it's secret" });
//then
const encodedStr = subj.toDbString(str, "encoded_json");
expect(encodedStr).not.toBe('{"id":"123","amount":10}');
expect(subj.fromDbString(encodedStr, "encoded_json")).toEqual(str);
});
});
tdrive/backend/node/test/unit/core/services/database/services/orm/connectors/postgres/postgres-query-builder.test.ts
+149
View File
@@ -0,0 +1,149 @@
import {
PostgresQueryBuilder
} from "../../../../../../../../../src/core/platform/services/database/services/orm/connectors/postgres/postgres-query-builder";
import { expect, jest } from "@jest/globals";
import { randomInt, randomUUID } from "crypto";
import { newTestDbEntity, normalizeWhitespace, TestDbEntity } from "./utils";
import {
comparisonType
} from "../../../../../../../../../src/core/platform/services/database/services/orm/repository/repository";
describe('The PostgresQueryBuilder', () => {
const subj: PostgresQueryBuilder = new PostgresQueryBuilder("");
beforeEach(async () => {
});
afterEach(() => {
jest.clearAllMocks();
});
test('buildSelect query with filter', async () => {
//given
const filter = new TestDbEntity({id: randomUUID(), company_id: randomUUID() })
//when
const query = subj.buildSelect(TestDbEntity, filter as { [key: string]: any }, null);
//then
expect(normalizeWhitespace(query[0] as string)).toBe(`SELECT * FROM "test_table" WHERE id = $1 AND company_id = $2 ORDER BY id DESC LIMIT 100 OFFSET 0`);
expect(query[1]).toEqual([filter.id, filter.company_id]);
});
test('buildSelect query with array in filters', async () => {
//given
const filter = {id: randomUUID(), company_id: [randomUUID(), randomUUID()] };
//when
const query = subj.buildSelect(TestDbEntity, filter as { [key: string]: any }, null);
//then
expect(normalizeWhitespace(query[0] as string)).toBe(`SELECT * FROM "test_table" WHERE id = $1 AND company_id IN ($2,$3) ORDER BY id DESC LIMIT 100 OFFSET 0`);
expect(query[1]).toEqual([filter.id, filter.company_id[0], filter.company_id[1]]);
});
test('buildSelect query "lt" options', async () => {
//given
const options = { $lt: [["added", randomInt(10)] as comparisonType, ["added", randomInt(10)] as comparisonType]};
//when
const query = subj.buildSelect(TestDbEntity, null, options);
//then
expect(normalizeWhitespace(query[0] as string)).toBe(`SELECT * FROM "test_table" WHERE added < $1 AND added < $2 ORDER BY id DESC LIMIT 100 OFFSET 0`);
expect(query[1]).toEqual(options.$lt.map(e=> e[1]));
});
test('buildSelect query "gt" options', async () => {
//given
const options = { $gt: [["added", randomInt(10)] as comparisonType, ["added", randomInt(10)] as comparisonType]};
//when
const query = subj.buildSelect(TestDbEntity, null, options);
//then
expect(normalizeWhitespace(query[0] as string)).toBe(`SELECT * FROM "test_table" WHERE added > $1 AND added > $2 ORDER BY id DESC LIMIT 100 OFFSET 0`);
expect(query[1]).toEqual(options.$gt.map(e=> e[1]));
});
test('buildSelect query "in" options', async () => {
//given
const options = { $in: [["added", [randomInt(10), randomInt(10)]] as comparisonType,]};
//when
const query = subj.buildSelect(TestDbEntity, null, options);
//then
expect(normalizeWhitespace(query[0] as string)).toBe(`SELECT * FROM "test_table" WHERE added IN ($1,$2) ORDER BY id DESC LIMIT 100 OFFSET 0`);
expect(query[1]).toEqual(options.$in[0][1]);
});
test('buildSelect query "like" options', async () => {
//given
const options = { $like: [["id", randomUUID()] as comparisonType, ["company_id", randomUUID()] as comparisonType]};
//when
const query = subj.buildSelect(TestDbEntity, null, options);
//then
expect(normalizeWhitespace(query[0] as string)).toBe(`SELECT * FROM "test_table" WHERE id LIKE $1 AND company_id LIKE $2 ORDER BY id DESC LIMIT 100 OFFSET 0`);
expect(query[1]).toEqual(options.$like.map(e=> `%${e[1]}%`));
});
test('buildDelete query', async () => {
//given
const entity = newTestDbEntity();
//when
const query = subj.buildDelete(entity);
//then
expect(normalizeWhitespace(query[0] as string)).toBe(`DELETE FROM "test_table" WHERE company_id = $1 AND id = $2`);
expect(query[1]).toEqual([entity.company_id, entity.id]);
});
test('buildInsert', async () => {
//given
const entity = newTestDbEntity();
//when
const query = subj.buildInsert(entity);
//then
expect(normalizeWhitespace(query[0] as string)).toBe(`INSERT INTO "test_table" (company_id, id, parent_id, is_in_trash, tags, added) VALUES ($1, $2, $3, $4, $5, $6)`);
assertInsertQueryParams(entity, query[1] as any[])
});
test('buildUpdate', async () => {
//given
const entity = newTestDbEntity();
//when
const query = subj.buildUpdate(entity);
//then
expect(normalizeWhitespace(query[0] as string)).toBe(`UPDATE "test_table" SET parent_id = $1, is_in_trash = $2, tags = $3, added = $4 WHERE company_id = $5 AND id = $6`);
assertUpdateQueryParams(entity, query[1] as any[])
});
const assertInsertQueryParams = (actual: TestDbEntity, expected: any[]) => {
expect(expected[0]).toBe(actual.company_id);
expect(expected[1]).toBe(actual.id);
expect(expected[2]).toBe(actual.parent_id);
expect(expected[3]).toBe(actual.is_in_trash);
expect(expected[4]).toBe( JSON.stringify(actual.tags));
expect(expected[5]).toBe(actual.added);
}
const assertUpdateQueryParams = (actual: TestDbEntity, expected: string[]) => {
expect(expected[0]).toBe(actual.parent_id);
expect(expected[1]).toBe(actual.is_in_trash);
expect(expected[2]).toBe( JSON.stringify(actual.tags));
expect(expected[3]).toBe(actual.added);
expect(expected[4]).toBe(actual.company_id);
expect(expected[5]).toBe(actual.id);
}
});
tdrive/backend/node/test/unit/core/services/database/services/orm/connectors/postgres/postgres.debug.test.ts
+67
View File
@@ -0,0 +1,67 @@
import 'reflect-metadata';
import { DriveFile } from "../../../../../../../../../src/services/documents/entities/drive-file";
import { describe, it } from '@jest/globals';
import {
PostgresConnectionOptions,
PostgresConnector,
} from '../../../../../../../../../src/core/platform/services/database/services/orm/connectors/postgres/postgres';
import { getEntityDefinition } from '../../../../../../../../../src/core/platform/services/database/services/orm/utils';
import { randomUUID } from "crypto";
import { now } from "moment";
//THIS IS NOT TEST, IT'S JUST RUNNER TO TEST POSTGRESQL API
describe.skip('The Postgres Connector module', () => {
const subj: PostgresConnector = new PostgresConnector('postgres', {
user: "tdrive_user",
password: "tdrive_secret",
database: "tdrive",
port: 5432,
host: "localhost",
statement_timeout: 10000,
query_timeout: 10000
} as PostgresConnectionOptions, '');
it('createTable test ALTER TABLE query', async () => {
let driveFile = new DriveFile();
const definition = getEntityDefinition(driveFile);
await subj.connect();
//when
await subj.createTable(definition.entityDefinition, definition.columnsDefinition);
driveFile.id = randomUUID();
driveFile.name = "My Test File";
driveFile.company_id = randomUUID();
driveFile.access_info = {
entities: [],
public: {
token: randomUUID(),
password: randomUUID(),
expiration: now(),
level: "read",
}
}
driveFile.is_directory = false;
driveFile.is_in_trash = true;
driveFile.last_modified = now();
driveFile.tags = ["mytag", "ehey"]
await subj.upsert([driveFile], {action: "INSERT"} )
let files = await subj.find(DriveFile, null, null);
expect(files.getEntities().length).toBeGreaterThan(0);
await subj.remove(files.getEntities());
files = await subj.find(DriveFile, null, null);
expect(files.getEntities().length).toEqual(0);
await subj.drop();
//then
}, 3000000);
});
tdrive/backend/node/test/unit/core/services/database/services/orm/connectors/postgres/postgres.test.ts
+63
View File
@@ -0,0 +1,63 @@
import 'reflect-metadata';
import { describe, expect, jest } from '@jest/globals';
import {
PostgresConnectionOptions,
PostgresConnector, TableRowInfo
} from "../../../../../../../../../src/core/platform/services/database/services/orm/connectors/postgres/postgres";
import { getEntityDefinition } from '../../../../../../../../../src/core/platform/services/database/services/orm/utils';
import { TestDbEntity, normalizeWhitespace, newTestDbEntity } from "./utils";
describe('The Postgres Connector module', () => {
const NUMBER_OF_HEALTHCHECK_CALLS = 1;
const subj: PostgresConnector = new PostgresConnector('postgres', {} as PostgresConnectionOptions, '');
let dbQuerySpy;
beforeEach(async () => {
dbQuerySpy = jest.spyOn((subj as any).client, 'query');
//healthcheck mock
dbQuerySpy
.mockReturnValueOnce({rows: [{now: 1}], rowCount: 1});
jest.spyOn((subj as any).client, 'connect').mockImplementation(jest.fn);
await subj.connect();
});
afterEach(() => {
jest.clearAllMocks();
});
test('createTable generates table structure queries', async () => {
// given
const definition = getEntityDefinition(new TestDbEntity());
dbQuerySpy
//mock create table response
.mockReturnValueOnce({ rows: [], rowCount: 1})
//mock query for existing columns
.mockReturnValueOnce({ rows: [{ column_name: "parent_id"} as TableRowInfo, { column_name: "company_id"}], rowCount: 1 })
//mock alter table response
.mockReturnValueOnce({ rows: [], rowCount: 1 })
//mock alter table add primary key query response
.mockReturnValueOnce({ rows: [], rowCount: 1 })
//mock alter table create index query response
.mockReturnValueOnce({ rows: [], rowCount: 1 })
//mock alter table create index query response
.mockReturnValueOnce({ rows: [], rowCount: 1 })
//when
await subj.createTable(definition.entityDefinition, definition.columnsDefinition);
//then
expect(dbQuerySpy).toHaveBeenCalledTimes(6 + NUMBER_OF_HEALTHCHECK_CALLS);
expect(normalizeWhitespace(dbQuerySpy.mock.calls[1][0])).toBe(normalizeWhitespace(`CREATE TABLE IF NOT EXISTS "test_table" ( company_id UUID, id UUID, parent_id UUID, is_in_trash BOOLEAN, tags TEXT, added BIGINT );`))
expect(normalizeWhitespace(dbQuerySpy.mock.calls[2][0])).toBe(normalizeWhitespace("SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_name = $1"));
expect(dbQuerySpy.mock.calls[2][1]).toStrictEqual(["test_table"])
expect(normalizeWhitespace(dbQuerySpy.mock.calls[3][0])).toBe(`ALTER TABLE "test_table" ADD COLUMN id UUID, ADD COLUMN is_in_trash BOOLEAN, ADD COLUMN tags TEXT, ADD COLUMN added BIGINT`)
expect(normalizeWhitespace(dbQuerySpy.mock.calls[4][0])).toBe(`ALTER TABLE "test_table" ADD PRIMARY KEY ( company_id, id);`)
expect(normalizeWhitespace(dbQuerySpy.mock.calls[5][0])).toBe(`CREATE INDEX IF NOT EXISTS index_test_table_company_id_parent_id ON "test_table" ((company_id), parent_id)`)
expect(normalizeWhitespace(dbQuerySpy.mock.calls[6][0])).toBe(`CREATE INDEX IF NOT EXISTS index_test_table_company_id_is_in_trash ON "test_table" ((company_id), is_in_trash)`)
});
});
tdrive/backend/node/test/unit/core/services/database/services/orm/connectors/postgres/utils.ts
+61
View File
@@ -0,0 +1,61 @@
import { Column, Entity } from "../../../../../../../../../src/core/platform/services/database/services/orm/decorators";
import { Type } from "class-transformer";
import { randomInt, randomUUID } from "crypto";
export const normalizeWhitespace = (query: string) => query.replace(/\s+/g, ' ').trim();
@Entity("test_table", {
globalIndexes: [
["company_id", "parent_id"],
["company_id", "is_in_trash"],
],
primaryKey: [["company_id"], "id"],
type: "test_table",
})
// @ts-ignore
export class TestDbEntity {
@Type(() => String)
@Column("company_id", "uuid")
// @ts-ignore
company_id: string;
@Type(() => String)
@Column("id", "uuid", { generator: "uuid" })
// @ts-ignore
id: string;
@Type(() => String)
@Column("parent_id", "uuid")
// @ts-ignore
parent_id: string;
@Type(() => Boolean)
@Column("is_in_trash", "boolean")
// @ts-ignore
is_in_trash: boolean;
@Column("tags", "encoded_json")
// @ts-ignore
tags: string[];
@Type(() => Number)
@Column("added", "number")
// @ts-ignore
added: number;
public constructor(init?:Partial<TestDbEntity>) {
Object.assign(this, init);
}
}
export const newTestDbEntity = () => {
return new TestDbEntity({
company_id: randomUUID(),
id: randomUUID(),
parent_id: randomUUID(),
is_in_trash: true,
tags: [randomUUID(), randomUUID()],
added: randomInt(1, 1000)
})
}
tdrive/backend/node/test/unit/core/services/database/services/orm/manager.test.ts
+95
View File
@@ -0,0 +1,95 @@
import 'reflect-metadata';
import { describe, expect, it, jest } from '@jest/globals';
import EntityManager from "../../../../../../../src/core/platform/services/database/services/orm/manager";
import { Connector } from "../../../../../../../src/core/platform/services/database/services/orm/connectors";
import { randomUUID } from "crypto";
import { TestDbEntity } from "./connectors/postgres/utils";
import WorkspaceUser, { getInstance } from "../../../../../../../src/services/workspaces/entities/workspace_user";
describe('EntityManager', () => {
const subj: EntityManager<TestDbEntity> = new EntityManager<TestDbEntity>({ } as Connector);
beforeEach(async () => {
});
afterEach(() => {
jest.clearAllMocks();
subj.reset();
});
test ("persist should store entity to insert if all fields for pk is empty", () => {
//when
subj.persist(new TestDbEntity());
//then
expect((subj as any).toInsert.length).toEqual(1);
expect((subj as any).toUpdate.length).toEqual(0);
expect((subj as any).toRemove.length).toEqual(0);
expect((subj as any).toInsert[0].id).toBeDefined();
expect((subj as any).toInsert[0].company_id).toBeDefined();
});
test ("persist should store entity to insert if id is set", () => {
//when
subj.persist(new TestDbEntity({id: randomUUID()}));
//then
expect((subj as any).toInsert.length).toEqual(1);
expect((subj as any).toUpdate.length).toEqual(0);
expect((subj as any).toRemove.length).toEqual(0);
expect((subj as any).toInsert[0].id).toBeDefined();
expect((subj as any).toInsert[0].company_id).toBeDefined();
});
test ("persist should store entity to insert if company_id is set", () => {
//when
subj.persist(new TestDbEntity({company_id: randomUUID()}));
//then
expect((subj as any).toInsert.length).toEqual(1);
expect((subj as any).toUpdate.length).toEqual(0);
expect((subj as any).toRemove.length).toEqual(0);
expect((subj as any).toInsert[0].id).toBeDefined();
expect((subj as any).toInsert[0].company_id).toBeDefined();
});
test ("persist should store entity to update if all pk fields are set", () => {
//when
let entity = new TestDbEntity({id: randomUUID(), company_id: randomUUID()});
subj.persist(entity);
//then
expect((subj as any).toUpdate.length).toEqual(1);
expect((subj as any).toInsert.length).toEqual(0);
expect((subj as any).toRemove.length).toEqual(0);
expect((subj as any).toUpdate[0]).toEqual(entity)
});
test ("persist should store entity to update if all pk fields are set and column name is different from field name", () => {
//when
let entity = getInstance({id: randomUUID(), workspaceId: randomUUID(), userId: randomUUID()});
subj.persist(entity);
//then
expect((subj as any).toUpdate.length).toEqual(1);
expect((subj as any).toInsert.length).toEqual(0);
expect((subj as any).toRemove.length).toEqual(0);
expect((subj as any).toUpdate[0]).toEqual(entity)
});
test ("persist should store entity to insert if not all pk fields are set and column name is different from field name", () => {
//when
let entity = getInstance({id: randomUUID(), workspaceId: randomUUID()});
subj.persist(entity);
//then
expect((subj as any).toUpdate.length).toEqual(0);
expect((subj as any).toInsert.length).toEqual(1);
expect((subj as any).toRemove.length).toEqual(0);
expect((subj as any).toInsert[0]).toEqual(entity);
expect(entity.userId).toBeDefined();
});
});
tdrive/docker-compose.dev.tests.opensearch.yml
+16 -9
View File
@@ -29,13 +29,15 @@ services:
# networks:
# - opensearch-net # All of the containers will join the same Docker bridge network
mongo:
container_name: mongo
image: mongo
volumes:
- ./docker-data/mongo:/data/db
postgres:
image: postgres
restart: always
environment:
POSTGRES_DB: tdrive
POSTGRES_PASSWORD: tdrive_secret
POSTGRES_USER: tdrive_user
ports:
- 27017:27017
- "5432:5432"
node:
image: tdrive/tdrive-node:test
@@ -48,7 +50,12 @@ services:
environment:
- LOG_LEVEL=error
- NODE_ENV=test
- DB_DRIVER
- DB_DRIVER=postgres
- DB_POSTGRES_DBNAME=tdrive
- DB_POSTGRES_HOST=postgres
- DB_POSTGRES_PASSWORD=tdrive_secret
- DB_POSTGRES_PORT=5432
- DB_POSTGRES_USERNAME=tdrive_user
- PUBSUB_URLS=amqp://guest:guest@rabbitmq:5672
- SEARCH_OS_FLUSHINTERVAL=1
- SEARCH_OS_ENDPOINT=https://opensearch-node1:9200
@@ -56,8 +63,8 @@ services:
- SEARCH_OS_USE_AUTH=true
- SEARCH_OS_USERNAME=admin
depends_on:
- mongo
- postgres
- opensearch-node1
links:
- mongo
- postgres
- opensearch-node1
tdrive/docker-compose.tests.postgresql.yml
+13
View File
@@ -0,0 +1,13 @@
version: "3.4"
services:
postgres:
image: postgres
restart: always
environment:
POSTGRES_DB: tdrive
POSTGRES_PASSWORD: tdrive_secret
POSTGRES_USER: tdrive_user
ports:
- "5432:5432"