🌟 Support for PostgreSQL (#306)

tdrive/backend/node/src/core/platform/services/database/services/connector-factory.ts

@@ -5,6 +5,7 @@ import {
   CassandraConnector,
 } from "./orm/connectors/cassandra/cassandra";
 import { MongoConnectionOptions, MongoConnector } from "./orm/connectors/mongodb/mongodb";
+import { PostgresConnectionOptions, PostgresConnector } from "./orm/connectors/postgres/postgres";
 
 export class ConnectorFactory {
   public create(type: DatabaseType, options: ConnectionOptions, secret: string): Connector {
@@ -13,6 +14,8 @@ export class ConnectorFactory {
         return new CassandraConnector(type, options as CassandraConnectionOptions, secret);
       case "mongodb":
         return new MongoConnector(type, options as MongoConnectionOptions, secret);
+      case "postgres":
+        return new PostgresConnector(type, options as PostgresConnectionOptions, secret);
       default:
         throw new Error(`${type} is not supported`);
     }

tdrive/backend/node/src/core/platform/services/database/services/index.ts

@@ -7,6 +7,7 @@ import { CassandraConnectionOptions } from "./orm/connectors/cassandra/cassandra
 import { MongoConnectionOptions } from "./orm/connectors/mongodb/mongodb";
 import { EntityTarget } from "./orm/types";
 import { RepositoryManager } from "./orm/repository/manager";
+import { PostgresConnectionOptions } from "./orm/connectors/postgres/postgres";
 
 export default class DatabaseService implements DatabaseServiceAPI {
   version = "1";
@@ -40,6 +41,9 @@ export default class DatabaseService implements DatabaseServiceAPI {
   }
 }
 
-export declare type ConnectionOptions = MongoConnectionOptions | CassandraConnectionOptions;
+export declare type ConnectionOptions =
+  | MongoConnectionOptions
+  | CassandraConnectionOptions
+  | PostgresConnectionOptions;
 
-export declare type DatabaseType = "mongodb" | "cassandra";
+export declare type DatabaseType = "mongodb" | "cassandra" | "postgres";

tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/abstract-connector.ts

@@ -1,27 +1,23 @@
 /* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/explicit-module-boundary-types */
-import { Connector } from ".";
+import { Connector, UpsertOptions } from ".";
 import { ConnectionOptions, DatabaseType } from "../..";
 import { FindOptions } from "../repository/repository";
 import { ColumnDefinition, EntityDefinition } from "../types";
 import { ListResult } from "../../../../../framework/api/crud-service";
 
-export abstract class AbstractConnector<T extends ConnectionOptions, DatabaseClient>
-  implements Connector
-{
+export abstract class AbstractConnector<T extends ConnectionOptions> implements Connector {
   constructor(protected type: DatabaseType, protected options: T, protected secret: string) {}
 
   abstract connect(): Promise<this>;
 
   abstract drop(): Promise<this>;
 
-  abstract getClient(): DatabaseClient;
-
   abstract createTable(
     entity: EntityDefinition,
     columns: { [name: string]: ColumnDefinition },
   ): Promise<boolean>;
 
-  abstract upsert(entities: any[]): Promise<boolean[]>;
+  abstract upsert(entities: any[], _options: UpsertOptions): Promise<boolean[]>;
 
   abstract remove(entities: any[]): Promise<boolean[]>;
 
@@ -38,8 +34,4 @@ export abstract class AbstractConnector<T extends ConnectionOptions, DatabaseCli
   getType(): DatabaseType {
     return this.type;
   }
-
-  getSecret(): string {
-    return this.secret;
-  }
 }

tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/cassandra/cassandra.ts

@@ -1,4 +1,4 @@
-/* eslint-disable @typescript-eslint/no-explicit-any */
+/* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/explicit-module-boundary-types */
 import cassandra, { types } from "cassandra-driver";
 import { md5 } from "../../../../../../../crypto";
 import { defer, Subject, throwError, timer } from "rxjs";
@@ -48,10 +48,7 @@ export interface CassandraConnectionOptions {
   delay?: number;
 }
 
-export class CassandraConnector extends AbstractConnector<
-  CassandraConnectionOptions,
-  cassandra.Client
-> {
+export class CassandraConnector extends AbstractConnector<CassandraConnectionOptions> {
   private client: cassandra.Client;
   private keyspaceExists = false;
 

tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts

@@ -5,11 +5,14 @@ import { MongoConnectionOptions } from "./mongodb/mongodb";
 import { ColumnDefinition, EntityDefinition } from "../types";
 import { FindOptions } from "../repository/repository";
 import { ListResult } from "../../../../../framework/api/crud-service";
+import { PostgresConnectionOptions } from "./postgres/postgres";
 
 export * from "./mongodb/mongodb";
 export * from "./cassandra/cassandra";
 
-export type UpsertOptions = any;
+export type UpsertOptions = {
+  action?: "INSERT" | "UPDATE";
+};
 
 export type RemoveOptions = any;
 
@@ -41,7 +44,7 @@ export interface Connector extends Initializable {
    * Upsert
    * returns true if the object was created/updated, false otherwise
    */
-  upsert(entities: any[]): Promise<boolean[]>;
+  upsert(entities: any[], _options: UpsertOptions): Promise<boolean[]>;
 
   /**
    * Remove
@@ -60,4 +63,7 @@ export interface Connector extends Initializable {
   ): Promise<ListResult<EntityType>>;
 }
 
-export declare type ConnectionOptions = MongoConnectionOptions | CassandraConnectionOptions;
+export declare type ConnectionOptions =
+  | MongoConnectionOptions
+  | CassandraConnectionOptions
+  | PostgresConnectionOptions;

tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts

@@ -9,15 +9,13 @@ import { buildSelectQuery } from "./query-builder";
 import { transformValueFromDbString, transformValueToDbString } from "./typeTransforms";
 import { logger } from "../../../../../../framework";
 
-export { MongoPagination } from "./pagination";
-
 export interface MongoConnectionOptions {
   // TODO: More options
   uri: string;
   database: string;
 }
 
-export class MongoConnector extends AbstractConnector<MongoConnectionOptions, mongo.MongoClient> {
+export class MongoConnector extends AbstractConnector<MongoConnectionOptions> {
   private client: mongo.MongoClient;
 
   async init(): Promise<this> {
@@ -75,6 +73,7 @@ export class MongoConnector extends AbstractConnector<MongoConnectionOptions, mo
     return true;
   }
   async upsert(entities: any[], _options: UpsertOptions = {}): Promise<boolean[]> {
+    logger.trace(`services.database.orm.mongodb.upsert - entities count: ${entities.length}`);
     return new Promise(async resolve => {
       const promises: Promise<mongo.UpdateResult>[] = [];
 
@@ -83,6 +82,11 @@ export class MongoConnector extends AbstractConnector<MongoConnectionOptions, mo
       entities.forEach(entity => {
         const { columnsDefinition, entityDefinition } = getEntityDefinition(entity);
         const primaryKey = unwrapPrimarykey(entityDefinition);
+        logger.trace(
+          `services.database.orm.mongodb.upsert[${_options.action}] - Entity{${
+            entityDefinition.name
+          }: ${JSON.stringify(entity)}`,
+        );
 
         //Set updated content
         const set: any = {};

tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres-data-transform.ts

@@ -0,0 +1,156 @@
+import { ColumnType } from "../../../orm/types";
+import { isBoolean, isInteger, isNull, isString, isUndefined } from "lodash";
+import { decrypt, encrypt } from "../../../../../../../crypto";
+import { logger } from "../../../../../../framework";
+
+export type TransformOptions = {
+  secret?: string;
+};
+
+export const TypeMappings = {
+  encoded_string: "TEXT",
+  encoded_json: "TEXT",
+  json: "TEXT",
+  string: "TEXT",
+  number: "BIGINT",
+  timeuuid: "UUID",
+  uuid: "UUID",
+  boolean: "BOOLEAN",
+  counter: "BIGINT",
+
+  // backward compatibility
+  tdrive_boolean: "BOOLEAN",
+  tdrive_datetime: "BIGINT", //Deprecated
+};
+
+/**
+ * Possible data transformations:
+ *   string --> everything
+ *   encoded_string --> everything
+ *   json --> everything
+ *   encoded_json --> everything
+ *
+ *   number --> number, string
+ *   tdrive_int --> number, string
+ *   tdrive_datetime --> number. string
+ *
+ *   timeuuid --> string
+ *   uuid --> string
+ *
+ *   boolean --> boolean, number
+ *   tdrive_boolean --> boolean, number
+ *
+ *   tdrive_datetime --> number
+ *
+ *   null and undefined --> null
+ */
+export class PostgresDataTransformer {
+  constructor(readonly options: TransformOptions) {}
+
+  fromDbString(v: any, type: ColumnType): any {
+    if (isNull(v) || isUndefined(v)) {
+      return null;
+    }
+
+    this.checkType(type);
+
+    if (v !== null && (type === "encoded_string" || type === "encoded_json")) {
+      let decryptedValue: string;
+
+      if (typeof v === "string" && v.trim() === "") {
+        return v;
+      }
+
+      try {
+        decryptedValue = decrypt(v, this.options.secret).data;
+      } catch (err) {
+        logger.debug(`Can not decrypt data (${err.message}) %o of type ${type}`, v);
+
+        decryptedValue = v;
+      }
+
+      if (type === "encoded_json") {
+        try {
+          decryptedValue = JSON.parse(decryptedValue);
+        } catch (err) {
+          logger.debug(
+            { err },
+            `Can not parse JSON from decrypted data %o of type ${type}`,
+            decryptedValue,
+          );
+          decryptedValue = null;
+        }
+      }
+
+      return decryptedValue;
+    }
+
+    if (type === "number" || type === "tdrive_datetime" || type === "counter") {
+      const number = parseInt(v, 10);
+      if (isNaN(number)) throw new Error(`Can't parse ${v} to int`);
+      return number;
+    }
+
+    if (type === "json") {
+      return JSON.parse(v);
+    }
+
+    return v;
+  }
+
+  toDbString(v: any, type: ColumnType): any {
+    this.checkType(type);
+
+    if (isNull(v) || isUndefined(v)) {
+      return null;
+    }
+
+    if (type === "number" || type === "tdrive_datetime" || type === "counter") {
+      const number = parseInt(v, 10);
+      if (isNaN(number)) throw new Error(`Can't parse ${v} to int`);
+      return number;
+    }
+
+    if (type === "uuid" || type === "timeuuid") {
+      if (!isString(v))
+        throw new Error(`uuid or timeuuid could be only strings, and ${v} is not string`);
+      return v;
+    }
+
+    if (type === "boolean" || type === "tdrive_boolean") {
+      //Security to avoid string with "false" in it
+      if (!isInteger(v) && !isBoolean(v)) {
+        throw new Error(`'${v}' is not a ${type}`);
+      }
+      return !!v;
+    }
+
+    if (type === "string") {
+      if (isString(v)) return v;
+      else throw "We can't store in string fields only strings";
+    }
+
+    if (type === "json") {
+      return JSON.stringify(v);
+    }
+
+    if (type === "encoded_string" || type === "encoded_json") {
+      if (type === "encoded_string" && !isString(v))
+        throw "We can't store in string fields only strings";
+
+      if (type === "encoded_json") {
+        v = JSON.stringify(v);
+      }
+      const encrypted = encrypt(v, this.options.secret);
+      return `${(encrypted.data || "").toString().replace(/'/gm, "''")}`;
+    }
+
+    return v.toString();
+  }
+
+  private checkType(type: ColumnType) {
+    if (!Object.keys(TypeMappings).includes(type)) {
+      throw new Error(`Support of ${type} not implemented yet for PostgreSQL`);
+    }
+  }
+}

tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres-query-builder.ts

@@ -0,0 +1,185 @@
+import { comparisonType, FindOptions } from "../../repository/repository";
+import { ObjectType } from "../../types";
+import { getEntityDefinition, unwrapPrimarykey } from "../../utils";
+import { PostgresDataTransformer } from "./postgres-data-transform";
+import _ from "lodash";
+
+export type Query = [string, any[]];
+
+export class PostgresQueryBuilder {
+  private dataTransformer: PostgresDataTransformer;
+
+  constructor(private secret: string) {
+    this.dataTransformer = new PostgresDataTransformer({ secret: this.secret });
+  }
+
+  buildSelect<Entity>(
+    entityType: ObjectType<Entity>,
+    filters: Record<string, unknown>,
+    options: FindOptions,
+  ): Query {
+    const instance = new (entityType as any)();
+    const { columnsDefinition, entityDefinition } = getEntityDefinition(instance);
+
+    const query = (whereClause: string, orderByClause: string, limit: number, offset: number) => {
+      return `SELECT * FROM "${entityDefinition.name}" 
+            ${whereClause?.length ? "WHERE " + whereClause : ""} 
+            ${orderByClause?.length ? "ORDER BY " + orderByClause : ""}
+            LIMIT ${limit} OFFSET ${offset}`;
+    };
+
+    // === EQUAL or IN operator ===
+    let idx = 1;
+    const values = [];
+    let whereClause = "";
+    if (filters) {
+      Object.keys(filters)
+        .filter(key => !_.isUndefined(filters[key]))
+        .forEach(key => {
+          const filter = filters[key];
+          if (Array.isArray(filter)) {
+            if (filter.length) {
+              const inClause: string[] = filter.map(
+                value => `${this.dataTransformer.toDbString(value, columnsDefinition[key].type)}`,
+              );
+
+              whereClause += `${key} IN ($${inClause.map(() => idx++).join(",$")}) AND `;
+              values.push(...inClause);
+            }
+          } else {
+            const value = `${this.dataTransformer.toDbString(filter, columnsDefinition[key].type)}`;
+            whereClause += `${key} = $${idx++} AND `;
+            values.push(value);
+          }
+        });
+    }
+
+    if (options) {
+      // ==== Comparison operators ===
+      const appendComparison = (predicates: comparisonType[], operator: string) => {
+        if (predicates) {
+          predicates.forEach(element => {
+            whereClause += `${element[0]} ${operator} $${idx++} AND `;
+            values.push(
+              this.dataTransformer.toDbString(element[1], columnsDefinition[element[0]].type),
+            );
+          });
+        }
+      };
+
+      appendComparison(options.$lt, "<");
+      appendComparison(options.$lte, "<=");
+      appendComparison(options.$gt, ">");
+      appendComparison(options.$gte, ">=");
+
+      // === IN ===
+      options.$in?.forEach(e => {
+        whereClause += `${e[0]} IN ($${e[1].map(() => idx++).join(",$")}) AND `;
+        values.push(...e[1]);
+      });
+
+      // === LIKE ====
+      options.$like?.forEach(e => {
+        whereClause += `${e[0]} LIKE $${idx++} AND `;
+        values.push(`%${e[1]}%`);
+      });
+    }
+
+    if (whereClause && whereClause.endsWith("AND ")) whereClause = whereClause.slice(0, -4);
+
+    // ==== ORDER BY =====
+    const orderByClause = `${entityDefinition.options.primaryKey
+      .slice(1)
+      .map(
+        (key: string) =>
+          `${key} ${(columnsDefinition[key].options.order || "ASC") === "ASC" ? "DESC" : "ASC"}`,
+      )}`;
+
+    // ==== PAGING =====
+    let limit = 100;
+    let offset = 0;
+    if (options?.pagination) {
+      if (options.pagination.limitStr) {
+        limit = Number.parseInt(options.pagination.limitStr);
+      }
+      if (options.pagination.page_token) {
+        offset = (Number.parseInt(options.pagination.page_token) - 1) * limit;
+      }
+    }
+    return [query(whereClause, orderByClause, limit, offset), values];
+  }
+
+  buildDelete<Entity>(entity: Entity): Query {
+    const { columnsDefinition, entityDefinition } = getEntityDefinition(entity);
+    const primaryKey = unwrapPrimarykey(entityDefinition);
+
+    const toValueKeyDBStringPair = (key: string) => {
+      return [
+        key,
+        this.dataTransformer.toDbString(
+          entity[columnsDefinition[key].nodename],
+          columnsDefinition[key].type,
+        ),
+      ];
+    };
+
+    const where = primaryKey.map(key => toValueKeyDBStringPair(key));
+    const query = `DELETE FROM "${entityDefinition.name}" 
+                WHERE ${where.map((e, idx) => `${e[0]} = $${idx + 1}`).join(" AND ")}`;
+
+    return [query, where.map(f => f[1])];
+  }
+
+  buildInsert<Entity>(entity: Entity): Query {
+    const { columnsDefinition, entityDefinition } = getEntityDefinition(entity);
+    const toValueKeyDBStringPair = (key: string) => {
+      return [
+        key,
+        this.dataTransformer.toDbString(
+          entity[columnsDefinition[key].nodename],
+          columnsDefinition[key].type,
+        ),
+      ];
+    };
+
+    const fields = Object.keys(columnsDefinition)
+      .filter(key => entity[columnsDefinition[key].nodename] !== undefined)
+      .map(key => toValueKeyDBStringPair(key));
+    const query = `INSERT INTO "${entityDefinition.name}" (${fields.map(e => e[0]).join(", ")}) 
+              VALUES (${fields.map((e, idx) => `$${idx + 1}`).join(", ")})`;
+    return [query, fields.map(f => f[1])];
+  }
+
+  buildUpdate<Entity>(entity: Entity): Query {
+    const { columnsDefinition, entityDefinition } = getEntityDefinition(entity);
+    const primaryKey = unwrapPrimarykey(entityDefinition);
+
+    const toValueKeyDBStringPair = (key: string) => {
+      return [
+        key,
+        this.dataTransformer.toDbString(
+          entity[columnsDefinition[key].nodename],
+          columnsDefinition[key].type,
+        ),
+      ];
+    };
+
+    // Set updated content
+    const set = Object.keys(columnsDefinition)
+      .filter(key => primaryKey.indexOf(key) === -1)
+      .filter(key => entity[columnsDefinition[key].nodename] !== undefined)
+      .map(key => toValueKeyDBStringPair(key));
+    //Set primary key
+    const where = primaryKey.map(key => toValueKeyDBStringPair(key));
+    //Start index for where clause params
+    const whereIdx = set.length + 1;
+    const query = `UPDATE "${entityDefinition.name}" 
+              SET ${set.map((e, idx) => `${e[0]} = $${idx + 1}`).join(", ")} 
+              WHERE ${where.map((e, idx) => `${e[0]} = $${whereIdx + idx}`).join(" AND ")}`;
+    const values = [];
+    values.push(...set.map(f => f[1]));
+    values.push(...where.map(f => f[1]));
+
+    return [query, values];
+  }
+}

tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres.ts

@@ -0,0 +1,283 @@
+import { AbstractConnector } from "../abstract-connector";
+import { ColumnDefinition, EntityDefinition, ObjectType } from "../../types";
+import { ListResult, Paginable, Pagination } from "../../../../../../framework/api/crud-service";
+import { FindOptions } from "../../repository/repository";
+import { Client, QueryResult } from "pg";
+import { getLogger, logger } from "../../../../../../framework";
+import { getEntityDefinition } from "../../../orm/utils";
+import { UpsertOptions } from "src/core/platform/services/database/services/orm/connectors";
+import { PostgresDataTransformer, TypeMappings } from "./postgres-data-transform";
+import { PostgresQueryBuilder, Query } from "./postgres-query-builder";
+
+export interface PostgresConnectionOptions {
+  database: string;
+  user: string;
+  password: string;
+  port: number;
+  host: string;
+  ssl: false;
+  idleTimeoutMillis: 1000; // close idle clients after 1 second
+  connectionTimeoutMillis: 1000; // return an error after 1 second if connection could not be established
+  statement_timeout: number; // number of milliseconds before a statement in query will time out, default is no timeout
+  query_timeout: number; // number of milliseconds before a query call will time out, default is no timeout
+}
+
+export class PostgresConnector extends AbstractConnector<PostgresConnectionOptions> {
+  private logger = getLogger("PostgresConnector");
+  private client: Client = new Client(this.options);
+  private dataTransformer = new PostgresDataTransformer({ secret: this.secret });
+  private queryBuilder = new PostgresQueryBuilder(this.secret);
+
+  private async healthcheck(): Promise<void> {
+    const result: QueryResult = await this.client.query("SELECT NOW()");
+    if (!result || result.rowCount != 1) {
+      throw new Error("Connection Error");
+    }
+    this.logger.info(`DB connection is fine, current time is ${result.rows[0].now}`);
+  }
+
+  async connect(): Promise<this> {
+    if (this.client) {
+      this.logger.info("Connecting to DB");
+      this.client.on("error", err => {
+        this.logger.warn(err, "PostgreSQL connection error");
+      });
+
+      await this.client.connect();
+      this.logger.info("Connection pool created");
+      await this.healthcheck();
+    }
+    return this;
+  }
+
+  async init(): Promise<this> {
+    if (!this.client) {
+      await this.connect();
+    }
+    return this;
+  }
+
+  async createTable(
+    entity: EntityDefinition,
+    columns: { [p: string]: ColumnDefinition },
+  ): Promise<boolean> {
+    const columnsString = Object.keys(columns)
+      .map(colName => {
+        const definition = columns[colName];
+        return `${colName} ${TypeMappings[definition.type]}`;
+      })
+      .join(",\n");
+
+    // --- Generate final create table query --- //
+    const query = `
+        CREATE TABLE IF NOT EXISTS "${entity.name}"
+          (
+            ${columnsString}
+          );`;
+
+    try {
+      this.logger.debug(
+        `service.database.orm.createTable - Creating table ${entity.name} : ${query}`,
+      );
+      const result: QueryResult = await this.client.query(query);
+      this.logger.info(`Table is created with the result ${JSON.stringify(result)}`);
+    } catch (err) {
+      this.logger.warn(
+        { err },
+        `service.database.orm.createTable - creation error for table ${entity.name} : ${err.message}`,
+      );
+      return false;
+    }
+
+    //--- Alter table if not up to date --- //
+    await this.alterTable(entity, columns);
+
+    // --- Create primary key --- //
+    await this.alterTablePrimaryKey(entity);
+
+    // --- Create indexes --- //
+    return await this.alterTableIndexes(entity);
+  }
+
+  private async alterTableIndexes(entity: EntityDefinition) {
+    if (entity.options.globalIndexes) {
+      for (const globalIndex of entity.options.globalIndexes) {
+        const indexName = globalIndex.join("_");
+        const indexDbName = `index_${entity.name}_${indexName}`;
+
+        const query = `CREATE INDEX IF NOT EXISTS ${indexDbName} ON "${entity.name}" 
+        (${globalIndex.length === 1 ? globalIndex[0] : `(${globalIndex[0]}), ${globalIndex[1]}`})`;
+
+        try {
+          this.logger.debug(`Creating index ${indexName} (${indexDbName}) : ${query}`);
+          await this.client.query(query);
+        } catch (err) {
+          this.logger.warn(
+            err,
+            `Creation error for index ${indexName} (${indexDbName}) : ${err.message}`,
+          );
+          return false;
+        }
+      }
+    }
+  }
+
+  private async alterTablePrimaryKey(entity: EntityDefinition) {
+    if (entity.options.primaryKey) {
+      const query = `ALTER TABLE "${entity.name}" ADD PRIMARY KEY (
+        ${entity.options.primaryKey.join(", ")});`;
+      try {
+        await this.client.query(query);
+      } catch (err) {
+        this.logger.warn(err, `Error creating primary key for "${entity.name}"`);
+      }
+    }
+  }
+
+  private async alterTable(entity: EntityDefinition, columns: { [p: string]: ColumnDefinition }) {
+    const existingColumns = await this.getTableDefinition(entity.name);
+    if (existingColumns.length > 0) {
+      this.logger.debug(
+        `Existing columns for table "${entity.name}", generating alter table queries ...`,
+      );
+      const alterQueryColumns = Object.keys(columns)
+        .filter(colName => existingColumns.indexOf(colName) < 0)
+        .map(colName => `ADD COLUMN ${colName} ${TypeMappings[columns[colName].type]}`)
+        .join(", ");
+
+      if (alterQueryColumns.length > 0) {
+        const alterQuery = `ALTER TABLE "${entity.name}" ${alterQueryColumns}`;
+        const queryResult: QueryResult = await this.client.query(alterQuery);
+        this.logger.info(`Table is altered with the result ${queryResult}`);
+      }
+    }
+  }
+
+  async drop(): Promise<this> {
+    logger.info("Drop database is not implemented for security reasons.");
+    return this;
+  }
+
+  async dropTables(): Promise<this> {
+    const query = `
+        DO $$ 
+        DECLARE 
+          tablename text;
+        BEGIN 
+          FOR tablename IN (SELECT table_name FROM information_schema.tables WHERE table_schema = 'public') 
+          LOOP 
+            EXECUTE 'DROP TABLE IF EXISTS "' || tablename || '" CASCADE'; 
+          END LOOP; 
+        END $$;`;
+    logger.debug(`service.database.orm.postgres.dropTables - Query: "${query}"`);
+    await this.client.query(query);
+    return this;
+  }
+
+  async find<EntityType>(
+    entityType: ObjectType<EntityType>,
+    filters: Record<string, unknown>,
+    options: FindOptions,
+  ): Promise<ListResult<EntityType>> {
+    const query = this.queryBuilder.buildSelect(entityType, filters, options);
+
+    logger.debug(`services.database.orm.postgres.find - Query: ${query}`);
+
+    const results = await this.client.query(query[0] as string, query[1] as never[]);
+
+    const { columnsDefinition, entityDefinition } = getEntityDefinition(
+      new (entityType as ObjectType<EntityType>)(),
+    );
+    const entities: EntityType[] = [];
+    results.rows.forEach(row => {
+      const entity = new (entityType as ObjectType<EntityType>)();
+      Object.keys(row).forEach(key => {
+        if (columnsDefinition[key]) {
+          entity[columnsDefinition[key].nodename] = this.dataTransformer.fromDbString(
+            row[key],
+            columnsDefinition[key].type,
+          );
+        }
+      });
+      entities.push(entity);
+    });
+
+    const nextPageToken = options?.pagination?.page_token || "0";
+    const limit = parseInt(options?.pagination?.limitStr);
+    const nextToken = entities.length === limit && (parseInt(nextPageToken) + limit).toString(10);
+    const nextPage: Paginable = new Pagination(nextToken, options?.pagination?.limitStr || "100");
+    logger.debug(
+      `services.database.orm.postgres.find - Query Result (items=${entities.length}): ${query}`,
+    );
+
+    return new ListResult<EntityType>(entityDefinition.type, entities, nextPage);
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  remove<Entity>(entities: Entity[]): Promise<boolean[]> {
+    return Promise.all(entities.map(entity => this.removeOne(entity)));
+  }
+
+  async removeOne<Entity>(entity: Entity): Promise<boolean> {
+    const query = this.queryBuilder.buildDelete(entity);
+    logger.debug(`service.database.orm.postgres.remove - Query: "${query}"`);
+    const result = await this.client.query(query[0] as string, query[1] as never[]);
+    return result.rowCount > 0;
+  }
+
+  async upsert<Entity>(entities: Entity[], _options: UpsertOptions): Promise<boolean[]> {
+    if (!_options?.action) {
+      throw new Error("Can't perform unknown operation");
+    } else {
+      return Promise.all(entities.map(entity => this.upsertOne(entity, _options)));
+    }
+  }
+
+  private async upsertOne<Entity>(entity: Entity, _options: UpsertOptions): Promise<boolean> {
+    if (_options.action == "INSERT") {
+      const query = this.queryBuilder.buildInsert(entity);
+      return this.execute(query);
+    } else if (_options.action == "UPDATE") {
+      if (!(await this.execute(this.queryBuilder.buildUpdate(entity)))) {
+        return this.execute(this.queryBuilder.buildInsert(entity));
+      }
+    } else {
+      return false;
+    }
+  }
+
+  private async execute(query: Query) {
+    logger.debug(`service.database.orm.postgres - Query: "${query[0]}"`);
+    try {
+      const result: QueryResult = await this.client.query(query[0] as string, query[1] as never[]);
+      return result.rowCount > 0;
+    } catch (err) {
+      logger.error({ err }, `services.database.orm.postgres - Error with SQL query: ${query[0]}`);
+      return false;
+    }
+  }
+
+  async getTableDefinition(name: string): Promise<string[]> {
+    try {
+      const query = `SELECT 
+           table_name, 
+           column_name, 
+           data_type 
+        FROM 
+           information_schema.columns
+        WHERE 
+           table_name = $1`;
+      const dbResult: QueryResult<TableRowInfo> = await this.client.query(query, [name]);
+      return dbResult.rows.map(row => row.column_name);
+    } catch (err) {
+      this.logger.warn("Error querying table information", err);
+      throw err;
+    }
+  }
+}
+
+export type TableRowInfo = {
+  table_name: string;
+  column_name: string;
+  data_type: string;
+};

tdrive/backend/node/src/core/platform/services/database/services/orm/manager.ts

@@ -9,12 +9,16 @@ import { DatabaseEntitiesRemovedEvent, DatabaseEntitiesSavedEvent } from "./type
 import { localEventBus } from "../../../../framework/event-bus";
 
 export default class EntityManager<EntityType extends Record<string, any>> {
-  private toPersist: EntityType[] = [];
+  private toInsert: EntityType[] = [];
+  private toUpdate: EntityType[] = [];
   private toRemove: EntityType[] = [];
 
   constructor(readonly connector: Connector) {}
 
   public persist(entity: any): this {
+    logger.trace(
+      `services.database.orm.entity-manager.persist - entity: ${JSON.stringify(entity)}`,
+    );
     if (!entity.constructor.prototype._entity || !entity.constructor.prototype._columns) {
       logger.error("Can not persist this object %o", entity);
       throw Error("Cannot persist this object: it is not an entity.");
@@ -24,32 +28,7 @@ export default class EntityManager<EntityType extends Record<string, any>> {
     const { columnsDefinition, entityDefinition } = getEntityDefinition(entity);
     const primaryKey: string[] = unwrapPrimarykey(entityDefinition);
 
-    primaryKey.forEach(pk => {
-      if (entity[pk] === undefined) {
-        const definition = columnsDefinition[pk];
-
-        if (!definition) {
-          throw Error(`There is no definition for primary key ${pk}`);
-        }
-
-        //Create default value
-        switch (definition.options.generator || definition.type) {
-          case "uuid":
-            entity[pk] = uuidv4();
-            break;
-          case "timeuuid":
-            entity[pk] = uuidv1();
-            break;
-          case "number":
-            entity[pk] = 0;
-            break;
-          default:
-            entity[pk] = "";
-        }
-      }
-    });
-
-    //Apply the onUpsert
+    //apply on upsert(generating created_at, updated_at fields)
     for (const column in columnsDefinition) {
       const definition = columnsDefinition[column];
       if (definition.options.onUpsert) {
@@ -57,8 +36,39 @@ export default class EntityManager<EntityType extends Record<string, any>> {
       }
     }
 
-    this.toPersist = this.toPersist.filter(e => e !== entity);
-    this.toPersist.push(_.cloneDeep(entity));
+    // generate primary key
+    const emptyPkFields = primaryKey.filter(
+      pk => entity[columnsDefinition[pk].nodename] === undefined,
+    );
+    emptyPkFields.forEach(pk => {
+      const definition = columnsDefinition[pk];
+
+      if (!definition) {
+        throw Error(`There is no definition for primary key ${pk}`);
+      }
+      //Create default value
+      switch (definition.options.generator || definition.type) {
+        case "uuid":
+          entity[columnsDefinition[pk].nodename] = uuidv4();
+          break;
+        case "timeuuid":
+          entity[columnsDefinition[pk].nodename] = uuidv1();
+          break;
+        case "number":
+          entity[columnsDefinition[pk].nodename] = 0;
+          break;
+        default:
+          entity[columnsDefinition[pk].nodename] = "";
+      }
+    });
+
+    if (emptyPkFields.length > 0) {
+      this.toInsert = this.toInsert.filter(e => e !== entity);
+      this.toInsert.push(_.cloneDeep(entity));
+    } else {
+      this.toUpdate = this.toUpdate.filter(e => e !== entity);
+      this.toUpdate.push(_.cloneDeep(entity));
+    }
 
     return this;
   }
@@ -78,25 +88,38 @@ export default class EntityManager<EntityType extends Record<string, any>> {
   }
 
   public async flush(): Promise<this> {
-    this.toPersist = _.uniqWith(this.toPersist, _.isEqual);
+    this.toInsert = _.uniqWith(this.toInsert, _.isEqual);
+    this.toUpdate = _.uniqWith(this.toUpdate, _.isEqual);
     this.toRemove = _.uniqWith(this.toRemove, _.isEqual);
 
     localEventBus.publish("database:entities:saved", {
-      entities: this.toPersist.map(e => _.cloneDeep(e)),
+      entities: this.toInsert.map(e => _.cloneDeep(e)),
+    } as DatabaseEntitiesSavedEvent);
+
+    localEventBus.publish("database:entities:saved", {
+      entities: this.toUpdate.map(e => _.cloneDeep(e)),
     } as DatabaseEntitiesSavedEvent);
 
     localEventBus.publish("database:entities:saved", {
       entities: this.toRemove.map(e => _.cloneDeep(e)),
     } as DatabaseEntitiesRemovedEvent);
 
-    await this.connector.upsert(this.toPersist);
-    await this.connector.remove(this.toRemove);
+    if (this.toInsert.length > 0) {
+      await this.connector.upsert(this.toInsert, { action: "INSERT" });
+    }
+    if (this.toUpdate.length > 0) {
+      await this.connector.upsert(this.toUpdate, { action: "UPDATE" });
+    }
+    if (this.toRemove.length > 0) {
+      await this.connector.remove(this.toRemove);
+    }
 
     return this;
   }
 
   public reset(): void {
-    this.toPersist = [];
+    this.toInsert = [];
+    this.toUpdate = [];
     this.toRemove = [];
   }
 }

tdrive/backend/node/src/core/platform/services/database/services/orm/repository/repository.ts

@@ -96,18 +96,20 @@ export default class Repository<EntityType> {
   }
 
   async save(entity: EntityType, _context?: ExecutionContext): Promise<void> {
-    (await this.manager.persist(entity).flush()).reset();
+    this.manager.persist(entity);
+    return this.manager.flush().then(manager => manager.reset());
   }
 
   async saveAll(entities: EntityType[] = [], _context?: ExecutionContext): Promise<void> {
     logger.debug("services.database.repository - Saving entities");
 
     entities.forEach(entity => this.manager.persist(entity));
-    await (await this.manager.flush()).reset();
+    return this.manager.flush().then(manager => manager.reset());
   }
 
   async remove(entity: EntityType, _context?: ExecutionContext): Promise<void> {
-    await (await this.manager.remove(entity).flush()).reset();
+    this.manager.remove(entity);
+    return this.manager.flush().then(manager => manager.reset());
   }
 
   //Avoid using this except when no choice

tdrive/backend/node/src/core/platform/services/database/services/orm/types.ts

@@ -46,7 +46,7 @@ export type ColumnType =
 export type EntityTarget<Entity> = ObjectType<Entity>;
 
 // eslint-disable-next-line @typescript-eslint/ban-types
-export type ObjectType<T> = { new (): T } | Function;
+export type ObjectType<T> = { new (): T };
 
 /** Local Event bus */
 

tdrive/backend/node/src/core/platform/services/database/services/orm/utils.ts

@@ -6,10 +6,10 @@ export function getEntityDefinition(instance: any): {
   entityDefinition: EntityDefinition;
   columnsDefinition: { [name: string]: ColumnDefinition };
 } {
-  const entityConfituration = _.cloneDeep(instance.constructor.prototype._entity);
+  const entityConfiguration = _.cloneDeep(instance.constructor.prototype._entity);
   const entityColumns = _.cloneDeep(instance.constructor.prototype._columns);
   return {
-    entityDefinition: entityConfituration,
+    entityDefinition: entityConfiguration,
     columnsDefinition: entityColumns,
   };
 }
@@ -17,11 +17,10 @@ export function getEntityDefinition(instance: any): {
 export function unwrapPrimarykey(entityDefinition: EntityDefinition): string[] {
   const initial = [...entityDefinition.options.primaryKey];
   const partitionKey = initial.shift();
-  const primaryKey: string[] = [
+  return [
     ...(typeof partitionKey === "string" ? [partitionKey] : partitionKey),
     ...(initial as string[]),
   ];
-  return primaryKey;
 }
 
 export function unwrapIndexes(entityDefinition: EntityDefinition): string[] {
@@ -89,8 +88,7 @@ export function fromMongoDbOrderable(orderable: string): string {
     return null;
   }
   const uuid_arr = orderable.split("-");
-  const timeuuid = [uuid_arr[2], uuid_arr[1], uuid_arr[0], uuid_arr[3], uuid_arr[4]].join("-");
-  return timeuuid;
+  return [uuid_arr[2], uuid_arr[1], uuid_arr[0], uuid_arr[3], uuid_arr[4]].join("-");
 }
 
 /**
@@ -103,8 +101,7 @@ export function toMongoDbOrderable(timeuuid?: string): string {
     return null;
   }
   const uuid_arr = timeuuid.split("-");
-  const time_str = [uuid_arr[2], uuid_arr[1], uuid_arr[0], uuid_arr[3], uuid_arr[4]].join("-");
-  return time_str;
+  return [uuid_arr[2], uuid_arr[1], uuid_arr[0], uuid_arr[3], uuid_arr[4]].join("-");
 }
 
 /**

tdrive/backend/node/src/core/platform/services/storage/connectors/local/service.ts

@@ -4,6 +4,7 @@ import p from "path";
 import { rm } from "fs/promises"; // Do not change the import, this is not the same function import { rm } from "fs"
 import { StorageConnectorAPI, WriteMetadata } from "../../provider";
 import fs from "fs";
+import { logger } from "../../../../framework/logger";
 
 export type LocalConfiguration = {
   path: string;
@@ -18,6 +19,7 @@ export default class LocalConnectorService implements StorageConnectorAPI {
 
   write(relativePath: string, stream: Readable): Promise<WriteMetadata> {
     const path = this.getFullPath(relativePath);
+    logger.trace(`Writing file ${path}`);
 
     const directory = p.dirname(path);
     if (!existsSync(directory)) {
@@ -30,10 +32,12 @@ export default class LocalConnectorService implements StorageConnectorAPI {
       const file = createWriteStream(path);
       file
         .on("error", function (err) {
+          logger.error(`Error ${err.message} during writing file ${path}`);
           reject(err);
         })
         .on("finish", () => {
           const stats = statSync(path);
+          logger.trace(`File ${path} have been written`);
           resolve({
             size: stats.size,
           });
@@ -43,6 +47,7 @@ export default class LocalConnectorService implements StorageConnectorAPI {
   }
 
   async read(path: string): Promise<Readable> {
+    logger.trace(`Reading file ... ${path}`);
     const fullPath = this.getFullPath(path);
     if (!fs.existsSync(fullPath)) {
       throw new Error(`File doesn't exists ${fullPath}`);

tdrive/backend/node/src/core/platform/services/storage/index.ts

@@ -51,7 +51,8 @@ export default class StorageService extends TdriveService<StorageAPI> implements
     } else {
       logger.info("Using 'local' connector for storage.");
       const defaultHomeDir = this.configuration.get<string>("local.path");
-      if (defaultHomeDir) this.homeDir = `/${defaultHomeDir}`;
+      if (defaultHomeDir) this.homeDir = `${defaultHomeDir}`;
+      logger.trace(`Home directory for the storage: ${this.homeDir}`);
     }
     logger.info(
       `Using 'local' connector for storage${

tdrive/backend/node/src/services/applications-api/index.ts

@@ -5,6 +5,7 @@ import { Prefix, TdriveService } from "../../core/platform/framework";
 import WebServerAPI from "../../core/platform/services/webserver/provider";
 import Application from "../applications/entities/application";
 import web from "./web/index";
+import { logger } from "../../core/platform/framework/logger";
 
 @Prefix("/api")
 export default class ApplicationsApiService extends TdriveService<undefined> {
@@ -26,7 +27,7 @@ export default class ApplicationsApiService extends TdriveService<undefined> {
       if (domain && prefix) {
         try {
           fastify.all("/" + prefix + "/*", async (req, rep) => {
-            console.log("Proxying", req.method, req.url, "to", domain);
+            logger.info("Proxying", req.method, req.url, "to", domain);
             try {
               const response = await axios.request({
                 url: domain + req.url,
@@ -54,15 +55,15 @@ export default class ApplicationsApiService extends TdriveService<undefined> {
 
               rep.send(response.data);
             } catch (err) {
-              console.error(`${err}`);
+              logger.error(`${err}`);
               rep.raw.statusCode = 500;
               rep.raw.end(JSON.stringify({ error: err.message }));
             }
           });
-          console.log("Listening at ", "/" + prefix + "/*");
+          logger.info("Listening at ", "/" + prefix + "/*");
         } catch (e) {
-          console.log(e);
-          console.log("Can't listen to ", "/" + prefix + "/*");
+          logger.error(e);
+          logger.info("Can't listen to ", "/" + prefix + "/*");
         }
       }
     }

tdrive/backend/node/src/services/documents/web/controllers/documents.ts

@@ -25,8 +25,6 @@ import {
 } from "../../types";
 import { DriveFileDTO } from "../dto/drive-file-dto";
 import { DriveFileDTOBuilder } from "../../services/drive-file-dto-builder";
-import { ExecutionContext } from "../../../../core/platform/framework/api/crud-service";
-import gr from "../../../global-resolver";
 import config from "config";
 
 export class DocumentsController {
@@ -35,12 +33,6 @@ export class DocumentsController {
     ? config.get("drive.rootAdmins")
     : [];
 
-  private getCompanyUserRole(companyId: string, userId: string, context?: ExecutionContext) {
-    return gr.services.companies
-      .getCompanyUser({ id: companyId }, { id: userId }, context)
-      .then(a => (a ? a.level : null));
-  }
-
   /**
    * Creates a DriveFile item
    *

tdrive/backend/node/src/services/user/entities/company_user.ts

@@ -35,22 +35,6 @@ export default class CompanyUser {
 
   @Column("last_update_day", "number")
   lastUpdateDay: number;
-
-  /**
-   * 0: member,
-   * 1, 2, 3: admin,
-   */
-  @Column("level", "number")
-  level: number; //Depreciated
-
-  @Column("is_externe", "tdrive_boolean")
-  isExterne: boolean; //Depreciated
-
-  @Column("did_connect_today", "tdrive_boolean")
-  didConnectToday: boolean; //Depreciated
-
-  @Column("app_used_today", "json")
-  appUsedToday: Array<string>; //Depreciated
 }
 
 export type CompanyUserPrimaryKey = Partial<Pick<CompanyUser, "group_id" | "user_id">>;

tdrive/backend/node/src/services/user/services/companies.ts

@@ -276,9 +276,6 @@ export class CompanyServiceImpl {
     }
 
     entity.role = role;
-    if (level) {
-      entity.level = level;
-    }
     entity.applications = applications;
     await this.companyUserRepository.save(entity, context);
 

tdrive/backend/node/src/services/workspaces/entities/workspace_user.ts

@@ -29,9 +29,6 @@ export default class WorkspaceUser {
 
   @Column("last_access", "number")
   lastAccess: number;
-
-  @Column("is_externe", "tdrive_boolean")
-  isExternal: boolean; //Depreciated
 }
 
 export type WorkspaceUserPrimaryKey = Partial<Pick<WorkspaceUser, "workspaceId" | "userId">>;