🌟 Support for PostgreSQL (#306)

This commit is contained in:
Anton Shepilov
2024-01-13 23:49:34 +03:00
committed by GitHub
parent 81777154af
commit 70c033aabe
38 changed files with 2202 additions and 364 deletions
@@ -25,7 +25,7 @@ export class DriveFileMockClass {
extension: string;
description: string;
tags: string[];
last_modified: string;
last_modified: number;
access_info: MockAccessInformation;
creator: string;
is_directory: boolean;
@@ -81,7 +81,7 @@ describe("The /users API", () => {
//Wait for indexation to happen
await new Promise(r => setTimeout(r, 5000));
let resources = await search("ha");
let resources = await search("ha", platform.workspace.company_id);
expect(resources.length).toBe(2);
resources = await search("bob rabiot");
@@ -285,10 +285,6 @@ describe("The /workspace users API", () => {
expect(response.statusCode).toBe(201);
const resource = response.json()["resource"];
checkUserObject(resource);
workspaceUsersCount = await testDbService.getWorkspaceUsersCountFromDb(workspaceId);
expect(workspaceUsersCount).toBe(5);
});
});
@@ -0,0 +1,173 @@
import {
PostgresDataTransformer, TypeMappings
} from "../../../../../../../../../src/core/platform/services/database/services/orm/connectors/postgres/postgres-data-transform";
import { randomInt, randomUUID } from "crypto";
import { ColumnType } from "../../../../../../../../../src/core/platform/services/database/services/orm/types";
import { v4 } from "uuid";
import { DriveScope } from "../../../../../../../../../src/services/documents/entities/drive-file";
describe('The PostgresDataTransformer', () => {
test('toDbString anything to counter, blob and other not implemented types should throw an error', async () => {
//given
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
//then
expect(() => subj.toDbString(randomInt(1000), "blob")).toThrow(Error);
expect(() => subj.toDbString(null, "blob")).toThrow(Error);
});
test.each(Object.keys(TypeMappings))('toDbString for "%s" and null or undefined --> to null', async (type: ColumnType) => {
//given
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
//then
expect(subj.toDbString(null, type)).toBeNull();
expect(subj.toDbString(undefined, type)).toBeNull();
});
test('toDbString for tdrive_int, tdrive_datetime should return the same number', async () => {
//given
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
const number = randomInt(1000);
//then
expect(subj.toDbString(number, "number")).toBe(number);
expect(subj.toDbString(number, "counter")).toBe(number);
expect(subj.toDbString(number, "tdrive_datetime")).toBe(number);
expect(subj.toDbString(String(number), "tdrive_datetime")).toBe(number);
});
test('toDbString for tdrive_int, tdrive_datetime should throw error if its not a number', async () => {
//given
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
//then
expect(() => subj.toDbString("", "number")).toThrow(Error);
expect(() => subj.toDbString("str_" + randomUUID(), "tdrive_datetime")).toThrow(Error);
expect(() => subj.toDbString(true, "tdrive_datetime")).toThrow(Error);
expect(() => subj.toDbString({ idx: 1} , "tdrive_datetime")).toThrow(Error);
});
test('toDbString for uuid and timeuuid should throw exception if value is not string', async () => {
//given
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
//then
expect(() => subj.toDbString({ }, "uuid")).toThrow(Error);
expect(() => subj.toDbString({ }, "timeuuid")).toThrow(Error);
expect(() => subj.toDbString(true, "uuid")).toThrow(Error);
expect(() => subj.toDbString(1 , "timeuuid")).toThrow(Error);
});
test('toDbString for uuid and timeuuid should be transformed to the same string', async () => {
//given
const uuid = v4()
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
//then
expect(subj.toDbString(uuid, "uuid")).toBe(uuid);
expect(subj.toDbString(uuid, "timeuuid")).toBe(uuid);
});
test('toDbString for boolean and to tdrive_boolean should return boolean', async () => {
//given
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
const number = randomInt(1000) + 1;
//then
expect(subj.toDbString(number, "tdrive_boolean")).toBe(true);
expect(subj.toDbString(number, "boolean")).toBe(true);
expect(subj.toDbString(0, "tdrive_boolean")).toBe(false);
expect(subj.toDbString(-number, "boolean")).toBe(true);
expect(subj.toDbString(true, "boolean")).toBe(true);
expect(subj.toDbString(false, "boolean")).toBe(false);
});
test('toDbString for boolean and to tdrive_boolean should throw error for any not boolean', async () => {
//given
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
//then
expect(() => subj.toDbString("true", "tdrive_boolean")).toThrow(Error);
expect(() => subj.toDbString("true", "boolean")).toThrow(Error);
expect(() => subj.toDbString("false", "tdrive_boolean")).toThrow(Error);
expect(() => subj.toDbString("false", "boolean")).toThrow(Error);
expect(() => subj.toDbString({ }, "boolean")).toThrow(Error);
expect(() => subj.toDbString([true], "tdrive_boolean")).toThrow(Error);
});
test('toDbString for string should return boolean the result of toString() method', async () => {
//given
const scope: DriveScope = "personal";
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
//then
expect(subj.toDbString("test_string", "string")).toBe("test_string");
expect(subj.toDbString(scope, "string")).toBe("personal");
});
test('toDbString for json should return stringified object', async () => {
//given
const file = { id: "123", amount: 10};
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
//then
expect(subj.toDbString(file, "json")).toBe('{"id":"123","amount":10}');
expect(subj.toDbString(10, "json")).toBe("10");
});
test('toDbString for encoded_string and empty secret should return plain string', async () => {
//given
const str = randomUUID();
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
//then
expect(subj.toDbString(str, "encoded_string")).toBe(str);
});
test('toDbString for encoded_json and empty secret should return plain string', async () => {
//then
//given
const file = { id: "123", amount: 10};
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "" });
//then
expect(subj.toDbString(file, "encoded_json")).toBe('{"id":"123","amount":10}');
expect(subj.toDbString(10, "encoded_json")).toBe("10");
});
test('toDbString for encoded_json should return decodable string different from the source one', async () => {
//then
//given
const file = { id: "123", amount: 10};
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "hey it's secret" });
//then
const encodedJson = subj.toDbString(file, "encoded_json");
expect(encodedJson).not.toBe('{"id":"123","amount":10}');
const encodedNumber = subj.toDbString(10, "encoded_json");
expect(encodedNumber).not.toBe("10");
expect(subj.fromDbString(encodedJson, "encoded_json")).toEqual(file);
expect(subj.fromDbString(encodedNumber, "encoded_json")).toEqual(10);
});
test('toDbString for encoded_string should return decodable string different from the source one', async () => {
//then
//given
const str = randomUUID();
const subj: PostgresDataTransformer = new PostgresDataTransformer({ secret: "hey it's secret" });
//then
const encodedStr = subj.toDbString(str, "encoded_json");
expect(encodedStr).not.toBe('{"id":"123","amount":10}');
expect(subj.fromDbString(encodedStr, "encoded_json")).toEqual(str);
});
});
@@ -0,0 +1,149 @@
import {
PostgresQueryBuilder
} from "../../../../../../../../../src/core/platform/services/database/services/orm/connectors/postgres/postgres-query-builder";
import { expect, jest } from "@jest/globals";
import { randomInt, randomUUID } from "crypto";
import { newTestDbEntity, normalizeWhitespace, TestDbEntity } from "./utils";
import {
comparisonType
} from "../../../../../../../../../src/core/platform/services/database/services/orm/repository/repository";
describe('The PostgresQueryBuilder', () => {
const subj: PostgresQueryBuilder = new PostgresQueryBuilder("");
beforeEach(async () => {
});
afterEach(() => {
jest.clearAllMocks();
});
test('buildSelect query with filter', async () => {
//given
const filter = new TestDbEntity({id: randomUUID(), company_id: randomUUID() })
//when
const query = subj.buildSelect(TestDbEntity, filter as { [key: string]: any }, null);
//then
expect(normalizeWhitespace(query[0] as string)).toBe(`SELECT * FROM "test_table" WHERE id = $1 AND company_id = $2 ORDER BY id DESC LIMIT 100 OFFSET 0`);
expect(query[1]).toEqual([filter.id, filter.company_id]);
});
test('buildSelect query with array in filters', async () => {
//given
const filter = {id: randomUUID(), company_id: [randomUUID(), randomUUID()] };
//when
const query = subj.buildSelect(TestDbEntity, filter as { [key: string]: any }, null);
//then
expect(normalizeWhitespace(query[0] as string)).toBe(`SELECT * FROM "test_table" WHERE id = $1 AND company_id IN ($2,$3) ORDER BY id DESC LIMIT 100 OFFSET 0`);
expect(query[1]).toEqual([filter.id, filter.company_id[0], filter.company_id[1]]);
});
test('buildSelect query "lt" options', async () => {
//given
const options = { $lt: [["added", randomInt(10)] as comparisonType, ["added", randomInt(10)] as comparisonType]};
//when
const query = subj.buildSelect(TestDbEntity, null, options);
//then
expect(normalizeWhitespace(query[0] as string)).toBe(`SELECT * FROM "test_table" WHERE added < $1 AND added < $2 ORDER BY id DESC LIMIT 100 OFFSET 0`);
expect(query[1]).toEqual(options.$lt.map(e=> e[1]));
});
test('buildSelect query "gt" options', async () => {
//given
const options = { $gt: [["added", randomInt(10)] as comparisonType, ["added", randomInt(10)] as comparisonType]};
//when
const query = subj.buildSelect(TestDbEntity, null, options);
//then
expect(normalizeWhitespace(query[0] as string)).toBe(`SELECT * FROM "test_table" WHERE added > $1 AND added > $2 ORDER BY id DESC LIMIT 100 OFFSET 0`);
expect(query[1]).toEqual(options.$gt.map(e=> e[1]));
});
test('buildSelect query "in" options', async () => {
//given
const options = { $in: [["added", [randomInt(10), randomInt(10)]] as comparisonType,]};
//when
const query = subj.buildSelect(TestDbEntity, null, options);
//then
expect(normalizeWhitespace(query[0] as string)).toBe(`SELECT * FROM "test_table" WHERE added IN ($1,$2) ORDER BY id DESC LIMIT 100 OFFSET 0`);
expect(query[1]).toEqual(options.$in[0][1]);
});
test('buildSelect query "like" options', async () => {
//given
const options = { $like: [["id", randomUUID()] as comparisonType, ["company_id", randomUUID()] as comparisonType]};
//when
const query = subj.buildSelect(TestDbEntity, null, options);
//then
expect(normalizeWhitespace(query[0] as string)).toBe(`SELECT * FROM "test_table" WHERE id LIKE $1 AND company_id LIKE $2 ORDER BY id DESC LIMIT 100 OFFSET 0`);
expect(query[1]).toEqual(options.$like.map(e=> `%${e[1]}%`));
});
test('buildDelete query', async () => {
//given
const entity = newTestDbEntity();
//when
const query = subj.buildDelete(entity);
//then
expect(normalizeWhitespace(query[0] as string)).toBe(`DELETE FROM "test_table" WHERE company_id = $1 AND id = $2`);
expect(query[1]).toEqual([entity.company_id, entity.id]);
});
test('buildInsert', async () => {
//given
const entity = newTestDbEntity();
//when
const query = subj.buildInsert(entity);
//then
expect(normalizeWhitespace(query[0] as string)).toBe(`INSERT INTO "test_table" (company_id, id, parent_id, is_in_trash, tags, added) VALUES ($1, $2, $3, $4, $5, $6)`);
assertInsertQueryParams(entity, query[1] as any[])
});
test('buildUpdate', async () => {
//given
const entity = newTestDbEntity();
//when
const query = subj.buildUpdate(entity);
//then
expect(normalizeWhitespace(query[0] as string)).toBe(`UPDATE "test_table" SET parent_id = $1, is_in_trash = $2, tags = $3, added = $4 WHERE company_id = $5 AND id = $6`);
assertUpdateQueryParams(entity, query[1] as any[])
});
const assertInsertQueryParams = (actual: TestDbEntity, expected: any[]) => {
expect(expected[0]).toBe(actual.company_id);
expect(expected[1]).toBe(actual.id);
expect(expected[2]).toBe(actual.parent_id);
expect(expected[3]).toBe(actual.is_in_trash);
expect(expected[4]).toBe( JSON.stringify(actual.tags));
expect(expected[5]).toBe(actual.added);
}
const assertUpdateQueryParams = (actual: TestDbEntity, expected: string[]) => {
expect(expected[0]).toBe(actual.parent_id);
expect(expected[1]).toBe(actual.is_in_trash);
expect(expected[2]).toBe( JSON.stringify(actual.tags));
expect(expected[3]).toBe(actual.added);
expect(expected[4]).toBe(actual.company_id);
expect(expected[5]).toBe(actual.id);
}
});
@@ -0,0 +1,67 @@
import 'reflect-metadata';
import { DriveFile } from "../../../../../../../../../src/services/documents/entities/drive-file";
import { describe, it } from '@jest/globals';
import {
PostgresConnectionOptions,
PostgresConnector,
} from '../../../../../../../../../src/core/platform/services/database/services/orm/connectors/postgres/postgres';
import { getEntityDefinition } from '../../../../../../../../../src/core/platform/services/database/services/orm/utils';
import { randomUUID } from "crypto";
import { now } from "moment";
//THIS IS NOT TEST, IT'S JUST RUNNER TO TEST POSTGRESQL API
describe.skip('The Postgres Connector module', () => {
const subj: PostgresConnector = new PostgresConnector('postgres', {
user: "tdrive_user",
password: "tdrive_secret",
database: "tdrive",
port: 5432,
host: "localhost",
statement_timeout: 10000,
query_timeout: 10000
} as PostgresConnectionOptions, '');
it('createTable test ALTER TABLE query', async () => {
let driveFile = new DriveFile();
const definition = getEntityDefinition(driveFile);
await subj.connect();
//when
await subj.createTable(definition.entityDefinition, definition.columnsDefinition);
driveFile.id = randomUUID();
driveFile.name = "My Test File";
driveFile.company_id = randomUUID();
driveFile.access_info = {
entities: [],
public: {
token: randomUUID(),
password: randomUUID(),
expiration: now(),
level: "read",
}
}
driveFile.is_directory = false;
driveFile.is_in_trash = true;
driveFile.last_modified = now();
driveFile.tags = ["mytag", "ehey"]
await subj.upsert([driveFile], {action: "INSERT"} )
let files = await subj.find(DriveFile, null, null);
expect(files.getEntities().length).toBeGreaterThan(0);
await subj.remove(files.getEntities());
files = await subj.find(DriveFile, null, null);
expect(files.getEntities().length).toEqual(0);
await subj.drop();
//then
}, 3000000);
});
@@ -0,0 +1,63 @@
import 'reflect-metadata';
import { describe, expect, jest } from '@jest/globals';
import {
PostgresConnectionOptions,
PostgresConnector, TableRowInfo
} from "../../../../../../../../../src/core/platform/services/database/services/orm/connectors/postgres/postgres";
import { getEntityDefinition } from '../../../../../../../../../src/core/platform/services/database/services/orm/utils';
import { TestDbEntity, normalizeWhitespace, newTestDbEntity } from "./utils";
describe('The Postgres Connector module', () => {
const NUMBER_OF_HEALTHCHECK_CALLS = 1;
const subj: PostgresConnector = new PostgresConnector('postgres', {} as PostgresConnectionOptions, '');
let dbQuerySpy;
beforeEach(async () => {
dbQuerySpy = jest.spyOn((subj as any).client, 'query');
//healthcheck mock
dbQuerySpy
.mockReturnValueOnce({rows: [{now: 1}], rowCount: 1});
jest.spyOn((subj as any).client, 'connect').mockImplementation(jest.fn);
await subj.connect();
});
afterEach(() => {
jest.clearAllMocks();
});
test('createTable generates table structure queries', async () => {
// given
const definition = getEntityDefinition(new TestDbEntity());
dbQuerySpy
//mock create table response
.mockReturnValueOnce({ rows: [], rowCount: 1})
//mock query for existing columns
.mockReturnValueOnce({ rows: [{ column_name: "parent_id"} as TableRowInfo, { column_name: "company_id"}], rowCount: 1 })
//mock alter table response
.mockReturnValueOnce({ rows: [], rowCount: 1 })
//mock alter table add primary key query response
.mockReturnValueOnce({ rows: [], rowCount: 1 })
//mock alter table create index query response
.mockReturnValueOnce({ rows: [], rowCount: 1 })
//mock alter table create index query response
.mockReturnValueOnce({ rows: [], rowCount: 1 })
//when
await subj.createTable(definition.entityDefinition, definition.columnsDefinition);
//then
expect(dbQuerySpy).toHaveBeenCalledTimes(6 + NUMBER_OF_HEALTHCHECK_CALLS);
expect(normalizeWhitespace(dbQuerySpy.mock.calls[1][0])).toBe(normalizeWhitespace(`CREATE TABLE IF NOT EXISTS "test_table" ( company_id UUID, id UUID, parent_id UUID, is_in_trash BOOLEAN, tags TEXT, added BIGINT );`))
expect(normalizeWhitespace(dbQuerySpy.mock.calls[2][0])).toBe(normalizeWhitespace("SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_name = $1"));
expect(dbQuerySpy.mock.calls[2][1]).toStrictEqual(["test_table"])
expect(normalizeWhitespace(dbQuerySpy.mock.calls[3][0])).toBe(`ALTER TABLE "test_table" ADD COLUMN id UUID, ADD COLUMN is_in_trash BOOLEAN, ADD COLUMN tags TEXT, ADD COLUMN added BIGINT`)
expect(normalizeWhitespace(dbQuerySpy.mock.calls[4][0])).toBe(`ALTER TABLE "test_table" ADD PRIMARY KEY ( company_id, id);`)
expect(normalizeWhitespace(dbQuerySpy.mock.calls[5][0])).toBe(`CREATE INDEX IF NOT EXISTS index_test_table_company_id_parent_id ON "test_table" ((company_id), parent_id)`)
expect(normalizeWhitespace(dbQuerySpy.mock.calls[6][0])).toBe(`CREATE INDEX IF NOT EXISTS index_test_table_company_id_is_in_trash ON "test_table" ((company_id), is_in_trash)`)
});
});
@@ -0,0 +1,61 @@
import { Column, Entity } from "../../../../../../../../../src/core/platform/services/database/services/orm/decorators";
import { Type } from "class-transformer";
import { randomInt, randomUUID } from "crypto";
export const normalizeWhitespace = (query: string) => query.replace(/\s+/g, ' ').trim();
@Entity("test_table", {
globalIndexes: [
["company_id", "parent_id"],
["company_id", "is_in_trash"],
],
primaryKey: [["company_id"], "id"],
type: "test_table",
})
// @ts-ignore
export class TestDbEntity {
@Type(() => String)
@Column("company_id", "uuid")
// @ts-ignore
company_id: string;
@Type(() => String)
@Column("id", "uuid", { generator: "uuid" })
// @ts-ignore
id: string;
@Type(() => String)
@Column("parent_id", "uuid")
// @ts-ignore
parent_id: string;
@Type(() => Boolean)
@Column("is_in_trash", "boolean")
// @ts-ignore
is_in_trash: boolean;
@Column("tags", "encoded_json")
// @ts-ignore
tags: string[];
@Type(() => Number)
@Column("added", "number")
// @ts-ignore
added: number;
public constructor(init?:Partial<TestDbEntity>) {
Object.assign(this, init);
}
}
export const newTestDbEntity = () => {
return new TestDbEntity({
company_id: randomUUID(),
id: randomUUID(),
parent_id: randomUUID(),
is_in_trash: true,
tags: [randomUUID(), randomUUID()],
added: randomInt(1, 1000)
})
}
@@ -0,0 +1,95 @@
import 'reflect-metadata';
import { describe, expect, it, jest } from '@jest/globals';
import EntityManager from "../../../../../../../src/core/platform/services/database/services/orm/manager";
import { Connector } from "../../../../../../../src/core/platform/services/database/services/orm/connectors";
import { randomUUID } from "crypto";
import { TestDbEntity } from "./connectors/postgres/utils";
import WorkspaceUser, { getInstance } from "../../../../../../../src/services/workspaces/entities/workspace_user";
describe('EntityManager', () => {
const subj: EntityManager<TestDbEntity> = new EntityManager<TestDbEntity>({ } as Connector);
beforeEach(async () => {
});
afterEach(() => {
jest.clearAllMocks();
subj.reset();
});
test ("persist should store entity to insert if all fields for pk is empty", () => {
//when
subj.persist(new TestDbEntity());
//then
expect((subj as any).toInsert.length).toEqual(1);
expect((subj as any).toUpdate.length).toEqual(0);
expect((subj as any).toRemove.length).toEqual(0);
expect((subj as any).toInsert[0].id).toBeDefined();
expect((subj as any).toInsert[0].company_id).toBeDefined();
});
test ("persist should store entity to insert if id is set", () => {
//when
subj.persist(new TestDbEntity({id: randomUUID()}));
//then
expect((subj as any).toInsert.length).toEqual(1);
expect((subj as any).toUpdate.length).toEqual(0);
expect((subj as any).toRemove.length).toEqual(0);
expect((subj as any).toInsert[0].id).toBeDefined();
expect((subj as any).toInsert[0].company_id).toBeDefined();
});
test ("persist should store entity to insert if company_id is set", () => {
//when
subj.persist(new TestDbEntity({company_id: randomUUID()}));
//then
expect((subj as any).toInsert.length).toEqual(1);
expect((subj as any).toUpdate.length).toEqual(0);
expect((subj as any).toRemove.length).toEqual(0);
expect((subj as any).toInsert[0].id).toBeDefined();
expect((subj as any).toInsert[0].company_id).toBeDefined();
});
test ("persist should store entity to update if all pk fields are set", () => {
//when
let entity = new TestDbEntity({id: randomUUID(), company_id: randomUUID()});
subj.persist(entity);
//then
expect((subj as any).toUpdate.length).toEqual(1);
expect((subj as any).toInsert.length).toEqual(0);
expect((subj as any).toRemove.length).toEqual(0);
expect((subj as any).toUpdate[0]).toEqual(entity)
});
test ("persist should store entity to update if all pk fields are set and column name is different from field name", () => {
//when
let entity = getInstance({id: randomUUID(), workspaceId: randomUUID(), userId: randomUUID()});
subj.persist(entity);
//then
expect((subj as any).toUpdate.length).toEqual(1);
expect((subj as any).toInsert.length).toEqual(0);
expect((subj as any).toRemove.length).toEqual(0);
expect((subj as any).toUpdate[0]).toEqual(entity)
});
test ("persist should store entity to insert if not all pk fields are set and column name is different from field name", () => {
//when
let entity = getInstance({id: randomUUID(), workspaceId: randomUUID()});
subj.persist(entity);
//then
expect((subj as any).toUpdate.length).toEqual(0);
expect((subj as any).toInsert.length).toEqual(1);
expect((subj as any).toRemove.length).toEqual(0);
expect((subj as any).toInsert[0]).toEqual(entity);
expect(entity.userId).toBeDefined();
});
});