♻️Remove Cassandra Support
This commit is contained in:
Anton SHEPILOV
committed by
Anton Shepilov
Anton Shepilov
parent
8214a5f6d4
commit
8f75c04fdd
@@ -68,12 +68,6 @@ jobs:
|
||||
docker-compose -f docker-compose.dev.tests.opensearch.yml logs
|
||||
docker-compose -f docker-compose.dev.tests.opensearch.yml run -e NODE_OPTIONS=--unhandled-rejections=warn -e SEARCH_DRIVER=opensearch -e DB_DRIVER=postgres -e PUBSUB_TYPE=local node npm run test:all
|
||||
docker-compose -f docker-compose.dev.tests.opensearch.yml down
|
||||
- name: e2e-cassandra-test
|
||||
run: |
|
||||
cd tdrive
|
||||
docker-compose -f docker-compose.tests.yml up -d scylladb elasticsearch rabbitmq
|
||||
sleep 60
|
||||
docker-compose -f docker-compose.tests.yml run -e NODE_OPTIONS=--unhandled-rejections=warn -e SEARCH_DRIVER=elasticsearch -e DB_DRIVER=cassandra node npm run test:all
|
||||
- name: coverage
|
||||
uses: adRise/jest-cov-reporter@e13d7638888cad7788298f4022b203b31a10c7c8
|
||||
with:
|
||||
|
||||
@@ -43,11 +43,6 @@
|
||||
"uri": "DB_MONGO_URI",
|
||||
"database": "DB_MONGO_DATABASE"
|
||||
},
|
||||
"cassandra": {
|
||||
"contactPoints": "DB_CASSANDRA_URI",
|
||||
"localDataCenter": "DB_CASSANDRA_LOCALDATACENTER",
|
||||
"keyspace": "DB_CASSANDRA_KEYSPACE"
|
||||
},
|
||||
"postgres": {
|
||||
"database": "DB_POSTGRES_DBNAME",
|
||||
"host": "DB_POSTGRES_HOST",
|
||||
|
||||
@@ -63,20 +63,12 @@
|
||||
},
|
||||
"database": {
|
||||
"secret": "",
|
||||
"type": "cassandra",
|
||||
"type": "mongodb",
|
||||
"encryption": "aes-256-cbc",
|
||||
"mongodb": {
|
||||
"uri": "mongodb://mongo:27017",
|
||||
"database": "tdrive"
|
||||
},
|
||||
"cassandra": {
|
||||
"contactPoints": ["scylladb:9042"],
|
||||
"localDataCenter": "datacenter1",
|
||||
"keyspace": "tdrive",
|
||||
"wait": false,
|
||||
"retries": 10,
|
||||
"delay": 200
|
||||
}
|
||||
},
|
||||
"message-queue": {
|
||||
"// possible 'type' values are": "'amqp' or 'local'",
|
||||
|
||||
@@ -8,15 +8,6 @@
|
||||
"secret": "",
|
||||
"mongodb": {
|
||||
"uri": "mongodb://mongo:27017"
|
||||
},
|
||||
"cassandra": {
|
||||
"contactPoints": ["scylladb:9042"],
|
||||
"wait": true,
|
||||
"retries": 50,
|
||||
"delay": 500,
|
||||
"queryOptions": {
|
||||
"consistency": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"message-queue": {
|
||||
|
||||
@@ -129,7 +129,6 @@
|
||||
"archiver": "^5.3.1",
|
||||
"axios": "^1.6.8",
|
||||
"bcrypt": "^5.0.1",
|
||||
"cassandra-driver": "^4.6.0",
|
||||
"class-transformer": "^0.3.1",
|
||||
"cli-table": "^0.3.6",
|
||||
"config": "^3.3.2",
|
||||
|
||||
@@ -29,14 +29,6 @@ export default function printConfigSummary(useIcons: boolean = !!process.env.HAV
|
||||
"user",
|
||||
user,
|
||||
],
|
||||
cassandra: ({ contactPoints, localDataCenter, keyspace }) => [
|
||||
"host",
|
||||
contactPoints,
|
||||
"datacenter",
|
||||
localDataCenter,
|
||||
"keyspace",
|
||||
keyspace,
|
||||
],
|
||||
S3: ({ endPoint, port, accessKey, bucket }) => [
|
||||
"host",
|
||||
[endPoint, port],
|
||||
@@ -62,7 +54,6 @@ export default function printConfigSummary(useIcons: boolean = !!process.env.HAV
|
||||
elasticsearch: "ES",
|
||||
opensearch: "OS",
|
||||
mongodb: "mongo",
|
||||
cassandra: "Cass.",
|
||||
};
|
||||
if (useIcons) {
|
||||
const makeLyingLengthIcon = str => ({ length: 1, toString: _ => str } as string);
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
node_modules
|
||||
yarn.lock
|
||||
@@ -1,4 +0,0 @@
|
||||
Edit the file to your needs
|
||||
|
||||
yarn install
|
||||
node index.js
|
||||
@@ -1,214 +0,0 @@
|
||||
/* eslint-disable */
|
||||
var cassandra = require("cassandra-driver");
|
||||
|
||||
var fromAuthProvider = new cassandra.auth.PlainTextAuthProvider("", "");
|
||||
var fromContactPoints = [""];
|
||||
var fromKeyspace = "tdrive";
|
||||
|
||||
var toAuthProvider = new cassandra.auth.PlainTextAuthProvider("", "");
|
||||
var toContactPoints = [""];
|
||||
var toKeyspace = "tdrive";
|
||||
|
||||
var forceUpdateAll = false;
|
||||
var ignoreTables = ["notification"];
|
||||
var countersTables = ["statistics", "stats_counter", "channel_counters", "scheduled_queue_counter"];
|
||||
|
||||
var specialConversions = {
|
||||
"tdrive.group_user.date_added": value => {
|
||||
return new Date(value).getTime();
|
||||
},
|
||||
};
|
||||
|
||||
// -- start process
|
||||
|
||||
var fromClient = new cassandra.Client({
|
||||
localDataCenter: "datacenter1",
|
||||
contactPoints: fromContactPoints,
|
||||
authProvider: fromAuthProvider,
|
||||
keyspace: fromKeyspace,
|
||||
});
|
||||
|
||||
var toClient = new cassandra.Client({
|
||||
localDataCenter: "datacenter1",
|
||||
contactPoints: toContactPoints,
|
||||
authProvider: toAuthProvider,
|
||||
keyspace: toKeyspace,
|
||||
queryOptions: {
|
||||
consistency: cassandra.types.consistencies.quorum,
|
||||
},
|
||||
});
|
||||
|
||||
// -- Get all tables and copy schema
|
||||
|
||||
async function client(origin, query, parameters, options) {
|
||||
return await new Promise((resolve, reject) => {
|
||||
origin.execute(query, [], {}, function (err, result) {
|
||||
if (err) {
|
||||
reject({ err, result });
|
||||
} else {
|
||||
resolve(result);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const result = await client(
|
||||
fromClient,
|
||||
"SELECT table_name from system_schema.tables WHERE keyspace_name = '" + fromKeyspace + "'",
|
||||
[],
|
||||
{},
|
||||
);
|
||||
for (row of result.rows) {
|
||||
try {
|
||||
const fromTable = fromKeyspace + "." + row.table_name;
|
||||
const toTable = toKeyspace + "." + row.table_name;
|
||||
|
||||
if (ignoreTables.includes(row.table_name)) {
|
||||
console.log(fromTable.padEnd(50) + " | " + "ignored".padEnd(20) + " | ⏺");
|
||||
continue;
|
||||
}
|
||||
|
||||
let fromCount = 0;
|
||||
|
||||
const destColumns = (
|
||||
await client(
|
||||
toClient,
|
||||
"SELECT column_name from system_schema.columns where keyspace_name = '" +
|
||||
toKeyspace +
|
||||
"' and table_name = '" +
|
||||
row.table_name +
|
||||
"'",
|
||||
[],
|
||||
{},
|
||||
)
|
||||
).rows.map(r => r.column_name);
|
||||
|
||||
try {
|
||||
const fromResult = await client(
|
||||
fromClient,
|
||||
"SELECT count(*) from " + fromTable + "",
|
||||
[],
|
||||
{},
|
||||
);
|
||||
fromCount = parseInt(fromResult.rows[0].count);
|
||||
} catch (err) {
|
||||
fromCount = NaN;
|
||||
}
|
||||
|
||||
if (destColumns.length === 0) {
|
||||
if (fromCount > 0)
|
||||
console.log(
|
||||
fromTable.padEnd(50) + " | " + ("not_in_dest" + "/" + fromCount).padEnd(20) + " | ⏺",
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const toResult = await client(toClient, "SELECT count(*) from " + toTable + "", [], {});
|
||||
const toCount = parseInt(toResult.rows[0].count);
|
||||
|
||||
console.log(
|
||||
fromTable.padEnd(50) +
|
||||
" | " +
|
||||
(toCount + "/" + fromCount).padEnd(20) +
|
||||
" | " +
|
||||
(toCount >= fromCount ? "✅" : "❌"),
|
||||
);
|
||||
|
||||
if (row.table_name.indexOf("counter") >= 0 || countersTables.includes(row.table_name)) {
|
||||
console.log(
|
||||
fromTable.padEnd(50) + " | " + ("counter_table" + "/" + fromCount).padEnd(20) + " | 🧮",
|
||||
);
|
||||
if (fromCount > toCount || !fromCount || forceUpdateAll) {
|
||||
await new Promise(r => {
|
||||
fromClient.eachRow(
|
||||
"SELECT JSON * from " + fromTable,
|
||||
[],
|
||||
{ prepare: true, fetchSize: 1000 },
|
||||
async function (n, row) {
|
||||
//
|
||||
//TODO handle counters (it is special !)
|
||||
},
|
||||
async function (err, result) {
|
||||
if (result && result.nextPage) {
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
result.nextPage();
|
||||
} else {
|
||||
r();
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (fromCount > toCount || !fromCount || forceUpdateAll) {
|
||||
await new Promise(r => {
|
||||
fromClient.eachRow(
|
||||
"SELECT JSON * from " + fromTable,
|
||||
[],
|
||||
{ prepare: true, fetchSize: 1000 },
|
||||
async function (n, row) {
|
||||
try {
|
||||
const json = JSON.parse(row["[json]"]);
|
||||
|
||||
//The from table can have additional depreciated fields, we need to remove them
|
||||
const filteredJson = {};
|
||||
for (const col of destColumns) {
|
||||
if (
|
||||
typeof json[col] == "string" &&
|
||||
(json[col] || "").match(
|
||||
/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]+/,
|
||||
)
|
||||
) {
|
||||
json[col] = json[col].split(".")[0];
|
||||
}
|
||||
|
||||
if (specialConversions[fromTable + "." + col]) {
|
||||
json[col] = specialConversions[fromTable + "." + col](json[col]);
|
||||
}
|
||||
|
||||
if (json[col] !== undefined) filteredJson[col] = json[col];
|
||||
}
|
||||
|
||||
await client(
|
||||
toClient,
|
||||
"INSERT INTO " +
|
||||
toTable +
|
||||
" JSON '" +
|
||||
JSON.stringify(filteredJson).replace(/'/g, "'$&") +
|
||||
"'",
|
||||
[],
|
||||
{},
|
||||
);
|
||||
} catch (err) {
|
||||
console.log(err);
|
||||
}
|
||||
},
|
||||
async function (err, result) {
|
||||
if (result && result.nextPage) {
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
result.nextPage();
|
||||
} else {
|
||||
r();
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
console.log(
|
||||
fromTable.padEnd(50) + " | " + ("error" + "/" + fromCount).padEnd(20) + " | ❌",
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
console.log(err);
|
||||
continue;
|
||||
}
|
||||
|
||||
//TODO copy content
|
||||
}
|
||||
})();
|
||||
@@ -1,9 +0,0 @@
|
||||
{
|
||||
"name": "copy-cluster",
|
||||
"version": "1.0.0",
|
||||
"main": "index.js",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"cassandra-driver": "^4.6.3"
|
||||
}
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
Edit the file to your needs
|
||||
|
||||
yarn install
|
||||
node index.js
|
||||
@@ -1,88 +0,0 @@
|
||||
/* eslint-disable */
|
||||
var cassandra = require("cassandra-driver");
|
||||
|
||||
var authProvider = new cassandra.auth.PlainTextAuthProvider("", "");
|
||||
var contactPoints = [""];
|
||||
var from = "table_a";
|
||||
var to = "table_b";
|
||||
|
||||
var client = new cassandra.Client({
|
||||
localDataCenter: "datacenter1",
|
||||
contactPoints: contactPoints,
|
||||
authProvider: authProvider,
|
||||
keyspace: "tdrive",
|
||||
});
|
||||
|
||||
//Copy object to other table
|
||||
(async () => {
|
||||
const query = "SELECT * FROM " + from;
|
||||
const parameters = [];
|
||||
|
||||
let count = 0;
|
||||
let max = 0;
|
||||
let min = new Date().getTime();
|
||||
|
||||
let rows = [];
|
||||
|
||||
const options = { prepare: true, fetchSize: 1000 };
|
||||
|
||||
await new Promise(r => {
|
||||
client.eachRow(
|
||||
query,
|
||||
parameters,
|
||||
options,
|
||||
function (n, row) {
|
||||
min = Math.min(parseInt(row.created_at.toString()), min);
|
||||
max = Math.max(parseInt(row.created_at.toString()), max);
|
||||
count++;
|
||||
if (count % 100 == 0) {
|
||||
console.log(count);
|
||||
}
|
||||
rows.push(row);
|
||||
},
|
||||
function (err, result) {
|
||||
if (result && result.nextPage) {
|
||||
result.nextPage();
|
||||
} else {
|
||||
r();
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
console.log("Downloaded ", count, " rows of ", from, ". Starting copy...");
|
||||
|
||||
let copyCount = 0;
|
||||
|
||||
for (let i = 0; i < count; i++) {
|
||||
copyCount++;
|
||||
if (copyCount % 100 == 0) {
|
||||
console.log(copyCount, "of", count);
|
||||
}
|
||||
|
||||
const row = rows[i];
|
||||
|
||||
await new Promise(r => {
|
||||
let query =
|
||||
"UPDATE " +
|
||||
to +
|
||||
" SET answers=?, created_at=?, created_by=?, last_activity=?, participants=?, updated_at=? WHERE id=?";
|
||||
client.execute(
|
||||
query,
|
||||
[
|
||||
row.answers,
|
||||
row.created_at,
|
||||
row.created_by,
|
||||
row.last_activity,
|
||||
row.participants,
|
||||
row.updated_at,
|
||||
row.id,
|
||||
],
|
||||
() => {
|
||||
r();
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
console.log("Ended with ", count, "threads");
|
||||
})();
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"name": "copy-table",
|
||||
"version": "1.0.0",
|
||||
"main": "index.js",
|
||||
"license": "MIT"
|
||||
}
|
||||
@@ -1,17 +1,11 @@
|
||||
import { DatabaseType } from ".";
|
||||
import { ConnectionOptions, Connector } from "./orm/connectors";
|
||||
import {
|
||||
CassandraConnectionOptions,
|
||||
CassandraConnector,
|
||||
} from "./orm/connectors/cassandra/cassandra";
|
||||
import { MongoConnectionOptions, MongoConnector } from "./orm/connectors/mongodb/mongodb";
|
||||
import { PostgresConnectionOptions, PostgresConnector } from "./orm/connectors/postgres/postgres";
|
||||
|
||||
export class ConnectorFactory {
|
||||
public create(type: DatabaseType, options: ConnectionOptions, secret: string): Connector {
|
||||
switch (type) {
|
||||
case "cassandra":
|
||||
return new CassandraConnector(type, options as CassandraConnectionOptions, secret);
|
||||
case "mongodb":
|
||||
return new MongoConnector(type, options as MongoConnectionOptions, secret);
|
||||
case "postgres":
|
||||
|
||||
@@ -3,7 +3,6 @@ import { ConnectorFactory } from "./connector-factory";
|
||||
import { Connector } from "./orm/connectors";
|
||||
import Manager from "./orm/manager";
|
||||
import Repository from "./orm/repository/repository";
|
||||
import { CassandraConnectionOptions } from "./orm/connectors/cassandra/cassandra";
|
||||
import { MongoConnectionOptions } from "./orm/connectors/mongodb/mongodb";
|
||||
import { EntityTarget } from "./orm/types";
|
||||
import { RepositoryManager } from "./orm/repository/manager";
|
||||
@@ -48,9 +47,6 @@ export default class DatabaseService implements DatabaseServiceAPI {
|
||||
}
|
||||
}
|
||||
|
||||
export declare type ConnectionOptions =
|
||||
| MongoConnectionOptions
|
||||
| CassandraConnectionOptions
|
||||
| PostgresConnectionOptions;
|
||||
export declare type ConnectionOptions = MongoConnectionOptions | PostgresConnectionOptions;
|
||||
|
||||
export declare type DatabaseType = "mongodb" | "cassandra" | "postgres";
|
||||
export declare type DatabaseType = "mongodb" | "postgres";
|
||||
|
||||
-565
@@ -1,565 +0,0 @@
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/explicit-module-boundary-types */
|
||||
import cassandra, { types } from "cassandra-driver";
|
||||
import { md5 } from "../../../../../../../crypto";
|
||||
import { defer, Subject, throwError, timer } from "rxjs";
|
||||
import { concat, delayWhen, retryWhen, take, tap } from "rxjs/operators";
|
||||
import { UpsertOptions } from "..";
|
||||
import { logger } from "../../../../../../framework";
|
||||
import { getEntityDefinition, unwrapPrimarykey } from "../../utils";
|
||||
import { EntityDefinition, ColumnDefinition, ObjectType } from "../../types";
|
||||
import { AbstractConnector } from "../abstract-connector";
|
||||
import {
|
||||
transformValueToDbString,
|
||||
cassandraType,
|
||||
transformValueFromDbString,
|
||||
} from "./typeTransforms";
|
||||
import { FindOptions } from "../../repository/repository";
|
||||
import { ListResult, Pagination } from "../../../../../../framework/api/crud-service";
|
||||
import { Paginable } from "../../../../../../framework/api/crud-service";
|
||||
import { buildSelectQuery } from "./query-builder";
|
||||
|
||||
export { CassandraPagination } from "./pagination";
|
||||
|
||||
export interface CassandraConnectionOptions {
|
||||
contactPoints: string[];
|
||||
localDataCenter: string;
|
||||
username: string;
|
||||
password: string;
|
||||
keyspace: string;
|
||||
|
||||
/**
|
||||
* Consistency level
|
||||
*/
|
||||
queryOptions: { consistency: number };
|
||||
|
||||
/**
|
||||
* Wait for keyspace and tables to be created at init
|
||||
*/
|
||||
wait: boolean;
|
||||
|
||||
/**
|
||||
* When wait = true, retry to get the resources N times where N = retries
|
||||
*/
|
||||
retries?: number;
|
||||
|
||||
/**
|
||||
* Delay in ms between the retries. The delay is growing each time a retry fails like delay = retryCount * delay
|
||||
*/
|
||||
delay?: number;
|
||||
}
|
||||
|
||||
export class CassandraConnector extends AbstractConnector<CassandraConnectionOptions> {
|
||||
private client: cassandra.Client;
|
||||
private keyspaceExists = false;
|
||||
|
||||
getClient(): cassandra.Client {
|
||||
return this.client;
|
||||
}
|
||||
|
||||
async init(): Promise<this> {
|
||||
if (!this.client) {
|
||||
await this.connect();
|
||||
}
|
||||
|
||||
try {
|
||||
await this.createKeyspace();
|
||||
} catch (err) {
|
||||
logger.warn("services.database.orm.cassandra - Keyspace can not be created", err);
|
||||
}
|
||||
|
||||
if (this.options.wait) {
|
||||
await this.waitForKeyspace(this.options.delay, this.options.retries);
|
||||
}
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
createKeyspace(): Promise<cassandra.types.ResultSet> {
|
||||
const query = `CREATE KEYSPACE IF NOT EXISTS ${this.options.keyspace} WITH replication = {'class': 'NetworkTopologyStrategy', 'datacenter1': '2'} AND durable_writes = true;`;
|
||||
logger.info(query);
|
||||
return this.client.execute(query);
|
||||
}
|
||||
|
||||
async isKeyspaceCreated(): Promise<boolean> {
|
||||
let result;
|
||||
|
||||
if (this.keyspaceExists) {
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
result = await this.client.execute(
|
||||
`SELECT * FROM system_schema.keyspaces where keyspace_name = '${this.options.keyspace}'`,
|
||||
);
|
||||
|
||||
if (!result) {
|
||||
throw new Error("No result for keyspace query");
|
||||
}
|
||||
} catch (err) {
|
||||
throw new Error("Keyspace query error");
|
||||
}
|
||||
|
||||
if (result) {
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
this.keyspaceExists = true;
|
||||
logger.info(`Keyspace '${this.options.keyspace}' found.`);
|
||||
}
|
||||
|
||||
return result ? Promise.resolve(true) : Promise.reject(new Error("Keyspace not found"));
|
||||
}
|
||||
|
||||
waitForKeyspace(delay: number = 500, retries: number = 10): Promise<boolean> {
|
||||
const subject = new Subject<boolean>();
|
||||
const obs$ = defer(() => this.isKeyspaceCreated());
|
||||
|
||||
obs$
|
||||
.pipe(
|
||||
retryWhen(errors =>
|
||||
errors.pipe(
|
||||
delayWhen((_, i) => timer(i * delay)),
|
||||
//delay(1000), if we want fixed delay
|
||||
tap(() => logger.debug("services.database.orm.cassandra - Retrying...")),
|
||||
take(retries),
|
||||
concat(throwError("Maximum number of retries reached")),
|
||||
),
|
||||
),
|
||||
)
|
||||
.subscribe(
|
||||
() => logger.debug("services.database.orm.cassandra - Keyspace has been found"),
|
||||
err => {
|
||||
logger.error(
|
||||
{ err },
|
||||
"services.database.orm.cassandra - Error while getting keyspace information",
|
||||
);
|
||||
subject.error(new Error("Can not find keyspace information"));
|
||||
},
|
||||
() => subject.complete(),
|
||||
);
|
||||
|
||||
return subject.toPromise();
|
||||
}
|
||||
|
||||
async drop(): Promise<this> {
|
||||
logger.info("Drop database is not implemented for security reasons.");
|
||||
return this;
|
||||
}
|
||||
|
||||
async connect(): Promise<this> {
|
||||
if (this.client) {
|
||||
return this;
|
||||
}
|
||||
|
||||
// Environment variable format is comma separated string
|
||||
const contactPoints =
|
||||
typeof this.options.contactPoints === "string"
|
||||
? (this.options.contactPoints as string).split(",")
|
||||
: this.options.contactPoints;
|
||||
|
||||
const cassandraOptions: cassandra.DseClientOptions = {
|
||||
contactPoints: contactPoints,
|
||||
localDataCenter: this.options.localDataCenter,
|
||||
queryOptions: {},
|
||||
};
|
||||
|
||||
if (this.options.username && this.options.password) {
|
||||
cassandraOptions.authProvider = new cassandra.auth.PlainTextAuthProvider(
|
||||
this.options.username,
|
||||
this.options.password,
|
||||
);
|
||||
}
|
||||
|
||||
//Set default consistency level to quorum
|
||||
cassandraOptions.queryOptions.consistency =
|
||||
this.options?.queryOptions?.consistency || types.consistencies.quorum;
|
||||
|
||||
this.client = new cassandra.Client(cassandraOptions);
|
||||
await this.client.connect();
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
async disconnect(): Promise<this> {
|
||||
if (this.client) await this.client.shutdown();
|
||||
this.client = null;
|
||||
return this;
|
||||
}
|
||||
|
||||
async getTableDefinition(name: string): Promise<string[]> {
|
||||
let result: string[];
|
||||
|
||||
try {
|
||||
const query = `SELECT column_name from system_schema.columns WHERE keyspace_name = '${this.options.keyspace}' AND table_name='${name}';`;
|
||||
const dbResult = await this.client.execute(query);
|
||||
result = dbResult.rows.map(row => row.values()[0]);
|
||||
} catch (err) {
|
||||
throw "Table query error";
|
||||
}
|
||||
|
||||
return result ? Promise.resolve(result || []) : Promise.resolve([]);
|
||||
}
|
||||
|
||||
async createTable(
|
||||
entity: EntityDefinition,
|
||||
columns: { [name: string]: ColumnDefinition },
|
||||
): Promise<boolean> {
|
||||
await this.waitForKeyspace(this.options.delay, this.options.retries);
|
||||
|
||||
let result = true;
|
||||
|
||||
// --- Generate column and key definition --- //
|
||||
const primaryKey = entity.options.primaryKey || [];
|
||||
|
||||
if (primaryKey.length === 0) {
|
||||
logger.error(
|
||||
`services.database.orm.cassandra - Primary key was not defined for table ${entity.name}`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
const partitionKeyPart = primaryKey.shift();
|
||||
const partitionKey: string[] =
|
||||
typeof partitionKeyPart === "string" ? [partitionKeyPart] : partitionKeyPart;
|
||||
const clusteringKeys: string[] = primaryKey as string[];
|
||||
if ([...partitionKey, ...clusteringKeys].some(key => columns[key] === undefined)) {
|
||||
logger.error(
|
||||
`services.database.orm.cassandra - One primary key item doesn't exists in entity columns for table ${entity.name}`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
const clusteringOrderBy = clusteringKeys.map(key => {
|
||||
const direction: "ASC" | "DESC" = columns[key].options.order || "ASC";
|
||||
return `${key} ${direction}`;
|
||||
});
|
||||
const clusteringOrderByString =
|
||||
clusteringOrderBy.length > 0
|
||||
? `WITH CLUSTERING ORDER BY (${clusteringOrderBy.join(", ")})`
|
||||
: "";
|
||||
|
||||
const allKeys = [`(${partitionKey.join(", ")})`, ...clusteringKeys];
|
||||
const primaryKeyString = `(${allKeys.join(", ")})`;
|
||||
|
||||
const columnsString = Object.keys(columns)
|
||||
.map(colName => {
|
||||
const definition = columns[colName];
|
||||
return `${colName} ${cassandraType[definition.type]}`;
|
||||
})
|
||||
.join(",\n");
|
||||
|
||||
// --- Generate final create table query --- //
|
||||
let query = `
|
||||
CREATE TABLE IF NOT EXISTS ${this.options.keyspace}.${entity.name}
|
||||
(
|
||||
${columnsString},
|
||||
PRIMARY KEY ${primaryKeyString}
|
||||
) ${clusteringOrderByString};`;
|
||||
|
||||
// --- Alter table if not up to date --- //
|
||||
const existingColumns = await this.getTableDefinition(entity.name);
|
||||
if (existingColumns.length > 0) {
|
||||
logger.debug(
|
||||
`services.database.orm.cassandra - Existing columns for table ${entity.name}, generating altertable queries`,
|
||||
);
|
||||
const alterQueryColumns = Object.keys(columns)
|
||||
.filter(colName => existingColumns.indexOf(colName) < 0)
|
||||
.map(colName => `${colName} ${cassandraType[columns[colName].type]}`);
|
||||
query = `ALTER TABLE ${this.options.keyspace}.${entity.name} ADD (${alterQueryColumns.join(
|
||||
", ",
|
||||
)})`;
|
||||
|
||||
if (alterQueryColumns.length === 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// --- Write table --- //
|
||||
try {
|
||||
logger.debug(`service.database.orm.createTable - Creating table ${entity.name} : ${query}`);
|
||||
await this.client.execute(query);
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{ err },
|
||||
`service.database.orm.createTable - creation error for table ${entity.name} : ${err.message}`,
|
||||
);
|
||||
result = false;
|
||||
}
|
||||
|
||||
// --- Create indexes --- //
|
||||
if (entity.options.globalIndexes) {
|
||||
for (const globalIndex of entity.options.globalIndexes) {
|
||||
const indexName = globalIndex.join("_");
|
||||
const indexDbName = `index_${md5(indexName)}`;
|
||||
|
||||
const query = `CREATE INDEX IF NOT EXISTS ${indexDbName} ON ${this.options.keyspace}."${
|
||||
entity.name
|
||||
}" (${
|
||||
globalIndex.length === 1 ? globalIndex[0] : `(${globalIndex[0]}), ${globalIndex[1]}`
|
||||
})`;
|
||||
|
||||
try {
|
||||
logger.debug(
|
||||
`service.database.orm.createTable - Creating index ${indexName} (${indexDbName}) : ${query}`,
|
||||
);
|
||||
await this.client.execute(query);
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{ err },
|
||||
`service.database.orm.createTable - creation error for index ${indexName} (${indexDbName}) : ${err.message}`,
|
||||
);
|
||||
result = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async upsert(entities: any[], _options: UpsertOptions = {}): Promise<boolean[]> {
|
||||
return new Promise(resolve => {
|
||||
const promises: Promise<boolean>[] = [];
|
||||
|
||||
entities.forEach(entity => {
|
||||
const { columnsDefinition, entityDefinition } = getEntityDefinition(entity);
|
||||
const primaryKey = unwrapPrimarykey(entityDefinition);
|
||||
|
||||
//Set updated content
|
||||
const set = Object.keys(columnsDefinition)
|
||||
.filter(key => primaryKey.indexOf(key) === -1)
|
||||
.filter(key => entity[columnsDefinition[key].nodename] !== undefined)
|
||||
.map(key => [
|
||||
`${key}`,
|
||||
`${transformValueToDbString(
|
||||
entity[columnsDefinition[key].nodename],
|
||||
columnsDefinition[key].type,
|
||||
{
|
||||
columns: columnsDefinition[key].options,
|
||||
secret: this.secret,
|
||||
column: { key },
|
||||
},
|
||||
)}`,
|
||||
]);
|
||||
//Set primary key
|
||||
const where = primaryKey.map(key => [
|
||||
`${key}`,
|
||||
`${transformValueToDbString(
|
||||
entity[columnsDefinition[key].nodename],
|
||||
columnsDefinition[key].type,
|
||||
{
|
||||
columns: columnsDefinition[key].options,
|
||||
secret: this.secret,
|
||||
disableSalts: true,
|
||||
column: { key },
|
||||
},
|
||||
)}`,
|
||||
]);
|
||||
|
||||
// Add time-to-live options
|
||||
let ttlOptions = "";
|
||||
if (entityDefinition.options.ttl && entityDefinition.options.ttl > 0) {
|
||||
ttlOptions = `USING TTL ${entityDefinition.options.ttl}`;
|
||||
}
|
||||
|
||||
// Insert and update are equivalent for most of Cassandra
|
||||
// Update is prefered because the only solution for counters
|
||||
let query = `UPDATE ${this.options.keyspace}.${
|
||||
entityDefinition.name
|
||||
} ${ttlOptions} SET ${set.map(e => `${e[0]} = ${e[1]}`).join(", ")} WHERE ${where
|
||||
.map(e => `${e[0]} = ${e[1]}`)
|
||||
.join(" AND ")}`;
|
||||
|
||||
// If no "set" part, we cannot do an update, so insert
|
||||
if (set.length === 0) {
|
||||
query = `INSERT INTO ${this.options.keyspace}.${
|
||||
entityDefinition.name
|
||||
} ${ttlOptions} (${where.map(e => e[0]).join(", ")}) VALUES (${where
|
||||
.map(e => e[1])
|
||||
.join(", ")})`;
|
||||
}
|
||||
|
||||
logger.debug(`service.database.orm.upsert - Query: "${query}"`);
|
||||
|
||||
promises.push(
|
||||
new Promise(async resolve => {
|
||||
try {
|
||||
await this.getClient().execute(query);
|
||||
resolve(true);
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
{ err },
|
||||
`services.database.orm.cassandra - Error with CQL query: ${query}`,
|
||||
);
|
||||
resolve(false);
|
||||
}
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
Promise.all(promises).then(resolve);
|
||||
});
|
||||
}
|
||||
|
||||
async remove(entities: any[]): Promise<boolean[]> {
|
||||
return new Promise(resolve => {
|
||||
const promises: Promise<boolean>[] = [];
|
||||
|
||||
entities.forEach(entity => {
|
||||
const { columnsDefinition, entityDefinition } = getEntityDefinition(entity);
|
||||
const primaryKey = unwrapPrimarykey(entityDefinition);
|
||||
|
||||
//Set primary key
|
||||
const where = primaryKey.map(
|
||||
key =>
|
||||
`${key} = ${transformValueToDbString(
|
||||
entity[columnsDefinition[key].nodename],
|
||||
columnsDefinition[key].type,
|
||||
{
|
||||
columns: columnsDefinition[key].options,
|
||||
secret: this.secret,
|
||||
disableSalts: true,
|
||||
column: { key },
|
||||
},
|
||||
)}`,
|
||||
);
|
||||
|
||||
const query = `DELETE FROM ${this.options.keyspace}.${
|
||||
entityDefinition.name
|
||||
} WHERE ${where.join(" AND ")}`;
|
||||
logger.debug(`services.database.orm.cassandra.remove - Query: ${query}`);
|
||||
|
||||
promises.push(
|
||||
new Promise(async resolve => {
|
||||
try {
|
||||
await this.getClient().execute(query);
|
||||
resolve(true);
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
{ err },
|
||||
`services.database.orm.cassandra.remove - Error with CQL query: ${query}`,
|
||||
);
|
||||
resolve(false);
|
||||
}
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
Promise.all(promises).then(resolve);
|
||||
});
|
||||
}
|
||||
|
||||
async find<Table>(
|
||||
entityType: Table,
|
||||
filters: any,
|
||||
options: FindOptions = {},
|
||||
): Promise<ListResult<Table>> {
|
||||
const instance = new (entityType as any)();
|
||||
const { columnsDefinition, entityDefinition } = getEntityDefinition(instance);
|
||||
|
||||
const query = buildSelectQuery<Table>(
|
||||
entityType as unknown as ObjectType<Table>,
|
||||
filters,
|
||||
options,
|
||||
{
|
||||
keyspace: this.options.keyspace,
|
||||
secret: this.secret,
|
||||
},
|
||||
);
|
||||
|
||||
logger.debug(`services.database.orm.cassandra.find - Query: ${query}`);
|
||||
|
||||
const results = await this.getClient().execute(query, [], {
|
||||
fetchSize: parseInt(options.pagination.limitStr) || 100,
|
||||
pageState: options.pagination.page_token || undefined,
|
||||
prepare: false,
|
||||
});
|
||||
|
||||
const entities: Table[] = [];
|
||||
results.rows.forEach(row => {
|
||||
const entity = new (entityType as any)();
|
||||
Object.keys(row).forEach(key => {
|
||||
if (columnsDefinition[key]) {
|
||||
entity[columnsDefinition[key].nodename] = transformValueFromDbString(
|
||||
row[key],
|
||||
columnsDefinition[key].type,
|
||||
{ column: { key: key, ...columnsDefinition[key].options }, secret: this.secret },
|
||||
);
|
||||
}
|
||||
});
|
||||
entities.push(entity);
|
||||
});
|
||||
|
||||
const nextPage: Paginable = new Pagination(
|
||||
results.pageState,
|
||||
options.pagination.limitStr || "100",
|
||||
);
|
||||
logger.debug(
|
||||
`services.database.orm.cassandra.find - Query Result (items=${entities.length}): ${query}`,
|
||||
);
|
||||
|
||||
return new ListResult<Table>(entityDefinition.type, entities, nextPage);
|
||||
}
|
||||
}
|
||||
|
||||
export function waitForTable(
|
||||
client: cassandra.Client,
|
||||
keyspace: string,
|
||||
table: string,
|
||||
retries: number = 10,
|
||||
delay: number = 500,
|
||||
): Promise<boolean> {
|
||||
const subject = new Subject<boolean>();
|
||||
const obs$ = defer(() => checkForTable(client, keyspace, table));
|
||||
|
||||
obs$
|
||||
.pipe(
|
||||
retryWhen(errors =>
|
||||
errors.pipe(
|
||||
delayWhen((_, i) => timer(i * delay)),
|
||||
//delay(1000),
|
||||
tap(() =>
|
||||
logger.debug("services.database.orm.cassandra - Retrying to get table metadata..."),
|
||||
),
|
||||
take(retries),
|
||||
concat(throwError("Maximum number of retries reached")),
|
||||
),
|
||||
),
|
||||
)
|
||||
.subscribe(
|
||||
() => logger.debug(`services.database.orm.cassandra - Table ${table} has been found`),
|
||||
err => {
|
||||
logger.debug({ err }, `services.database.orm.cassandra - Table ${table} error`);
|
||||
subject.error(new Error("Can not find table"));
|
||||
},
|
||||
() => subject.complete(),
|
||||
);
|
||||
|
||||
return subject.toPromise();
|
||||
}
|
||||
|
||||
async function checkForTable(
|
||||
client: cassandra.Client,
|
||||
keyspace: string,
|
||||
table: string,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const result: cassandra.types.ResultSet = await client.execute(
|
||||
`SELECT * FROM system_schema.tables WHERE keyspace_name='${keyspace}' AND table_name='${table}'`,
|
||||
);
|
||||
|
||||
const tableMetadata = result.rows[0];
|
||||
|
||||
logger.debug(
|
||||
"services.database.orm.cassandra.checkForTable - Table metadata %o",
|
||||
tableMetadata,
|
||||
);
|
||||
|
||||
if (!tableMetadata) {
|
||||
throw new Error("Can not find table metadata");
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
{ err },
|
||||
"services.database.orm.cassandra.checkForTable - Error while getting table metadata",
|
||||
);
|
||||
throw new Error("Error while getting table metadata");
|
||||
}
|
||||
}
|
||||
-19
@@ -1,19 +0,0 @@
|
||||
import { Paginable } from "../../../../../../framework/api/crud-service";
|
||||
import { Pagination } from "../../../../../../framework/api/crud-service";
|
||||
|
||||
export class CassandraPagination extends Pagination {
|
||||
limit = 100;
|
||||
|
||||
private constructor(readonly page_token: string, readonly limitStr = "100") {
|
||||
super(page_token, limitStr);
|
||||
this.limit = Number.parseInt(limitStr, 10);
|
||||
}
|
||||
|
||||
static from(pagination: Paginable): CassandraPagination {
|
||||
return new CassandraPagination(pagination.page_token, pagination.limitStr);
|
||||
}
|
||||
|
||||
static next(current: Pagination, pageState: string): Paginable {
|
||||
return new Pagination(pageState, current.limitStr);
|
||||
}
|
||||
}
|
||||
-136
@@ -1,136 +0,0 @@
|
||||
import { FindOptions } from "../../repository/repository";
|
||||
import { ObjectType } from "../../types";
|
||||
import { getEntityDefinition, secureOperators, filteringRequired } from "../../utils";
|
||||
import { transformValueToDbString } from "./typeTransforms";
|
||||
|
||||
export function buildSelectQuery<Entity>(
|
||||
entityType: ObjectType<Entity>,
|
||||
filters: Record<string, unknown>,
|
||||
findOptions: FindOptions,
|
||||
options: {
|
||||
secret?: string;
|
||||
keyspace: string;
|
||||
} = {
|
||||
secret: "",
|
||||
keyspace: "tdrive",
|
||||
},
|
||||
): string {
|
||||
const instance = new (entityType as any)();
|
||||
const { columnsDefinition, entityDefinition } = getEntityDefinition(instance);
|
||||
let allowFiltering = false;
|
||||
|
||||
const where = Object.keys(filters)
|
||||
.map(key => {
|
||||
let result: string;
|
||||
const filter = filters[key];
|
||||
|
||||
if (filteringRequired(key)) {
|
||||
allowFiltering = true;
|
||||
}
|
||||
|
||||
if (!filter) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (Array.isArray(filter)) {
|
||||
if (!filter.length) {
|
||||
return;
|
||||
}
|
||||
const inClause: string[] = filter.map(
|
||||
value =>
|
||||
`${transformValueToDbString(value, columnsDefinition[key].type, {
|
||||
columns: columnsDefinition[key].options,
|
||||
secret: options.secret || "",
|
||||
disableSalts: true,
|
||||
})}`,
|
||||
);
|
||||
|
||||
result = `${key} IN (${inClause.join(",")})`;
|
||||
} else {
|
||||
result = `${key} = ${transformValueToDbString(filter, columnsDefinition[key].type, {
|
||||
columns: columnsDefinition[key].options,
|
||||
secret: options.secret || "",
|
||||
disableSalts: true,
|
||||
})}`;
|
||||
}
|
||||
|
||||
return result;
|
||||
})
|
||||
.filter(Boolean);
|
||||
|
||||
secureOperators(transformValueToDbString, findOptions, entityType, options);
|
||||
|
||||
const whereClause = `${[
|
||||
...where,
|
||||
...(buildComparison(findOptions) || []),
|
||||
...(buildIn(findOptions) || []),
|
||||
...(buildLike(findOptions) || []),
|
||||
].join(" AND ")}`.trimEnd();
|
||||
|
||||
let orderByClause = "";
|
||||
if (findOptions.pagination?.reversed) {
|
||||
orderByClause = `${entityDefinition.options.primaryKey
|
||||
.slice(1)
|
||||
.map(
|
||||
(key: string) =>
|
||||
`${key} ${(columnsDefinition[key].options.order || "ASC") === "ASC" ? "DESC" : "ASC"}`,
|
||||
)
|
||||
.join(", ")}`;
|
||||
}
|
||||
|
||||
const query = `SELECT * FROM ${options.keyspace}.${entityDefinition.name} ${
|
||||
whereClause.trim().length ? "WHERE " + whereClause : ""
|
||||
} ${orderByClause.trim().length ? "ORDER BY " + orderByClause : ""}`
|
||||
.trimEnd()
|
||||
.concat(allowFiltering ? " ALLOW FILTERING;" : ";");
|
||||
|
||||
return query;
|
||||
}
|
||||
|
||||
export function buildComparison(options: FindOptions = {}): string[] {
|
||||
let lessClause;
|
||||
let lessEqualClause;
|
||||
let greaterClause;
|
||||
let greaterEqualClause;
|
||||
|
||||
if (options.$lt) {
|
||||
lessClause = options.$lt.map(element => `${element[0]} < ${element[1]}`);
|
||||
}
|
||||
|
||||
if (options.$lte) {
|
||||
lessEqualClause = options.$lte.map(element => `${element[0]} <= ${element[1]}`);
|
||||
}
|
||||
|
||||
if (options.$gt) {
|
||||
greaterClause = options.$gt.map(element => `${element[0]} > ${element[1]}`);
|
||||
}
|
||||
|
||||
if (options.$gte) {
|
||||
greaterEqualClause = options.$gte.map(element => `${element[0]} >= ${element[1]}`);
|
||||
}
|
||||
|
||||
return [
|
||||
...(lessClause || []),
|
||||
...(lessEqualClause || []),
|
||||
...(greaterClause || []),
|
||||
...(greaterEqualClause || []),
|
||||
];
|
||||
}
|
||||
|
||||
export function buildIn(options: FindOptions = {}): string[] {
|
||||
let inClauses: string[];
|
||||
if (options.$in) {
|
||||
inClauses = options.$in.map(element => `${element[0]} IN (${element[1].join(",")})`);
|
||||
}
|
||||
|
||||
return inClauses || [];
|
||||
}
|
||||
|
||||
export function buildLike(options: FindOptions = {}): string[] {
|
||||
let likeClauses: string[];
|
||||
if (options.$like) {
|
||||
likeClauses = options.$like.map(element => `${element[0]} LIKE '%${element[1]}%`);
|
||||
}
|
||||
|
||||
return likeClauses || [];
|
||||
}
|
||||
-174
@@ -1,174 +0,0 @@
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
/* eslint-disable @typescript-eslint/no-unused-vars */
|
||||
/* eslint-disable @typescript-eslint/explicit-module-boundary-types */
|
||||
import { isBoolean, isInteger, isNull, isUndefined } from "lodash";
|
||||
import { ColumnOptions, ColumnType } from "../../types";
|
||||
import { decrypt, encrypt } from "../../../../../../../crypto";
|
||||
import { logger } from "../../../../../../../../core/platform/framework";
|
||||
|
||||
export const cassandraType = {
|
||||
encoded_string: "TEXT",
|
||||
encoded_json: "TEXT",
|
||||
string: "TEXT",
|
||||
json: "TEXT",
|
||||
number: "BIGINT",
|
||||
timeuuid: "TIMEUUID",
|
||||
uuid: "UUID",
|
||||
counter: "COUNTER",
|
||||
blob: "BLOB",
|
||||
boolean: "BOOLEAN",
|
||||
|
||||
// backward compatibility
|
||||
tdrive_boolean: "TINYINT",
|
||||
tdrive_int: "INT", //Depreciated
|
||||
tdrive_datetime: "TIMESTAMP", //Depreciated
|
||||
};
|
||||
|
||||
type TransformOptions = {
|
||||
secret?: any;
|
||||
disableSalts?: boolean;
|
||||
columns?: ColumnOptions;
|
||||
column?: any;
|
||||
};
|
||||
|
||||
export const transformValueToDbString = (
|
||||
v: any,
|
||||
type: ColumnType,
|
||||
options: TransformOptions = {},
|
||||
): string => {
|
||||
if (type === "tdrive_datetime") {
|
||||
if (isNaN(v) || isNull(v)) {
|
||||
return "null";
|
||||
}
|
||||
return `${v}`;
|
||||
}
|
||||
|
||||
if (type === "number" || type === "tdrive_int") {
|
||||
if (isNull(v)) {
|
||||
return "null";
|
||||
}
|
||||
if (isNaN(v)) {
|
||||
return "null";
|
||||
}
|
||||
return `${parseInt(v)}`;
|
||||
}
|
||||
if (type === "uuid" || type === "timeuuid") {
|
||||
if (isNull(v)) {
|
||||
return "null";
|
||||
}
|
||||
|
||||
v = (v || "").toString().replace(/[^a-zA-Z0-9-]/g, "");
|
||||
return `${v}`;
|
||||
}
|
||||
if (type === "boolean") {
|
||||
//Security to avoid string with "false" in it
|
||||
if (!isInteger(v) && !isBoolean(v) && !isNull(v) && !isUndefined(v)) {
|
||||
throw new Error(`'${v}' is not a ${type}`);
|
||||
}
|
||||
return `${!!v}`;
|
||||
}
|
||||
if (type === "tdrive_boolean") {
|
||||
if (!isBoolean(v)) {
|
||||
throw new Error(`'${v}' is not a ${type}`);
|
||||
}
|
||||
return v ? "1" : "0";
|
||||
}
|
||||
if (type === "encoded_string" || type === "encoded_json") {
|
||||
if (type === "encoded_json") {
|
||||
try {
|
||||
v = JSON.stringify(v);
|
||||
} catch (err) {
|
||||
v = null;
|
||||
}
|
||||
}
|
||||
const encrypted = encrypt(v, options.secret, { disableSalts: options.disableSalts });
|
||||
return `'${(encrypted.data || "").toString().replace(/'/gm, "''")}'`;
|
||||
}
|
||||
if (type === "blob") {
|
||||
return "''"; //Not implemented yet
|
||||
}
|
||||
if (type === "string" || type === "json") {
|
||||
if (type === "json" && v !== null) {
|
||||
try {
|
||||
v = JSON.stringify(v);
|
||||
} catch (err) {
|
||||
v = null;
|
||||
}
|
||||
}
|
||||
return `'${(v || "").toString().replace(/'/gm, "''")}'`;
|
||||
}
|
||||
if (type === "counter") {
|
||||
if (isNaN(v)) throw new Error("Counter value should be a number");
|
||||
return `${options.column.key} + ${v}`;
|
||||
}
|
||||
return `'${(v || "").toString().replace(/'/gm, "''")}'`;
|
||||
};
|
||||
|
||||
export const transformValueFromDbString = (
|
||||
v: any,
|
||||
type: string,
|
||||
options: TransformOptions = {},
|
||||
): any => {
|
||||
logger.trace(`Transform value %o of type ${type}`, v);
|
||||
|
||||
if (type === "tdrive_datetime") {
|
||||
return new Date(`${v}`).getTime();
|
||||
}
|
||||
|
||||
if (v !== null && (type === "encoded_string" || type === "encoded_json")) {
|
||||
let decryptedValue: any;
|
||||
|
||||
if (typeof v === "string" && v.trim() === "") {
|
||||
return v;
|
||||
}
|
||||
|
||||
try {
|
||||
decryptedValue = decrypt(v, options.secret).data;
|
||||
} catch (err) {
|
||||
logger.debug(`Can not decrypt data (${err.message}) %o of type ${type}`, v);
|
||||
|
||||
decryptedValue = v;
|
||||
}
|
||||
|
||||
if (type === "encoded_json") {
|
||||
try {
|
||||
decryptedValue = JSON.parse(decryptedValue);
|
||||
} catch (err) {
|
||||
logger.debug(
|
||||
{ err },
|
||||
`Can not parse JSON from decrypted data %o of type ${type}`,
|
||||
decryptedValue,
|
||||
);
|
||||
decryptedValue = null;
|
||||
}
|
||||
}
|
||||
|
||||
return decryptedValue;
|
||||
}
|
||||
|
||||
if (type === "tdrive_boolean" || type === "boolean") {
|
||||
return Boolean(v).valueOf();
|
||||
}
|
||||
|
||||
if (type === "json") {
|
||||
try {
|
||||
return JSON.parse(v);
|
||||
} catch (err) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
if (type === "uuid" || type === "timeuuid") {
|
||||
return v ? String(v).valueOf() : null;
|
||||
}
|
||||
|
||||
if (type === "number") {
|
||||
return new Number(v).valueOf();
|
||||
}
|
||||
|
||||
if (type === "counter") {
|
||||
return new Number(v).valueOf();
|
||||
}
|
||||
|
||||
return v;
|
||||
};
|
||||
+1
-6
@@ -1,6 +1,5 @@
|
||||
import { Initializable } from "../../../../../framework";
|
||||
import { DatabaseType } from "../..";
|
||||
import { CassandraConnectionOptions } from "./cassandra/cassandra";
|
||||
import { MongoConnectionOptions } from "./mongodb/mongodb";
|
||||
import { ColumnDefinition, EntityDefinition } from "../types";
|
||||
import { FindOptions } from "../repository/repository";
|
||||
@@ -8,7 +7,6 @@ import { ListResult } from "../../../../../framework/api/crud-service";
|
||||
import { PostgresConnectionOptions } from "./postgres/postgres";
|
||||
|
||||
export * from "./mongodb/mongodb";
|
||||
export * from "./cassandra/cassandra";
|
||||
|
||||
export type UpsertOptions = {
|
||||
action?: "INSERT" | "UPDATE";
|
||||
@@ -68,7 +66,4 @@ export interface Connector extends Initializable {
|
||||
): Promise<ListResult<EntityType>>;
|
||||
}
|
||||
|
||||
export declare type ConnectionOptions =
|
||||
| MongoConnectionOptions
|
||||
| CassandraConnectionOptions
|
||||
| PostgresConnectionOptions;
|
||||
export declare type ConnectionOptions = MongoConnectionOptions | PostgresConnectionOptions;
|
||||
|
||||
-166
@@ -1,166 +0,0 @@
|
||||
import "reflect-metadata";
|
||||
import { describe, expect, it } from "@jest/globals";
|
||||
import {
|
||||
buildSelectQuery,
|
||||
buildComparison,
|
||||
buildIn,
|
||||
} from "../../../../../../../../../src/core/platform/services/database/services/orm/connectors/cassandra/query-builder";
|
||||
import { DriveFile } from "../../../../../../../../../src/services/documents/entities/drive-file";
|
||||
|
||||
describe("The QueryBuilder module", () => {
|
||||
describe("The buildSelectQuery function", () => {
|
||||
it("should build a valid query from primary key parameters", () => {
|
||||
const filters = {
|
||||
company_id: "comp1",
|
||||
parent_id: "parent1",
|
||||
};
|
||||
const result = buildSelectQuery<DriveFile>(DriveFile, filters, {}, { keyspace: "tdrive" });
|
||||
|
||||
expect(result).toEqual(
|
||||
"SELECT * FROM tdrive.drive_files WHERE company_id = comp1 AND parent_id = 'parent1';",
|
||||
);
|
||||
});
|
||||
|
||||
it("should build a valid query from primary key parameters and comparison", () => {
|
||||
const filters = {
|
||||
company_id: "comp1",
|
||||
parent_id: "parent1",
|
||||
};
|
||||
const result = buildSelectQuery<DriveFile>(
|
||||
DriveFile,
|
||||
filters,
|
||||
{
|
||||
$lt: [["size", 1000]],
|
||||
},
|
||||
{ keyspace: "tdrive" },
|
||||
);
|
||||
|
||||
expect(result).toEqual(
|
||||
"SELECT * FROM tdrive.drive_files WHERE company_id = comp1 AND parent_id = 'parent1' AND size < 1000;",
|
||||
);
|
||||
});
|
||||
|
||||
it("should build IN query from array parameters", () => {
|
||||
const filters = {
|
||||
company_id: "comp1",
|
||||
parent_id: "parent1",
|
||||
creator: ["u1", "u2", "u3"],
|
||||
};
|
||||
const result = buildSelectQuery<DriveFile>(DriveFile, filters, {}, { keyspace: "tdrive" });
|
||||
|
||||
expect(result).toEqual(
|
||||
"SELECT * FROM tdrive.drive_files WHERE company_id = comp1 AND parent_id = 'parent1' AND creator IN (u1,u2,u3);",
|
||||
);
|
||||
});
|
||||
|
||||
it("should not build IN query from array parameters when array is empty", () => {
|
||||
const filters = {
|
||||
company_id: "comp1",
|
||||
parent_id: "parent1",
|
||||
user_id: [],
|
||||
};
|
||||
const result = buildSelectQuery<DriveFile>(DriveFile, filters, {}, { keyspace: "tdrive" });
|
||||
|
||||
expect(result).toEqual(
|
||||
"SELECT * FROM tdrive.drive_files WHERE company_id = comp1 AND parent_id = 'parent1';",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("The buildComparison function", () => {
|
||||
it("should create a valid < string", () => {
|
||||
expect(
|
||||
buildComparison({
|
||||
$lt: [["foo", 1]],
|
||||
}),
|
||||
).toContain("foo < 1");
|
||||
|
||||
const result = buildComparison({
|
||||
$lt: [
|
||||
["foo", 1],
|
||||
["bar", 2],
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toContain("foo < 1");
|
||||
expect(result).toContain("bar < 2");
|
||||
});
|
||||
|
||||
it("should create a valid <= string", () => {
|
||||
expect(
|
||||
buildComparison({
|
||||
$lte: [["foo", 1]],
|
||||
}),
|
||||
).toContain("foo <= 1");
|
||||
|
||||
const result = buildComparison({
|
||||
$lte: [
|
||||
["foo", 1],
|
||||
["bar", 2],
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toContain("foo <= 1");
|
||||
expect(result).toContain("bar <= 2");
|
||||
});
|
||||
|
||||
it("should create a valid > string", () => {
|
||||
expect(
|
||||
buildComparison({
|
||||
$gt: [["foo", 1]],
|
||||
}),
|
||||
).toContain("foo > 1");
|
||||
|
||||
const result = buildComparison({
|
||||
$gt: [
|
||||
["foo", 1],
|
||||
["bar", 2],
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toContain("foo > 1");
|
||||
expect(result).toContain("bar > 2");
|
||||
});
|
||||
|
||||
it("should create a valid >= string", () => {
|
||||
expect(
|
||||
buildComparison({
|
||||
$gte: [["foo", 1]],
|
||||
}),
|
||||
).toContain("foo >= 1");
|
||||
|
||||
const result = buildComparison({
|
||||
$gte: [
|
||||
["foo", 1],
|
||||
["bar", 2],
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toContain("foo >= 1");
|
||||
expect(result).toContain("bar >= 2");
|
||||
});
|
||||
|
||||
it("should combine conditions", () => {
|
||||
const result = buildComparison({
|
||||
$gt: [["foo", 1]],
|
||||
$gte: [["bar", 2]],
|
||||
$lt: [["baz", 3]],
|
||||
$lte: [["qix", 4]],
|
||||
});
|
||||
expect(result).toContain("foo > 1");
|
||||
expect(result).toContain("bar >= 2");
|
||||
expect(result).toContain("baz < 3");
|
||||
expect(result).toContain("qix <= 4");
|
||||
});
|
||||
});
|
||||
|
||||
describe("The buildIn function", () => {
|
||||
it("should create a id IN (ids) string", () => {
|
||||
expect(
|
||||
buildIn({
|
||||
$in: [["id", ["1", "2", "3"]]],
|
||||
}),
|
||||
).toContain("id IN (1,2,3)");
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,36 +0,0 @@
|
||||
{
|
||||
"websocket": {
|
||||
"path": "/socket/",
|
||||
"adapters": {
|
||||
"types": []
|
||||
},
|
||||
"auth": {
|
||||
"jwt": {
|
||||
"secret": "supersecret"
|
||||
}
|
||||
}
|
||||
},
|
||||
"auth": {
|
||||
"jwt": {
|
||||
"secret": "supersecret"
|
||||
}
|
||||
},
|
||||
"database": {
|
||||
"secret": "ab63bb3e90c0271c9a1c06651a7c0967eab8851a7a897766",
|
||||
"type": "cassandra",
|
||||
"cassandra": {
|
||||
"contactPoints": ["scylladb:9042"],
|
||||
"localDataCenter": "datacenter1",
|
||||
"keyspace": "tdrive",
|
||||
"wait": false,
|
||||
"retries": 10,
|
||||
"delay": 200
|
||||
}
|
||||
},
|
||||
"pubsub": {
|
||||
"type": "amqp",
|
||||
"amqp": {
|
||||
"urls": ["amqp://guest:guest@rabbitmq:5672"]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
{
|
||||
"websocket": {
|
||||
"path": "/socket/",
|
||||
"adapters": {
|
||||
"types": []
|
||||
},
|
||||
"auth": {
|
||||
"jwt": {
|
||||
"secret": "supersecret (use the same as in php)"
|
||||
}
|
||||
}
|
||||
},
|
||||
"auth": {
|
||||
"jwt": {
|
||||
"secret": "supersecret (use the same as in php)"
|
||||
}
|
||||
},
|
||||
"database": {
|
||||
"secret": "GET YOUR SECRET FROM PHP Parameters.php: db.secret",
|
||||
"type": "cassandra",
|
||||
"cassandra": {
|
||||
"contactPoints": [
|
||||
"scylladb:9042"
|
||||
],
|
||||
"localDataCenter": "datacenter1",
|
||||
"keyspace": "tdrive",
|
||||
"wait": false,
|
||||
"retries": 10,
|
||||
"delay": 200
|
||||
}
|
||||
},
|
||||
"pubsub": {
|
||||
"urls": [
|
||||
"amqp://admin:admin@rabbitmq:5672"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -1,220 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace Configuration;
|
||||
|
||||
class Parameters extends \Common\Configuration
|
||||
{
|
||||
|
||||
public $configuration = [];
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this->configuration = [
|
||||
"env" => [
|
||||
"type" => "prod",
|
||||
"timer" => false,
|
||||
"admin_api_token" => "",
|
||||
"licence_key" => "",
|
||||
"standalone" => true,
|
||||
//"frontend_server_name" => "http://localhost:8000/", // define only if needed
|
||||
"server_name" => "http://localhost:8000/",
|
||||
"internal_server_name" => "http://nginx/"
|
||||
],
|
||||
"jwt" => [
|
||||
"secret" => "supersecret",
|
||||
"expiration" => 60*60, //1 hour
|
||||
"refresh_expiration" => 60*60*24*31 //1 month
|
||||
],
|
||||
"node" => [
|
||||
"api" => "http://node:3000/private/",
|
||||
"secret" => "api_supersecret"
|
||||
],
|
||||
"db" => [
|
||||
"driver" => "pdo_cassandra",
|
||||
"host" => "scylladb",
|
||||
"port" => 9042,
|
||||
"dbname" => "tdrive",
|
||||
"user" => "root",
|
||||
"password" => "root",
|
||||
"encryption_key" => "ab63bb3e90c0271c9a1c06651a7c0967eab8851a7a897766",
|
||||
"replication" => "{'class': 'SimpleStrategy', 'replication_factor': '1'}"
|
||||
],
|
||||
"es" => [
|
||||
"host" => false
|
||||
],
|
||||
"queues" => [
|
||||
"rabbitmq" => [
|
||||
"use" => true,
|
||||
"host" => "rabbitmq",
|
||||
"port" => 5672,
|
||||
"username" => "guest",
|
||||
"password" => "guest",
|
||||
"vhost" => false
|
||||
]
|
||||
],
|
||||
"storage" => [
|
||||
"drive_previews_tmp_folder" => "/tmp/",
|
||||
"drive_tmp_folder" => "/tmp/",
|
||||
"drive_salt" => "SecretPassword",
|
||||
"providers" => [
|
||||
[
|
||||
"label" => "someOpenStack",
|
||||
"type" => "openstack",
|
||||
"use" => false,
|
||||
"project_id" => "",
|
||||
"auth_url" => "https//auth.cloud.ovh.net/v3",
|
||||
"buckets_prefix" => "",
|
||||
"disable_encryption" => false,
|
||||
"buckets" => [
|
||||
"fr" => [
|
||||
"public" => "",
|
||||
"private" => "",
|
||||
"region" => "SBG5"
|
||||
]
|
||||
],
|
||||
"user" => [
|
||||
"id" => "",
|
||||
"password" => "",
|
||||
"domain_name" => "default"
|
||||
]
|
||||
],
|
||||
[
|
||||
"label" => "someS3",
|
||||
"type" => "S3",
|
||||
"base_url" => "http://127.0.0.1:9000",
|
||||
"use" => false,
|
||||
"version" => "latest",
|
||||
"disable_encryption" => false,
|
||||
"buckets_prefix" => "dev.",
|
||||
"buckets" => [
|
||||
"fr" => "eu-west-3"
|
||||
],
|
||||
"credentials" => [
|
||||
"key" => "",
|
||||
"secret" => ""
|
||||
]
|
||||
],
|
||||
[
|
||||
"label" => "someLocal",
|
||||
"type" => "local",
|
||||
"use" => true,
|
||||
"disable_encryption" => false,
|
||||
"location" => "../drive/",
|
||||
"preview_location" => "../web/medias/",
|
||||
"preview_public_path" => "/medias/"
|
||||
],
|
||||
],
|
||||
],
|
||||
"mail" => [
|
||||
"sender" => [
|
||||
"host" => "",
|
||||
"port" => "",
|
||||
"username" => "",
|
||||
"password" => "",
|
||||
"auth_mode" => "plain"
|
||||
],
|
||||
"template_dir" => "/src/Tdrive/Core/Resources/views/",
|
||||
"tdrive_domain_url" => "https://tdriveapp.com/",
|
||||
"from" => "noreply@tdriveapp.com",
|
||||
"from_name" => "Tdrive",
|
||||
"tdrive_address" => "Tdrive, 54000 Nancy, France",
|
||||
"dkim" => [
|
||||
"private_key" => "",
|
||||
"domain_name" => '',
|
||||
"selector" => ''
|
||||
],
|
||||
"mailjet"=> [
|
||||
"contact_list_subscribe"=> false,
|
||||
"contact_list_newsletter"=> false
|
||||
]
|
||||
],
|
||||
"push_notifications" => [
|
||||
"apns_certificate" => __DIR__ . "/certs/apns_prod.pem",
|
||||
"firebase_api_key" => "KEY",
|
||||
],
|
||||
|
||||
//Defaults values for all clients but editable in database
|
||||
"defaults" => [
|
||||
"applications" => [
|
||||
"tdrive_drive" => [ "default" => true ], //False to not install
|
||||
"tdrive_calendar" => [ "default" => true ],
|
||||
"tdrive_tasks" => [ "default" => true ],
|
||||
"connectors" => [
|
||||
"jitsi" => [ "default" => true ],
|
||||
]
|
||||
],
|
||||
"connectors" => [
|
||||
],
|
||||
"branding" => [
|
||||
"name" => "Tdrive",
|
||||
"enable_newsletter" => false,
|
||||
/*
|
||||
"header" => [
|
||||
"logo" => 'https://openpaas.linagora.com/images/white-logo.svg',
|
||||
"apps" => [
|
||||
[
|
||||
"name"=> 'Accueil',
|
||||
"url"=> 'https://openpaas.linagora.com/',
|
||||
"icon"=> 'https://openpaas.linagora.com/images/application-menu/home-icon.svg',
|
||||
],
|
||||
[
|
||||
"name"=> 'Inbox',
|
||||
"url"=> 'https://openpaas.linagora.com/#/unifiedinbox/inbox',
|
||||
"icon"=> 'https://openpaas.linagora.com/unifiedinbox/images/inbox-icon.svg',
|
||||
],
|
||||
[
|
||||
"name"=> 'Calendrier',
|
||||
"url"=> 'https://openpaas.linagora.com/#/calendar',
|
||||
"icon"=> 'https://openpaas.linagora.com/calendar/images/calendar-icon.svg',
|
||||
],
|
||||
[
|
||||
"name"=> 'Contacts',
|
||||
"url"=> 'https://openpaas.linagora.com/#/contact/addressbooks/',
|
||||
"icon"=> 'https://openpaas.linagora.com/contact/images/contacts-icon.svg',
|
||||
],
|
||||
],
|
||||
],
|
||||
"style" => [
|
||||
"color" => '#2196F3',
|
||||
"default_border_radius" => '2',
|
||||
],
|
||||
"link" => "https://open-paas.org/",
|
||||
"logo" => "https://open-paas.org/wp-content/uploads/2019/10/openpaas.png"
|
||||
*/
|
||||
],
|
||||
"auth" => [
|
||||
"internal" => [
|
||||
"use" => true,
|
||||
"disable_account_creation" => false,
|
||||
"disable_email_verification" => true
|
||||
],
|
||||
"openid" => [
|
||||
"use" => false,
|
||||
"provider_uri" => 'https://auth0.com',
|
||||
"client_id" => '',
|
||||
"client_secret" => '',
|
||||
//"disable_logout_redirect" => false
|
||||
"provider_config" => [
|
||||
//token_endpoint
|
||||
//token_endpoint_auth_methods_supported
|
||||
//userinfo_endpoint
|
||||
//end_session_endpoint
|
||||
//authorization_endpoint
|
||||
]
|
||||
],
|
||||
"cas" => [
|
||||
"use" => false,
|
||||
"base_url" => '',
|
||||
"email_key" => '',
|
||||
"lastname_key" => '',
|
||||
"firstname_key" => ''
|
||||
],
|
||||
"console" => [
|
||||
"use" => false
|
||||
]
|
||||
]
|
||||
],
|
||||
];
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
version: "3.4"
|
||||
|
||||
services:
|
||||
scylladb:
|
||||
image: scylladb/scylla:4.1.0
|
||||
command: --smp 1 --memory 400M
|
||||
ports:
|
||||
- 9042:9042
|
||||
volumes:
|
||||
- ./docker-data/scylladb:/var/lib/scylla
|
||||
rabbitmq:
|
||||
image: rabbitmq:3
|
||||
|
||||
node:
|
||||
image: tdrive/tdrive-node
|
||||
ports:
|
||||
- 9229:9229
|
||||
environment:
|
||||
- DEV=dev
|
||||
- DB_DRIVER=cassandra
|
||||
- DB_CASSANDRA_URI=scylladb:9042
|
||||
- DB_CASSANDRA_KEYSPACE=tdrive
|
||||
- PUBSUB_URLS=amqp://guest:guest@rabbitmq:5672
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/tdrive-node/Dockerfile
|
||||
target: development
|
||||
volumes:
|
||||
- ./backend/node:/usr/src/app
|
||||
- ./docker-data/documents/:/storage/
|
||||
depends_on:
|
||||
- scylladb
|
||||
- rabbitmq
|
||||
- elasticsearch
|
||||
links:
|
||||
- scylladb
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
|
||||
elasticsearch:
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.4
|
||||
environment:
|
||||
- cluster.name=docker-cluster
|
||||
- discovery.type=single-node
|
||||
- bootstrap.memory_lock=true
|
||||
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
|
||||
ulimits:
|
||||
memlock:
|
||||
soft: -1
|
||||
hard: -1
|
||||
volumes:
|
||||
- ./docker-data/es:/usr/share/elasticsearch/data
|
||||
|
||||
nginx:
|
||||
image: tdrive/tdrive-frontend
|
||||
environment:
|
||||
- DEV=dev
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/tdrive-frontend/Dockerfile
|
||||
ports:
|
||||
- 8000:80
|
||||
depends_on:
|
||||
- node
|
||||
volumes:
|
||||
- ./docker-data/logs/nginx/:/var/log/nginx
|
||||
- ./docker-data/letsencrypt/:/etc/letsencrypt/
|
||||
- ./frontend/:/tdrive-react/
|
||||
- ./docker-data/drive-preview/:/tdrive-core/web/medias/
|
||||
- ./docker-data/uploads/:/tdrive-core/web/upload/
|
||||
Reference in New Issue
Block a user