eduvia/workavia-drive
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ref: remove unused backed code

Browse Source
This commit is contained in:
montaghanmy
2023-04-03 08:56:49 +01:00
parent 57c996e030
commit 1458a11e7c
97 changed files with 263 additions and 7200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
twake/backend/node/src/cli/cmds/applications_cmds/ls.ts
-54
View File
@@ -1,54 +0,0 @@
import yargs from "yargs";
import ora from "ora";
import twake from "../../../twake";
import Table from "cli-table";
import * as process from "process";
import gr from "../../../services/global-resolver";
/**
* Merge command parameters. Check the builder definition below for more details.
*/
type CLIArgs = Record<string, unknown>;
const services = [
"storage",
"counter",
"message-queue",
"platform-services",
"applications",
"auth",
"realtime",
"websocket",
// "user",
"search",
"database",
"webserver",
// "statistics",
];
const command: yargs.CommandModule<unknown, CLIArgs> = {
command: "ls",
describe: "command that allow you to list applications (unpublished only)",
handler: async _argv => {
const spinner = ora({ text: "Retrieving applications" }).start();
const platform = await twake.run(services);
await gr.doInit(platform);
//
const unpublished = await gr.services.applications.marketplaceApps.listUnpublished(undefined);
//
const table = new Table({
head: ["ID", "Name", "Description"],
colWidths: [40, 20, 40],
});
unpublished.forEach((app: any) => {
table.push([app.id, app.identity.name, app.identity.description]);
});
spinner.stop();
console.log(table.toString());
await platform.stop();
process.exit(0);
},
};
export default command;
twake/backend/node/src/cli/cmds/applications_cmds/publish.ts
-89
View File
@@ -1,89 +0,0 @@
import yargs from "yargs";
import ora from "ora";
import twake from "../../../twake";
import Table from "cli-table";
import * as process from "process";
import gr from "../../../services/global-resolver";
/**
* Merge command parameters. Check the builder definition below for more details.
*/
type CLIArgs = {
id: string;
};
const services = [
"storage",
"counter",
"message-queue",
"platform-services",
"applications",
"auth",
"realtime",
"websocket",
"user",
"search",
"database",
"webserver",
"statistics",
];
const command: yargs.CommandModule<unknown, CLIArgs> = {
command: "publish <id>",
describe:
"command that allow you to validate an application and make it available on the marketplace",
builder: {
force: {
default: false,
type: "boolean",
description: "Force update unrequested application",
},
},
handler: async argv => {
if (argv.id) {
let spinner = ora({ text: "Retrieving application" }).start();
const platform = await twake.run(services);
await gr.doInit(platform);
let app = await gr.services.applications.marketplaceApps.get({ id: argv.id }, undefined);
spinner.stop();
if (!app) {
console.error(`Application ${argv.id} not found`);
process.exit(1);
}
let table = new Table();
table.push(app);
table.push({ name: app.identity.name });
table.push({ description: app.identity.description });
table.push({ website: app.identity.website });
table.push({ requested: app.publication.requested });
table.push({ published: app.publication.published });
console.log(table.toString());
if (app.publication.published) {
console.error("Application already published");
process.exit(1);
}
if (!app.publication.requested && !argv.force) {
console.error(
"Application is not requested to be published. Use --force if you want to publish it anyway",
);
process.exit(1);
}
spinner = ora({ text: "Publishing application" }).start();
await gr.services.applications.marketplaceApps.publish({ id: argv.id }, undefined);
app = await gr.services.applications.marketplaceApps.get({ id: argv.id }, undefined);
spinner.stop();
console.log("Application published");
table = new Table();
table.push(app);
table.push({ published: app.publication.published });
console.log(table.toString());
}
process.exit(0);
},
};
export default command;
twake/backend/node/src/cli/cmds/applications_cmds/unpublish.ts
-73
View File
@@ -1,73 +0,0 @@
import yargs from "yargs";
import ora from "ora";
import twake from "../../../twake";
import Table from "cli-table";
import * as process from "process";
import gr from "../../../services/global-resolver";
/**
* Merge command parameters. Check the builder definition below for more details.
*/
type CLIArgs = {
id: string;
};
const services = [
"storage",
"counter",
"message-queue",
"platform-services",
"applications",
"auth",
"realtime",
"websocket",
"user",
"search",
"database",
"webserver",
"statistics",
];
const command: yargs.CommandModule<unknown, CLIArgs> = {
command: "unpublish <id>",
describe: "command that allow you to make application unavailable on the marketplace",
handler: async argv => {
if (argv.id) {
let spinner = ora({ text: "Retrieving application" }).start();
const platform = await twake.run(services);
await gr.doInit(platform);
let app = await gr.services.applications.marketplaceApps.get({ id: argv.id }, undefined);
spinner.stop();
if (!app) {
console.error(`Application ${argv.id} not found`);
process.exit(1);
}
let table = new Table();
table.push(app);
table.push({ name: app.identity.name });
table.push({ description: app.identity.description });
table.push({ website: app.identity.website });
table.push({ requested: app.publication.requested });
table.push({ published: app.publication.published });
console.log(table.toString());
if (!app.publication.published) {
console.error("Application is not published");
process.exit(1);
}
spinner = ora({ text: "Unpublishing application" }).start();
await gr.services.applications.marketplaceApps.unpublish({ id: argv.id }, undefined);
app = await gr.services.applications.marketplaceApps.get({ id: argv.id }, undefined);
spinner.stop();
console.log("Application unpublished");
table = new Table();
table.push(app);
table.push({ published: app.publication.published });
console.log(table.toString());
}
process.exit(0);
},
};
export default command;
twake/backend/node/src/cli/cmds/export_cmds/export_company.ts
-69
View File
@@ -5,8 +5,6 @@ import { Pagination } from "../../../core/platform/framework/api/crud-service";
import WorkspaceUser from "../../../services/workspaces/entities/workspace_user"; import WorkspaceUser from "../../../services/workspaces/entities/workspace_user";
import { ChannelVisibility } from "../../../services/channels/types"; import { ChannelVisibility } from "../../../services/channels/types";
import { Channel, ChannelMember } from "../../../services/channels/entities"; import { Channel, ChannelMember } from "../../../services/channels/entities";
import { Message } from "../../../services/messages/entities/messages";
import { MessageWithReplies } from "../../../services/messages/types";
import { formatCompany } from "../../../services/user/utils"; import { formatCompany } from "../../../services/user/utils";
import gr from "../../../services/global-resolver"; import gr from "../../../services/global-resolver";
import { formatUser } from "../../../utils/users"; import { formatUser } from "../../../utils/users";
@@ -107,15 +105,6 @@ const command: yargs.CommandModule<unknown, CLIArgs> = {
} }
writeFileSync(`${output}/users.json`, JSON.stringify(users)); writeFileSync(`${output}/users.json`, JSON.stringify(users));
//Applications
console.log("- Create applications json file");
const applications = await gr.services.applications.companyApps.list(
new Pagination(),
{},
{ company: { id: company.id }, user: { id: "", server_request: true } },
);
writeFileSync(`${output}/applications.json`, JSON.stringify(applications));
//Channels //Channels
console.log("- Create channels json file"); console.log("- Create channels json file");
const directChannels: Channel[] = []; const directChannels: Channel[] = [];
@@ -201,64 +190,6 @@ const command: yargs.CommandModule<unknown, CLIArgs> = {
); );
} }
//Messages
console.log("- Create messages json file");
//Note: direct channels content is private and not needed for R&D
for (const channel of [...allPublicChannels /*, ...directChannels*/]) {
let threads: MessageWithReplies[] = [];
let messages: Message[] = [];
let pagination = new Pagination();
try {
do {
const page = await gr.services.messages.views.listChannel(
pagination,
{
include_users: false,
replies_per_thread: 10000,
emojis: false,
},
{
user: { id: "", server_request: true },
channel: {
company_id: channel.company_id,
workspace_id: channel.workspace_id,
id: channel.id,
},
},
);
for (const thread of page.getEntities()) {
messages = [...messages, ...thread.last_replies];
}
threads = [
...threads,
...page.getEntities().map(thread => {
thread.last_replies = undefined;
return thread;
}),
];
pagination = page.nextPage as Pagination;
} while (pagination.page_token);
} catch (err) {
console.log(`-- Error on the channel ${channel.id}`);
console.log(err);
}
mkdirSync(`${output}/workspaces/${channel.workspace_id}/channels/${channel.id}`, {
recursive: true,
});
writeFileSync(
`${output}/workspaces/${channel.workspace_id}/channels/${channel.id}/threads.json`,
JSON.stringify(threads),
);
writeFileSync(
`${output}/workspaces/${channel.workspace_id}/channels/${channel.id}/messages.json`,
JSON.stringify(messages),
);
}
await platform.stop(); await platform.stop();
}, },
}; };
twake/backend/node/src/cli/cmds/migration_cmds/application.ts
-264
View File
@@ -1,264 +0,0 @@
import yargs from "yargs";
import twake from "../../../twake";
import ora from "ora";
import { TwakePlatform } from "../../../core/platform/platform";
import { DatabaseServiceAPI } from "../../../core/platform/services/database/api";
import PhpApplication, {
DepreciatedDisplayConfiguration,
TYPE as phpTYPE,
} from "./php-application/php-application-entity";
import { Pagination } from "../../../core/platform/framework/api/crud-service";
import Application, { TYPE } from "../../../services/applications/entities/application";
import gr from "../../../services/global-resolver";
type Options = {
onlyApplication?: string;
replaceExisting?: boolean;
};
class ApplicationMigrator {
database: DatabaseServiceAPI;
constructor(readonly platform: TwakePlatform) {
this.database = this.platform.getProvider<DatabaseServiceAPI>("database");
}
public async run(options: Options = {}): Promise<void> {
const phpRepository = await this.database.getRepository(phpTYPE, PhpApplication);
const repository = await this.database.getRepository(TYPE, Application);
let page: Pagination = { limitStr: "100" };
do {
const applicationListResult = await phpRepository.find({}, { pagination: page }, undefined);
page = applicationListResult.nextPage as Pagination;
for (const application of applicationListResult.getEntities()) {
if (
!(await repository.findOne(
{
id: application.id,
},
{},
undefined,
)) ||
options.replaceExisting
) {
const newApplication = importDepreciatedFields(application);
await repository.save(newApplication, undefined);
}
}
} while (page.page_token);
}
}
const services = [
"storage",
"counter",
"platform-services",
"user",
"search",
"channels",
"database",
"webserver",
"message-queue",
"applications",
"console",
"auth",
"statistics",
"realtime",
];
const command: yargs.CommandModule<unknown, unknown> = {
command: "application",
describe: "command that allow you to migrate php applications to node",
builder: {
onlyApplication: {
default: null,
type: "string",
description: "Migrate only this application ID",
},
replaceExisting: {
default: false,
type: "boolean",
description: "Replace already migrated applications",
},
},
// eslint-disable-next-line @typescript-eslint/no-unused-vars
handler: async argv => {
const spinner = ora({ text: "Migrating php applications - " }).start();
const platform = await twake.run(services);
await gr.doInit(platform);
const migrator = new ApplicationMigrator(platform);
const onlyApplication = argv.onlyApplication as string | null;
const replaceExisting = (argv.replaceExisting || false) as boolean;
await migrator.run({
onlyApplication,
replaceExisting,
});
return spinner.stop();
},
};
export default command;
export const importDepreciatedFields = (application: PhpApplication): Application => {
const newApplication = new Application();
newApplication.id = application.id;
newApplication.company_id = application.group_id;
newApplication.is_default = application.is_default;
if (!newApplication.identity?.name) {
newApplication.identity = {
code:
application.depreciated_simple_name ||
(application.depreciated_name || "").toLocaleLowerCase(),
name: application.depreciated_name,
icon: application.depreciated_icon_url,
description: application.depreciated_description,
website: "http://twake.app/",
categories: [],
compatibility: ["twake"],
};
}
if (newApplication.publication?.published === undefined) {
newApplication.publication = newApplication.publication || {
published: false,
requested: false,
};
newApplication.publication.published = application.depreciated_is_available_to_public;
newApplication.publication.requested =
application.depreciated_public && !application.depreciated_twake_team_validation;
}
if (!newApplication.stats?.version) {
newApplication.stats = newApplication.stats || {
created_at: null,
updated_at: null,
version: null,
};
newApplication.stats.version = 1;
newApplication.stats.created_at = Date.now();
newApplication.stats.updated_at = Date.now();
}
if (!newApplication.api?.private_key) {
newApplication.api = newApplication.api || {
hooks_url: null,
allowed_ips: null,
private_key: null,
};
newApplication.api.hooks_url = application.depreciated_api_events_url;
newApplication.api.allowed_ips = application.depreciated_api_allowed_ip;
newApplication.api.private_key = application.depreciated_api_private_key;
}
if (newApplication.access?.write === undefined) {
newApplication.access = newApplication.access || {
read: null,
write: null,
delete: null,
hooks: null,
};
try {
newApplication.access.write = JSON.parse(application.depreciated_capabilities || "[]") || [];
newApplication.access.delete = JSON.parse(application.depreciated_capabilities || "[]") || [];
} catch (e) {
newApplication.access.write = [];
newApplication.access.delete = [];
}
try {
newApplication.access.read = JSON.parse(application.depreciated_privileges || "[]") || [];
} catch (e) {
newApplication.access.read = [];
}
try {
newApplication.access.hooks = JSON.parse(application.depreciated_hooks || "[]") || [];
} catch (e) {
newApplication.access.hooks = [];
}
}
newApplication.display = importDepreciatedDisplayFields(
newApplication,
JSON.parse(application.depreciated_display_configuration),
);
return newApplication;
};
export const importDepreciatedDisplayFields = (
application: Application,
depreciatedDisplay: DepreciatedDisplayConfiguration,
): Application["display"] => {
let display = application.display;
if (!display?.twake) {
display = display || { twake: {} };
display.twake = display.twake || {};
}
display.twake.tab = depreciatedDisplay?.channel_tab
? { url: depreciatedDisplay?.channel_tab?.iframe } || true
: undefined;
display.twake.standalone = depreciatedDisplay?.app
? { url: depreciatedDisplay?.app?.iframe } || true
: undefined;
display.twake.configuration = [];
if (depreciatedDisplay?.configuration?.can_configure_in_workspace)
display.twake.configuration.push("global");
if (depreciatedDisplay?.configuration?.can_configure_in_channel)
display.twake.configuration.push("channel");
display.twake.direct = depreciatedDisplay?.member_app
? { name: application.identity.name, icon: application.identity.icon } || true
: undefined;
if (depreciatedDisplay?.drive_module) {
display.twake.files = {
editor: undefined,
actions: [],
};
display.twake.files.editor = {
preview_url: depreciatedDisplay?.drive_module?.can_open_files?.preview_url,
edition_url: depreciatedDisplay?.drive_module?.can_open_files?.url,
extensions: [
...(depreciatedDisplay?.drive_module?.can_open_files?.main_ext || []),
...(depreciatedDisplay?.drive_module?.can_open_files?.other_ext || []),
],
empty_files: (depreciatedDisplay?.drive_module?.can_create_files as any) || [],
};
}
if (depreciatedDisplay?.messages_module) {
display.twake.chat = {
input:
depreciatedDisplay?.messages_module?.in_plus ||
depreciatedDisplay?.messages_module?.right_icon
? {
icon: application.identity.icon,
}
: undefined,
commands:
(depreciatedDisplay?.messages_module
?.commands as Application["display"]["twake"]["chat"]["commands"]) || undefined,
actions: depreciatedDisplay?.messages_module?.action
? [
{
name: depreciatedDisplay?.messages_module?.action.description,
id: "default",
},
]
: undefined,
};
}
return display;
};
twake/backend/node/src/cli/cmds/migration_cmds/channel-member-read-cursors.ts
-173
View File
@@ -1,173 +0,0 @@
import { DatabaseServiceAPI } from "../../../core/platform/services/database/api";
import { TwakePlatform } from "../../../core/platform/platform";
import Repository from "../../../core/platform/services/database/services/orm/repository/repository";
import { ChannelMember, ChannelMemberReadCursors } from "../../../services/channels/entities";
import {
ExecutionContext,
Paginable,
Pagination,
} from "../../../core/platform/framework/api/crud-service";
import gr from "../../../services/global-resolver";
import yargs from "yargs";
import ora from "ora";
import twake from "../../../twake";
class ChannelMemberReadCursorsMigrator {
readSectionRepository: Repository<ChannelMemberReadCursors>;
database: DatabaseServiceAPI;
constructor(readonly _platform: TwakePlatform) {
this.database = this._platform.getProvider<DatabaseServiceAPI>("database");
}
public async run(_options = {}, context?: ExecutionContext): Promise<void> {
const companyPagination: Paginable = new Pagination(null, "100");
const companies = await gr.services.companies.getCompanies(companyPagination);
context.user.server_request = true;
for (const company of companies.getEntities()) {
const { id: CompanyId } = company;
const workspaces = await gr.services.workspaces.getAllForCompany(CompanyId);
const workspaceIds = [...workspaces.map(({ id }) => id), "direct"];
for (const workspaceId of workspaceIds) {
const channels = await gr.services.channels.channels.getAllChannelsInWorkspace(
CompanyId,
workspaceId,
);
for (const channel of channels) {
const { id: channelId } = channel;
const membersPagination: Pagination = new Pagination(null, "100");
const threadsPagination: Pagination = new Pagination(null, "1");
let members: ChannelMember[];
try {
const membersList = await gr.services.channels.members.list(
membersPagination,
{},
{
...context,
channel: {
company_id: CompanyId,
id: channelId,
workspace_id: workspaceId,
},
},
);
members = membersList.getEntities();
} catch (error) {
continue;
}
let lastMessage = await gr.services.messages.views.listChannel(
threadsPagination,
{},
{
...context,
channel: {
company_id: CompanyId,
id: channelId,
workspace_id: workspaceId,
},
user: {
id: null,
},
},
);
const firstMessage = await gr.services.messages.views.listChannel(
{
...threadsPagination,
reversed: true,
},
{},
{
...context,
channel: {
company_id: CompanyId,
id: channelId,
workspace_id: workspaceId,
},
user: {
id: null,
},
},
);
if (!firstMessage.getEntities().length) {
continue;
}
if (!lastMessage.getEntities().length) {
lastMessage = firstMessage;
}
const firstMessageId = firstMessage.getEntities()[0].id;
const lastMessageId = lastMessage.getEntities()[0].id;
if (!firstMessageId) {
continue;
} else {
for (const member of members) {
await gr.services.channels.members.setChannelMemberReadSections(
{
start: firstMessageId,
end: lastMessageId,
},
{
...context,
channel_id: channelId,
workspace_id: workspaceId,
company: {
id: CompanyId,
},
user: {
id: member.user_id,
},
},
);
}
}
}
}
}
}
}
const services = [
"search",
"database",
"webserver",
"auth",
"counter",
"cron",
"message-queue",
"push",
"realtime",
"storage",
"tracker",
"websocket",
"email-pusher",
];
const command: yargs.CommandModule<unknown, unknown> = {
command: "channel-member-read-cursor-repair",
describe: "fixes the channel members read cursors for old messages",
builder: {},
handler: async () => {
const spinner = ora({ text: "Fixing channel members read cursors" }).start();
const platform = await twake.run(services);
await gr.doInit(platform);
const migrator = new ChannelMemberReadCursorsMigrator(platform);
await migrator.run({});
return spinner.stop();
},
};
export default command;
twake/backend/node/src/cli/cmds/migration_cmds/devices.ts
-103
View File
@@ -1,103 +0,0 @@
import yargs from "yargs";
import twake from "../../../twake";
import ora from "ora";
import { TwakePlatform } from "../../../core/platform/platform";
import { DatabaseServiceAPI } from "../../../core/platform/services/database/api";
import { PhpDevice, TYPE as phpTYPE } from "./php-device/php-device-entity";
import { Pagination } from "../../../core/platform/framework/api/crud-service";
import Device, { TYPE } from "../../../services/user/entities/device";
import User, { TYPE as userTYPE } from "../../../services/user/entities/user";
import _ from "lodash";
import gr from "../../../services/global-resolver";
type Options = {
replaceExisting?: boolean;
};
class DeviceMigrator {
database: DatabaseServiceAPI;
constructor(readonly platform: TwakePlatform) {
this.database = this.platform.getProvider<DatabaseServiceAPI>("database");
}
public async run(options: Options = {}): Promise<void> {
const phpRepository = await this.database.getRepository(phpTYPE, PhpDevice);
const userRepository = await this.database.getRepository(userTYPE, User);
const repository = await this.database.getRepository(TYPE, Device);
// Get all companies
let page: Pagination = { limitStr: "100" };
// For each devices
do {
const deviceListResult = await phpRepository.find({}, { pagination: page }, undefined);
page = deviceListResult.nextPage as Pagination;
for (const device of deviceListResult.getEntities()) {
if (
!(await repository.findOne({ id: device.value }, {}, undefined)) ||
options.replaceExisting
) {
if (device.type === "FCM" || device.type === "fcm") {
const newDevice = new Device();
newDevice.id = device.value;
newDevice.type = "FCM";
newDevice.user_id = device.user_id;
newDevice.version = device.version;
await repository.save(newDevice, undefined);
const user = await userRepository.findOne({ id: device.user_id }, {}, undefined);
if (user) {
user.devices = _.uniq([...(user.devices || []), newDevice.id]);
await userRepository.save(user, undefined);
}
}
}
}
} while (page.page_token);
}
}
const services = [
"storage",
"counter",
"platform-services",
"user",
"search",
"channels",
"database",
"webserver",
"message-queue",
"workspaces",
"console",
"auth",
];
const command: yargs.CommandModule<unknown, unknown> = {
command: "device",
describe: "command that allow you to migrate php devices to node",
builder: {
replaceExisting: {
default: false,
type: "boolean",
description: "Replace already migrated devices",
},
},
// eslint-disable-next-line @typescript-eslint/no-unused-vars
handler: async argv => {
const spinner = ora({ text: "Migrating php devices - " }).start();
const platform = await twake.run(services);
await gr.doInit(platform);
const migrator = new DeviceMigrator(platform);
const replaceExisting = (argv.replaceExisting || false) as boolean;
await migrator.run({
replaceExisting,
});
return spinner.stop();
},
};
export default command;
twake/backend/node/src/cli/cmds/migration_cmds/drive.ts
-40
View File
@@ -1,40 +0,0 @@
import ora from "ora";
import globalResolver from "../../../services/global-resolver";
import twake from "../../../twake";
import yargs from "yargs";
import DriveMigrator from "./php-drive-file/drive-migrator-service";
const services = [
"search",
"database",
"webserver",
"auth",
"counter",
"cron",
"message-queue",
"push",
"realtime",
"storage",
"tracker",
"websocket",
"email-pusher",
];
const command: yargs.CommandModule<unknown, unknown> = {
command: "drive",
describe: "migrate php drive items to node",
builder: {},
handler: async _argv => {
console.log("test");
const spinner = ora({ text: "Migrating php drive - " }).start();
const platform = await twake.run(services);
await globalResolver.doInit(platform);
const migrator = new DriveMigrator(platform);
await migrator.run();
return spinner.stop();
},
};
export default command;
twake/backend/node/src/cli/cmds/migration_cmds/message-channel-repair.ts
-96
View File
@@ -1,96 +0,0 @@
import yargs from "yargs";
import twake from "../../../twake";
import ora from "ora";
import { DatabaseServiceAPI } from "../../../core/platform/services/database/api";
import { TwakePlatform } from "../../../core/platform/platform";
import gr from "../../../services/global-resolver";
type Options = {
from?: string;
onlyCompany?: string;
onlyWorkspace?: string;
onlyChannel?: string;
dryRun?: boolean;
};
class MessageReferenceRepair {
database: DatabaseServiceAPI;
constructor(readonly platform: TwakePlatform) {
this.database = this.platform.getProvider<DatabaseServiceAPI>("database");
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
public async run(_options: Options = {}): Promise<void> {
//TODO repair messages
}
}
const services = [
"storage",
"counter",
"platform-services",
"user",
"search",
"channels",
"database",
"webserver",
"message-queue",
"messages",
];
const command: yargs.CommandModule<unknown, unknown> = {
command: "message-channel-repair",
describe: "command that allow you to repair messages references in channels",
builder: {
from: {
default: null,
type: "string",
description: "Start migration from this company ID",
},
onlyCompany: {
default: null,
type: "string",
description: "Migrate only this company ID",
},
onlyWorkspace: {
default: null,
type: "string",
description: "Migrate only this workspace ID",
},
onlyChannel: {
default: null,
type: "string",
description: "Migrate only this channel ID",
},
dryRun: {
default: false,
type: "boolean",
description: "Do not save anything and show missing references",
},
},
handler: async argv => {
const spinner = ora({ text: "Fixing messages references - " }).start();
const platform = await twake.run(services);
await gr.doInit(platform);
const migrator = new MessageReferenceRepair(platform);
const from = argv.from as string | null;
const onlyCompany = argv.onlyCompany as string | null;
const onlyWorkspace = argv.onlyWorkspace as string | null;
const onlyChannel = argv.onlyChannel as string | null;
const dryRun = (argv.dryRun || false) as boolean;
await migrator.run({
from,
onlyCompany,
onlyWorkspace,
onlyChannel,
dryRun,
});
return spinner.stop();
},
};
export default command;
twake/backend/node/src/cli/cmds/migration_cmds/message-files-cache.ts
-190
View File
@@ -1,190 +0,0 @@
import yargs from "yargs";
import twake from "../../../twake";
import ora from "ora";
import { TwakePlatform } from "../../../core/platform/platform";
import { DatabaseServiceAPI } from "../../../core/platform/services/database/api";
import {
ExecutionContext,
Paginable,
Pagination,
} from "../../../core/platform/framework/api/crud-service";
import gr from "../../../services/global-resolver";
import { getInstance, MessageFileRef } from "../../../services/messages/entities/message-file-refs";
import Repository from "../../../core/platform/services/database/services/orm/repository/repository";
import uuid from "node-uuid";
import { MessageFile } from "../../../services/messages/entities/message-files";
import { ThreadExecutionContext } from "../../../services/messages/types";
type Options = Record<string, unknown>;
class MessageFilesCacheMigrator {
database: DatabaseServiceAPI;
repository: Repository<MessageFileRef>;
messageFileRepository: Repository<MessageFile>;
constructor(readonly platform: TwakePlatform) {
this.database = this.platform.getProvider<DatabaseServiceAPI>("database");
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
public async run(_options: Options = {}, context?: ExecutionContext): Promise<void> {
this.repository = await gr.database.getRepository<MessageFileRef>(
"message_file_refs",
MessageFileRef,
);
this.messageFileRepository = await gr.database.getRepository<MessageFile>(
"message_files",
MessageFile,
);
let count = 0;
let companyPagination: Paginable = new Pagination(null, "100");
do {
const companyList = await gr.services.companies.getCompanies(companyPagination);
companyPagination = companyList.nextPage;
for (const company of companyList.getEntities()) {
const companyId: string = company.id;
const workspaceList = await gr.services.workspaces.getAllForCompany(companyId);
for (const workspaceId of [...workspaceList.map(w => w.id), "direct"]) {
const channelsList = await gr.services.channels.channels.getAllChannelsInWorkspace(
companyId,
workspaceId,
context,
);
for (const channel of channelsList) {
const channelId = channel.id;
let threadPagination: Paginable = new Pagination(null, "100");
do {
const threadList = await gr.services.messages.views.listChannel(
threadPagination,
{},
{
channel: {
company_id: companyId,
workspace_id: workspaceId,
id: channelId,
},
user: { id: null },
},
);
for (const thread of threadList.getEntities()) {
let messagesPagination: Paginable = new Pagination(null, "100");
do {
const messagesList = await gr.services.messages.messages.list(
messagesPagination,
{},
{
thread: { id: thread.id },
company: { id: companyId },
workspace: { id: workspaceId },
channel: { id: channelId },
user: { id: null },
} as ThreadExecutionContext,
);
messagesPagination = messagesList.nextPage;
messagesPagination.page_token = messagesList.getEntities()[0]?.id;
for (const message of messagesList.getEntities()) {
if (message.files && message.files.length > 0) {
for (const _messageFile of message.files) {
count++;
try {
const messageFile = await this.messageFileRepository.findOne(
{
message_id: message.id,
id: _messageFile.id,
},
{},
undefined,
);
if (messageFile) {
console.log(messageFile.metadata.name);
//Update user uploads
const fileRef = getInstance({
target_type: "user_upload",
target_id: message.user_id,
id: uuid.v1({ msecs: message.created_at }),
created_at: message.created_at,
company_id: companyId,
workspace_id: workspaceId,
channel_id: channelId,
thread_id: message.thread_id,
message_id: message.id,
message_file_id: messageFile.id,
file_id: messageFile.metadata.external_id,
});
await this.repository.save(fileRef, undefined);
//Update messageFileRepository
messageFile.cache = {
company_id: companyId,
workspace_id: workspaceId,
channel_id: channelId,
user_id: message.user_id,
};
messageFile.thread_id = message.thread_id;
await this.messageFileRepository.save(messageFile, undefined);
}
} catch (e) {}
}
}
}
} while (messagesPagination.page_token);
}
threadPagination = threadList.nextPage;
threadPagination.page_token =
threadPagination.page_token &&
threadList.getEntities()?.[threadList.getEntities().length - 1]?.thread_id;
} while (false); // && threadPagination.page_token && threadList.getEntities().length > 50);
}
}
}
console.log("updated messages: ", count);
} while (companyPagination.page_token);
}
}
const services = [
"search",
"database",
"webserver",
"auth",
"counter",
"cron",
"message-queue",
"push",
"realtime",
"storage",
"tracker",
"websocket",
];
const command: yargs.CommandModule<unknown, unknown> = {
command: "message-files-cache",
describe: "command that allow you to fix cache for each message-file refs",
builder: {},
handler: async () => {
const spinner = ora({ text: "Migrating messages - " }).start();
const platform = await twake.run(services);
await gr.doInit(platform);
const migrator = new MessageFilesCacheMigrator(platform);
await migrator.run({});
return spinner.stop();
},
};
export default command;
twake/backend/node/src/cli/cmds/migration_cmds/message.ts
-91
View File
@@ -1,91 +0,0 @@
import yargs from "yargs";
import twake from "../../../twake";
import ora from "ora";
import MessageMigrator from "./php-message/message-migrator-service";
import gr from "../../../services/global-resolver";
const services = [
"storage",
"counter",
"platform-services",
"user",
"search",
"channels",
"database",
"webserver",
"message-queue",
"messages",
"statistics",
];
const command: yargs.CommandModule<unknown, unknown> = {
command: "message",
describe: "command that allow you to migrate php messages to node",
builder: {
from: {
default: null,
type: "string",
description: "Start migration from this company ID",
},
onlyCompany: {
default: null,
type: "string",
description: "Migrate only this company ID",
},
onlyWorkspace: {
default: null,
type: "string",
description: "Migrate only this workspace ID",
},
onlyChannel: {
default: null,
type: "string",
description: "Migrate only this channel ID",
},
ignoreExisting: {
default: false,
type: "boolean",
description: "Skip existing message ids",
},
backToPhp: {
default: false,
type: "boolean",
description:
"Run the migration back, put node messages into php table (only the one that are not already here)",
},
dryRun: {
default: false,
type: "boolean",
description: "Do not save anything",
},
},
// eslint-disable-next-line @typescript-eslint/no-unused-vars
handler: async argv => {
const spinner = ora({ text: "Migrating php messages - " }).start();
const platform = await twake.run(services);
await gr.doInit(platform);
const migrator = new MessageMigrator(platform);
const from = argv.from as string | null;
const onlyCompany = argv.onlyCompany as string | null;
const onlyWorkspace = argv.onlyWorkspace as string | null;
const onlyChannel = argv.onlyChannel as string | null;
const ignoreExisting = (argv.ignoreExisting || false) as boolean;
const backToPhp = (argv.backToPhp || false) as boolean;
const dryRun = (argv.dryRun || false) as boolean;
await migrator.run({
from,
onlyCompany,
onlyWorkspace,
onlyChannel,
ignoreExisting,
backToPhp,
dryRun,
});
return spinner.stop();
},
};
export default command;
twake/backend/node/src/cli/cmds/migration_cmds/messages-files-medias-separation.ts
-125
View File
@@ -1,125 +0,0 @@
import yargs from "yargs";
import twake from "../../../twake";
import ora from "ora";
import { DatabaseServiceAPI } from "../../../core/platform/services/database/api";
import { TwakePlatform } from "../../../core/platform/platform";
import gr from "../../../services/global-resolver";
import { MessageFileRef } from "../../../services/messages/entities/message-file-refs";
import { Paginable } from "../../../core/platform/framework/api/crud-service";
import { Pagination } from "../../../core/platform/framework/api/crud-service";
import { fileIsMedia } from "../../../services/files/utils";
import { MessageFile } from "../../../services/messages/entities/message-files";
import _ from "lodash";
type Options = Record<string, unknown>;
class MessageReferenceRepair {
database: DatabaseServiceAPI;
constructor(readonly platform: TwakePlatform) {
this.database = this.platform.getProvider<DatabaseServiceAPI>("database");
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
public async run(_options: Options = {}): Promise<void> {
const repository = await gr.database.getRepository<MessageFileRef>(
"message_file_refs",
MessageFileRef,
);
const repositoryMessageFile = await gr.database.getRepository<MessageFile>(
"message_files",
MessageFile,
);
let count = 0;
let companyPagination: Paginable = new Pagination(null, "100");
do {
const companyList = await gr.services.companies.getCompanies(companyPagination);
companyPagination = companyList.nextPage;
for (const company of companyList.getEntities()) {
const companyId: string = company.id;
const workspaceList = await gr.services.workspaces.getAllForCompany(companyId);
for (const workspaceId of [...workspaceList.map(w => w.id), "direct"]) {
const channelsList = await gr.services.channels.channels.getAllChannelsInWorkspace(
companyId,
workspaceId,
);
for (const channel of channelsList) {
const channelId = channel.id;
let filePagination: Pagination = new Pagination(null, "100");
do {
const items = await repository.find(
{
target_type: "channel",
target_id: channelId,
company_id: companyId,
},
{ pagination: filePagination },
);
for (const item of items.getEntities()) {
try {
const msgFile = await repositoryMessageFile.findOne({
message_id: item.message_id,
id: item.message_file_id,
});
if (msgFile) {
count++;
const isMedia = fileIsMedia(msgFile);
const ref = _.cloneDeep(item);
ref.target_type = isMedia ? "channel_media" : "channel_file";
await repository.save(ref);
}
} catch (e) {
console.log("Error", e);
}
}
filePagination = new Pagination(items.nextPage.page_token, "100");
} while (filePagination.page_token);
}
}
}
console.log("updated files refs: ", count);
} while (companyPagination.page_token);
}
}
const services = [
"search",
"database",
"webserver",
"auth",
"counter",
"cron",
"message-queue",
"push",
"realtime",
"storage",
"tracker",
"websocket",
];
const command: yargs.CommandModule<unknown, unknown> = {
command: "messages-files-medias-separation",
describe: "command to separate medias and files in messages-files channels refs",
builder: {},
handler: async () => {
const spinner = ora({ text: "Fixing messages references - " }).start();
const platform = await twake.run(services);
await gr.doInit(platform);
const migrator = new MessageReferenceRepair(platform);
await migrator.run({});
return spinner.stop();
},
};
export default command;
twake/backend/node/src/cli/cmds/migration_cmds/php-application/php-application-entity.ts
-133
View File
@@ -1,133 +0,0 @@
import { Type } from "class-transformer";
import {
Column,
Entity,
} from "../../../../core/platform/services/database/services/orm/decorators";
export const TYPE = "application";
@Entity(TYPE, {
primaryKey: [["group_id"], "app_group_name", "id"],
type: TYPE,
})
export default class PhpApplication {
@Type(() => String)
@Column("id", "timeuuid", { generator: "timeuuid" })
id: string;
@Type(() => String)
@Column("group_id", "timeuuid")
group_id: string;
@Type(() => String)
@Column("app_group_name", "string")
app_group_name = "";
@Column("name", "encoded_string")
depreciated_name: string;
@Column("simple_name", "encoded_string")
depreciated_simple_name: string;
@Column("description", "encoded_string")
depreciated_description: string;
@Column("icon_url", "encoded_string")
depreciated_icon_url: string;
@Column("is_default", "twake_boolean")
is_default: boolean;
@Column("public", "twake_boolean")
depreciated_public: boolean;
@Column("twake_team_validation", "twake_boolean")
depreciated_twake_team_validation: boolean;
@Column("is_available_to_public", "twake_boolean")
depreciated_is_available_to_public: boolean; //Vrai si $public ET $twake_team_validation
@Column("api_events_url", "encoded_string")
depreciated_api_events_url: string;
@Column("api_allowed_ip", "encoded_string")
depreciated_api_allowed_ip: string;
@Column("api_private_key", "encoded_string")
depreciated_api_private_key: string;
@Column("privileges", "encoded_string")
depreciated_privileges = "[]";
@Column("capabilities", "encoded_string")
depreciated_capabilities = "[]";
@Column("hooks", "encoded_string")
depreciated_hooks = "[]";
@Column("display_configuration", "encoded_string")
depreciated_display_configuration = "{}";
}
export type DepreciatedDisplayConfiguration = {
version?: 0; //Legacy
tasks_module?: {
can_connect_to_tasks?: boolean;
};
calendar_module?: {
can_connect_to_calendar?: boolean;
};
drive_module?: {
can_connect_to_directory?: boolean;
can_open_files?: {
url?: string; //Une url à appeler pour éditer le fichier (ouvert dans un onglet)
preview_url?: string; //Une url à appeler pour prévisualiser un fichier (iframe)
main_ext?: string[]; //Extensions principales
other_ext?: string[]; //Extensions secondaires
};
can_create_files?: {
url?: string;
filename?: string;
name?: string;
}[];
};
member_app?: boolean; // Si défini, votre application génèrera un membre
// virtuel dans l'espace de travail avec lequel les
// utilisateurs pourront discuter.
messages_module?: {
in_plus?: {
should_wait_for_popup: boolean;
};
right_icon?: {
icon_url?: string; //If defined replace original icon url of your app
should_wait_for_popup?: boolean;
type?: string; //"file" | "call"
};
action?: {
should_wait_for_popup?: boolean;
description?: string; //Description de l'action, sinon remplacé par le nom de l'app
};
commands?: {
command?: string; // my_app mycommand
description?: string;
}[];
};
channel?: {
can_connect_to_channel?: string;
};
channel_tab?: {
iframe?: string;
};
app?: {
iframe?: string;
plus_btn: {
should_wait_for_popup?: boolean;
};
};
configuration?: {
can_configure_in_workspace?: boolean;
can_configure_in_channel?: boolean;
can_configure_in_calendar?: boolean;
can_configure_in_tasks?: boolean;
};
};
twake/backend/node/src/cli/cmds/migration_cmds/php-device/php-device-entity.ts
-26
View File
@@ -1,26 +0,0 @@
import {
Column,
Entity,
} from "../../../../core/platform/services/database/services/orm/decorators";
export const TYPE = "device";
@Entity(TYPE, {
primaryKey: [["id"]],
type: TYPE,
})
export class PhpDevice {
@Column("id", "timeuuid")
id: string;
@Column("user_id", "timeuuid")
user_id: string;
@Column("type", "string")
type: string;
@Column("version", "string")
version: string;
@Column("value", "encoded_string")
value: string;
}
twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/drive-migrator-service.ts
-535
View File
@@ -1,535 +0,0 @@
import { logger } from "../../../../core/platform/framework";
import { ExecutionContext, Pagination } from "../../../../core/platform/framework/api/crud-service";
import { TwakePlatform } from "../../../../core/platform/platform";
import Repository from "../../../../core/platform/services/database/services/orm/repository/repository";
import { DriveFile, AccessInformation } from "../../../../services/documents/entities/drive-file";
import {
generateAccessToken,
getDefaultDriveItem,
getDefaultDriveItemVersion,
} from "../../../../services/documents/utils";
import globalResolver from "../../../../services/global-resolver";
import Company from "../../../../services/user/entities/company";
import Workspace from "../../../../services/workspaces/entities/workspace";
import { PhpDriveFile } from "./php-drive-file-entity";
import { PhpDriveFileService } from "./php-drive-service";
import mimes from "../../../../utils/mime";
import WorkspaceUser from "../../../../services/workspaces/entities/workspace_user";
import CompanyUser from "src/services/user/entities/company_user";
interface CompanyExecutionContext extends ExecutionContext {
company: {
id: string;
};
}
interface WorkspaceExecutionContext extends CompanyExecutionContext {
workspace_id: string;
}
class DriveMigrator {
private phpDriveService: PhpDriveFileService;
private nodeRepository: Repository<DriveFile>;
constructor(readonly _platform: TwakePlatform) {
this.phpDriveService = new PhpDriveFileService();
}
/**
* Run
*/
public run = async (): Promise<void> => {
await globalResolver.doInit(this._platform);
await this.phpDriveService.init();
this.nodeRepository = await globalResolver.database.getRepository<DriveFile>(
"drive_files",
DriveFile,
);
let page: Pagination = { limitStr: "100" };
const context: ExecutionContext = {
user: {
id: null,
server_request: true,
},
};
do {
const companyListResult = await globalResolver.services.companies.getCompanies(page);
page = companyListResult.nextPage as Pagination;
for (const company of companyListResult.getEntities()) {
await this.migrateCompany(company, {
...context,
company: { id: company.id },
});
}
} while (page.page_token);
};
/**
* Migrate a company drive files.
*
* @param {Company} company - the company to migrate
*/
private migrateCompany = async (
company: Company,
context: CompanyExecutionContext,
): Promise<void> => {
logger.info(`Migrating company ${company.id}`);
const companyAdminOrOwnerId = await this.getCompanyOwnerOrAdminId(company.id, context);
const workspaceList = await globalResolver.services.workspaces.getAllForCompany(company.id);
for (const workspace of workspaceList) {
const wsContext = {
...context,
workspace_id: workspace.id,
user: { id: companyAdminOrOwnerId, server_request: true },
};
const access = await this.getWorkspaceAccess(workspace, company, wsContext);
await this.migrateWorkspace(workspace, access, wsContext);
}
};
/**
* Migrates a workspace drive files.
*
* @param {Workspace} workspace - the workspace to migrate
*/
private migrateWorkspace = async (
workspace: Workspace,
access: AccessInformation,
context: WorkspaceExecutionContext,
): Promise<void> => {
let page: Pagination = { limitStr: "100" };
console.debug(`Migrating workspace ${workspace.id} of company ${context.company.id}`);
logger.info(`Migrating workspace ${workspace.id} root folder`);
const workspaceFolder = await this.createWorkspaceFolder(workspace, access, context);
// Migrate the root folder.
do {
const phpDriveFiles = await this.phpDriveService.listDirectory(
page,
"",
workspace.id,
context,
);
page = phpDriveFiles.nextPage as Pagination;
for (const phpDriveFile of phpDriveFiles.getEntities()) {
await this.migrateDriveFile(phpDriveFile, workspaceFolder.id, access, context);
}
} while (page.page_token);
logger.info(`Migrating workspace ${workspace.id} trash`);
// Migrate the trash.
page = { limitStr: "100" };
do {
const phpDriveFiles = await this.phpDriveService.listDirectory(page, "trash", workspace.id);
page = phpDriveFiles.nextPage as Pagination;
for (const phpDriveFile of phpDriveFiles.getEntities()) {
await this.migrateDriveFile(phpDriveFile, "trash", access, context);
}
} while (page.page_token);
};
/**
* Migrates a php drive item to a Node drive file
*
* @param {PhpDriveItem} item - the php drive file to migrate.
*/
private migrateDriveFile = async (
item: PhpDriveFile,
parentId: string,
access: AccessInformation,
context: WorkspaceExecutionContext,
): Promise<void> => {
logger.info(`Migrating php drive item ${item.id} - parent: ${parentId ?? "root"}`);
try {
const migrationRecord = await this.phpDriveService.getMigrationRecord(
item.id,
context.company.id,
);
const newDriveItem = getDefaultDriveItem(
{
name: item.name || item.id,
extension: item.extension,
added: item.added.toString(),
content_keywords:
item.content_keywords && item.content_keywords.length
? item.content_keywords.join(",")
: "",
creator: item.creator || context.user.id,
is_directory: item.isdirectory,
is_in_trash: item.isintrash,
description: item.description,
tags: item.tags || [],
parent_id: parentId,
company_id: context.company.id,
access_info: access,
},
context,
);
if (migrationRecord && migrationRecord.company_id === context.company.id) {
console.debug(`${item.id} is already migrated`);
} else {
await this.nodeRepository.save(newDriveItem);
}
if (item.isdirectory) {
const newParentId =
migrationRecord && migrationRecord.company_id === context.company.id
? migrationRecord.new_id
: newDriveItem.id;
let page: Pagination = { limitStr: "100" };
do {
const directoryChildren = await this.phpDriveService.listDirectory(
page,
item.id,
context.workspace_id,
);
page = directoryChildren.nextPage as Pagination;
for (const child of directoryChildren.getEntities()) {
try {
await this.migrateDriveFile(child, newParentId, access, context);
} catch (error) {
logger.error(`Failed to migrate drive item ${child.id}`);
console.error(`Failed to migrate drive item ${child.id}`);
}
}
} while (page.page_token);
} else {
let versionPage: Pagination = { limitStr: "100" };
if (
migrationRecord &&
migrationRecord.item_id === item.id &&
migrationRecord.company_id === context.company.id
) {
logger.info(`item is already migrated - ${item.id} - skipping`);
console.log(`item is already migrated - ${item.id} - skipping`);
return;
}
const mime = mimes[item.extension];
let createdVersions = 0;
do {
const itemVersions = await this.phpDriveService.listItemVersions(
versionPage,
item.id,
context,
);
versionPage = itemVersions.nextPage as Pagination;
for (const version of itemVersions.getEntities()) {
try {
const newVersion = getDefaultDriveItemVersion(
{
creator_id: version.creator_id || context.user.id,
data: version.data,
date_added: +version.date_added,
drive_item_id: newDriveItem.id,
file_size: version.file_size,
filename: version.filename,
key: version.key,
provider: version.provider,
realname: version.realname,
mode: version.mode,
},
context,
);
const file = await this.phpDriveService.migrate(
version.file_id,
item.workspace_id,
version.id,
{
filename: version.filename,
userId: version.creator_id || context.user.id,
totalSize: version.file_size,
waitForThumbnail: true,
chunkNumber: 1,
totalChunks: 1,
type: mime,
},
context,
);
if (!file) {
throw Error("cannot download file version");
}
newVersion.file_metadata = {
external_id: file.id,
mime: file.metadata.mime,
name: file.metadata.name || version.filename,
size: file.upload_data.size || version.file_size,
};
await globalResolver.services.documents.documents.createVersion(
newDriveItem.id,
newVersion,
context,
);
createdVersions++;
} catch (error) {
logger.error(`Failed to migrate version ${version.id} for drive item ${item.id}`);
console.error(`Failed to migrate version ${version.id} for drive item ${item.id}`);
}
}
} while (versionPage.page_token);
if (createdVersions === 0) {
await this.nodeRepository.remove(newDriveItem);
return;
}
}
if (!migrationRecord) {
await this.phpDriveService.markAsMigrated(item.id, newDriveItem.id, context.company.id);
}
} catch (error) {
logger.error(
`Failed to migrate Drive item ${item.id} / workspace ${item.workspace_id} / company_id: ${context.company.id}`,
error,
);
console.error(`Failed to migrate Drive item ${item.id}`, error);
}
};
/**
* Fetches the first found company owner or admin identifier.
*
* @param {string} companyId - the companyId
* @param {ExecutionContext} context - the execution context
* @returns {Promise<string>}
*/
private getCompanyOwnerOrAdminId = async (
companyId: string,
context: ExecutionContext,
): Promise<string> => {
let pagination: Pagination = { limitStr: "100" };
let companyOwnerOrAdminId = null;
do {
const companyUsers = await globalResolver.services.companies.companyUserRepository.find(
{ group_id: companyId },
{ pagination },
context,
);
pagination = companyUsers.nextPage as Pagination;
const companyAdminOrOwner = companyUsers
.getEntities()
.find(({ role }) => ["admin", "owner"].includes(role));
if (companyAdminOrOwner) {
companyOwnerOrAdminId = companyAdminOrOwner.id;
}
} while (pagination && !companyOwnerOrAdminId);
return companyOwnerOrAdminId;
};
/**
* Compute the Access Information for the workspace folder to be created.
*
* @param {Workspace} workspace - the target workspace
* @param {Company} company - the target company
* @param {WorkspaceExecutionContext} context - the execution context
* @returns {Promise<AccessInformation>}
*/
private getWorkspaceAccess = async (
workspace: Workspace,
company: Company,
context: WorkspaceExecutionContext,
): Promise<AccessInformation> => {
const companyUsersCount = await globalResolver.services.companies.getUsersCount(company.id);
const workspaceUsersCount = await globalResolver.services.workspaces.getUsersCount(
workspace.id,
);
if (companyUsersCount === workspaceUsersCount) {
return {
entities: [
{
id: "parent",
type: "folder",
level: "manage",
},
{
id: company.id,
type: "company",
level: "none",
},
{
id: context.user?.id,
type: "user",
level: "manage",
},
],
public: {
level: "none",
token: generateAccessToken(),
},
};
}
let workspaceUsers: WorkspaceUser[] = [];
let wsUsersPagination: Pagination = { limitStr: "100" };
do {
const wsUsersQuery = await globalResolver.services.workspaces.getUsers(
{ workspaceId: workspace.id },
wsUsersPagination,
context,
);
wsUsersPagination = wsUsersQuery.nextPage as Pagination;
workspaceUsers = [...workspaceUsers, ...wsUsersQuery.getEntities()];
} while (wsUsersPagination.page_token);
if (companyUsersCount < 30 || workspaceUsersCount < 30) {
return {
entities: [
{
id: "parent",
type: "folder",
level: "none",
},
{
id: company.id,
type: "company",
level: "none",
},
{
id: context.user?.id,
type: "user",
level: "manage",
},
...workspaceUsers.reduce((acc, curr) => {
acc = [
...acc,
{
id: curr.userId,
type: "user",
level: "manage",
},
];
return acc;
}, []),
],
public: {
level: "none",
token: generateAccessToken(),
},
};
}
let companyUsers: CompanyUser[] = [];
let companyUsersPaginations: Pagination = { limitStr: "100" };
do {
const companyUsersQuery = await globalResolver.services.companies.getUsers(
{ group_id: company.id },
companyUsersPaginations,
{},
context,
);
companyUsersPaginations = companyUsersQuery.nextPage as Pagination;
companyUsers = [...companyUsers, ...companyUsersQuery.getEntities()];
} while (companyUsersPaginations.page_token);
return {
entities: [
{
id: "parent",
type: "folder",
level: "none",
},
{
id: company.id,
type: "company",
level: "manage",
},
{
id: context.user?.id,
type: "user",
level: "manage",
},
...companyUsers.reduce((acc, curr) => {
if (workspaceUsers.find(({ userId }) => curr.user_id === userId)) {
return acc;
}
acc = [
...acc,
{
id: curr.user_id,
type: "user",
level: "none",
},
];
return acc;
}, []),
],
public: {
level: "none",
token: generateAccessToken(),
},
};
};
/**
* Creates a folder for the workspace to migrate.
*
* @param {Workspace} workspace - the workspace to migrate.
* @param {AccessInformation} access - the access information.
* @param {WorkspaceExecutionContext} context - the execution context
* @returns {Promise<DriveFile>}
*/
private createWorkspaceFolder = async (
workspace: Workspace,
access: AccessInformation,
context: WorkspaceExecutionContext,
): Promise<DriveFile> => {
const workspaceFolder = getDefaultDriveItem(
{
name: workspace.name || workspace.id,
extension: "",
content_keywords: "",
creator: context.user.id,
is_directory: true,
is_in_trash: false,
description: "",
tags: [],
parent_id: "root",
company_id: context.company.id,
access_info: access,
},
context,
);
await this.nodeRepository.save(workspaceFolder);
await this.phpDriveService.markAsMigrated(workspace.id, workspaceFolder.id, context.company.id);
return workspaceFolder;
};
}
export default DriveMigrator;
twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-file-entity.ts
-64
View File
@@ -1,64 +0,0 @@
import { Type } from "class-transformer";
import {
Column,
Entity,
} from "../../../../core/platform/services/database/services/orm/decorators";
export const TYPE = "drive_file";
@Entity(TYPE, {
primaryKey: [["workspace_id"], "parent_id", "id"],
type: TYPE,
})
export class PhpDriveFile {
@Type(() => String)
@Column("workspace_id", "timeuuid")
workspace_id: string;
@Type(() => String)
@Column("parent_id", "string")
parent_id: string | "";
@Type(() => String)
@Column("id", "timeuuid", { generator: "timeuuid" })
id: string;
@Column("isintrash", "boolean")
isintrash: boolean;
@Column("added", "string")
added: string;
@Column("attachements", "encoded_json")
attachements: unknown;
@Column("content_keywords", "encoded_json")
content_keywords: string[] | null;
@Column("creator", "string")
creator: string;
@Column("description", "string")
description: string;
@Column("extension", "string")
extension: string;
@Column("hidden_data", "encoded_json")
hidden_data: unknown;
@Column("isdirectory", "boolean")
isdirectory: boolean;
@Column("last_modified", "string")
last_modified: string;
@Column("name", "encoded_string")
name: string;
@Column("size", "number")
size: number;
@Column("tags", "encoded_json")
tags: string[] | null;
}
twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-file-version-entity.ts
-58
View File
@@ -1,58 +0,0 @@
import { Type } from "class-transformer";
import {
Column,
Entity,
} from "../../../../core/platform/services/database/services/orm/decorators";
export const TYPE = "drive_file_version";
@Entity(TYPE, {
primaryKey: [["file_id"], "id"],
type: TYPE,
})
export class PhpDriveFileVersion {
@Type(() => String)
@Column("id", "timeuuid", { generator: "timeuuid" })
id: string;
@Type(() => String)
@Column("file_id", "string")
file_id: string;
@Type(() => String)
@Column("creator_id", "string")
creator_id: string;
@Type(() => String)
@Column("realname", "encoded_string")
realname: string;
@Type(() => String)
@Column("key", "string")
key: string;
@Type(() => String)
@Column("mode", "string")
mode: string;
@Type(() => String)
@Column("file_size", "number")
file_size: number;
@Type(() => String)
@Column("date_added", "string")
date_added: string;
@Type(() => String)
@Column("filename", "encoded_string")
filename: string;
@Type(() => String)
@Column("provider", "string")
provider: string;
@Column("data", "encoded_json")
data: unknown;
}
export type PhpDriveFileVersionPrimaryKey = Pick<PhpDriveFileVersion, "file_id" | "id">;
twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-migration-record-entity.ts
-25
View File
@@ -1,25 +0,0 @@
import { Type } from "class-transformer";
import {
Column,
Entity,
} from "../../../../core/platform/services/database/services/orm/decorators";
export const TYPE = "php_drive_migration_record";
@Entity(TYPE, {
primaryKey: [["company_id"], "item_id"],
type: TYPE,
})
export class phpDriveMigrationRecord {
@Type(() => String)
@Column("item_id", "timeuuid")
item_id: string;
@Type(() => String)
@Column("company_id", "uuid")
company_id: string;
@Type(() => String)
@Column("new_id", "string")
new_id: string;
}
twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-service.ts
-189
View File
@@ -1,189 +0,0 @@
import {
ExecutionContext,
ListResult,
Pagination,
} from "../../../../core/platform/framework/api/crud-service";
import Repository from "../../../../core/platform/services/database/services/orm/repository/repository";
import globalResolver from "../../../../services/global-resolver";
import { PhpDriveFile, TYPE as DRIVE_FILE_TABLE } from "./php-drive-file-entity";
import { Initializable, logger, TwakeServiceProvider } from "../../../../core/platform/framework";
import {
PhpDriveFileVersion,
TYPE as DRIVE_FILE_VERSION_TABLE,
} from "./php-drive-file-version-entity";
import axios from "axios";
import { Multipart } from "fastify-multipart";
import { CompanyExecutionContext } from "../../../../services/files/web/types";
import { File } from "../../../../services/files/entities/file";
import { UploadOptions } from "../../../../services/files/types";
import {
phpDriveMigrationRecord,
TYPE as MIGRATION_RECORD_TABLE,
} from "./php-drive-migration-record-entity";
export interface MigrateOptions extends UploadOptions {
userId: string;
}
export interface PhpDriveServiceAPI extends TwakeServiceProvider, Initializable {}
export class PhpDriveFileService implements PhpDriveServiceAPI {
version: "1";
public repository: Repository<PhpDriveFile>;
public versionRepository: Repository<PhpDriveFileVersion>;
public migrationRepository: Repository<phpDriveMigrationRecord>;
/**
* Init the service.
*
* @returns {PhpDriveFileService}
*/
async init(): Promise<this> {
this.repository = await globalResolver.database.getRepository<PhpDriveFile>(
DRIVE_FILE_TABLE,
PhpDriveFile,
);
this.versionRepository = await globalResolver.database.getRepository<PhpDriveFileVersion>(
DRIVE_FILE_VERSION_TABLE,
PhpDriveFileVersion,
);
this.migrationRepository = await globalResolver.database.getRepository<phpDriveMigrationRecord>(
MIGRATION_RECORD_TABLE,
phpDriveMigrationRecord,
);
return this;
}
/**
* Lists the drive item directory children.
*
* @param {Pagination} pagination - the page.
* @param {string} directory - the drive item / directory id to search within.
* @param {string} workspaceId - the workspace id
* @param {ExecutionContext} context - the execution context.
* @returns {Promise<ListResult<PhpDriveFile>>} - the drive item children.
*/
listDirectory = async (
pagination: Pagination,
directory: string | "" | "trash",
workspaceId: string,
context?: ExecutionContext,
): Promise<ListResult<PhpDriveFile>> =>
await this.repository.find(
{
workspace_id: workspaceId,
parent_id: directory,
},
{ pagination },
context,
);
/**
* Lists the versions of a drive item.
*
* @param {Pagination} pagination - the page.
* @param {string} itemId - the drive item id.
* @param {ExecutionContext} context - the execution context.
* @returns {Promise<ListResult<PhpDriveFileVersion>>} - the list of the item versions.
*/
listItemVersions = async (
pagination: Pagination,
itemId: string,
context?: ExecutionContext,
): Promise<ListResult<PhpDriveFileVersion>> =>
await this.versionRepository.find(
{
file_id: itemId,
},
{ pagination },
context,
);
/**
* Downloads a file version from the old drive and uploads it to the new Drive.
*
* @param {string} fileId - the old file id
* @param {string} workspaceId - the workspace id
* @param {string} versionId - the version id
* @param {MigrateOptions} options - the file upload / migration options.
* @param {CompanyExecutionContext} context - the company execution context.
* @param {string} public_access_key - the file public access key.
* @returns {Promise<File>} - the uploaded file information.
*/
migrate = async (
fileId: string,
workspaceId: string,
versionId: string,
options: MigrateOptions,
context: CompanyExecutionContext,
public_access_key?: string,
): Promise<File> => {
try {
const url = `https://web.twake.app/ajax/drive/download?workspace_id=${workspaceId}&element_id=${fileId}&version_id=${versionId}&download=1${
public_access_key ? `&public_access_key=${public_access_key}` : ""
}`;
const response = await axios.get(url, {
responseType: "stream",
});
if (!response.data) {
throw Error("invalid download response");
}
const file = {
file: response.data,
};
return await globalResolver.services.files.save(null, file as Multipart, options, {
...context,
user: {
id: options.userId,
},
});
} catch (error) {
logger.error(`Failed to migrate file ${fileId} on workspace ${workspaceId}`, error);
throw Error(error);
}
};
/**
* Saves a drive item.
*
* @param {PhpDriveFile} item - the php drive item.
* @returns {Promise<void>}
*/
save = async (item: PhpDriveFile): Promise<void> => await this.repository.save(item);
/**
* Marks a drive item as migrated.
*
* @param {string} itemId - the drive item.
* @param {string} newId - the new drive item id.
* @param {string} companyId - the company id.
*/
markAsMigrated = async (itemId: string, newId: string, companyId: string): Promise<void> => {
const migrationRecord = new phpDriveMigrationRecord();
migrationRecord.item_id = itemId;
migrationRecord.new_id = newId;
migrationRecord.company_id = companyId;
await this.migrationRepository.save(migrationRecord);
};
/**
* Fetches the drive item migration record.
*
* @param {string} itemId - the drive item id.
* @param {string} companyId - the company id.
* @returns {Promise<boolean>}
*/
getMigrationRecord = async (
itemId: string,
companyId: string,
): Promise<phpDriveMigrationRecord> =>
await this.migrationRepository.findOne({ item_id: itemId, company_id: companyId });
}
twake/backend/node/src/cli/cmds/migration_cmds/php-message/message-migrator-service.ts
-560
View File
@@ -1,560 +0,0 @@
import { PhpMessagesService } from "./php-message-service";
import { convertUuidV4ToV1 } from "./utils";
import Company from "../../../../services/user/entities/company";
import { Pagination } from "../../../../core/platform/framework/api/crud-service";
import { PhpMessage } from "./php-message-entity";
import { TwakePlatform } from "../../../../core/platform/platform";
import {
Message,
MessageEdited,
MessageOverride,
MessagePinnedInfo,
MessageReaction,
} from "../../../../services/messages/entities/messages";
import { MessageChannelRef } from "../../../../services/messages/entities/message-channel-refs";
import { ParticipantObject, Thread } from "../../../../services/messages/entities/threads";
import { Block } from "../../../../services/messages/blocks-types";
import { WorkspaceExecutionContext } from "../../../../services/workspaces/types";
import gr from "../../../../services/global-resolver";
type MigratedChannel = {
id: string;
workspace_id: string;
company_id: string;
owner?: string;
};
type Options = {
from?: string;
onlyCompany?: string;
onlyWorkspace?: string;
onlyChannel?: string;
ignoreExisting?: boolean;
backToPhp?: boolean;
dryRun?: boolean;
};
class MessageMigrator {
private phpMessageService: PhpMessagesService;
private migratedMessages = 0;
private options: Options = {};
constructor(readonly _platform: TwakePlatform) {
this.phpMessageService = new PhpMessagesService();
}
public async run(options: Options = {}): Promise<void> {
await gr.doInit(this._platform);
this.options = options;
await this.phpMessageService.init();
if (this.options.onlyCompany) {
const company = await gr.services.companies.getCompany({ id: options.onlyCompany });
await this.migrateCompanyMessages(company);
} else {
let waitForCompany = false;
if (this.options.from) {
waitForCompany = true;
}
// Get all companies
let page: Pagination = { limitStr: "100" };
// For each companies find workspaces
do {
const companyListResult = await gr.services.companies.getCompanies(page);
page = companyListResult.nextPage as Pagination;
for (const company of companyListResult.getEntities()) {
if (waitForCompany && this.options.from == `${company.id}`) {
waitForCompany = false;
}
if (!waitForCompany) {
await this.migrateCompanyMessages(company);
}
}
} while (page.page_token);
}
console.log(
`Php Messages successfully migrated to node: (${this.migratedMessages} messages) !`,
);
}
private async migrateCompanyMessages(company: Company) {
console.log(`Start migration for ${company.id}...`);
if (!this.options.onlyWorkspace || this.options.onlyWorkspace === "direct") {
await this.migrateCompanyDirectMessages(company);
console.log(
`${company.id} - (1/2) migrated direct messages (total: ${this.migratedMessages} messages) ✅`,
);
}
if (!this.options.onlyWorkspace || this.options.onlyWorkspace !== "direct") {
await this.migrateCompanyChannelsMessages(company);
console.log(`${company.id} - (2/2) completed (total: ${this.migratedMessages} messages) ✅`);
}
}
/**
* Set all direct messages in company and set them to channelPhpMessages
*/
private async migrateCompanyDirectMessages(company: Company) {
await gr.doInit(this._platform);
let pageDirectChannels: Pagination = { limitStr: "100" };
// For each directChannels find messages
do {
const directChannelsInCompanyResult = await gr.services.channels.channels.list(
pageDirectChannels,
{},
{
workspace: {
workspace_id: "direct",
company_id: company.id,
},
user: { id: null, server_request: true },
},
);
pageDirectChannels = directChannelsInCompanyResult.nextPage as Pagination;
for (const directChannel of directChannelsInCompanyResult.getEntities()) {
await this.migrateChannelsMessages(company, {
id: directChannel.id,
workspace_id: "direct",
company_id: directChannel.company_id,
});
}
} while (pageDirectChannels.page_token);
}
/**
* Set all messages in company and set them to channelPhpMessages
*/
private async migrateCompanyChannelsMessages(company: Company) {
await gr.doInit(this._platform);
// Get all workspaces in company
const workspacesInCompany = (
await gr.services.workspaces.list({ limitStr: "" }, {}, {
user: {
id: null,
server_request: true,
},
company_id: company.id,
} as WorkspaceExecutionContext)
).getEntities();
// For each workspaces find channels
for (const workspace of workspacesInCompany) {
if (this.options.onlyWorkspace && `${workspace.id}` !== this.options.onlyWorkspace) {
continue;
}
// Get all channels in workspace
let pageChannels: Pagination = { limitStr: "1" };
do {
const channelsInWorkspace = await gr.services.channels.channels.list(
pageChannels,
{},
{
workspace: {
workspace_id: workspace.id,
company_id: workspace.company_id,
},
user: { id: null, server_request: true },
},
);
pageChannels = channelsInWorkspace.nextPage as Pagination;
// For each channels find messages
for (const channel of channelsInWorkspace.getEntities()) {
await this.migrateChannelsMessages(company, {
id: channel.id,
workspace_id: channel.workspace_id,
company_id: channel.company_id,
owner: channel.owner,
});
}
} while (pageChannels.page_token);
}
}
//Params: company, channel
private async migrateChannelsMessages(company: Company, channel: MigratedChannel) {
if (this.options.onlyChannel && `${channel.id}` !== this.options.onlyChannel) {
return;
}
if (this.options.backToPhp) {
await this.migrateChannelsMessagesBackToPhp(company, channel);
return;
}
//This function will migrate all messages in a channel
let pagePhpMessages: Pagination = { limitStr: "100" };
do {
const messages = await this.phpMessageService.list(
pagePhpMessages,
{},
{
channel_id: convertUuidV4ToV1(channel.id),
user: { id: null, server_request: true },
},
);
for (const message of messages.getEntities()) {
try {
await this.migrateMessage(company, channel, message);
} catch (err) {
console.log(err);
}
}
pagePhpMessages = messages.nextPage as Pagination;
} while (pagePhpMessages.page_token);
}
private migratedThreads: string[] = [];
/**
* Migrate php message to node
* @param company
* @param channel
* @param message
*/
private async migrateMessage(company: Company, channel: MigratedChannel, message: PhpMessage) {
if (!message.id) {
return;
}
await gr.doInit(this._platform);
//Create thread first if not exists
const threadId = message.parent_message_id || message.id;
if (this.options.ignoreExisting) {
const msg = await gr.services.messages.messages.get({
thread_id: threadId,
id: message.id,
});
if (msg) {
return;
}
}
const threadDoesNotExists = !this.migratedThreads.includes(threadId);
if (threadDoesNotExists) {
await this.migratePhpMessageToNodeThread(message, channel, company);
this.migratedThreads.push(threadId);
}
//Migrate message itself
await this.migratePhpMessageToNodeMessage(threadId, message, company);
this.migratedMessages++;
if (this.migratedMessages % 100 == 0) {
console.log(`${company.id} - ... (total: ${this.migratedMessages} messages)`);
}
//Force delay between channels
await new Promise(r => {
setTimeout(r, 40);
});
}
private async migrateChannelsMessagesBackToPhp(company: Company, channel: MigratedChannel) {
await gr.doInit(this._platform);
const channelRefRepository = await gr.database.getRepository(
"message_channel_refs",
MessageChannelRef,
);
const messageRepository = await gr.database.getRepository("messages", Message);
//This function will migrate all messages in a channel
let pageMessages: Pagination = { limitStr: "100" };
do {
const messages = await channelRefRepository.find(
{
company_id: company.id,
workspace_id: channel.workspace_id,
channel_id: channel.id,
},
{ pagination: pageMessages },
undefined,
);
for (const messageRef of messages.getEntities()) {
const messages = await messageRepository.find(
{
thread_id: messageRef.thread_id,
},
{},
undefined,
);
for (const message of messages.getEntities()) {
const uuidv1_channel_id =
channel.id.substring(0, 14) + "1" + channel.id.substring(14 + 1);
let phpMessage = await this.phpMessageService.get({
id: message.id,
});
if (!phpMessage) {
phpMessage = await this.phpMessageService.get({
channel_id: uuidv1_channel_id,
parent_message_id: message.thread_id,
id: message.id,
});
}
if (!phpMessage && message.subtype !== "deleted") {
//This message doesn't exists in php, move it to php
const newPhpMessage = new PhpMessage();
newPhpMessage.id = message.id;
newPhpMessage.channel_id = uuidv1_channel_id;
newPhpMessage.parent_message_id = message.thread_id;
newPhpMessage.application_id = message.application_id || null;
newPhpMessage.modification_date = Math.floor(
message.edited?.edited_at || message.created_at,
);
newPhpMessage.creation_date = Math.floor(message.created_at);
newPhpMessage.sender = message.user_id;
newPhpMessage.pinned = !!message.pinned_info?.pinned_at;
newPhpMessage.edited = !!message.edited?.edited_at;
newPhpMessage.message_type =
message.subtype === "application" ? 1 : message.subtype === "system" ? 2 : 0;
newPhpMessage.hidden_data = message.context;
newPhpMessage.reactions = "{}";
newPhpMessage.responses_count = 0;
let prepared: any[] = [];
(message.blocks || []).map(block => {
if (block.type === "twacode") {
prepared = block["elements"];
}
if (block.type === "section" && prepared.length > 0) {
prepared = [
{
type: "twacode",
content: block?.text?.text || "",
},
];
}
});
newPhpMessage.content = {
fallback_string: message.text,
original_str: message.text,
files: (message.files || []).map(f => {
return {
content: f,
mode: "mini",
type: "file",
};
}),
prepared: prepared,
};
if (!this.options.dryRun) {
await this.phpMessageService.repository.save(newPhpMessage, undefined);
}
this.migratedMessages++;
if (this.migratedMessages % 100 == 0) {
console.log(`${company.id} - ... (total: ${this.migratedMessages} messages)`);
}
}
}
}
pageMessages = messages.nextPage as Pagination;
} while (pageMessages.page_token);
}
/**
* Migrate php message to node thread
* @param message PhpMessage
* @param channel MigratedChannel
*/
private async migratePhpMessageToNodeThread(
message: PhpMessage,
channel: MigratedChannel,
company: Company,
) {
const thread = new Thread();
// Set nodeThread values
thread.id = message.parent_message_id || message.id;
thread.created_at = message.creation_date;
thread.last_activity = message.modification_date;
thread.answers = 0;
thread.participants = [
{
type: "channel",
id: channel.id,
company_id: channel.company_id,
workspace_id: channel.workspace_id,
created_at: message.creation_date,
created_by: message.sender,
} as ParticipantObject,
];
if (this.options.dryRun) {
return;
}
// Create nodeThread
return await gr.services.messages.threads.save(
thread,
{},
{ user: { id: null, server_request: true }, company },
);
}
/**
* Set message string type
* @param integer 0 = null | 1 = system | 2 = application
*/
private setMessageType(integer: number): { type: Message["type"]; subtype: Message["subtype"] } {
switch (integer) {
case 1:
return {
type: "message",
subtype: "application",
};
case 2:
return {
type: "message",
subtype: "system",
};
case 0:
default:
return {
type: "message",
subtype: null,
};
}
}
/**
* Set blocks array
* @param content { [key: string]: any }
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
private setBlocks(content: { [key: string]: any }): Block[] {
const blocks: Block[] = [];
if (!content) return blocks;
const new_block_format: Block = {
type: "twacode",
elements: content.formatted || content.prepared || content || [],
};
blocks.push(new_block_format);
return blocks;
}
/**
* Set edited message Object
* @param modification_date timestamp
*/
private setMessageEditedObject(message: PhpMessage): MessageEdited {
return message.edited ? { edited_at: message.modification_date } : null;
}
/**
* Set pinned message Object
* @param message PhpMessage
*/
private setMessagePinnedObject(message: PhpMessage): MessagePinnedInfo {
if (message.pinned) {
return { pinned_at: 0, pinned_by: message.sender };
}
return null;
}
/**
* Set reactions message object
* @param reactions JSON
*/
private setMessageReactionsObject(reactions: string): MessageReaction[] {
const parsed_reactions = JSON.parse(reactions);
if (!parsed_reactions) return [];
const new_reactions_array: MessageReaction[] = [];
for (const reaction_name in parsed_reactions) {
const new_reaction_object: MessageReaction = {
name: reaction_name,
users: parsed_reactions[reaction_name].users,
count: parsed_reactions[reaction_name].count,
};
new_reactions_array.push(new_reaction_object);
}
return new_reactions_array;
}
/**
* Set override message Object
* @param message PhpMessage
*/
private setMessageOverrideObject(_message: PhpMessage): MessageOverride {
return null;
}
/**
* Migrate php message to node message
* @param message PhpMessage
*/
private async migratePhpMessageToNodeMessage(
threadId: string,
message: PhpMessage,
company: Company,
) {
const nodeMessage = new Message();
// Set nodeMessage values
nodeMessage.id = message.id;
nodeMessage.thread_id = threadId;
nodeMessage.type = this.setMessageType(message.message_type).type;
nodeMessage.subtype = this.setMessageType(message.message_type).subtype;
nodeMessage.created_at = message.creation_date;
nodeMessage.user_id = message.sender;
nodeMessage.application_id = message.application_id;
nodeMessage.text = message.content?.original_str || "";
nodeMessage.blocks = this.setBlocks(message.content);
nodeMessage.context = message.hidden_data || {};
nodeMessage.edited = this.setMessageEditedObject(message);
nodeMessage.pinned_info = this.setMessagePinnedObject(message);
nodeMessage.reactions = this.setMessageReactionsObject(message.reactions);
nodeMessage.override = this.setMessageOverrideObject(message);
nodeMessage.context._front_id = message.id;
if (this.options.dryRun) {
return;
}
// Create nodeMessage then add it to thread
return await gr.services.messages.messages.save(
nodeMessage,
{
enforceViewPropagation: true,
},
{
user: { id: null, server_request: true },
thread: { id: threadId },
company: { id: company.id },
},
);
}
}
export default MessageMigrator;
twake/backend/node/src/cli/cmds/migration_cmds/php-message/php-message-entity.ts
-67
View File
@@ -1,67 +0,0 @@
import { Type } from "class-transformer";
import { merge } from "lodash";
import {
Column,
Entity,
} from "../../../../core/platform/services/database/services/orm/decorators";
export const TYPE = "message";
@Entity(TYPE, {
primaryKey: [["channel_id"], "parent_message_id", "id"],
type: TYPE,
})
export class PhpMessage {
@Type(() => String)
@Column("id", "timeuuid", { generator: "timeuuid" })
id: string;
@Type(() => String)
@Column("channel_id", "timeuuid")
channel_id: string;
@Type(() => String)
@Column("parent_message_id", "encoded_string")
parent_message_id: string;
@Type(() => String)
@Column("application_id", "encoded_string")
application_id: string | null;
@Column("modification_date", "twake_datetime")
modification_date: number | null;
@Column("creation_date", "twake_datetime")
creation_date: number;
@Column("message_type", "number")
message_type: number;
@Column("sender_id", "timeuuid")
sender: string;
@Column("content", "encoded_json")
// eslint-disable-next-line @typescript-eslint/no-explicit-any
content: { [key: string]: any } | null;
@Column("hidden_data", "encoded_json")
// eslint-disable-next-line @typescript-eslint/no-explicit-any
hidden_data: { [key: string]: any } | null;
@Column("reactions", "encoded_string")
reactions: string | null;
@Column("pinned", "twake_boolean")
pinned: boolean;
@Column("edited", "twake_boolean")
edited: boolean;
@Column("responses_count", "twake_int")
responses_count: number | null;
}
export type PhpMessagePrimaryKey = Pick<PhpMessage, "parent_message_id" | "channel_id" | "id">;
export function getInstance(message: PhpMessage): PhpMessage {
return merge(new PhpMessage(), message);
}
twake/backend/node/src/cli/cmds/migration_cmds/php-message/php-message-service.ts
-59
View File
@@ -1,59 +0,0 @@
import {
DeleteResult,
ExecutionContext,
ListResult,
Pagination,
} from "../../../../core/platform/framework/api/crud-service";
import Repository, {
FindFilter,
} from "../../../../core/platform/services/database/services/orm/repository/repository";
import { PhpMessagesServiceAPI } from "./types";
import { PhpMessage, PhpMessagePrimaryKey } from "./php-message-entity";
import gr from "../../../../services/global-resolver";
export interface PhpMessageExecutionContext extends ExecutionContext {
channel_id: string;
parent_message_id?: string;
id?: string;
}
export class PhpMessagesService implements PhpMessagesServiceAPI {
version: "1";
public repository: Repository<PhpMessage>;
async init(): Promise<this> {
this.repository = await gr.database.getRepository<PhpMessage>("message", PhpMessage);
return this;
}
get(pk: { parent_message_id?: string; channel_id?: string; id: string }): Promise<PhpMessage> {
if (pk.channel_id) {
pk.channel_id = `${pk.channel_id}`;
pk.channel_id.substring(0, 14) + "1" + pk.channel_id.substring(14 + 1);
}
return this.repository.findOne(pk, {}, undefined);
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async delete(pk: PhpMessagePrimaryKey): Promise<DeleteResult<PhpMessage>> {
throw Error("not implemented");
}
async list<ListOptions>(
// eslint-disable-next-line @typescript-eslint/no-unused-vars
pagination: Pagination,
// eslint-disable-next-line @typescript-eslint/no-unused-vars
options?: ListOptions,
// eslint-disable-next-line @typescript-eslint/no-unused-vars
context?: PhpMessageExecutionContext,
): Promise<ListResult<PhpMessage>> {
const findFilter: FindFilter = {
channel_id: context.channel_id,
parent_message_id: context.parent_message_id,
//id: context.id,
};
const list = await this.repository.find(findFilter, { pagination }, undefined);
return list;
}
}
twake/backend/node/src/cli/cmds/migration_cmds/php-message/types.ts
-3
View File
@@ -1,3 +0,0 @@
import { Initializable, TwakeServiceProvider } from "../../../../core/platform/framework";
export interface PhpMessagesServiceAPI extends TwakeServiceProvider, Initializable {}
twake/backend/node/src/cli/cmds/migration_cmds/php-message/utils.ts
-11
View File
@@ -1,11 +0,0 @@
export const convertUuidV4ToV1 = (str: string): string => {
const uuid = [...str];
uuid[14] = "1";
return uuid.join("");
};
export const convertUuidV1ToV4 = (str: string): string => {
const uuid = [...str];
uuid[14] = "4";
return uuid.join("");
};
twake/backend/node/src/cli/cmds/migration_cmds/php-workspace/php-workspace-entity.ts
-45
View File
@@ -1,45 +0,0 @@
import { merge } from "lodash";
import {
Column,
Entity,
} from "../../../../core/platform/services/database/services/orm/decorators";
export const TYPE = "workspace";
@Entity(TYPE, {
primaryKey: [["id"]],
type: TYPE,
})
export class PhpWorkspace {
@Column("id", "timeuuid")
id: string;
@Column("group_id", "timeuuid")
group_id: string;
@Column("name", "encoded_string")
name: string;
@Column("logo", "encoded_string")
logo: string;
@Column("stats", "encoded_string")
stats: string;
@Column("is_deleted", "boolean")
isDeleted: boolean;
@Column("is_archived", "boolean")
isArchived: boolean;
@Column("is_default", "boolean")
isDefault: boolean;
@Column("date_added", "number")
dateAdded: number;
}
export type PhpMessagePrimaryKey = Pick<PhpWorkspace, "id">;
export function getInstance(workspace: PhpWorkspace): PhpWorkspace {
return merge(new PhpWorkspace(), workspace);
}
twake/backend/node/src/cli/cmds/migration_cmds/workspace.ts
-148
View File
@@ -1,148 +0,0 @@
import yargs from "yargs";
import twake from "../../../twake";
import ora from "ora";
import { TwakePlatform } from "../../../core/platform/platform";
import { DatabaseServiceAPI } from "../../../core/platform/services/database/api";
import { PhpWorkspace, TYPE as phpTYPE } from "./php-workspace/php-workspace-entity";
import { Pagination } from "../../../core/platform/framework/api/crud-service";
import Workspace, { TYPE, getInstance } from "../../../services/workspaces/entities/workspace";
import _ from "lodash";
import gr from "../../../services/global-resolver";
type Options = {
from?: string;
onlyCompany?: string;
onlyWorkspace?: string;
replaceExisting?: boolean;
};
class WorkspaceMigrator {
database: DatabaseServiceAPI;
constructor(readonly platform: TwakePlatform) {
this.database = this.platform.getProvider<DatabaseServiceAPI>("database");
}
public async run(options: Options = {}): Promise<void> {
const phpRepository = await this.database.getRepository(phpTYPE, PhpWorkspace);
const repository = await this.database.getRepository(TYPE, Workspace);
let waitForCompany = false;
if (options.from) {
waitForCompany = true;
}
// Get all companies
let page: Pagination = { limitStr: "100" };
// For each companies find workspaces
do {
const workspaceListResult = await phpRepository.find({}, { pagination: page }, undefined);
page = workspaceListResult.nextPage as Pagination;
for (const workspace of workspaceListResult.getEntities()) {
if (waitForCompany && options.from == `${workspace.group_id}`) {
waitForCompany = false;
}
if (!waitForCompany) {
if (
(!options.onlyCompany && !options.onlyWorkspace) ||
options.onlyCompany == `${workspace.group_id}`
) {
if (
!(await repository.findOne(
{ company_id: workspace.group_id, id: workspace.id },
{},
undefined,
)) ||
options.replaceExisting
) {
const newWorkspace = getInstance(
_.pick(
workspace,
"id",
"company_id",
"name",
"logo",
"stats",
"is_deleted",
"is_archived",
"is_default",
"date_added",
),
);
newWorkspace.company_id = workspace.group_id;
await repository.save(newWorkspace, undefined);
}
}
}
}
} while (page.page_token);
}
}
const services = [
"storage",
"counter",
"platform-services",
"applications",
"user",
"search",
"channels",
"database",
"webserver",
"message-queue",
"workspaces",
"console",
"auth",
];
const command: yargs.CommandModule<unknown, unknown> = {
command: "workspace",
describe: "command that allow you to migrate php workspaces to node",
builder: {
from: {
default: null,
type: "string",
description: "Start migration from this workspace ID",
},
onlyCompany: {
default: null,
type: "string",
description: "Migrate only this workspace ID",
},
onlyWorkspace: {
default: null,
type: "string",
description: "Migrate only this workspace ID",
},
replaceExisting: {
default: false,
type: "boolean",
description: "Replace already migrated workspaces",
},
},
// eslint-disable-next-line @typescript-eslint/no-unused-vars
handler: async argv => {
const spinner = ora({ text: "Migrating php worskpaces - " }).start();
const platform = await twake.run(services);
await gr.doInit(platform);
const migrator = new WorkspaceMigrator(platform);
const from = argv.from as string | null;
const onlyCompany = argv.onlyCompany as string | null;
const onlyWorkspace = argv.onlyWorkspace as string | null;
const replaceExisting = (argv.replaceExisting || false) as boolean;
await migrator.run({
from,
onlyCompany,
onlyWorkspace,
replaceExisting,
});
return spinner.stop();
},
};
export default command;
twake/backend/node/src/cli/cmds/search_cmds/index_all.ts
-17
View File
@@ -7,18 +7,10 @@ import { Pagination } from "../../../core/platform/framework/api/crud-service";
import User, { TYPE as UserTYPE } from "../../../services/user/entities/user"; import User, { TYPE as UserTYPE } from "../../../services/user/entities/user";
import { Channel } from "../../../services/channels/entities"; import { Channel } from "../../../services/channels/entities";
import Application, {
TYPE as ApplicationTYPE,
} from "../../../services/applications/entities/application";
import Repository from "../../../core/platform/services/database/services/orm/repository/repository"; import Repository from "../../../core/platform/services/database/services/orm/repository/repository";
import { SearchServiceAPI } from "../../../core/platform/services/search/api"; import { SearchServiceAPI } from "../../../core/platform/services/search/api";
import CompanyUser, { TYPE as CompanyUserTYPE } from "../../../services/user/entities/company_user"; import CompanyUser, { TYPE as CompanyUserTYPE } from "../../../services/user/entities/company_user";
import { Message, TYPE as MessageTYPE } from "../../../services/messages/entities/messages";
import gr from "../../../services/global-resolver"; import gr from "../../../services/global-resolver";
import {
MessageFile,
TYPE as MessageFileTYPE,
} from "../../../services/messages/entities/message-files";
type Options = { type Options = {
repository?: string; repository?: string;
@@ -36,17 +28,8 @@ class SearchIndexAll {
public async run(options: Options = {}): Promise<void> { public async run(options: Options = {}): Promise<void> {
const repositories: Map<string, Repository<any>> = new Map(); const repositories: Map<string, Repository<any>> = new Map();
repositories.set("messages", await this.database.getRepository(MessageTYPE, Message));
repositories.set(
"message_files",
await this.database.getRepository(MessageFileTYPE, MessageFile),
);
repositories.set("users", await this.database.getRepository(UserTYPE, User)); repositories.set("users", await this.database.getRepository(UserTYPE, User));
repositories.set("channels", await this.database.getRepository("channels", Channel)); repositories.set("channels", await this.database.getRepository("channels", Channel));
repositories.set(
"applications",
await this.database.getRepository(ApplicationTYPE, Application),
);
const repository = repositories.get(options.repository); const repository = repositories.get(options.repository);
if (!repository) { if (!repository) {
twake/backend/node/src/core/platform/services/email-pusher/types.ts
-2
View File
@@ -1,5 +1,4 @@
import { Channel } from "../../../../services/channels/entities/channel"; import { Channel } from "../../../../services/channels/entities/channel";
import { Message } from "../../../../services/messages/entities/messages";
import Company from "../../../../services/user/entities/company"; import Company from "../../../../services/user/entities/company";
import Workspace from "../../../../services/workspaces/entities/workspace"; import Workspace from "../../../../services/workspaces/entities/workspace";
import User from "../../../../services/user/entities/user"; import User from "../../../../services/user/entities/user";
@@ -11,7 +10,6 @@ export type EmailBuilderDataPayload = {
notifications: { notifications: {
channel: Channel; channel: Channel;
workspace: Workspace; workspace: Workspace;
message: Message & { user: UserObject };
}[]; }[];
}; };
twake/backend/node/src/core/platform/services/knowledge-graph/api-client.ts
-36
View File
@@ -5,7 +5,6 @@ import { KnowledgeGraphCreateBodyRequest, KnowledgeGraphCreateMessageObjectData
import { md5 } from "../../../../core/crypto"; import { md5 } from "../../../../core/crypto";
import { Channel } from "../../../../services/channels/entities"; import { Channel } from "../../../../services/channels/entities";
import gr from "../../../../services/global-resolver"; import gr from "../../../../services/global-resolver";
import { Message } from "../../../../services/messages/entities/messages";
import Company from "../../../../services/user/entities/company"; import Company from "../../../../services/user/entities/company";
import User from "../../../../services/user/entities/user"; import User from "../../../../services/user/entities/user";
import Workspace from "../../../../services/workspaces/entities/workspace"; import Workspace from "../../../../services/workspaces/entities/workspace";
@@ -139,41 +138,6 @@ export default class KnowledgeGraphAPIClient {
} }
} }
public async onMessageUpsert(
channelId: string,
message: Partial<Message>,
sensitiveData: boolean,
): Promise<void> {
const response = await this.send({
records: [
{
key: "null",
value: {
id: "Message",
properties: {
_kg_user_id: await this.getUserKGId(message.user_id),
_kg_email_id: await this.getUserKGMailId(message.user_id),
_kg_company_id: await this.getCompanyKGId(message.cache?.company_id),
message_thread_id: message.thread_id,
message_content: sensitiveData ? message.text : "",
type_message: message.type,
message_created_at: message.created_at,
message_updated_at: message.updated_at,
user_id: message.user_id,
channel_id: message.cache?.channel_id || channelId,
workspace_id: message.cache?.workspace_id,
company_id: message.cache?.company_id,
},
},
},
],
});
if (response.statusText === "OK") {
this.logger.info("onMessageUpsert %o", response.config.data);
}
}
private async send(data: any) { private async send(data: any) {
return await this.axiosInstance.post< return await this.axiosInstance.post<
KnowledgeGraphCreateBodyRequest<KnowledgeGraphCreateMessageObjectData[]> KnowledgeGraphCreateBodyRequest<KnowledgeGraphCreateMessageObjectData[]>
twake/backend/node/src/core/platform/services/knowledge-graph/index.ts
-23
View File
@@ -5,7 +5,6 @@ import Workspace from "../../../../services/workspaces/entities/workspace";
import Company from "../../../../services/user/entities/company"; import Company from "../../../../services/user/entities/company";
import User from "../../../../services/user/entities/user"; import User from "../../../../services/user/entities/user";
import { Channel } from "../../../../services/channels/entities"; import { Channel } from "../../../../services/channels/entities";
import { Message } from "../../../../services/messages/entities/messages";
import { import {
KnowledgeGraphGenericEventPayload, KnowledgeGraphGenericEventPayload,
KnowledgeGraphEvents, KnowledgeGraphEvents,
@@ -48,11 +47,6 @@ export default class KnowledgeGraphService
this.onChannelCreated.bind(this), this.onChannelCreated.bind(this),
); );
localEventBus.subscribe<KnowledgeGraphGenericEventPayload<Message>>(
KnowledgeGraphEvents.MESSAGE_UPSERT,
this.onMessageUpsert.bind(this),
);
localEventBus.subscribe<KnowledgeGraphGenericEventPayload<User>>( localEventBus.subscribe<KnowledgeGraphGenericEventPayload<User>>(
KnowledgeGraphEvents.USER_UPSERT, KnowledgeGraphEvents.USER_UPSERT,
this.onUserCreated.bind(this), this.onUserCreated.bind(this),
@@ -94,23 +88,6 @@ export default class KnowledgeGraphService
} }
} }
async onMessageUpsert(data: KnowledgeGraphGenericEventPayload<Message>): Promise<void> {
this.logger.debug(`${KnowledgeGraphEvents.MESSAGE_UPSERT} %o`, data);
const allowedToShare = await this.shouldForwardEvent(
[data.resource.cache.company_id],
data.resource.user_id,
);
if (this.kgAPIClient && allowedToShare) {
this.kgAPIClient.onMessageUpsert(
data.resource.cache.company_id,
data.resource,
allowedToShare === "all",
);
}
}
async onUserCreated(data: KnowledgeGraphGenericEventPayload<User>): Promise<void> { async onUserCreated(data: KnowledgeGraphGenericEventPayload<User>): Promise<void> {
this.logger.info(`${KnowledgeGraphEvents.USER_UPSERT} %o`, data); this.logger.info(`${KnowledgeGraphEvents.USER_UPSERT} %o`, data);
twake/backend/node/src/core/platform/services/knowledge-graph/provider.ts
-2
View File
@@ -3,13 +3,11 @@ import { Channel } from "../../../../services/channels/entities";
import { TwakeServiceProvider } from "../../framework"; import { TwakeServiceProvider } from "../../framework";
import { KnowledgeGraphGenericEventPayload } from "./types"; import { KnowledgeGraphGenericEventPayload } from "./types";
import Workspace from "../../../../services/workspaces/entities/workspace"; import Workspace from "../../../../services/workspaces/entities/workspace";
import { Message } from "../../../../services/messages/entities/messages";
import User from "../../../../services/user/entities/user"; import User from "../../../../services/user/entities/user";
export default interface KnowledgeGraphAPI extends TwakeServiceProvider { export default interface KnowledgeGraphAPI extends TwakeServiceProvider {
onCompanyCreated(data: KnowledgeGraphGenericEventPayload<Company>): void; onCompanyCreated(data: KnowledgeGraphGenericEventPayload<Company>): void;
onWorkspaceCreated(data: KnowledgeGraphGenericEventPayload<Workspace>): void; onWorkspaceCreated(data: KnowledgeGraphGenericEventPayload<Workspace>): void;
onChannelCreated(data: KnowledgeGraphGenericEventPayload<Channel>): void; onChannelCreated(data: KnowledgeGraphGenericEventPayload<Channel>): void;
onMessageUpsert(data: KnowledgeGraphGenericEventPayload<Message>): void;
onUserCreated(data: KnowledgeGraphGenericEventPayload<User>): void; onUserCreated(data: KnowledgeGraphGenericEventPayload<User>): void;
} }
twake/backend/node/src/core/platform/services/tracker/index.ts
-17
View File
@@ -43,23 +43,6 @@ export default class Tracker extends TwakeService<TrackerAPI> implements Tracker
); );
}); });
const messageSentEvent = "channel:message_sent";
localEventBus.subscribe<ResourceEventsPayload>(messageSentEvent, data => {
logger.debug(`Tracker - New ${messageSentEvent} event`);
this.track(
{
user: data.user,
event: messageSentEvent,
properties: {
is_direct: data.message.workspace_id === "direct" ? true : false,
is_thread_reply: data.message.thread_id ? true : false,
},
},
(err: Error) =>
err ? logger.error({ err }, "Tracker - Error while tracking", messageSentEvent) : false,
);
});
const channelCreatedEvent = "channel:created"; const channelCreatedEvent = "channel:created";
localEventBus.subscribe<ResourceEventsPayload>(channelCreatedEvent, data => { localEventBus.subscribe<ResourceEventsPayload>(channelCreatedEvent, data => {
logger.debug(`Tracker - New ${channelCreatedEvent} event`); logger.debug(`Tracker - New ${channelCreatedEvent} event`);
twake/backend/node/src/services/applications-api/index.ts
-34
View File
@@ -1,34 +0,0 @@
import { Prefix, TwakeService } from "../../core/platform/framework";
import WebServerAPI from "../../core/platform/services/webserver/provider";
import web from "./web/index";
import FastProxy from "fast-proxy";
@Prefix("/api")
export default class ApplicationsApiService extends TwakeService<undefined> {
version = "1";
name = "applicationsapi";
public async doInit(): Promise<this> {
const fastify = this.context.getProvider<WebServerAPI>("webserver").getServer();
fastify.register((instance, _opts, next) => {
web(instance, { prefix: this.prefix });
next();
});
//Redirect requests from /plugins/* to the plugin server (if installed)
const { proxy, close } = FastProxy({
base: this.configuration.get("plugins.server"),
});
fastify.addHook("onClose", close);
fastify.all("/plugins/*", (req, rep) => {
proxy(req.raw, rep.raw, req.url, {});
});
return this;
}
// TODO: remove
api(): undefined {
return undefined;
}
}
twake/backend/node/src/services/applications-api/types.ts
-13
View File
@@ -1,13 +0,0 @@
import { Channel } from "../channels/entities";
import { Message } from "../messages/entities/messages";
import { Thread } from "../messages/entities/threads";
export type HookType = {
type: "message";
application_id: string;
company_id: string;
channel?: Channel;
thread: Thread;
message: Message;
};
twake/backend/node/src/services/applications-api/web/controllers/index.ts
-181
View File
@@ -1,181 +0,0 @@
import { FastifyInstance, FastifyReply, FastifyRequest, HTTPMethods } from "fastify";
import { ApplicationObject } from "../../../applications/entities/application";
import {
ApplicationApiExecutionContext,
ApplicationLoginRequest,
ApplicationLoginResponse,
ConfigureRequest,
} from "../types";
import { ResourceGetResponse } from "../../../../utils/types";
import { CrudException } from "../../../../core/platform/framework/api/crud-service";
import { localEventBus } from "../../../../core/platform/framework/event-bus";
import {
RealtimeApplicationEvent,
RealtimeBaseBusEvent,
} from "../../../../core/platform/services/realtime/types";
import gr from "../../../global-resolver";
import _ from "lodash";
import { v4 } from "uuid";
export class ApplicationsApiController {
async token(
request: FastifyRequest<{ Body: ApplicationLoginRequest }>,
): Promise<ResourceGetResponse<ApplicationLoginResponse>> {
const context = getExecutionContext(request);
if (!request.body.id || !request.body.secret) {
throw CrudException.forbidden("Application not found");
}
const app = await gr.services.applications.marketplaceApps.get(
{
id: request.body.id,
},
context,
);
if (!app) {
throw CrudException.forbidden("Application not found");
}
if (!app.api.private_key || app.api.private_key !== request.body.secret) {
throw CrudException.forbidden("Secret key is not valid");
}
return {
resource: {
access_token: gr.platformServices.auth.generateJWT(request.body.id, null, {
track: false,
provider_id: "",
application_id: request.body.id,
}),
},
};
}
async me(
request: FastifyRequest,
_reply: FastifyReply,
): Promise<ResourceGetResponse<ApplicationObject>> {
const context = getExecutionContext(request);
const entity = await gr.services.applications.marketplaceApps.get(
{
id: context.application_id,
},
context,
);
if (!entity) {
throw CrudException.notFound("Application not found");
}
return { resource: entity.getApplicationObject() };
}
async configure(
request: FastifyRequest<{ Body: ConfigureRequest }>,
_reply: FastifyReply,
): Promise<Record<string, string>> {
const app_id = request.currentUser.application_id;
const context = getExecutionContext(request);
const application = await gr.services.applications.marketplaceApps.get({ id: app_id }, context);
if (!application) {
throw CrudException.forbidden("Application not found");
}
const body = request.body;
const data = {
action: "configure",
application: {
id: app_id,
identity: application.identity,
},
form: body.form,
connection_id: body.connection_id,
hidden_data: {},
configurator_id: v4(),
};
localEventBus.publish("realtime:event", {
room: "/me/" + body.user_id,
type: "application",
data,
} as RealtimeBaseBusEvent<RealtimeApplicationEvent>);
return { status: "ok" };
}
async proxy(
request: FastifyRequest<{ Params: { company_id: string; service: string; version: string } }>,
reply: FastifyReply,
fastify: FastifyInstance,
): Promise<void> {
// Check the application has access to this company
const company_id = request.params.company_id;
const companyApplication = gr.services.applications.companyApps.get({
company_id,
application_id: request.currentUser.application_id,
id: undefined,
});
if (!companyApplication) {
throw CrudException.forbidden("This application is not installed in the requested company");
}
const context = getExecutionContext(request);
const app = await gr.services.applications.marketplaceApps.get(
{
id: request.currentUser.application_id,
},
context,
);
// Check call can be done from this IP
if (
app.api.allowed_ips.trim() &&
app.api.allowed_ips !== "*" &&
!_.includes(
app.api.allowed_ips
.split(",")
.map(a => a.trim())
.filter(a => a),
request.ip,
)
) {
throw CrudException.forbidden(
`This application is not allowed to access from this IP (${request.ip})`,
);
}
//TODO Check application access rights (write, read, remove for each micro services)
const _access = app.access;
//TODO save some statistics about API usage for application and per companies
const route = request.url.replace("/api/", "/internal/services/");
fastify.inject(
{
method: request.method as HTTPMethods,
url: route,
payload: request.body as any,
headers: _.pick(request.headers, "authorization"),
},
(err, response) => {
reply.headers(response.headers);
reply.send(response.payload);
},
);
}
}
function getExecutionContext(request: FastifyRequest): ApplicationApiExecutionContext {
return {
application_id: request.currentUser?.application_id,
user: request.currentUser,
url: request.url,
method: request.routerMethod,
transport: "http",
};
}
twake/backend/node/src/services/applications-api/web/index.ts
-10
View File
@@ -1,10 +0,0 @@
import { FastifyInstance, FastifyRegisterOptions } from "fastify";
import routes from "./routes";
export default (
fastify: FastifyInstance,
options: FastifyRegisterOptions<{ prefix: string }>,
): void => {
fastify.log.debug("Configuring /internal/services/applications/v1 routes");
fastify.register(routes, options);
};
twake/backend/node/src/services/applications-api/web/routes.ts
-54
View File
@@ -1,54 +0,0 @@
import { FastifyInstance, FastifyPluginCallback, FastifyRequest } from "fastify";
import { ApplicationsApiController } from "./controllers";
import { ApplicationApiBaseRequest } from "./types";
import { logger as log } from "../../../core/platform/framework";
import { configureRequestSchema } from "./schemas";
const routes: FastifyPluginCallback = (fastify: FastifyInstance, options, next) => {
const controller = new ApplicationsApiController();
const checkApplication = async (request: FastifyRequest<{ Body: ApplicationApiBaseRequest }>) => {
if (!request.currentUser.application_id) {
log.debug(request.currentUser);
throw fastify.httpErrors.forbidden("You should log in as application");
}
};
//Authenticate the application
fastify.route({
method: "POST",
url: "/console/v1/login",
handler: controller.token.bind(controller),
});
//Get myself as an application
fastify.route({
method: "GET",
url: "/console/v1/me",
preValidation: [fastify.authenticate],
preHandler: [checkApplication],
handler: controller.me.bind(controller),
});
//Open a configuration popup on the client side
fastify.route({
method: "POST",
url: "/console/v1/configure",
preValidation: [fastify.authenticate],
schema: configureRequestSchema,
handler: controller.configure.bind(controller),
});
//Get myself as an application
fastify.route({
method: ["POST", "GET", "DELETE", "PUT"],
url: "/:service/:version/companies/:company_id/*",
preValidation: [fastify.authenticate],
handler: (request, reply) => controller.proxy.bind(controller)(request, reply, fastify),
});
next();
};
export default routes;
twake/backend/node/src/services/applications-api/web/schemas.ts
-16
View File
@@ -1,16 +0,0 @@
export const applicationsSchema = {
type: "object",
properties: {},
};
export const configureRequestSchema = {
body: {
type: "object",
properties: {
user_id: { type: "string" },
connection_id: { type: "string" },
form: {},
},
required: ["user_id", "connection_id"],
},
};
twake/backend/node/src/services/applications-api/web/types.ts
-23
View File
@@ -1,23 +0,0 @@
import { AccessToken } from "../../../utils/types";
import { ExecutionContext } from "../../../core/platform/framework/api/crud-service";
export interface ApplicationApiBaseRequest {
id: string;
secret: string;
}
export type ApplicationLoginRequest = ApplicationApiBaseRequest;
export interface ApplicationLoginResponse {
access_token: AccessToken;
}
export interface ApplicationApiExecutionContext extends ExecutionContext {
application_id: string;
}
export interface ConfigureRequest {
user_id: string;
connection_id: string;
form?: any;
}
twake/backend/node/src/services/applications/entities/application.search.ts
-33
View File
@@ -1,33 +0,0 @@
import Application, { TYPE } from "./application";
export default {
index: TYPE,
source: (entity: Application) => {
return {
company_id: entity.company_id,
name: entity.identity.name,
description: entity.identity.description,
categories: entity.identity.categories,
compatibility: entity.identity.compatibility,
published: entity.publication.published,
created_at: entity.stats.created_at,
};
},
mongoMapping: {
text: {
name: "text",
description: "text",
},
},
esMapping: {
properties: {
company_id: { type: "keyword" },
name: { type: "text", index_prefixes: { min_chars: 1 } },
description: { type: "text" },
categories: { type: "keyword" },
compatibility: { type: "keyword" },
published: { type: "boolean" },
created_at: { type: "number" },
},
},
};
twake/backend/node/src/services/applications/entities/application.ts
-211
View File
@@ -1,211 +0,0 @@
import { Type } from "class-transformer";
import _, { merge } from "lodash";
import { Column, Entity } from "../../../core/platform/services/database/services/orm/decorators";
import search from "./application.search";
export const TYPE = "applications";
@Entity(TYPE, {
primaryKey: ["id"],
type: TYPE,
search,
})
export default class Application {
@Type(() => String)
@Column("id", "timeuuid", { generator: "timeuuid" })
id: string;
@Type(() => String)
@Column("group_id", "timeuuid")
company_id: string;
@Column("is_default", "boolean")
is_default: boolean;
@Column("identity", "json")
identity: ApplicationIdentity;
//This information is private to the application, make sure not to disclose it
@Column("api", "encoded_json")
api: ApplicationApi;
@Column("access", "json")
access: ApplicationAccess;
@Column("display", "json")
display: ApplicationDisplay;
@Column("publication", "json")
publication: ApplicationPublication;
@Column("stats", "json")
stats: ApplicationStatistics;
getPublicObject(): PublicApplicationObject {
const i = _.pick(
this,
"id",
"company_id",
"is_default",
"identity",
"access",
"display",
"publication",
"stats",
);
i.is_default = !!i.is_default;
return i;
}
getApplicationObject(): ApplicationObject {
const i = _.pick(
this,
"id",
"company_id",
"is_default",
"identity",
"access",
"display",
"publication",
"stats",
"api",
);
i.is_default = !!i.is_default;
return i;
}
}
export type PublicApplicationObject = Pick<
Application,
"id" | "company_id" | "is_default" | "identity" | "access" | "display" | "publication" | "stats"
>;
export type ApplicationObject = Pick<
Application,
| "id"
| "company_id"
| "is_default"
| "identity"
| "access"
| "display"
| "publication"
| "stats"
| "api"
>;
export type ApplicationPrimaryKey = { id: string };
export function getInstance(message: Application): Application {
return merge(new Application(), message);
}
export type ApplicationIdentity = {
code: string;
name: string;
icon: string;
description: string;
website: string;
categories: string[];
compatibility: "twake"[];
};
export type ApplicationPublication = {
published: boolean; //Publication accepted // RO
requested: boolean; //Publication requested
};
export type ApplicationStatistics = {
created_at: number; // RO
updated_at: number; // RO
version: number; // RO
};
export type ApplicationApi = {
hooks_url: string;
allowed_ips: string;
private_key: string; // RO
};
type ApplicationScopes =
| "files"
| "applications"
| "workspaces"
| "users"
| "messages"
| "channels";
export type ApplicationAccess = {
read: ApplicationScopes[];
write: ApplicationScopes[];
delete: ApplicationScopes[];
hooks: ApplicationScopes[];
};
export type ApplicationDisplay = {
twake: {
files?: {
editor?: {
preview_url: string; //Open a preview inline (iframe)
edition_url: string; //Url to edit the file (full screen)
extensions?: string[]; //Main extensions app can read
// if file was created by the app, then the app is able to edit with or without extension
empty_files?: {
url: string; // "https://[...]/empty.docx";
filename: string; // "Untitled.docx";
name: string; // "Word Document";
}[];
};
actions?: //List of action that can apply on a file
{
name: string;
id: string;
}[];
};
//Chat plugin
chat?: {
input?:
| true
| {
icon?: string; //If defined replace original icon url of your app
type?: "file" | "call"; //To add in existing apps folder / default icon
};
commands?: {
command: string; // my_app mycommand
description: string;
}[];
actions?: //List of action that can apply on a message
{
name: string;
id: string;
}[];
};
//Allow app to appear as a bot user in direct chat
direct?:
| true
| {
name?: string;
icon?: string; //If defined replace original icon url of your app
};
//Display app as a standalone application in a tab
tab?:
| {
url: string;
}
| true;
//Display app as a standalone application on the left bar
standalone?:
| {
url: string;
}
| true;
//Define where the app can be configured from
configuration?: ("global" | "channel")[];
};
};
twake/backend/node/src/services/applications/entities/company-application.ts
-40
View File
@@ -1,40 +0,0 @@
import { Type } from "class-transformer";
import { Column, Entity } from "../../../core/platform/services/database/services/orm/decorators";
import { PublicApplicationObject } from "./application";
export const TYPE = "group_app";
@Entity(TYPE, {
primaryKey: [["group_id"], "app_id", "id"],
type: TYPE,
})
export default class CompanyApplication {
@Type(() => String)
@Column("group_id", "timeuuid")
company_id: string;
@Type(() => String)
@Column("app_id", "timeuuid")
application_id: string;
@Type(() => String)
@Column("id", "timeuuid", { generator: "timeuuid" })
id: string;
@Column("created_at", "number")
created_at: number;
@Type(() => String)
@Column("created_by", "string")
created_by: string; //Will be the default delegated user when doing actions on Twake
}
export type CompanyApplicationPrimaryKey = Pick<
CompanyApplication,
"company_id" | "application_id" | "id"
>;
export class CompanyApplicationWithApplication extends CompanyApplication {
//Not in database but attached to this object
application?: PublicApplicationObject;
}
twake/backend/node/src/services/applications/index.ts
-24
View File
@@ -1,24 +0,0 @@
import { Prefix, TwakeService } from "../../core/platform/framework";
import WebServerAPI from "../../core/platform/services/webserver/provider";
import web from "./web";
@Prefix("/internal/services/applications/v1")
export default class ApplicationsService extends TwakeService<undefined> {
version = "1";
name = "applications";
public async doInit(): Promise<this> {
const fastify = this.context.getProvider<WebServerAPI>("webserver").getServer();
fastify.register((instance, _opts, next) => {
web(instance, { prefix: this.prefix });
next();
});
return this;
}
// TODO: remove
api(): undefined {
return undefined;
}
}
twake/backend/node/src/services/applications/realtime.ts
-13
View File
@@ -1,13 +0,0 @@
import { WebsocketMetadata } from "../../utils/types";
export function getCompanyApplicationRooms(companyId: string): WebsocketMetadata[] {
return [
{
room: getCompanyApplicationRoom(companyId),
},
];
}
export function getCompanyApplicationRoom(companyApplicationId: string): string {
return `/company-application/${companyApplicationId}`;
}
twake/backend/node/src/services/applications/services/applications.ts
-143
View File
@@ -1,143 +0,0 @@
import Application, {
ApplicationPrimaryKey,
getInstance as getApplicationInstance,
PublicApplicationObject,
TYPE,
} from "../entities/application";
import Repository from "../../../core/platform/services/database/services/orm/repository/repository";
import { Initializable, logger, TwakeServiceProvider } from "../../../core/platform/framework";
import {
DeleteResult,
ExecutionContext,
ListResult,
OperationType,
Pagination,
SaveResult,
} from "../../../core/platform/framework/api/crud-service";
import SearchRepository from "../../../core/platform/services/search/repository";
import assert from "assert";
import gr from "../../global-resolver";
import { InternalToHooksProcessor } from "./internal-event-to-hooks";
export class ApplicationServiceImpl implements TwakeServiceProvider, Initializable {
version: "1";
repository: Repository<Application>;
searchRepository: SearchRepository<Application>;
async init(): Promise<this> {
try {
this.searchRepository = gr.platformServices.search.getRepository<Application>(
TYPE,
Application,
);
this.repository = await gr.database.getRepository<Application>(TYPE, Application);
} catch (err) {
console.log(err);
logger.error("Error while initializing applications service");
}
gr.platformServices.messageQueue.processor.addHandler(new InternalToHooksProcessor());
return this;
}
async get(pk: ApplicationPrimaryKey, context: ExecutionContext): Promise<Application> {
return await this.repository.findOne(pk, {}, context);
}
async list(
pagination: Pagination,
options?: { search?: string },
context?: ExecutionContext,
): Promise<ListResult<PublicApplicationObject>> {
let entities: ListResult<Application>;
if (options.search) {
entities = await this.searchRepository.search(
{},
{
pagination,
$text: {
$search: options.search,
},
},
context,
);
} else {
entities = await this.repository.find({}, { pagination }, context);
}
entities.filterEntities(app => app.publication.published);
const applications = entities
.getEntities()
.filter(app => app)
.map(app => app.getPublicObject());
return new ListResult(entities.type, applications, entities.nextPage);
}
async listUnpublished(context: ExecutionContext): Promise<Application[]> {
const entities = await this.repository.find({}, {}, context);
entities.filterEntities(app => !app.publication.published);
return entities.getEntities();
}
async listDefaults(context: ExecutionContext): Promise<ListResult<PublicApplicationObject>> {
const entities = [];
let page: Pagination = { limitStr: "100" };
do {
const applicationListResult = await this.repository.find({}, { pagination: page }, context);
page = applicationListResult.nextPage as Pagination;
applicationListResult.filterEntities(app => app.publication.published && app.is_default);
for (const application of applicationListResult.getEntities()) {
if (application) entities.push(application.getPublicObject());
}
} while (page.page_token);
return new ListResult(TYPE, entities);
}
async save<SaveOptions>(
item: Application,
options?: SaveOptions,
context?: ExecutionContext,
): Promise<SaveResult<Application>> {
assert(item.company_id, "company_id is not defined");
try {
const entity = getApplicationInstance(item);
await this.repository.save(entity, context);
return new SaveResult<Application>("application", entity, OperationType.UPDATE);
} catch (e) {
throw e;
}
}
async delete(
pk: ApplicationPrimaryKey,
context?: ExecutionContext,
): Promise<DeleteResult<Application>> {
const entity = await this.get(pk, context);
await this.repository.remove(entity, context);
return new DeleteResult<Application>("application", entity, true);
}
async publish(pk: ApplicationPrimaryKey, context: ExecutionContext): Promise<void> {
const entity = await this.get(pk, context);
if (!entity) {
throw new Error("Entity not found");
}
entity.publication.published = true;
await this.repository.save(entity, context);
}
async unpublish(pk: ApplicationPrimaryKey, context: ExecutionContext): Promise<void> {
const entity = await this.get(pk, context);
if (!entity) {
throw new Error("Entity not found");
}
entity.publication.published = false;
await this.repository.save(entity, context);
}
}
twake/backend/node/src/services/applications/services/company-applications.ts
-188
View File
@@ -1,188 +0,0 @@
import CompanyApplication, {
CompanyApplicationPrimaryKey,
CompanyApplicationWithApplication,
TYPE,
} from "../entities/company-application";
import Repository from "../../../core/platform/services/database/services/orm/repository/repository";
import {
Initializable,
logger,
RealtimeDeleted,
RealtimeSaved,
TwakeServiceProvider,
} from "../../../core/platform/framework";
import {
DeleteResult,
ListResult,
OperationType,
Paginable,
Pagination,
SaveResult,
} from "../../../core/platform/framework/api/crud-service";
import { CompanyExecutionContext } from "../web/types";
import { getCompanyApplicationRoom } from "../realtime";
import gr from "../../global-resolver";
export class CompanyApplicationServiceImpl implements TwakeServiceProvider, Initializable {
version: "1";
repository: Repository<CompanyApplication>;
async init(): Promise<this> {
try {
this.repository = await gr.database.getRepository<CompanyApplication>(
TYPE,
CompanyApplication,
);
} catch (err) {
console.log(err);
logger.error("Error while initializing applications service");
}
return this;
}
// TODO: remove logic from context
async get(
pk: Pick<CompanyApplicationPrimaryKey, "company_id" | "application_id"> & { id?: string },
context?: CompanyExecutionContext,
): Promise<CompanyApplicationWithApplication> {
const companyApplication = await this.repository.findOne(
{
group_id: context ? context.company.id : pk.company_id,
app_id: pk.application_id,
},
{},
context,
);
const application = await gr.services.applications.marketplaceApps.get(
{
id: pk.application_id,
},
context,
);
return {
...companyApplication,
application: application,
};
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
@RealtimeSaved<CompanyApplication>((companyApplication, _context) => {
return [
{
room: getCompanyApplicationRoom(companyApplication.id),
resource: companyApplication,
},
];
})
async save<SaveOptions>(
item: Pick<CompanyApplicationPrimaryKey, "company_id" | "application_id">,
_?: SaveOptions,
context?: CompanyExecutionContext,
): Promise<SaveResult<CompanyApplication>> {
if (!context?.user?.id && !context?.user?.server_request) {
throw new Error("Only an user of a company can add an application to a company.");
}
let operation = OperationType.UPDATE;
let companyApplication = await this.repository.findOne(
{
group_id: context?.company.id,
app_id: item.application_id,
},
{},
context,
);
if (!companyApplication) {
operation = OperationType.CREATE;
companyApplication = new CompanyApplication();
companyApplication.company_id = context.company.id;
companyApplication.application_id = item.application_id;
companyApplication.created_at = new Date().getTime();
companyApplication.created_by = context?.user?.id || "";
await this.repository.save(companyApplication, context);
}
return new SaveResult(TYPE, companyApplication, operation);
}
async initWithDefaultApplications(
companyId: string,
context: CompanyExecutionContext,
): Promise<void> {
const defaultApps = await gr.services.applications.marketplaceApps.listDefaults(context);
for (const defaultApp of defaultApps.getEntities()) {
await this.save({ company_id: companyId, application_id: defaultApp.id }, {}, context);
}
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
@RealtimeDeleted<CompanyApplication>((companyApplication, _context) => {
return [
{
room: getCompanyApplicationRoom(companyApplication.id),
resource: companyApplication,
},
];
})
async delete(
pk: CompanyApplicationPrimaryKey,
context?: CompanyExecutionContext,
): Promise<DeleteResult<CompanyApplication>> {
const companyApplication = await this.repository.findOne(
{
group_id: context.company.id,
app_id: pk.application_id,
},
{},
context,
);
let deleted = false;
if (companyApplication) {
this.repository.remove(companyApplication, context);
deleted = true;
}
return new DeleteResult(TYPE, companyApplication, deleted);
}
async list<ListOptions>(
pagination: Paginable,
options?: ListOptions,
context?: CompanyExecutionContext,
): Promise<ListResult<CompanyApplicationWithApplication>> {
const companyApplications = await this.repository.find(
{
group_id: context.company.id,
},
{ pagination: Pagination.fromPaginable(pagination) },
context,
);
const applications = [];
for (const companyApplication of companyApplications.getEntities()) {
const application = await gr.services.applications.marketplaceApps.get(
{
id: companyApplication.application_id,
},
context,
);
if (application)
applications.push({
...companyApplication,
application: application,
});
}
return new ListResult<CompanyApplicationWithApplication>(
TYPE,
applications,
companyApplications.nextPage,
);
}
}
twake/backend/node/src/services/applications/services/hooks.ts
-96
View File
@@ -1,96 +0,0 @@
import Application from "../entities/application";
import Repository from "../../../core/platform/services/database/services/orm/repository/repository";
import {
Initializable,
logger as log,
TwakeServiceProvider,
} from "../../../core/platform/framework";
import { CrudException, ExecutionContext } from "../../../core/platform/framework/api/crud-service";
import SearchRepository from "../../../core/platform/services/search/repository";
import axios from "axios";
import * as crypto from "crypto";
import { isObject } from "lodash";
import gr from "../../global-resolver";
export class ApplicationHooksService implements TwakeServiceProvider, Initializable {
version: "1";
repository: Repository<Application>;
searchRepository: SearchRepository<Application>;
async init() {
return this;
}
async notifyApp(
application_id: string,
connection_id: string,
user_id: string,
type: string,
name: string,
content: any,
company_id: string,
workspace_id: string,
context: ExecutionContext,
): Promise<void> {
const app = await gr.services.applications.marketplaceApps.get({ id: application_id }, context);
if (!app) {
throw CrudException.notFound("Application not found");
}
if (!app.api.hooks_url) {
throw CrudException.badRequest("Application hooks_url is not defined");
}
const payload = {
type,
name,
content,
connection_id: connection_id,
user_id: user_id,
company_id,
workspace_id,
};
const signature = crypto
.createHmac("sha256", app.api.private_key)
.update(JSON.stringify(payload))
.digest("hex");
return await axios
.post(app.api.hooks_url, payload, {
headers: {
"Content-Type": "application/json",
"X-Twake-Signature": signature,
},
})
.then(({ data }) => data)
.catch(e => {
log.error(e.message);
const r = e.response;
if (!r) {
throw CrudException.badGateway("Can't connect remote application");
}
let msg = r.data;
if (isObject(msg)) {
// parse typical responses
if (r.data.message) {
msg = r.data.message;
} else if (r.data.error) {
msg = r.data.error;
} else {
msg = JSON.stringify(r.data);
}
}
if (r.status == 403) {
throw CrudException.forbidden(msg);
} else {
throw CrudException.badRequest(msg);
}
});
}
}
twake/backend/node/src/services/applications/services/internal-event-to-hooks.ts
-64
View File
@@ -1,64 +0,0 @@
import { logger } from "../../../core/platform/framework";
import { HookType } from "../../applications-api/types";
import { MessageQueueHandler } from "../../../core/platform/services/message-queue/api";
import { MessageHook } from "../../messages/types";
import gr from "../../global-resolver";
import { ExecutionContext } from "../../../core/platform/framework/api/crud-service";
export class InternalToHooksProcessor implements MessageQueueHandler<MessageHook, void> {
readonly topics = {
in: "application:hook:message",
};
readonly options = {
unique: true,
ack: true,
queue: "application:hook:message:consumer1",
};
readonly name = "Application::InternalToHooksProcessor";
validate(_: HookType): boolean {
return true;
}
async process(message: HookType, context?: ExecutionContext): Promise<void> {
logger.debug(`${this.name} - Receive hook of type ${message.type}`);
const application = await gr.services.applications.marketplaceApps.get(
{
id: message.application_id,
},
context,
);
//TODO Check application access rights (hooks)
const _access = application.access;
// Check application still exists in the company
if (
!(await gr.services.applications.companyApps.get({
company_id: message.company_id,
application_id: message.application_id,
id: undefined,
}))
) {
logger.error(
`${this.name} - Application ${message.application_id} not found in company ${message.company_id}`,
);
return;
}
await gr.services.applications.hooks.notifyApp(
message.application_id,
null,
null,
"hook",
null,
{ message },
null,
null,
context,
);
}
}
twake/backend/node/src/services/applications/web/controllers/applications.ts
-265
View File
@@ -1,265 +0,0 @@
import { FastifyReply, FastifyRequest } from "fastify";
import { CrudController } from "../../../../core/platform/services/webserver/types";
import {
PaginationQueryParameters,
ResourceCreateResponse,
ResourceDeleteResponse,
ResourceGetResponse,
ResourceListResponse,
ResourceUpdateResponse,
} from "../../../../utils/types";
import Application, {
ApplicationObject,
PublicApplicationObject,
} from "../../entities/application";
import {
CrudException,
ExecutionContext,
Pagination,
} from "../../../../core/platform/framework/api/crud-service";
import _ from "lodash";
import { randomBytes } from "crypto";
import { ApplicationEventRequestBody } from "../types";
import { logger as log } from "../../../../core/platform/framework";
import { hasCompanyAdminLevel } from "../../../../utils/company";
import gr from "../../../global-resolver";
export class ApplicationController
implements
CrudController<
ResourceGetResponse<PublicApplicationObject>,
ResourceUpdateResponse<PublicApplicationObject>,
ResourceListResponse<PublicApplicationObject>,
ResourceDeleteResponse
>
{
async get(
request: FastifyRequest<{ Params: { application_id: string } }>,
): Promise<ResourceGetResponse<ApplicationObject | PublicApplicationObject>> {
const context = getExecutionContext(request);
const entity = await gr.services.applications.marketplaceApps.get(
{
id: request.params.application_id,
},
context,
);
const companyUser = await gr.services.companies.getCompanyUser(
{ id: entity.company_id },
{ id: context.user.id },
);
const isAdmin = companyUser && companyUser.role == "admin";
return {
resource: isAdmin ? entity.getApplicationObject() : entity.getPublicObject(),
};
}
async list(
request: FastifyRequest<{
Querystring: PaginationQueryParameters & { search: string };
}>,
): Promise<ResourceListResponse<PublicApplicationObject>> {
const entities = await gr.services.applications.marketplaceApps.list(new Pagination(), {
search: request.query.search,
});
return {
resources: entities.getEntities(),
next_page_token: entities.nextPage.page_token,
};
}
async save(
request: FastifyRequest<{
Params: { application_id: string };
Body: { resource: Application };
}>,
_reply: FastifyReply,
): Promise<ResourceGetResponse<ApplicationObject | PublicApplicationObject>> {
const context = getExecutionContext(request);
try {
const app = request.body.resource;
const now = new Date().getTime();
let entity: Application;
if (request.params.application_id) {
entity = await gr.services.applications.marketplaceApps.get(
{
id: request.params.application_id,
},
context,
);
if (!entity) {
throw CrudException.notFound("Application not found");
}
entity.publication.requested = app.publication.requested;
if (app.publication.requested === false) {
entity.publication.published = false;
}
if (entity.publication.published) {
if (
!_.isEqual(
_.pick(entity, "identity", "api", "access", "display"),
_.pick(app, "identity", "api", "access", "display"),
)
) {
throw CrudException.badRequest(
"You can't update applications details while it published",
);
}
}
entity.identity = app.identity;
entity.api.hooks_url = app.api.hooks_url;
entity.api.allowed_ips = app.api.allowed_ips;
entity.access = app.access;
entity.display = app.display;
entity.stats.updated_at = now;
entity.stats.version++;
const res = await gr.services.applications.marketplaceApps.save(entity);
entity = res.entity;
} else {
// INSERT
app.is_default = false;
app.publication.published = false;
app.api.private_key = randomBytes(32).toString("base64");
app.stats = {
created_at: now,
updated_at: now,
version: 0,
};
const res = await gr.services.applications.marketplaceApps.save(app);
entity = res.entity;
}
return {
resource: entity.getApplicationObject(),
};
} catch (e) {
log.error(e);
throw e;
}
}
async delete(
request: FastifyRequest<{ Params: { application_id: string } }>,
reply: FastifyReply,
): Promise<ResourceDeleteResponse> {
const context = getExecutionContext(request);
const application = await gr.services.applications.marketplaceApps.get(
{
id: request.params.application_id,
},
context,
);
const compUser = await gr.services.companies.getCompanyUser(
{ id: application.company_id },
{ id: context.user.id },
);
if (!compUser || !hasCompanyAdminLevel(compUser.role)) {
throw CrudException.forbidden("You don't have the rights to delete this application");
}
const deleteResult = await gr.services.applications.marketplaceApps.delete(
{
id: request.params.application_id,
},
context,
);
if (deleteResult.deleted) {
reply.code(204);
return {
status: "success",
};
}
return {
status: "error",
};
}
async event(
request: FastifyRequest<{
Body: ApplicationEventRequestBody;
Params: { application_id: string };
}>,
_reply: FastifyReply,
): Promise<ResourceCreateResponse<any>> {
const context = getExecutionContext(request);
const content = request.body.data;
const applicationEntity = await gr.services.applications.marketplaceApps.get(
{
id: request.params.application_id,
},
context,
);
if (!applicationEntity) {
throw CrudException.notFound("Application not found");
}
const companyUser = gr.services.companies.getCompanyUser(
{ id: request.body.company_id },
{ id: context.user.id },
);
if (!companyUser) {
throw CrudException.badRequest(
"You cannot send event to an application from another company",
);
}
const applicationInCompany = await gr.services.applications.companyApps.get({
company_id: request.body.company_id,
application_id: request.params.application_id,
id: undefined,
});
if (!applicationInCompany) {
throw CrudException.badRequest("Application isn't installed in this company");
}
const hookResponse = await gr.services.applications.hooks.notifyApp(
request.params.application_id,
request.body.connection_id,
context.user.id,
request.body.type,
request.body.name,
content,
request.body.company_id,
request.body.workspace_id,
context,
);
return {
resource: hookResponse,
};
}
}
function getExecutionContext(request: FastifyRequest): ExecutionContext {
return {
user: request.currentUser,
url: request.url,
method: request.routerMethod,
transport: "http",
};
}
twake/backend/node/src/services/applications/web/controllers/company-applications.ts
-119
View File
@@ -1,119 +0,0 @@
import { FastifyReply, FastifyRequest } from "fastify";
import {
PaginationQueryParameters,
ResourceDeleteResponse,
ResourceGetResponse,
ResourceListResponse,
ResourceUpdateResponse,
} from "../../../../utils/types";
import { PublicApplicationObject } from "../../entities/application";
import { CompanyExecutionContext } from "../types";
import { CrudController } from "../../../../core/platform/services/webserver/types";
import { getCompanyApplicationRooms } from "../../realtime";
import gr from "../../../global-resolver";
export class CompanyApplicationController
implements
CrudController<
ResourceGetResponse<PublicApplicationObject>,
ResourceUpdateResponse<PublicApplicationObject>,
ResourceListResponse<PublicApplicationObject>,
ResourceDeleteResponse
>
{
async get(
request: FastifyRequest<{ Params: { company_id: string; application_id: string } }>,
): Promise<ResourceGetResponse<PublicApplicationObject>> {
const context = getCompanyExecutionContext(request);
const resource = await gr.services.applications.companyApps.get(
{
application_id: request.params.application_id,
company_id: context.company.id,
id: undefined,
},
context,
);
return {
resource: resource?.application,
};
}
async list(
request: FastifyRequest<{
Params: { company_id: string };
Querystring: PaginationQueryParameters & { search: string };
}>,
): Promise<ResourceListResponse<PublicApplicationObject>> {
const context = getCompanyExecutionContext(request);
const resources = await gr.services.applications.companyApps.list(
request.query,
{ search: request.query.search },
context,
);
return {
resources: resources.getEntities().map(ca => ca.application),
next_page_token: resources.nextPage.page_token,
websockets:
gr.platformServices.realtime.sign(
getCompanyApplicationRooms(request.params.company_id),
context.user.id,
) || [],
};
}
async save(
request: FastifyRequest<{
Params: { company_id: string; application_id: string };
Body: PublicApplicationObject;
}>,
): Promise<ResourceGetResponse<PublicApplicationObject>> {
const context = getCompanyExecutionContext(request);
const resource = await gr.services.applications.companyApps.save(
{ application_id: request.params.application_id, company_id: context.company.id },
{},
context,
);
const app = await gr.services.applications.companyApps.get(resource.entity);
return {
resource: app.application,
};
}
async delete(
request: FastifyRequest<{ Params: { company_id: string; application_id: string } }>,
_reply: FastifyReply,
): Promise<ResourceDeleteResponse> {
const context = getCompanyExecutionContext(request);
const resource = await gr.services.applications.companyApps.delete(
{
application_id: request.params.application_id,
company_id: context.company.id,
id: undefined,
},
context,
);
return {
status: resource.deleted ? "success" : "error",
};
}
}
function getCompanyExecutionContext(
request: FastifyRequest<{
Params: { company_id: string };
}>,
): CompanyExecutionContext {
return {
user: request.currentUser,
company: { id: request.params.company_id },
url: request.url,
method: request.routerMethod,
reqId: request.id,
transport: "http",
};
}
twake/backend/node/src/services/applications/web/controllers/index.ts
-1
View File
@@ -1 +0,0 @@
export * from "./applications";
twake/backend/node/src/services/applications/web/index.ts
-12
View File
@@ -1,12 +0,0 @@
import { FastifyInstance, FastifyRegisterOptions } from "fastify";
import routes from "./routes";
export default (
fastify: FastifyInstance,
options: FastifyRegisterOptions<{
prefix: string;
}>,
): void => {
fastify.log.debug("Configuring /internal/services/applications/v1 routes");
fastify.register(routes, options);
};
twake/backend/node/src/services/applications/web/routes.ts
-161
View File
@@ -1,161 +0,0 @@
import { FastifyInstance, FastifyPluginCallback, FastifyRequest } from "fastify";
import { ApplicationController } from "./controllers/applications";
import { CompanyApplicationController } from "./controllers/company-applications";
import Application from "../entities/application";
import { applicationEventHookSchema, applicationPostSchema } from "./schemas";
import { logger as log } from "../../../core/platform/framework";
import { checkUserBelongsToCompany, hasCompanyAdminLevel } from "../../../utils/company";
import gr from "../../global-resolver";
const applicationsUrl = "/applications";
const companyApplicationsUrl = "/companies/:company_id/applications";
const routes: FastifyPluginCallback = (fastify: FastifyInstance, options, next) => {
const applicationController = new ApplicationController();
const companyApplicationController = new CompanyApplicationController();
const adminCheck = async (
request: FastifyRequest<{
Body: { resource: Application };
Params: { application_id: string };
}>,
) => {
try {
let companyId: string = request.body?.resource?.company_id;
if (request.params.application_id) {
const application = await gr.services.applications.marketplaceApps.get(
{
id: request.params.application_id,
},
undefined,
);
if (!application) {
throw fastify.httpErrors.notFound("Application is not defined");
}
companyId = application.company_id;
}
const userId = request.currentUser.id;
if (!companyId) {
throw fastify.httpErrors.forbidden(`Company ${companyId} not found`);
}
const companyUser = await checkUserBelongsToCompany(userId, companyId);
if (!hasCompanyAdminLevel(companyUser.role)) {
throw fastify.httpErrors.forbidden("You must be an admin of this company");
}
} catch (e) {
log.error(e);
throw e;
}
};
/**
* Applications collection
* Marketplace of applications
*/
//Get and search list of applications in the marketplace
fastify.route({
method: "GET",
url: `${applicationsUrl}`,
preValidation: [fastify.authenticate],
// schema: applicationGetSchema,
handler: applicationController.list.bind(applicationController),
});
//Get a single application in the marketplace
fastify.route({
method: "GET",
url: `${applicationsUrl}/:application_id`,
preValidation: [fastify.authenticate],
// schema: applicationGetSchema,
handler: applicationController.get.bind(applicationController),
});
//Create application (must be my company application and I must be company admin)
fastify.route({
method: "POST",
url: `${applicationsUrl}`,
preHandler: [adminCheck],
preValidation: [fastify.authenticate],
schema: applicationPostSchema,
handler: applicationController.save.bind(applicationController),
});
//Edit application (must be my company application and I must be company admin)
fastify.route({
method: "POST",
url: `${applicationsUrl}/:application_id`,
preHandler: [adminCheck],
preValidation: [fastify.authenticate],
schema: applicationPostSchema,
handler: applicationController.save.bind(applicationController),
});
// Delete application (must be my company application and I must be company admin)
fastify.route({
method: "DELETE",
url: `${applicationsUrl}/:application_id`,
preHandler: [adminCheck],
preValidation: [fastify.authenticate],
handler: applicationController.delete.bind(applicationController),
});
/**
* Company applications collection
* Company-wide available applications
* (must be my company application and I must be company admin)
*/
//Get list of applications for a company
fastify.route({
method: "GET",
url: `${companyApplicationsUrl}`,
preValidation: [fastify.authenticate],
handler: companyApplicationController.list.bind(companyApplicationController),
});
//Get one application of a company
fastify.route({
method: "GET",
url: `${companyApplicationsUrl}/:application_id`,
preValidation: [fastify.authenticate],
handler: companyApplicationController.get.bind(companyApplicationController),
});
//Remove an application from a company
fastify.route({
method: "DELETE",
url: `${companyApplicationsUrl}/:application_id`,
preValidation: [fastify.authenticate],
handler: companyApplicationController.delete.bind(companyApplicationController),
});
//Add an application to the company
fastify.route({
method: "POST",
url: `${companyApplicationsUrl}/:application_id`,
preValidation: [fastify.authenticate],
handler: companyApplicationController.save.bind(companyApplicationController),
});
//Application event triggered by a user
fastify.route({
method: "POST",
url: `${applicationsUrl}/:application_id/event`,
preValidation: [fastify.authenticate],
schema: applicationEventHookSchema,
handler: applicationController.event.bind(applicationController),
});
next();
};
export default routes;
twake/backend/node/src/services/applications/web/schemas.ts
-129
View File
@@ -1,129 +0,0 @@
export const applicationsSchema = {
type: "object",
properties: {},
};
const applicationIdentity = {
type: "object",
properties: {
code: { type: "string" },
name: { type: "string" },
icon: { type: "string" },
description: { type: "string" },
website: { type: "string" },
categories: { type: "array", items: { type: "string" } },
compatibility: { type: "array", items: { type: "string" } },
},
required: ["code", "name", "icon", "description", "website", "categories", "compatibility"],
};
const applicationAccess = {
type: "object",
properties: {
read: { type: "array", items: { type: "string" } },
write: { type: "array", items: { type: "string" } },
delete: { type: "array", items: { type: "string" } },
hooks: { type: "array", items: { type: "string" } },
},
required: ["read", "write", "delete", "hooks"],
};
const requestApplicationPublication = {
type: "object",
properties: {
requested: { type: "boolean" },
},
required: ["requested"],
};
const responseApplicationPublication = {
type: "object",
properties: {
published: { type: "boolean" },
requested: { type: "boolean" },
},
required: ["requested", "published"],
};
const applicationStats = {
type: "object",
properties: {
created_at: { type: "number" },
updated_at: { type: "number" },
version: { type: "number" },
},
required: ["created_at", "updated_at", "version"],
};
const apiObject = {
type: "object",
properties: {
hooks_url: { type: "string" },
allowed_ips: { type: "string" },
private_key: { type: "string" },
},
required: ["hooks_url", "allowed_ips"],
};
const requestApplicationObject = {
type: "object",
properties: {
company_id: { type: "string" },
identity: applicationIdentity,
access: applicationAccess,
display: {},
api: apiObject,
publication: requestApplicationPublication,
},
required: ["company_id", "identity", "access", "display", "api", "publication"],
additionalProperties: false,
};
const responseApplicationObject = {
type: "object",
properties: {
id: { type: "string" },
is_default: { type: "boolean" },
company_id: { type: "string" },
identity: applicationIdentity,
access: applicationAccess,
display: {},
publication: responseApplicationPublication,
api: apiObject,
stats: applicationStats,
},
required: [
"id",
"is_default",
"company_id",
"identity",
"access",
"display",
"publication",
"stats",
],
additionalProperties: false,
};
export const applicationPostSchema = {
body: { type: "object", properties: { resource: requestApplicationObject } },
response: {
"2xx": {
resource: responseApplicationObject,
},
},
};
export const applicationEventHookSchema = {
body: {
type: "object",
properties: {
company_id: { type: "string" },
workspace_id: { type: "string" },
type: { type: "string" },
name: { type: "string" },
content: {},
},
required: ["company_id", "workspace_id", "type", "content"],
},
};
twake/backend/node/src/services/applications/web/types.ts
-15
View File
@@ -1,15 +0,0 @@
import { ExecutionContext } from "../../../core/platform/framework/api/crud-service";
export interface CompanyExecutionContext extends ExecutionContext {
company: { id: string };
}
export interface ApplicationEventRequestBody {
company_id: string;
workspace_id: string;
connection_id: string;
type: string;
name?: string;
content: any;
data: any;
}
twake/backend/node/src/services/channels/services/member/service.ts
+1 -1
View File
@@ -26,6 +26,7 @@ import {
ChannelMemberType, ChannelMemberType,
ChannelVisibility, ChannelVisibility,
WorkspaceExecutionContext, WorkspaceExecutionContext,
CompanyExecutionContext,
} from "../../types"; } from "../../types";
import { Channel, ResourceEventsPayload, User } from "../../../../utils/types"; import { Channel, ResourceEventsPayload, User } from "../../../../utils/types";
import { cloneDeep, isNil, omitBy } from "lodash"; import { cloneDeep, isNil, omitBy } from "lodash";
@@ -51,7 +52,6 @@ import { WorkspacePrimaryKey } from "../../../workspaces/entities/workspace";
import gr from "../../../global-resolver"; import gr from "../../../global-resolver";
import uuidTime from "uuid-time"; import uuidTime from "uuid-time";
import { CompanyExecutionContext } from "../../../../services/messages/types";
import { ChannelObject } from "../channel/types"; import { ChannelObject } from "../channel/types";
const USER_CHANNEL_KEYS = [ const USER_CHANNEL_KEYS = [
twake/backend/node/src/services/channels/services/pubsub/index.ts
-2
View File
@@ -1,6 +1,5 @@
import { Initializable } from "../../../../core/platform/framework"; import { Initializable } from "../../../../core/platform/framework";
import { NewChannelActivityProcessor } from "./new-channel-activity"; import { NewChannelActivityProcessor } from "./new-channel-activity";
import { NewDirectChannelMessageProcessor } from "./new-direct-channel-message";
import { NewUserInWorkspaceJoinDefaultChannelsProcessor } from "./new-user-in-workspace-join-default-channels"; import { NewUserInWorkspaceJoinDefaultChannelsProcessor } from "./new-user-in-workspace-join-default-channels";
import { NewPendingEmailsInWorkspaceJoinChannelsProcessor } from "./new-pending-emails-in-workspace-join-channels"; import { NewPendingEmailsInWorkspaceJoinChannelsProcessor } from "./new-pending-emails-in-workspace-join-channels";
import { NewWorkspaceProcessor } from "./new-workspace"; import { NewWorkspaceProcessor } from "./new-workspace";
@@ -10,7 +9,6 @@ export class ChannelsMessageQueueListener implements Initializable {
async init(): Promise<this> { async init(): Promise<this> {
const channelActivityProcessor = await new NewChannelActivityProcessor().init(); const channelActivityProcessor = await new NewChannelActivityProcessor().init();
gr.platformServices.messageQueue.processor.addHandler(channelActivityProcessor); gr.platformServices.messageQueue.processor.addHandler(channelActivityProcessor);
gr.platformServices.messageQueue.processor.addHandler(new NewDirectChannelMessageProcessor());
gr.platformServices.messageQueue.processor.addHandler( gr.platformServices.messageQueue.processor.addHandler(
new NewUserInWorkspaceJoinDefaultChannelsProcessor(), new NewUserInWorkspaceJoinDefaultChannelsProcessor(),
); );
twake/backend/node/src/services/channels/services/pubsub/new-direct-channel-message.ts
-93
View File
@@ -1,93 +0,0 @@
import { without } from "lodash";
import { Channel, ChannelMember } from "../../entities";
import { getLogger } from "../../../../core/platform/framework";
import { MessageQueueHandler } from "../../../../core/platform/services/message-queue/api";
import { MessageNotification } from "../../../messages/types";
import gr from "../../../global-resolver";
import { ExecutionContext } from "../../../../core/platform/framework/api/crud-service";
const logger = getLogger("channel.message-queue.new-direct-channel-message");
export class NewDirectChannelMessageProcessor
implements MessageQueueHandler<MessageNotification, void>
{
readonly topics = {
in: "message:created",
};
readonly options = {
unique: true,
ack: true,
queue: "message:created:consumer1",
};
readonly name = "Channel::NewDirectChannelMessageProcessor";
validate(message: MessageNotification): boolean {
return !!(message && message.channel_id && message.company_id && message.workspace_id);
}
async process(message: MessageNotification, context?: ExecutionContext): Promise<void> {
logger.info(
`${this.name} - Processing notification for message ${message.thread_id}/${message.id} in channel ${message.channel_id}`,
);
logger.debug(`${this.name} - Notification message ${JSON.stringify(message)}`);
try {
const channel = await gr.services.channels.channels.get(
{
company_id: message.company_id,
id: message.channel_id,
workspace_id: message.workspace_id,
},
context,
);
if (!channel || !Channel.isDirectChannel(channel)) {
logger.debug(`${this.name} - Not a direct channel`);
return;
}
const memberIds = without(channel.members || [], String(message.sender));
if (!memberIds.length) {
logger.debug(`${this.name} - No members to notify. Original array was %o`, channel.members);
}
const members = memberIds.map(user_id => {
return {
user_id,
channel_id: channel.id,
workspace_id: channel.workspace_id,
company_id: channel.company_id,
} as ChannelMember;
});
await Promise.all(
members.map(async member => {
try {
logger.info(`${this.name} - Adding ${member.user_id} to channel ${message.channel_id}`);
const memberSaved = await gr.services.channels.members.save(member, {
channel,
user: { id: member.user_id },
});
logger.info(
`${this.name} - Member added to channel ${message.channel_id} - ${JSON.stringify(
memberSaved,
)}`,
);
} catch (err) {
logger.info(
{ err },
`${this.name} - Error while adding user ${member.user_id} to direct channel ${member.channel_id}`,
);
}
}),
);
} catch (err) {
logger.error(
{ err },
`${this.name} - Error while processing direct channel members for message ${message.thread_id}/${message.id} in channel ${message.channel_id}`,
);
}
}
}
twake/backend/node/src/services/channels/types.ts
+4
View File
@@ -52,3 +52,7 @@ export type ChannelActivityNotification = {
sender_name: string; sender_name: string;
body: string; body: string;
}; };
export interface CompanyExecutionContext extends ExecutionContext {
company: { id: string };
}
twake/backend/node/src/services/documents/services/engine/extract-keywords.ts
+1 -2
View File
@@ -2,8 +2,7 @@ import globalResolver from "../../../../services/global-resolver";
import { logger } from "../../../../core/platform/framework"; import { logger } from "../../../../core/platform/framework";
import { MessageQueueHandler } from "../../../../core/platform/services/message-queue/api"; import { MessageQueueHandler } from "../../../../core/platform/services/message-queue/api";
import { DocumentsMessageQueueCallback, DocumentsMessageQueueRequest } from "../../types"; import { DocumentsMessageQueueCallback, DocumentsMessageQueueRequest } from "../../types";
import { extractKeywords, officeFileToString, pdfFileToString } from "../../utils"; import { extractKeywords, officeFileToString, pdfFileToString, isFileType } from "../../utils";
import { isFileType } from "../../../../services/previews/utils";
import { officeExtensions, textExtensions, pdfExtensions } from "../../../../utils/mime"; import { officeExtensions, textExtensions, pdfExtensions } from "../../../../utils/mime";
import { readableToString } from "../../../../utils/files"; import { readableToString } from "../../../../utils/files";
twake/backend/node/src/services/documents/utils.ts
+8 -11
View File
@@ -1,3 +1,4 @@
import mimes from "../../utils/mime";
import { merge } from "lodash"; import { merge } from "lodash";
import { DriveFile } from "./entities/drive-file"; import { DriveFile } from "./entities/drive-file";
import { import {
@@ -347,17 +348,6 @@ export const getAccessLevel = async (
repository: Repository<DriveFile>, repository: Repository<DriveFile>,
context: CompanyExecutionContext & { public_token?: string; twake_tab_token?: string }, context: CompanyExecutionContext & { public_token?: string; twake_tab_token?: string },
): Promise<DriveFileAccessLevel | "none"> => { ): Promise<DriveFileAccessLevel | "none"> => {
if (
context.user?.application_id &&
(
await globalResolver.services.applications.companyApps.get({
company_id: context.company.id,
application_id: context.user.application_id,
})
)?.application?.access //TODO check precise access right for applications
) {
return "manage";
}
if (!id || id === "root") if (!id || id === "root")
return !context?.user?.id ? "none" : (await isCompanyGuest(context)) ? "read" : "manage"; return !context?.user?.id ? "none" : (await isCompanyGuest(context)) ? "read" : "manage";
@@ -804,3 +794,10 @@ export const canMoveItem = async (
return true; return true;
}; };
export function isFileType(fileMime: string, fileName: string, requiredExtensions: string[]): any {
const extension = fileName.split(".").pop();
const secondaryExtensions = Object.keys(mimes).filter(k => mimes[k] === fileMime);
const fileExtensions = [extension, ...secondaryExtensions];
return fileExtensions.some(e => requiredExtensions.includes(e));
}
twake/backend/node/src/services/files/services/index.ts
-76
View File
@@ -6,8 +6,6 @@ import { File } from "../entities/file";
import Repository from "../../../../src/core/platform/services/database/services/orm/repository/repository"; import Repository from "../../../../src/core/platform/services/database/services/orm/repository/repository";
import { CompanyExecutionContext } from "../web/types"; import { CompanyExecutionContext } from "../web/types";
import { logger } from "../../../core/platform/framework"; import { logger } from "../../../core/platform/framework";
import { PreviewClearMessageQueueRequest, PreviewMessageQueueRequest } from "../../previews/types";
import { PreviewFinishedProcessor } from "./preview";
import _ from "lodash"; import _ from "lodash";
import { getDownloadRoute, getThumbnailRoute } from "../web/routes"; import { getDownloadRoute, getThumbnailRoute } from "../web/routes";
import { import {
@@ -27,9 +25,6 @@ export class FileServiceImpl {
try { try {
await Promise.all([ await Promise.all([
(this.repository = await gr.database.getRepository<File>("files", File)), (this.repository = await gr.database.getRepository<File>("files", File)),
gr.platformServices.messageQueue.processor.addHandler(
new PreviewFinishedProcessor(this, this.repository),
),
]); ]);
} catch (err) { } catch (err) {
logger.error("Error while initializing files service", err); logger.error("Error while initializing files service", err);
@@ -116,61 +111,6 @@ export class FileServiceImpl {
await this.repository.save(entity, context); await this.repository.save(entity, context);
} }
//Fixme: detect in multichunk when all chunks are uploaded to trigger this. For now we do only single chunks for preview
if (entity.upload_data.chunks === 1 && totalUploadedSize) {
/** Send preview generation task */
if (entity.upload_data.size < this.max_preview_file_size) {
const document: PreviewMessageQueueRequest["document"] = {
id: JSON.stringify(_.pick(entity, "id", "company_id")),
provider: gr.platformServices.storage.getConnectorType(),
path: getFilePath(entity),
encryption_algo: this.algorithm,
encryption_key: entity.encryption_key,
chunks: entity.upload_data.chunks,
filename: entity.metadata.name,
mime: entity.metadata.mime,
};
const output = {
provider: gr.platformServices.storage.getConnectorType(),
path: `${getFilePath(entity)}/thumbnails/`,
encryption_algo: this.algorithm,
encryption_key: entity.encryption_key,
pages: 10,
};
entity.metadata.thumbnails_status = "waiting";
await this.repository.save(entity, context);
try {
await gr.platformServices.messageQueue.publish<PreviewMessageQueueRequest>(
"services:preview",
{
data: { document, output },
},
);
if (options.waitForThumbnail) {
entity = await gr.services.files.getFile(
{
id: entity.id,
company_id: context.company.id,
},
context,
{ waitForThumbnail: true },
);
}
} catch (err) {
entity.metadata.thumbnails_status = "error";
await this.repository.save(entity, context);
logger.warn({ err }, "Previewing - Error while sending ");
}
}
/** End preview generation task generation */
}
} }
return entity; return entity;
@@ -286,22 +226,6 @@ export class FileServiceImpl {
totalChunks: fileToDelete.upload_data.chunks, totalChunks: fileToDelete.upload_data.chunks,
}); });
if (fileToDelete.thumbnails.length > 0) {
await gr.platformServices.messageQueue.publish<PreviewClearMessageQueueRequest>(
"services:preview:clear",
{
data: {
document: {
id: JSON.stringify(_.pick(fileToDelete, "id", "company_id")),
provider: gr.platformServices.storage.getConnectorType(),
path: `${path}/thumbnails/`,
thumbnails_number: fileToDelete.thumbnails.length,
},
},
},
);
}
return new DeleteResult("files", fileToDelete, true); return new DeleteResult("files", fileToDelete, true);
} }
} }
twake/backend/node/src/services/files/services/preview.ts
-67
View File
@@ -1,67 +0,0 @@
import { logger, TwakeContext } from "../../../core/platform/framework";
import { PreviewMessageQueueCallback } from "../../../../src/services/previews/types";
import Repository from "../../../../src/core/platform/services/database/services/orm/repository/repository";
import { File } from "../entities/file";
import { FileServiceImpl } from "./index";
import { MessageQueueHandler } from "../../../core/platform/services/message-queue/api";
import { ExecutionContext } from "../../../core/platform/framework/api/crud-service";
/**
* Update the file metadata and upload the thumbnails in storage
*/
export class PreviewFinishedProcessor
implements MessageQueueHandler<PreviewMessageQueueCallback, string>
{
constructor(readonly service: FileServiceImpl, private repository: Repository<File>) {}
async init(_context?: TwakeContext): Promise<this> {
return this;
}
readonly topics = {
in: "services:preview:callback",
};
readonly options = {
unique: true,
ack: true,
};
name = "FilePreviewProcessor";
validate(message: PreviewMessageQueueCallback): boolean {
return !!(message && message.document && message.thumbnails);
}
async process(message: PreviewMessageQueueCallback, context?: ExecutionContext): Promise<string> {
logger.info(
`${this.name} - Updating file metadata with preview generation ${message.thumbnails.length}`,
);
const pk: { company_id: string; id: string } = JSON.parse(message.document.id);
const entity = await this.repository.findOne(pk, {}, context);
if (!entity) {
logger.info(`This file ${message.document.id} does not exists anymore.`);
return;
}
entity.thumbnails = message.thumbnails.map((thumb, index) => {
return {
index,
id: thumb.path.split("/").pop(),
size: thumb.size,
type: thumb.type,
width: thumb.width,
height: thumb.height,
url: `/internal/services/files/v1/companies/${entity.company_id}/files/${entity.id}/thumbnails/${index}`,
};
});
if (!entity.metadata) entity.metadata = {};
entity.metadata.thumbnails_status = "done";
await this.repository.save(entity, context);
return "done";
}
}
twake/backend/node/src/services/files/utils.ts
+1 -2
View File
@@ -1,4 +1,3 @@
import { MessageFile } from "../messages/entities/message-files";
import { File, PublicFile } from "./entities/file"; import { File, PublicFile } from "./entities/file";
export const formatPublicFile = (file: Partial<File | PublicFile>): PublicFile => { export const formatPublicFile = (file: Partial<File | PublicFile>): PublicFile => {
@@ -16,7 +15,7 @@ export const formatPublicFile = (file: Partial<File | PublicFile>): PublicFile =
} as PublicFile; } as PublicFile;
}; };
export const fileIsMedia = (file: Partial<File | MessageFile>): boolean => { export const fileIsMedia = (file: Partial<File>): boolean => {
return ( return (
file.metadata?.mime?.startsWith("video/") || file.metadata?.mime?.startsWith("video/") ||
file.metadata?.mime?.startsWith("audio/") || file.metadata?.mime?.startsWith("audio/") ||
twake/backend/node/src/services/global-resolver.ts
-50
View File
@@ -24,15 +24,7 @@ import { WorkspaceServiceImpl } from "./workspaces/services/workspace";
import { UserExternalLinksServiceImpl } from "./user/services/external_links"; import { UserExternalLinksServiceImpl } from "./user/services/external_links";
import { UserNotificationBadgeService } from "./notifications/services/bages"; import { UserNotificationBadgeService } from "./notifications/services/bages";
import { NotificationPreferencesService } from "./notifications/services/preferences"; import { NotificationPreferencesService } from "./notifications/services/preferences";
import { ThreadMessagesService } from "./messages/services/messages";
import { MessagesFilesService } from "./messages/services/messages-files";
import { ThreadsService } from "./messages/services/threads";
import { UserBookmarksService } from "./messages/services/user-bookmarks";
import { UserServiceImpl } from "./user/services/users/service"; import { UserServiceImpl } from "./user/services/users/service";
import { CompanyApplicationServiceImpl } from "./applications/services/company-applications";
import { ApplicationServiceImpl } from "./applications/services/applications";
import { ViewsServiceImpl } from "./messages/services/views";
import { MessagesEngine } from "./messages/services/engine";
import { FileServiceImpl } from "./files/services"; import { FileServiceImpl } from "./files/services";
import { ChannelServiceImpl } from "./channels/services/channel/service"; import { ChannelServiceImpl } from "./channels/services/channel/service";
import { MemberServiceImpl } from "./channels/services/member/service"; import { MemberServiceImpl } from "./channels/services/member/service";
@@ -44,13 +36,8 @@ import { NotificationEngine } from "./notifications/services/engine";
import { MobilePushService } from "./notifications/services/mobile-push"; import { MobilePushService } from "./notifications/services/mobile-push";
import { ChannelMemberPreferencesServiceImpl } from "./notifications/services/channel-preferences"; import { ChannelMemberPreferencesServiceImpl } from "./notifications/services/channel-preferences";
import { ChannelThreadUsersServiceImpl } from "./notifications/services/channel-thread-users"; import { ChannelThreadUsersServiceImpl } from "./notifications/services/channel-thread-users";
import { PreviewProcessService } from "./previews/services/files/processing/service";
import { ApplicationHooksService } from "./applications/services/hooks";
import OnlineServiceImpl from "./online/service"; import OnlineServiceImpl from "./online/service";
import { PreviewEngine } from "./previews/services/files/engine";
import { ChannelsMessageQueueListener } from "./channels/services/pubsub"; import { ChannelsMessageQueueListener } from "./channels/services/pubsub";
import { LinkPreviewProcessService } from "./previews/services/links/processing/service";
import { LinkPreviewEngine } from "./previews/services/links/engine";
import { UserNotificationDigestService } from "./notifications/services/digest"; import { UserNotificationDigestService } from "./notifications/services/digest";
import { DocumentsService } from "./documents/services"; import { DocumentsService } from "./documents/services";
import { DocumentsEngine } from "./documents/services/engine"; import { DocumentsEngine } from "./documents/services/engine";
@@ -88,23 +75,6 @@ type TwakeServices = {
mobilePush: MobilePushService; mobilePush: MobilePushService;
digest: UserNotificationDigestService; digest: UserNotificationDigestService;
}; };
preview: {
files: PreviewProcessService;
links: LinkPreviewProcessService;
};
messages: {
messages: ThreadMessagesService;
messagesFiles: MessagesFilesService;
threads: ThreadsService;
userBookmarks: UserBookmarksService;
views: ViewsServiceImpl;
engine: MessagesEngine;
};
applications: {
marketplaceApps: ApplicationServiceImpl;
companyApps: CompanyApplicationServiceImpl;
hooks: ApplicationHooksService;
};
files: FileServiceImpl; files: FileServiceImpl;
channels: { channels: {
channels: ChannelServiceImpl; channels: ChannelServiceImpl;
@@ -159,9 +129,6 @@ class GlobalResolver {
assert(service, `Platform service ${key} was not initialized`); assert(service, `Platform service ${key} was not initialized`);
}); });
await new PreviewEngine().init();
await new LinkPreviewEngine().init();
this.services = { this.services = {
workspaces: await new WorkspaceServiceImpl().init(), workspaces: await new WorkspaceServiceImpl().init(),
companies: await new CompanyServiceImpl().init(), companies: await new CompanyServiceImpl().init(),
@@ -178,23 +145,6 @@ class GlobalResolver {
mobilePush: await new MobilePushService().init(), mobilePush: await new MobilePushService().init(),
digest: await new UserNotificationDigestService().init(), digest: await new UserNotificationDigestService().init(),
}, },
preview: {
files: await new PreviewProcessService().init(),
links: await new LinkPreviewProcessService().init(),
},
messages: {
messages: await new ThreadMessagesService().init(platform),
messagesFiles: await new MessagesFilesService().init(),
threads: await new ThreadsService().init(),
userBookmarks: await new UserBookmarksService().init(),
views: await new ViewsServiceImpl().init(),
engine: await new MessagesEngine().init(),
},
applications: {
marketplaceApps: await new ApplicationServiceImpl().init(),
companyApps: await new CompanyApplicationServiceImpl().init(),
hooks: await new ApplicationHooksService().init(),
},
files: await new FileServiceImpl().init(), files: await new FileServiceImpl().init(),
channels: { channels: {
channels: await new ChannelServiceImpl().init(), channels: await new ChannelServiceImpl().init(),
twake/backend/node/src/services/notifications/services/bages.ts
+245 -286
View File
@@ -1,302 +1,261 @@
/* eslint-disable @typescript-eslint/no-unused-vars */ /* eslint-disable @typescript-eslint/no-unused-vars */
import { import {
Initializable, Initializable,
RealtimeDeleted, RealtimeDeleted,
RealtimeSaved, RealtimeSaved,
TwakeContext, TwakeContext,
TwakeServiceProvider, TwakeServiceProvider,
} from "../../../core/platform/framework"; } from "../../../core/platform/framework";
import { ResourcePath } from "../../../core/platform/services/realtime/types"; import { ResourcePath } from "../../../core/platform/services/realtime/types";
import { import {
CrudException, CrudException,
DeleteResult, DeleteResult,
ExecutionContext, ExecutionContext,
ListResult, ListResult,
OperationType, OperationType,
Pagination, Pagination,
SaveResult, SaveResult,
} from "../../../core/platform/framework/api/crud-service"; } from "../../../core/platform/framework/api/crud-service";
import { import {
getUserNotificationBadgeInstance, getUserNotificationBadgeInstance,
UserNotificationBadge, UserNotificationBadge,
UserNotificationBadgePrimaryKey, UserNotificationBadgePrimaryKey,
UserNotificationBadgeType, UserNotificationBadgeType,
} from "../entities"; } from "../entities";
import { NotificationExecutionContext } from "../types"; import { NotificationExecutionContext } from "../types";
import { getNotificationRoomName } from "./realtime"; import { getNotificationRoomName } from "./realtime";
import Repository from "../../../core/platform/services/database/services/orm/repository/repository"; import Repository from "../../../core/platform/services/database/services/orm/repository/repository";
import gr from "../../global-resolver"; import gr from "../../global-resolver";
import _, { pick, uniq } from "lodash"; import _, { pick, uniq } from "lodash";
export class UserNotificationBadgeService implements TwakeServiceProvider, Initializable { export class UserNotificationBadgeService implements TwakeServiceProvider, Initializable {
version: "1"; version: "1";
repository: Repository<UserNotificationBadge>; repository: Repository<UserNotificationBadge>;
async init(context: TwakeContext): Promise<this> { async init(context: TwakeContext): Promise<this> {
this.repository = await gr.database.getRepository<UserNotificationBadge>( this.repository = await gr.database.getRepository<UserNotificationBadge>(
UserNotificationBadgeType, UserNotificationBadgeType,
UserNotificationBadge, UserNotificationBadge,
);
return this;
}
async get(
pk: UserNotificationBadgePrimaryKey,
context: ExecutionContext,
): Promise<UserNotificationBadge> {
return await this.repository.findOne(pk, {}, context);
}
@RealtimeSaved<UserNotificationBadge>((badge, context) => {
return [
{
room: ResourcePath.get(getNotificationRoomName(badge.user_id)),
},
];
})
async save<SaveOptions>(
badge: UserNotificationBadge,
context: ExecutionContext,
): Promise<SaveResult<UserNotificationBadge>> {
//Initiate the digest
await gr.services.notifications.digest.putBadge(badge);
await this.repository.save(getUserNotificationBadgeInstance(badge), context);
return new SaveResult(UserNotificationBadgeType, badge, OperationType.CREATE);
}
@RealtimeDeleted<UserNotificationBadge>((badge, context) => {
return [
{
room: ResourcePath.get(getNotificationRoomName(badge.user_id)),
},
];
})
async delete(
pk: UserNotificationBadgePrimaryKey,
context?: NotificationExecutionContext,
): Promise<DeleteResult<UserNotificationBadge>> {
//Cancel the current digest as we just read the badges
await gr.services.notifications.digest.cancelDigest(pk.company_id, pk.user_id);
await this.repository.remove(pk as UserNotificationBadge, context);
return new DeleteResult(UserNotificationBadgeType, pk as UserNotificationBadge, true);
}
list(): Promise<ListResult<UserNotificationBadge>> {
throw new Error("Not implemented");
}
async listForUserPerCompanies(
user_id: string,
context: ExecutionContext,
): Promise<ListResult<UserNotificationBadge>> {
//We remove all badge from current company as next block will create dupicates
const companies_ids = (await gr.services.companies.getAllForUser(user_id)).map(
gu => gu.group_id,
);
let result: UserNotificationBadge[] = [];
let type = "";
for (const company_id of companies_ids) {
const find = await this.repository.find(
{
company_id,
user_id,
},
{
pagination: new Pagination("", "1"),
},
context,
); );
type = find.type;
result = result.concat(find.getEntities()); return this;
} }
const badges = new ListResult(type, result); async get(
await this.ensureBadgesAreReachable(badges, context); pk: UserNotificationBadgePrimaryKey,
context: ExecutionContext,
return badges; ): Promise<UserNotificationBadge> {
} return await this.repository.findOne(pk, {}, context);
async listForUser(
company_id: string,
user_id: string,
filter: Pick<UserNotificationBadgePrimaryKey, "workspace_id" | "channel_id" | "thread_id">,
context?: ExecutionContext,
): Promise<ListResult<UserNotificationBadge>> {
if (!company_id || !user_id) {
throw CrudException.badRequest("company_id and user_id are required");
} }
//Cancel the current digest as we just read the badges @RealtimeSaved<UserNotificationBadge>((badge, context) => {
await gr.services.notifications.digest.cancelDigest(company_id, user_id); return [
{
const badges = await this.repository.find( room: ResourcePath.get(getNotificationRoomName(badge.user_id)),
{
...{
company_id,
user_id,
}, },
...pick(filter, ["workspace_id", "channel_id", "thread_id"]), ];
}, })
{}, async save<SaveOptions>(
context, badge: UserNotificationBadge,
); context: ExecutionContext,
): Promise<SaveResult<UserNotificationBadge>> {
await this.ensureBadgesAreReachable(badges, context); //Initiate the digest
await gr.services.notifications.digest.putBadge(badge);
return badges;
} await this.repository.save(getUserNotificationBadgeInstance(badge), context);
return new SaveResult(UserNotificationBadgeType, badge, OperationType.CREATE);
// This will ensure we are still in the channels and if not, we'll remove the badge
// We need to also ensure more than that
// - Are we in the workspace?
// - Are we in the company?
async ensureBadgesAreReachable(
badges: ListResult<UserNotificationBadge>,
context?: ExecutionContext,
): Promise<ListResult<UserNotificationBadge>> {
if (badges.getEntities().length === 0) {
return badges;
} }
const userId = badges.getEntities()[0].user_id; @RealtimeDeleted<UserNotificationBadge>((badge, context) => {
return [
const channels = uniq(badges.getEntities().map(r => r.channel_id)); {
for (const channelId of channels) { room: ResourcePath.get(getNotificationRoomName(badge.user_id)),
const someBadge = badges.getEntities().find(b => b.channel_id === channelId); },
const channelMemberPk = { ];
company_id: someBadge.company_id, })
workspace_id: someBadge.workspace_id, async delete(
channel_id: channelId, pk: UserNotificationBadgePrimaryKey,
user_id: userId, context?: NotificationExecutionContext,
}; ): Promise<DeleteResult<UserNotificationBadge>> {
const context = { //Cancel the current digest as we just read the badges
user: { id: channelMemberPk.user_id, server_request: true }, await gr.services.notifications.digest.cancelDigest(pk.company_id, pk.user_id);
channel: { id: channelId, ...channelMemberPk },
}; await this.repository.remove(pk as UserNotificationBadge, context);
const exists = return new DeleteResult(UserNotificationBadgeType, pk as UserNotificationBadge, true);
(await gr.services.channels.channels.get( }
list(): Promise<ListResult<UserNotificationBadge>> {
throw new Error("Not implemented");
}
async listForUserPerCompanies(
user_id: string,
context: ExecutionContext,
): Promise<ListResult<UserNotificationBadge>> {
//We remove all badge from current company as next block will create dupicates
const companies_ids = (await gr.services.companies.getAllForUser(user_id)).map(
gu => gu.group_id,
);
let result: UserNotificationBadge[] = [];
let type = "";
for (const company_id of companies_ids) {
const find = await this.repository.find(
{ {
id: channelId, company_id,
..._.pick(channelMemberPk, "company_id", "workspace_id"), user_id,
},
{
pagination: new Pagination("", "1"),
}, },
context, context,
)) && (await gr.services.channels.members.get(channelMemberPk, context)); );
if (!exists) { type = find.type;
for (const badge of badges.getEntities()) { result = result.concat(find.getEntities());
if (badge.channel_id === channelId) this.removeUserChannelBadges(badge, context);
}
badges.filterEntities(b => b.channel_id !== channelId);
} }
const badges = new ListResult(type, result);
await this.ensureBadgesAreReachable(badges, context);
return badges;
} }
const badgePerWorkspace = _.uniqBy(badges.getEntities(), r => r.workspace_id); async listForUser(
for (const badge of badgePerWorkspace) { company_id: string,
const workspaceId = badge.workspace_id; user_id: string,
const companyId = badge.company_id; filter: Pick<UserNotificationBadgePrimaryKey, "workspace_id" | "channel_id" | "thread_id">,
if (!workspaceId || workspaceId === "direct") { context?: ExecutionContext,
continue; ): Promise<ListResult<UserNotificationBadge>> {
if (!company_id || !user_id) {
throw CrudException.badRequest("company_id and user_id are required");
} }
try {
const exists = //Cancel the current digest as we just read the badges
(await gr.services.workspaces.get({ await gr.services.notifications.digest.cancelDigest(company_id, user_id);
id: workspaceId,
company_id: companyId, const badges = await this.repository.find(
})) && {
(await gr.services.workspaces.getUser({ ...{
workspaceId, company_id,
userId, user_id,
})); },
if (!exists) { ...pick(filter, ["workspace_id", "channel_id", "thread_id"]),
await gr.services.channels.members.ensureUserNotInWorkspaceIsNotInChannel( },
{ id: userId },
{ id: workspaceId, company_id: companyId },
context,
);
for (const badge of badges.getEntities()) {
if (badge.workspace_id === workspaceId) this.removeUserChannelBadges(badge, context);
}
badges.filterEntities(b => b.workspace_id !== workspaceId);
}
} catch (e) {}
}
return badges;
}
/**
* FIXME: This is a temporary implementation which is sending as many websocket notifications as there are badges to remove
* A better implementation will be to do a bulk delete and have a single websocket notification event
* @param filter
* @param context
*/
async removeUserChannelBadges(
filter: Pick<
UserNotificationBadgePrimaryKey,
"workspace_id" | "company_id" | "channel_id" | "user_id"
>,
context?: ExecutionContext,
): Promise<number> {
const badges = (
await this.repository.find(
_.pick(filter, ["workspace_id", "company_id", "channel_id", "user_id"]),
{}, {},
context, context,
) );
).getEntities();
await this.ensureBadgesAreReachable(badges, context);
return (
await Promise.all( return badges;
badges.map(async badge => {
try {
return (await this.delete(badge)).deleted;
} catch (err) {}
}),
)
).filter(Boolean).length;
}
/**
* acknowledge a notification and set the message status to delivered.
*
* @param {UserNotificationBadgePrimaryKey} pk - The primary key of the badge to acknowledge
* @param {ExecutionContext} context - The context of the acknowledge
* @returns {Promise<boolean>} - The result of the acknowledge
*/
async acknowledge(
notification: UserNotificationBadgePrimaryKey & { message_id: string },
context: ExecutionContext,
): Promise<boolean> {
const { message_id, ...pk } = notification;
const badge = await this.repository.findOne(pk, {}, context);
const payload = badge || notification;
const ThreadExecutionContext = {
company: {
id: payload.company_id,
},
thread: {
id: payload.thread_id,
},
message_id,
...context,
};
const result = await gr.services.messages.messages.updateDeliveryStatus(
{
...payload,
status: "delivered",
},
ThreadExecutionContext,
);
if (result) {
return true;
} }
return false; // This will ensure we are still in the channels and if not, we'll remove the badge
} // We need to also ensure more than that
} // - Are we in the workspace?
// - Are we in the company?
async ensureBadgesAreReachable(
badges: ListResult<UserNotificationBadge>,
context?: ExecutionContext,
): Promise<ListResult<UserNotificationBadge>> {
if (badges.getEntities().length === 0) {
return badges;
}
const userId = badges.getEntities()[0].user_id;
const channels = uniq(badges.getEntities().map(r => r.channel_id));
for (const channelId of channels) {
const someBadge = badges.getEntities().find(b => b.channel_id === channelId);
const channelMemberPk = {
company_id: someBadge.company_id,
workspace_id: someBadge.workspace_id,
channel_id: channelId,
user_id: userId,
};
const context = {
user: { id: channelMemberPk.user_id, server_request: true },
channel: { id: channelId, ...channelMemberPk },
};
const exists =
(await gr.services.channels.channels.get(
{
id: channelId,
..._.pick(channelMemberPk, "company_id", "workspace_id"),
},
context,
)) && (await gr.services.channels.members.get(channelMemberPk, context));
if (!exists) {
for (const badge of badges.getEntities()) {
if (badge.channel_id === channelId) this.removeUserChannelBadges(badge, context);
}
badges.filterEntities(b => b.channel_id !== channelId);
}
}
const badgePerWorkspace = _.uniqBy(badges.getEntities(), r => r.workspace_id);
for (const badge of badgePerWorkspace) {
const workspaceId = badge.workspace_id;
const companyId = badge.company_id;
if (!workspaceId || workspaceId === "direct") {
continue;
}
try {
const exists =
(await gr.services.workspaces.get({
id: workspaceId,
company_id: companyId,
})) &&
(await gr.services.workspaces.getUser({
workspaceId,
userId,
}));
if (!exists) {
await gr.services.channels.members.ensureUserNotInWorkspaceIsNotInChannel(
{ id: userId },
{ id: workspaceId, company_id: companyId },
context,
);
for (const badge of badges.getEntities()) {
if (badge.workspace_id === workspaceId) this.removeUserChannelBadges(badge, context);
}
badges.filterEntities(b => b.workspace_id !== workspaceId);
}
} catch (e) {}
}
return badges;
}
/**
* FIXME: This is a temporary implementation which is sending as many websocket notifications as there are badges to remove
* A better implementation will be to do a bulk delete and have a single websocket notification event
* @param filter
* @param context
*/
async removeUserChannelBadges(
filter: Pick<
UserNotificationBadgePrimaryKey,
"workspace_id" | "company_id" | "channel_id" | "user_id"
>,
context?: ExecutionContext,
): Promise<number> {
const badges = (
await this.repository.find(
_.pick(filter, ["workspace_id", "company_id", "channel_id", "user_id"]),
{},
context,
)
).getEntities();
return (
await Promise.all(
badges.map(async badge => {
try {
return (await this.delete(badge)).deleted;
} catch (err) {}
}),
)
).filter(Boolean).length;
}
}
twake/backend/node/src/services/notifications/services/digest.ts
+1 -13
View File
@@ -1,10 +1,8 @@
/* eslint-disable @typescript-eslint/no-unused-vars */ /* eslint-disable @typescript-eslint/no-unused-vars */
import { MessageWithReplies } from "../../../services/messages/types";
import { Initializable, TwakeServiceProvider } from "../../../core/platform/framework"; import { Initializable, TwakeServiceProvider } from "../../../core/platform/framework";
import { Paginable, Pagination } from "../../../core/platform/framework/api/crud-service"; import { Paginable, Pagination } from "../../../core/platform/framework/api/crud-service";
import Repository from "../../../core/platform/services/database/services/orm/repository/repository"; import Repository from "../../../core/platform/services/database/services/orm/repository/repository";
import { Channel } from "../../../services/channels/entities"; import { Channel } from "../../../services/channels/entities";
import { Message } from "../../../services/messages/entities/messages";
import { UserObject } from "../../../services/user/web/types"; import { UserObject } from "../../../services/user/web/types";
import Workspace from "../../../services/workspaces/entities/workspace"; import Workspace from "../../../services/workspaces/entities/workspace";
import gr from "../../global-resolver"; import gr from "../../global-resolver";
@@ -104,7 +102,6 @@ export class UserNotificationDigestService implements TwakeServiceProvider, Init
const notifications: { const notifications: {
channel: Channel; channel: Channel;
workspace: Workspace; workspace: Workspace;
message: Message & { user: UserObject };
}[] = []; }[] = [];
const workspaces: { [key: string]: Workspace } = {}; const workspaces: { [key: string]: Workspace } = {};
const channels: { [key: string]: Channel } = {}; const channels: { [key: string]: Channel } = {};
@@ -112,15 +109,7 @@ export class UserNotificationDigestService implements TwakeServiceProvider, Init
for (const badge of badges.getEntities()) { for (const badge of badges.getEntities()) {
if (!badge.thread_id) continue; if (!badge.thread_id) continue;
try { try {
const message = await gr.services.messages.messages.includeUsersInMessageWithReplies(
(await gr.services.messages.messages.get({
id: badge.thread_id,
thread_id: badge.thread_id,
})) as MessageWithReplies,
);
if (message.created_at < digest.created_at - 60 * 1000) continue;
channels[badge.channel_id] = channels[badge.channel_id] =
channels[badge.channel_id] || channels[badge.channel_id] ||
(await gr.services.channels.channels.get({ (await gr.services.channels.channels.get({
@@ -141,7 +130,6 @@ export class UserNotificationDigestService implements TwakeServiceProvider, Init
notifications.push({ notifications.push({
channel: channels[badge.channel_id], channel: channels[badge.channel_id],
workspace: workspaces[badge.workspace_id], workspace: workspaces[badge.workspace_id],
message: { ...message, user: (message.users || []).find(u => u.id === message.user_id) },
}); });
} catch (e) {} } catch (e) {}
} }
twake/backend/node/src/services/notifications/services/engine/index.ts
-2
View File
@@ -1,7 +1,6 @@
import { Initializable } from "../../../../core/platform/framework"; import { Initializable } from "../../../../core/platform/framework";
import { MarkChannelAsReadMessageProcessor } from "./processors/mark-channel-as-read"; import { MarkChannelAsReadMessageProcessor } from "./processors/mark-channel-as-read";
import { MarkChannelAsUnreadMessageProcessor } from "./processors/mark-channel-as-unread"; import { MarkChannelAsUnreadMessageProcessor } from "./processors/mark-channel-as-unread";
import { NewChannelMessageProcessor } from "./processors/new-channel-message";
import { PushNotificationMessageProcessor } from "./processors/mobile-push-notifications"; import { PushNotificationMessageProcessor } from "./processors/mobile-push-notifications";
import { PushNotificationToUsersMessageProcessor } from "./processors/push-to-users"; import { PushNotificationToUsersMessageProcessor } from "./processors/push-to-users";
import { LeaveChannelMessageProcessor } from "./processors/channel-member-deleted"; import { LeaveChannelMessageProcessor } from "./processors/channel-member-deleted";
@@ -24,7 +23,6 @@ export class NotificationEngine implements Initializable {
gr.platformServices.messageQueue.processor.addHandler( gr.platformServices.messageQueue.processor.addHandler(
new MarkChannelAsUnreadMessageProcessor(), new MarkChannelAsUnreadMessageProcessor(),
); );
gr.platformServices.messageQueue.processor.addHandler(new NewChannelMessageProcessor());
gr.platformServices.messageQueue.processor.addHandler(new PushNotificationMessageProcessor()); gr.platformServices.messageQueue.processor.addHandler(new PushNotificationMessageProcessor());
gr.platformServices.messageQueue.processor.addHandler( gr.platformServices.messageQueue.processor.addHandler(
new PushNotificationToUsersMessageProcessor(), new PushNotificationToUsersMessageProcessor(),
twake/backend/node/src/services/notifications/services/engine/processors/new-channel-message.ts
-259
View File
@@ -1,259 +0,0 @@
import { isDirectChannel } from "../../../../channels/utils";
import { logger } from "../../../../../core/platform/framework";
import { MessageNotification } from "../../../../messages/types";
import {
ChannelMemberNotificationPreference,
ChannelThreadUsers,
getChannelThreadUsersInstance,
} from "../../../entities";
import { ChannelMemberNotificationLevel } from "../../../../channels/types";
import { MentionNotification, NotificationMessageQueueHandler } from "../../../types";
import { ChannelType } from "../../../../../utils/types";
import gr from "../../../../global-resolver";
import { ExecutionContext } from "../../../../../core/platform/framework/api/crud-service";
export class NewChannelMessageProcessor
implements NotificationMessageQueueHandler<MessageNotification, MentionNotification>
{
readonly topics = {
in: "message:created",
out: "notification:mentions",
};
readonly options = {
unique: true,
ack: true,
queue: "message:created:consumer2",
};
readonly name = "NewChannelMessageProcessor";
validate(message: MessageNotification): boolean {
return !!(
message &&
message.channel_id &&
message.company_id &&
message.workspace_id &&
message.creation_date > Date.now() - 5 * 60 * 1000
);
}
async process(message: MessageNotification): Promise<MentionNotification> {
logger.info(
`${this.name} - Processing notification for message ${message.thread_id}/${message.id} in channel ${message.channel_id}`,
);
logger.debug(`${this.name} - Notification message ${JSON.stringify(message)}`);
try {
if (message.workspace_id === ChannelType.DIRECT) {
//Fixme: Monkey fix until we find a way to add user to channel BEFORE to add the badge to this channel
await new Promise(r => setTimeout(r, 1000));
}
const usersToNotify = await this.getUsersToNotify(message);
if (!usersToNotify?.length) {
logger.info(
`${this.name} - No users to notify for message ${message.thread_id}/${message.id} in channel ${message.channel_id}`,
);
return;
}
logger.info(
`${this.name} - Users to notify for message ${message.thread_id}/${message.id} in channel ${
message.channel_id
} : ['${usersToNotify.join("', '")}']`,
);
//Fixme Add user full names if known, need to import microservice
const users_names = {};
//Fixme add channel full names, need to import microservice
const channels_names = {};
return {
channel_id: message.channel_id,
company_id: message.company_id,
message_id: message.id,
thread_id: message.thread_id,
workspace_id: message.workspace_id,
creation_date: message.creation_date,
mentions: {
users: usersToNotify || [],
},
object_names: {
users: users_names,
channels: channels_names,
},
//Temp: should not be used like this when migrating messages to node
//But we don't remember why, so keeping this like this as it works well
text: message.text,
title: message.title,
} as MentionNotification;
} catch (err) {
logger.error(
{ err },
`${this.name} - Error while gettings users to notify for message ${message.thread_id}/${message.id} in channel ${message.channel_id}`,
);
}
}
async getUsersToNotify(
message: MessageNotification,
context?: ExecutionContext,
): Promise<string[]> {
let channelPreferencesForUsers: ChannelMemberNotificationPreference[];
const threadId = message.thread_id || message.id;
const isNewThread = !message.thread_id || `${message.thread_id}` === `${message.id}`;
const isDirect = isDirectChannel({ workspace_id: message.workspace_id });
const isAllOrHereMention = this.isAllOrHereMention(message);
const users: ChannelThreadUsers[] = [
// message sender is a user in the thread
...[
getChannelThreadUsersInstance({
company_id: message.company_id,
channel_id: message.channel_id,
thread_id: threadId,
user_id: message.sender,
}),
],
// mentionned users are users in the thread
...(message?.mentions?.users?.length
? message.mentions.users.map(user_id =>
getChannelThreadUsersInstance({
company_id: message.company_id,
channel_id: message.channel_id,
thread_id: threadId,
user_id,
}),
)
: []),
];
await gr.services.notifications.channelThreads.bulkSave(users, context);
if (isNewThread || isDirect || isAllOrHereMention) {
//get the channel level preferences
channelPreferencesForUsers = (
await gr.services.notifications.channelPreferences.getChannelPreferencesForUsers({
company_id: message.company_id,
channel_id: message.channel_id,
})
).getEntities();
return this.filterMembersToNotify(message, channelPreferencesForUsers).map(m => m.user_id);
}
// get the preferences of the users involved in the thread
channelPreferencesForUsers = await this.getAllInvolvedUsersPreferences(
{
channel_id: message.channel_id,
company_id: message.company_id,
thread_id: threadId,
},
context,
);
return this.filterThreadMembersToNotify(message, channelPreferencesForUsers).map(
m => m.user_id,
);
}
protected filterMembersToNotify(
message: MessageNotification,
membersPreferences: ChannelMemberNotificationPreference[],
): ChannelMemberNotificationPreference[] {
logger.debug(`${this.name} - Filter members ${JSON.stringify(membersPreferences)}`);
const isAllOrHere = this.isAllOrHereMention(message);
const isDirect = isDirectChannel({ workspace_id: message.workspace_id });
return (
membersPreferences
// 1. Remove the ones which does not want any notification (preference === NONE)
.filter(preference => preference.preferences !== ChannelMemberNotificationLevel.NONE)
// 2. Remove the sender
.filter(preference => String(preference.user_id) !== String(message.sender))
// 3. Filter based on truth table based on user preferences and current message
.filter(memberPreference => {
const userIsMentionned = this.userIsMentionned(memberPreference.user_id, message);
const truthTable = [
isDirect,
// all
memberPreference.preferences === ChannelMemberNotificationLevel.ALL,
// mentions
memberPreference.preferences === ChannelMemberNotificationLevel.MENTIONS &&
(isAllOrHere || userIsMentionned),
// me
memberPreference.preferences === ChannelMemberNotificationLevel.ME && userIsMentionned,
];
logger.debug(
`${this.name} - ${
memberPreference.user_id
} truth table [direct, all, mentions, me] : ${JSON.stringify(truthTable)}`,
);
return truthTable.includes(true);
})
);
}
protected filterThreadMembersToNotify(
message: MessageNotification,
membersPreferences: ChannelMemberNotificationPreference[],
): ChannelMemberNotificationPreference[] {
logger.debug(`${this.name} - Filter thread members ${JSON.stringify(membersPreferences)}`);
return (
membersPreferences
// 1. Remove the ones which does not want any notification (preference === NONE)
.filter(preference => preference.preferences !== ChannelMemberNotificationLevel.NONE)
//2. Remove the sender
.filter(preference => String(preference.user_id) !== String(message.sender))
);
}
/**
* When message is a response in a thread, get all the users involved in the thread
* ie the ones who where initially mentionned, mentionned in children messages, and the ones who replied
*/
protected async getAllInvolvedUsersPreferences(
thread: {
company_id: string;
channel_id: string;
thread_id: string;
},
context: ExecutionContext,
): Promise<ChannelMemberNotificationPreference[]> {
const usersIds: string[] = (
await gr.services.notifications.channelThreads.getUsersInThread(thread, context)
)
.getEntities()
.map(thread => thread.user_id);
return (
await gr.services.notifications.channelPreferences.getChannelPreferencesForUsers(
{ company_id: thread.company_id, channel_id: thread.channel_id },
usersIds,
undefined,
context,
)
).getEntities();
}
private isAllOrHereMention(message: MessageNotification): boolean {
return (
message.mentions &&
message.mentions.specials &&
(message.mentions.specials.includes("all") ||
message.mentions.specials.includes("here") ||
message.mentions.specials.includes("channel") ||
message.mentions.specials.includes("everyone"))
);
}
private userIsMentionned(user: string, message: MessageNotification) {
return message?.mentions?.users?.includes(String(user));
}
}
twake/backend/node/src/services/notifications/services/engine/processors/push-to-users.ts
-6
View File
@@ -168,12 +168,6 @@ export class PushNotificationToUsersMessageProcessor
message_id: badge.message_id, message_id: badge.message_id,
user_id: user, user_id: user,
mention_type: mentions.users.includes(user) mention_type: mentions.users.includes(user)
? "me"
: mentions.specials.length > 0
? "global"
: badge.thread_id !== badge.message_id
? "reply"
: null,
}); });
return this.saveBadge(badgeEntity, context); return this.saveBadge(badgeEntity, context);
}), }),
twake/backend/node/src/services/notifications/services/engine/processors/reaction-notification.ts
+2 -7
View File
@@ -102,12 +102,7 @@ export class PushReactionNotification
}), }),
gr.services.users.get({ id: reaction_user_id }), gr.services.users.get({ id: reaction_user_id }),
]); ]);
const msg = await gr.services.messages.messages.get({
thread_id: thread_id,
id: message.message_id,
});
const companyName = company?.name || ""; const companyName = company?.name || "";
const workspaceName = workspace_id === "direct" ? "Direct" : workspace?.name || ""; const workspaceName = workspace_id === "direct" ? "Direct" : workspace?.name || "";
const userName = this.getUserName(user) || "Twake"; const userName = this.getUserName(user) || "Twake";
@@ -120,7 +115,7 @@ export class PushReactionNotification
return { return {
title, title,
text: `${userName}: ${reaction} to ${msg?.text || "your message"}`, text: `${userName}: ${reaction}`,
}; };
} }
twake/backend/node/src/services/notifications/types.ts
-2
View File
@@ -1,7 +1,6 @@
import { ExecutionContext } from "../../core/platform/framework/api/crud-service"; import { ExecutionContext } from "../../core/platform/framework/api/crud-service";
import { Channel, ChannelMember } from "../channels/entities"; import { Channel, ChannelMember } from "../channels/entities";
import { PaginationQueryParameters } from "../channels/web/types"; import { PaginationQueryParameters } from "../channels/web/types";
import { SpecialMention } from "../messages/types";
import { uuid } from "../../utils/types"; import { uuid } from "../../utils/types";
import { MessageQueueHandler } from "../../core/platform/services/message-queue/api"; import { MessageQueueHandler } from "../../core/platform/services/message-queue/api";
@@ -26,7 +25,6 @@ export type MentionNotification = {
creation_date: number; creation_date: number;
mentions?: { mentions?: {
users: uuid[]; users: uuid[];
specials?: SpecialMention[];
}; };
object_names?: { object_names?: {
users: { [id: string]: string }; users: { [id: string]: string };
twake/backend/node/src/services/notifications/web/controllers/notification.ts
-39
View File
@@ -66,45 +66,6 @@ export class NotificationController
}), }),
}; };
} }
/**
* Acknowledge a notification
*
* @param {FastifyRequest} request - The request object
* @param {FastifyReply} reply - The reply object
* @returns {Promise<boolean>} - The response object
*/
async acknowledge(
request: FastifyRequest<{
Params: {
company_id: string;
};
Body: NotificationAcknowledgeBody;
}>,
reply: FastifyReply,
): Promise<boolean> {
const context = getExecutionContext(request);
const { company_id } = request.params;
const { workspace_id, channel_id, thread_id, message_id } = request.body;
try {
await gr.services.notifications.badges.acknowledge(
{
channel_id,
company_id,
thread_id,
user_id: context.user.id,
workspace_id,
message_id,
},
context,
);
return reply.send(true);
} catch (err) {
return reply.send(false);
}
}
} }
function getExecutionContext(request: FastifyRequest): WorkspaceExecutionContext { function getExecutionContext(request: FastifyRequest): WorkspaceExecutionContext {
twake/backend/node/src/services/notifications/web/routes.ts
-7
View File
@@ -31,13 +31,6 @@ const routes: FastifyPluginCallback = (fastify: FastifyInstance, options, next)
handler: notificationPreferencesController.save.bind(notificationPreferencesController), handler: notificationPreferencesController.save.bind(notificationPreferencesController),
}); });
fastify.route({
method: "POST",
url: `${badgesUrl}/:company_id/acknowledge`,
preValidation: [fastify.authenticate],
handler: notificationController.acknowledge.bind(notificationController),
});
next(); next();
}; };
twake/backend/node/src/services/previews/index.ts
-23
View File
@@ -1,23 +0,0 @@
import WebServerAPI from "../../core/platform/services/webserver/provider";
import { Prefix, TwakeService } from "../../core/platform/framework";
import web from "./web";
@Prefix("/internal/services/previews/v1")
export default class PreviewsService extends TwakeService<undefined> {
version = "1";
name = "previews";
public async doInit(): Promise<this> {
const fastify = this.context.getProvider<WebServerAPI>("webserver").getServer();
fastify.register((instance, _opts, next) => {
web(instance, { prefix: this.prefix });
next();
});
return this;
}
// TODO: remove
api(): undefined {
return undefined;
}
}
twake/backend/node/src/services/previews/services/files/engine/clear.ts
-48
View File
@@ -1,48 +0,0 @@
import { logger, TwakeContext } from "../../../../../core/platform/framework";
import { PreviewClearMessageQueueRequest, PreviewMessageQueueCallback } from "../../../types";
import gr from "../../../../global-resolver";
import { MessageQueueHandler } from "../../../../../core/platform/services/message-queue/api";
/**
* Clear thumbnails when the delete task is called
*/
export class ClearProcessor
implements MessageQueueHandler<PreviewClearMessageQueueRequest, PreviewMessageQueueCallback>
{
readonly name = "ClearProcessor";
readonly topics = {
in: "services:preview:clear",
out: "services:preview:callback",
};
readonly options = {
unique: true,
ack: true,
};
// eslint-disable-next-line @typescript-eslint/no-unused-vars
init?(context?: TwakeContext): Promise<this> {
throw new Error("Method not implemented.");
}
validate(message: PreviewClearMessageQueueRequest): boolean {
return !!(message && message.document);
}
async process(message: PreviewClearMessageQueueRequest): Promise<PreviewMessageQueueCallback> {
logger.info(`${this.name} - Processing preview generation ${message.document.id}`);
if (!this.validate(message)) {
throw new Error("Missing required fields");
}
for (let i = 0; i < message.document.thumbnails_number; i++) {
await gr.platformServices.storage.remove(
`${message.document.path.replace(/\/$/, "")}/${i}.png`,
);
}
return { document: message.document, thumbnails: [] };
}
}
twake/backend/node/src/services/previews/services/files/engine/index.ts
-15
View File
@@ -1,15 +0,0 @@
import { Initializable } from "../../../../../core/platform/framework";
import { ClearProcessor } from "./clear";
import { PreviewProcessor } from "./service";
import gr from "../../../../global-resolver";
/**
* The notification engine is in charge of processing data and delivering user notifications on the right place
*/
export class PreviewEngine implements Initializable {
async init(): Promise<this> {
gr.platformServices.messageQueue.processor.addHandler(new PreviewProcessor());
gr.platformServices.messageQueue.processor.addHandler(new ClearProcessor());
return this;
}
}
twake/backend/node/src/services/previews/services/files/engine/service.ts
-118
View File
@@ -1,118 +0,0 @@
import fs, { promises as fsPromise } from "fs";
import { logger } from "../../../../../core/platform/framework";
import {
PreviewMessageQueueCallback,
PreviewMessageQueueRequest,
ThumbnailResult,
} from "../../../types";
import gr from "../../../../global-resolver";
import { MessageQueueHandler } from "../../../../../core/platform/services/message-queue/api";
import { getTmpFile } from "../../../../../utils/files";
const { unlink } = fsPromise;
/**
* Generate thumbnails when the upload is finished
*/
export class PreviewProcessor
implements MessageQueueHandler<PreviewMessageQueueRequest, PreviewMessageQueueCallback>
{
readonly name = "PreviewProcessor";
init?(): Promise<this> {
throw new Error("Method not implemented.");
}
readonly topics = {
in: "services:preview",
out: "services:preview:callback",
};
readonly options = {
unique: true,
ack: true,
};
validate(message: PreviewMessageQueueRequest): boolean {
return !!(message && message.document && message.output);
}
async process(message: PreviewMessageQueueRequest): Promise<PreviewMessageQueueCallback> {
logger.info(`${this.name} - Processing preview generation ${message.document.id}`);
let res: PreviewMessageQueueCallback = { document: message.document, thumbnails: [] };
try {
res = await this.generate(message);
} catch (err) {
logger.error(`${this.name} - Can't generate thumbnails ${err}`);
}
logger.info(
`${this.name} - Generated ${res.thumbnails.length} thumbnails from ${
message.document.filename || message.document.id
}`,
);
return res;
}
async generate(message: PreviewMessageQueueRequest): Promise<PreviewMessageQueueCallback> {
//Download original file
const readable = await gr.platformServices.storage.read(message.document.path, {
totalChunks: message.document.chunks,
encryptionAlgo: message.document.encryption_algo,
encryptionKey: message.document.encryption_key,
});
if (!readable) {
return { document: message.document, thumbnails: [] };
}
const inputPath = getTmpFile();
const writable = fs.createWriteStream(inputPath);
readable.pipe(writable);
await new Promise(r => {
writable.on("finish", r);
});
writable.end();
//Generate previews
let localThumbnails: ThumbnailResult[] = [];
try {
localThumbnails = await gr.services.preview.files.generateThumbnails(
{ path: inputPath, mime: message.document.mime, filename: message.document.filename },
message.output,
);
} catch (err) {
logger.error(`${this.name} - Can't generate thumbnails ${err}`);
localThumbnails = [];
throw Error("Can't generate thumbnails.");
}
const thumbnails: PreviewMessageQueueCallback["thumbnails"] = [];
for (let i = 0; i < localThumbnails.length; i++) {
const uploadThumbnailPath = `${message.output.path.replace(/\/$/, "")}/${i}.png`;
const uploadThumbnail = fs.createReadStream(localThumbnails[i].path);
await gr.platformServices.storage.write(uploadThumbnailPath, uploadThumbnail, {
encryptionAlgo: message.output.encryption_algo,
encryptionKey: message.output.encryption_key,
});
thumbnails.push({
path: uploadThumbnailPath,
size: localThumbnails[i].size,
type: localThumbnails[i].type,
width: localThumbnails[i].width,
height: localThumbnails[i].height,
});
await unlink(localThumbnails[i].path);
}
return { document: message.document, thumbnails };
}
}
twake/backend/node/src/services/previews/services/files/processing/image.ts
-54
View File
@@ -1,54 +0,0 @@
import sharp from "sharp";
import { cleanFiles, getTmpFile } from "../../../../../utils/files";
import { PreviewMessageQueueRequest, ThumbnailResult } from "../../../types";
import { logger } from "../../../../../core/platform/framework/logger";
export async function generatePreview(
inputPaths: string[],
options: PreviewMessageQueueRequest["output"],
): Promise<{
output: ThumbnailResult[];
done: boolean;
error?: string;
}> {
const output: ThumbnailResult[] = [];
for (const inputPath of inputPaths) {
let result: sharp.OutputInfo;
const outputPath = getTmpFile();
try {
const inputMetadata = await sharp(inputPath).metadata();
const outputFormat = computeNewFormat(inputMetadata, options);
result = await sharp(inputPath).rotate().resize(outputFormat).toFile(outputPath);
output.push({
path: outputPath,
width: result.width,
height: result.height,
type: "image/png",
size: result.size,
});
} catch (error) {
logger.info(`sharp cant process ${error}`);
cleanFiles([outputPath]);
throw Error("Can't resize thumnail with Sharp");
}
}
return {
output,
done: true,
};
}
function computeNewFormat(
inputMetadata: sharp.Metadata,
options?: PreviewMessageQueueRequest["output"],
): { width: number; height: number } {
const maxOutputWidth = options?.width || 600;
const maxOutputHeight = options?.height || 400;
const inputWidth = inputMetadata.width;
const inputHeight = inputMetadata.height;
const scale = Math.max(inputWidth / maxOutputWidth, inputHeight / maxOutputHeight);
return { width: Math.round(inputWidth / scale), height: Math.round(inputHeight / scale) };
}
twake/backend/node/src/services/previews/services/files/processing/office.ts
-29
View File
@@ -1,29 +0,0 @@
// eslint-disable-next-line @typescript-eslint/no-var-requires
const unoconv = require("unoconv-promise");
import { cleanFiles } from "../../../../../utils/files";
import { logger } from "../../../../../core/platform/framework/logger";
export async function convertFromOffice(
path: string,
options: {
numberOfPages?: number;
},
): Promise<{ output: string; done: boolean }> {
if (options.numberOfPages >= 1) {
const outputPath = `${path}.pdf`;
try {
await unoconv.run({
file: path,
output: outputPath,
export: `PageRange=1-${options.numberOfPages}`,
});
} catch (err) {
logger.error(`unoconv cant process ${err}`);
cleanFiles([outputPath]);
throw Error("Can't convert file with unoconv");
}
return { output: outputPath, done: true };
} else {
logger.error("Unoconv can't processe, number of pages lower than 1");
}
}
twake/backend/node/src/services/previews/services/files/processing/pdf.ts
-43
View File
@@ -1,43 +0,0 @@
import { fromPath } from "pdf2pic";
import { mkdirSync } from "fs";
import { cleanFiles, getTmpFile } from "../../../../../utils/files";
export async function convertFromPdf(
inputPath: string,
options: {
numberOfPages: number;
},
): Promise<{ output: string[]; done: boolean }> {
const pages: string[] = [];
try {
const pdfOptions = {
density: 100,
saveFilename: "output",
savePath: getTmpFile(),
format: "png",
};
mkdirSync(pdfOptions.savePath, { recursive: true });
const storeAsImage = fromPath(inputPath, pdfOptions);
try {
for (let i = 1; i <= options.numberOfPages; i++) {
const image = await storeAsImage(i);
pages.push(
`${pdfOptions.savePath}/${pdfOptions.saveFilename}.${image.page}.${pdfOptions.format}`,
);
}
} catch (err) {
if (!pages.length) {
throw err;
}
//Just no more page to convert
}
} catch (error) {
console.error(error);
for (const file of pages) {
cleanFiles([file]);
}
throw Error("Can't convert file with pdf-image.");
}
return { output: pages, done: true };
}
twake/backend/node/src/services/previews/services/files/processing/service.ts
-71
View File
@@ -1,71 +0,0 @@
import { generatePreview as thumbnailsFromImages } from "./image";
import { convertFromOffice } from "./office";
import { convertFromPdf } from "./pdf";
import { isFileType } from "../../../utils";
import {
imageExtensions,
officeExtensions,
pdfExtensions,
videoExtensions,
} from "../../../../../utils/mime";
import { PreviewMessageQueueRequest, ThumbnailResult } from "../../../types";
import { generateVideoPreview } from "./video";
import { Initializable, TwakeServiceProvider } from "../../../../../core/platform/framework";
import { cleanFiles } from "../../../../../utils/files";
export class PreviewProcessService implements TwakeServiceProvider, Initializable {
name: "PreviewProcessService";
version: "1";
async init(): Promise<this> {
return this;
}
async generateThumbnails(
document: Pick<PreviewMessageQueueRequest["document"], "filename" | "mime" | "path">,
options: PreviewMessageQueueRequest["output"],
): Promise<ThumbnailResult[]> {
if (isFileType(document.mime, document.filename, officeExtensions)) {
const pdfPath = await convertFromOffice(document.path, {
numberOfPages: options.pages,
});
await cleanFiles([document.path]);
const thumbnailPath = await convertFromPdf(pdfPath.output, {
numberOfPages: options.pages,
});
await cleanFiles([pdfPath.output]);
const images = (await thumbnailsFromImages(thumbnailPath.output, options)).output;
await cleanFiles(thumbnailPath.output);
return images;
}
if (isFileType(document.mime, document.filename, pdfExtensions)) {
const thumbnailPath = await convertFromPdf(document.path, {
numberOfPages: options.pages,
});
await cleanFiles([document.path]);
const images = (await thumbnailsFromImages(thumbnailPath.output, options)).output;
await cleanFiles(thumbnailPath.output);
return images;
}
if (isFileType(document.mime, document.filename, imageExtensions)) {
const images = (await thumbnailsFromImages([document.path], options)).output;
await cleanFiles([document.path]);
return images;
}
if (isFileType(document.mime, document.filename, videoExtensions)) {
try {
const images = await generateVideoPreview([document.path]);
await cleanFiles([document.path]);
return images;
} catch (error) {
throw Error("failed to generate video preview");
}
}
throw "Can not proccess, file type can't be defined";
}
}
twake/backend/node/src/services/previews/services/files/processing/video.ts
-167
View File
@@ -1,167 +0,0 @@
import ffmpeg from "fluent-ffmpeg";
import { temporaryThumbnailFile, ThumbnailResult } from "../../../types";
import fs from "fs";
import { cleanFiles, getTmpFile } from "../../../../../utils/files";
/**
* Generate thumbnails for given video files.
*
* @param {String[]} videoPaths - the input video paths
* @returns {Promise<ThumbnailResult[]>} - resolves when the thumbnails are generated
*/
export async function generateVideoPreview(videoPaths: string[]): Promise<ThumbnailResult[]> {
const output: ThumbnailResult[] = [];
for (const videoPath of videoPaths) {
const { fileName, folder, filePath } = getTemporaryThumbnailFile();
try {
const { width, height } = await takeVideoScreenshot(videoPath, folder, fileName);
output.push({
...getThumbnailInformation(filePath),
width,
height,
});
} catch (error) {
cleanFiles([filePath]);
throw Error(`failed to generate video preview: ${error}`);
}
}
return output;
}
/**
* Generate one thumbnail from a single video file.
* the thumbnail is the first frame from the video
*
* @param {String} inputPath - the input video Path
* @param {String} outputFolder - the output folder
* @param {String} outputFile - the output file name
* @returns {Primise} - resolves when the thumbnail is generated
*/
const takeVideoScreenshot = async (
inputPath: string,
outputFolder: string,
outputFile: string,
): Promise<{ width: number; height: number }> => {
return new Promise(async (resolve, reject) => {
try {
const { width, height } = await getVideoDimensions(inputPath);
const { width: outputWidth, height: outputHeight } = calculateThumbnailDimensions(
width,
height,
);
ffmpeg(inputPath)
.screenshot({
count: 1,
filename: outputFile,
folder: outputFolder,
timemarks: ["0"],
size: `${outputWidth}x${outputHeight}`,
})
.on("end", () => {
resolve({ width: outputWidth, height: outputHeight });
})
.on("error", error => {
reject(error);
});
} catch (error) {
reject(error);
}
});
};
/**
* Get the generated thumbnail information.
*
* @param {String} path - the path to the thumbnail
* @param {PreviewMessageQueueRequest["output"]} options - the options for the thumbnails
* @returns { { path: string, type: string, size: number } } - the thumbnail information
*/
const getThumbnailInformation = (
path: string,
): {
path: string;
type: string;
size: number;
} => {
const stats = fs.statSync(path);
return {
size: stats.size,
type: "image/jpg",
path,
};
};
/**
* generate a temporary thumbnail file.
*
* @returns {temporaryThumbnailFile} - the temporary thumbnail file information
*/
const getTemporaryThumbnailFile = (): temporaryThumbnailFile => {
const filePath = `${getTmpFile()}.jpg`;
const fileName = filePath.split("/").pop();
const folder = filePath.substring(0, filePath.lastIndexOf("/"));
return {
folder,
fileName,
filePath,
};
};
/**
* Detect video dimensions
*
* @param {String} videoPath - the video path
* @returns {Promise<{ width: number, height: number }>} - the video dimensions
*/
async function getVideoDimensions(videoPath: string): Promise<{ width: number; height: number }> {
return new Promise((resolve, reject) => {
ffmpeg.ffprobe(videoPath, (err, metadata) => {
if (err) {
reject(err);
}
const { width, height } = metadata.streams[0];
resolve({ width, height });
});
});
}
/**
* Calculate the thumbnail dimensions.
* maximum dimension is 1080p and the minimum dimension is 320x240.
* The aspect ratio is preserved.
*
* @param {Number} width - the video width
* @param {Number} height - the video height
* @returns { { width: number, height: number } } - the thumbnail dimensions
*/
function calculateThumbnailDimensions(
width: number,
height: number,
): {
width: number;
height: number;
} {
let newWidth = Math.min(width, 1920);
let newHeight = Math.min(height, 1080);
const ratio = width / height;
if (width > 1920 || height > 1080) {
newWidth = Math.min(1920, width);
newHeight = Math.min(1080, newWidth / ratio);
}
if (width < 320 || height < 240) {
newWidth = Math.max(320, width);
newHeight = Math.max(240, newWidth / ratio);
}
return { width: newWidth, height: newHeight };
}
twake/backend/node/src/services/previews/services/links/engine/index.ts
-11
View File
@@ -1,11 +0,0 @@
import { Initializable } from "../../../../../core/platform/framework";
import { LinkPreviewProcessor } from "./service";
import gr from "../../../../global-resolver";
export class LinkPreviewEngine implements Initializable {
async init(): Promise<this> {
gr.platformServices.messageQueue.processor.addHandler(new LinkPreviewProcessor());
return this;
}
}
twake/backend/node/src/services/previews/services/links/engine/service.ts
-80
View File
@@ -1,80 +0,0 @@
import {
LinkPreview,
LinkPreviewMessageQueueCallback,
LinkPreviewMessageQueueRequest,
} from "../../../types";
import { logger } from "../../../../../core/platform/framework";
import gr from "../../../../global-resolver";
import { MessageQueueHandler } from "../../../../../core/platform/services/message-queue/api";
export class LinkPreviewProcessor
implements MessageQueueHandler<LinkPreviewMessageQueueRequest, LinkPreviewMessageQueueCallback>
{
readonly name = "LinkPreviewProcessor";
readonly topics = {
in: "services:preview:links",
out: "services:preview:links:callback",
};
readonly options = {
unique: true,
ack: true,
};
init?(): Promise<this> {
throw new Error("Method not implemented.");
}
/**
* Checks if the message is valid
*
* @param {LinkPreviewMessageQueueRequest} message - The message to check
* @returns {Boolean} - true if the message is valid
*/
validate(message: LinkPreviewMessageQueueRequest): boolean {
return !!(message && message.links && message.links.length);
}
/**
* process the links preview generation message
*
* @param {LinkPreviewMessageQueueRequest} message - The message to process
* @returns {Promise<LinkPreviewMessageQueueCallback>} - links preview callback
*/
async process(message: LinkPreviewMessageQueueRequest): Promise<LinkPreviewMessageQueueCallback> {
logger.info(`${this.name} - Processing preview generation for ${message.links.length} links`);
let res: LinkPreviewMessageQueueCallback = { previews: [], message: message.message };
try {
res = await this.generate(message);
} catch (err) {
logger.error(`${this.name} - Can't generate link previews ${err}`);
}
logger.info(`${this.name} - Generated ${res.previews.length} link previews`);
return res;
}
/**
* Generate previews for links
*
* @param {LinkPreviewMessageQueueRequest} message - The message to process
* @returns {Promise<LinkPreviewMessageQueueCallback>} - links preview callback
*/
async generate(
message: LinkPreviewMessageQueueRequest,
): Promise<LinkPreviewMessageQueueCallback> {
let previews: LinkPreview[] = [];
try {
previews = await gr.services.preview.links.generatePreviews(message.links);
} catch (err) {
logger.error(`${this.name} - Can't generate link previews ${err}`);
throw Error(`cannot generate link previews: ${err}`);
}
return { previews, message: message.message };
}
}
twake/backend/node/src/services/previews/services/links/processing/image.ts
-37
View File
@@ -1,37 +0,0 @@
import { LinkPreview } from "../../../types";
import { logger } from "../../../../../core/platform/framework";
import imageProbe from "probe-image-size";
import { getUrlFavicon, getDomain } from "../../../utils";
/**
* Generate a preview for a given image url.
*
* @param {String} url - the input url
* @returns {Promise<LinkPreview>} - resolves when the preview is generated
*/
export const generateImageUrlPreview = async (url: string): Promise<LinkPreview | null> => {
try {
const favicon = (await getUrlFavicon(getWebsiteUrl(url))) || null;
const domain = getDomain(url);
const title = url.split("/").pop();
const { height: img_height, width: img_width } = await imageProbe(url);
return {
title,
domain,
favicon,
url,
img_height,
img_width,
description: null,
img: url,
};
} catch (error) {
logger.error(`failed to generate image url preview: ${error}`);
}
};
const getWebsiteUrl = (url: string) => {
const urlObj = new URL(url);
return `${urlObj.protocol}//${urlObj.hostname}`;
};
twake/backend/node/src/services/previews/services/links/processing/link.ts
-77
View File
@@ -1,77 +0,0 @@
import { parser } from "html-metadata-parser";
import { LinkPreview } from "../../../types";
import { logger } from "../../../../../core/platform/framework";
import imageProbe from "probe-image-size";
import { getUrlFavicon, getDomain, TIMEOUT, MAX_SIZE } from "../../../utils";
import ua from "random-useragent";
type HtmlImage = {
src: string;
};
/**
* Generate a thumbnail for a given url.
*
* @param {String[]} urls - the input urls
* @returns {Promise<LinkPreview>} - resolves when the preview is generated
*/
export async function generateLinkPreview(url: string): Promise<LinkPreview> {
try {
return await getUrlInformation(url);
} catch (error) {
logger.error(`failed to generate link preview: ${error}`);
}
}
/**
* get url information
*
* @param {String} url - the input url
* @returns {Promise<LinkPreview>} - resolves when the information is retrieved
*/
const getUrlInformation = async (url: string): Promise<LinkPreview> => {
try {
const parsedPage = await parser(url, {
timeout: TIMEOUT,
maxContentLength: MAX_SIZE,
headers: {
"User-Agent": ua.getRandom(),
},
});
const title = parsedPage.og?.title || parsedPage.meta?.title || null;
const description = parsedPage.og?.description || parsedPage.meta?.description || null;
let img = parsedPage.og?.image || parsedPage.meta?.image || parsedPage.images?.[0] || null;
if (!title && !description && !img) {
throw new Error("not enough information to generate link preview");
}
const favicon = (await getUrlFavicon(url)) || null;
const domain = getDomain(url);
let img_height: number | null = null,
img_width: number | null = null;
if (img) {
if (typeof img === "object") {
img = (img as HtmlImage).src;
}
const dimensions = await imageProbe(img);
img_height = dimensions.height;
img_width = dimensions.width;
}
return {
title,
domain,
description,
favicon,
img,
img_height,
img_width,
url,
};
} catch (error) {
throw Error(`failed to get url information: ${error}`);
}
};
twake/backend/node/src/services/previews/services/links/processing/service.ts
-52
View File
@@ -1,52 +0,0 @@
import { LinkPreview, LinkPreviewMessageQueueRequest } from "../../../types";
import { generateLinkPreview } from "./link";
import { checkUrlContents } from "../../../utils";
import { generateImageUrlPreview } from "./image";
import {
Initializable,
logger,
TwakeServiceProvider,
} from "../../../../../core/platform/framework";
export class LinkPreviewProcessService implements TwakeServiceProvider, Initializable {
name: "LinkPreviewProcessService";
version: "1";
async init(): Promise<this> {
return this;
}
/**
* Generate previews for links
*
* @param {LinkPreviewMessageQueueRequest["links"]} links - input urls
* @returns {Promise<LinkPreview[]>} - The generated url previews
*/
async generatePreviews(links: LinkPreviewMessageQueueRequest["links"]): Promise<LinkPreview[]> {
const result: LinkPreview[] = [];
for (const link of links) {
try {
const contentType = await checkUrlContents(link);
if (!contentType) {
continue;
}
if (contentType.includes("text/html")) {
result.push(await generateLinkPreview(link));
}
if (contentType.startsWith("image")) {
result.push(await generateImageUrlPreview(link));
}
} catch (error) {
logger.error(`failed to generate link preview: ${error}`);
}
}
return result.filter(
preview => preview && (preview.title || preview.description || preview.img),
);
}
}
twake/backend/node/src/services/previews/types.ts
-84
View File
@@ -1,84 +0,0 @@
import { MessageLocalEvent } from "../messages/types";
export type PreviewMessageQueueRequest = {
document: {
id: string;
provider: string;
path: string;
encryption_algo?: string;
encryption_key?: string;
chunks?: number;
mime?: string;
filename?: string;
};
output: {
provider: string;
path: string;
encryption_algo?: string;
encryption_key?: string;
pages?: number; //Max number of pages for the document
width?: number; //Max width for the thumbnails
height?: number; //Max height for the thumbnails
};
};
export type PreviewClearMessageQueueRequest = {
document: {
id: string;
provider: string;
path: string;
thumbnails_number: number;
};
};
export type PreviewMessageQueueCallback = {
document: {
id: string;
path: string;
provider: string;
};
thumbnails: {
path: string;
size: number;
type: string;
width: number;
height: number;
provider?: string;
index?: number;
}[];
};
export type ThumbnailResult = {
path: string;
width: number;
height: number;
size: number;
type: string;
};
export type temporaryThumbnailFile = {
filePath: string;
fileName: string;
folder: string;
};
export type LinkPreview = {
title: string;
description: string | null;
domain: string;
favicon: string | null;
img: string | null;
img_height: number | null;
img_width: number | null;
url: string;
};
export type LinkPreviewMessageQueueRequest = {
links: string[];
message: MessageLocalEvent;
};
export type LinkPreviewMessageQueueCallback = {
message: MessageLocalEvent;
previews: LinkPreview[];
};
twake/backend/node/src/services/previews/utils.ts
-67
View File
@@ -1,67 +0,0 @@
import mimes from "../../utils/mime";
import getFavicons from "get-website-favicon";
import { logger } from "../../core/platform/framework";
import axios from "axios";
export const TIMEOUT = 5 * 1000;
export const MAX_SIZE = 5 * 1024 * 1024;
export function isFileType(fileMime: string, fileName: string, requiredExtensions: string[]): any {
const extension = fileName.split(".").pop();
const secondaryExtensions = Object.keys(mimes).filter(k => mimes[k] === fileMime);
const fileExtensions = [extension, ...secondaryExtensions];
return fileExtensions.some(e => requiredExtensions.includes(e));
}
/**
* get url favicon
*
* @param {String} url - the input url
* @returns {Promise<String | null >} - resolves when the favicon is retrieved
*/
export const getUrlFavicon = async (url: string): Promise<string | null> => {
try {
const result = await getFavicons(url);
if (!result.icons || !result.icons.length) {
return;
}
return result.icons[0].src;
} catch (error) {
logger.error(`failed to get url favicon: ${error}`);
}
};
/**
* Get domain from a given url.
*
* @param {String} url - the input url
* @returns {String} - resolves when the domain is retrieved
*/
export const getDomain = (url: string): string => {
try {
const domain = new URL(url).hostname;
return domain.replace(/^www\./, "");
} catch (error) {
throw Error(`failed to get domain: ${error}`);
}
};
/**
* Get url content type headers
*
* @param {string} url - the input url
* @returns {Promise<string | void>} - the request headers
*/
export const checkUrlContents = async (url: string): Promise<string | void> => {
try {
const response = await axios(url, {
maxContentLength: MAX_SIZE,
timeout: TIMEOUT,
});
return response.headers["content-type"];
} catch (error) {
throw Error(`failed to check url contents: ${error}`);
}
};
twake/backend/node/src/services/previews/web/controllers/index.ts
-1
View File
@@ -1 +0,0 @@
export * from "./previews";
twake/backend/node/src/services/previews/web/controllers/previews.ts
-1
View File
@@ -1 +0,0 @@
export class PreviewController {}
twake/backend/node/src/services/previews/web/index.ts
-12
View File
@@ -1,12 +0,0 @@
import { FastifyInstance, FastifyRegisterOptions } from "fastify";
import routes from "./routes";
export default (
fastify: FastifyInstance,
options: FastifyRegisterOptions<{
prefix: string;
}>,
): void => {
fastify.log.debug("Configuring /internal/services/previews/v1 routes");
fastify.register(routes, options);
};
twake/backend/node/src/services/previews/web/routes.ts
-7
View File
@@ -1,7 +0,0 @@
import { FastifyInstance, FastifyPluginCallback } from "fastify";
const routes: FastifyPluginCallback = (fastify: FastifyInstance, options, next) => {
next();
};
export default routes;
twake/backend/node/src/services/statistics/pubsub/messages.ts
-36
View File
@@ -1,36 +0,0 @@
import { MessageQueueHandler } from "../../../core/platform/services/message-queue/api";
import { getLogger } from "../../../core/platform/framework";
import { MessageNotification } from "../../messages/types";
import gr from "../../global-resolver";
const logger = getLogger("statistics.messages");
export class StatisticsMessageProcessor implements MessageQueueHandler<MessageNotification, void> {
readonly topics = {
in: "message:created",
};
readonly options = {
unique: true,
ack: true,
queue: "message:created:statistics-consumer",
};
readonly name = "Channel::StatisticsMessageProcessor";
validate(message: MessageNotification): boolean {
return !!(message && message.channel_id && message.company_id && message.workspace_id);
}
async process(message: MessageNotification): Promise<void> {
logger.info(`${this.name} - Processing increasing messages counter for ${message.company_id}`);
try {
await gr.services.statistics.increase(message.company_id, "messages");
} catch (err) {
logger.error(
{ err },
`${this.name} - Error while processing direct channel members for message ${message.thread_id}/${message.id} in channel ${message.channel_id}`,
);
}
}
}
twake/backend/node/src/services/workspaces/services/workspace.ts
-8
View File
@@ -150,14 +150,6 @@ export class WorkspaceServiceImpl implements TwakeServiceProvider, Initializable
{ company_id: workspace.company_id, user: { id: userId, server_request: true } }, { company_id: workspace.company_id, user: { id: userId, server_request: true } },
); );
await gr.services.applications.companyApps.initWithDefaultApplications(
created.entity.company_id,
{
company: { id: created.entity.company_id },
user: { id: userId, server_request: true },
},
);
return new CreateResult<Workspace>(TYPE, created.entity); return new CreateResult<Workspace>(TYPE, created.entity);
} }
twake/backend/node/src/utils/types.ts
-2
View File
@@ -10,7 +10,6 @@ import {
ChannelTab, ChannelTab,
} from "../services/channels/entities"; } from "../services/channels/entities";
import { ChannelParameters } from "../services/channels/web/types"; import { ChannelParameters } from "../services/channels/web/types";
import { MessageNotification } from "../services/messages/types";
export type uuid = string; export type uuid = string;
@@ -107,7 +106,6 @@ export interface ResourceEventsPayload {
channelParameters?: ChannelParameters; channelParameters?: ChannelParameters;
guest?: ChannelPendingEmails; guest?: ChannelPendingEmails;
member?: ChannelMember; member?: ChannelMember;
message?: Pick<MessageNotification, "sender" | "workspace_id" | "thread_id">;
actor?: User; actor?: User;
resourcesBefore?: (User | ChannelEntity | ChannelTab | ChannelMember)[]; resourcesBefore?: (User | ChannelEntity | ChannelTab | ChannelMember)[];
resourcesAfter?: (User | ChannelEntity | ChannelTab | ChannelMember)[]; resourcesAfter?: (User | ChannelEntity | ChannelTab | ChannelMember)[];