diff --git a/twake/backend/node/src/cli/cmds/applications_cmds/ls.ts b/twake/backend/node/src/cli/cmds/applications_cmds/ls.ts deleted file mode 100644 index 25341c51..00000000 --- a/twake/backend/node/src/cli/cmds/applications_cmds/ls.ts +++ /dev/null @@ -1,54 +0,0 @@ -import yargs from "yargs"; -import ora from "ora"; -import twake from "../../../twake"; -import Table from "cli-table"; -import * as process from "process"; -import gr from "../../../services/global-resolver"; - -/** - * Merge command parameters. Check the builder definition below for more details. - */ -type CLIArgs = Record; - -const services = [ - "storage", - "counter", - "message-queue", - "platform-services", - "applications", - "auth", - "realtime", - "websocket", - // "user", - "search", - "database", - "webserver", - // "statistics", -]; - -const command: yargs.CommandModule = { - command: "ls", - describe: "command that allow you to list applications (unpublished only)", - - handler: async _argv => { - const spinner = ora({ text: "Retrieving applications" }).start(); - const platform = await twake.run(services); - await gr.doInit(platform); - // - const unpublished = await gr.services.applications.marketplaceApps.listUnpublished(undefined); - // - const table = new Table({ - head: ["ID", "Name", "Description"], - colWidths: [40, 20, 40], - }); - unpublished.forEach((app: any) => { - table.push([app.id, app.identity.name, app.identity.description]); - }); - spinner.stop(); - console.log(table.toString()); - await platform.stop(); - process.exit(0); - }, -}; - -export default command; diff --git a/twake/backend/node/src/cli/cmds/applications_cmds/publish.ts b/twake/backend/node/src/cli/cmds/applications_cmds/publish.ts deleted file mode 100644 index 18d88fc3..00000000 --- a/twake/backend/node/src/cli/cmds/applications_cmds/publish.ts +++ /dev/null @@ -1,89 +0,0 @@ -import yargs from "yargs"; -import ora from "ora"; -import twake from "../../../twake"; -import Table from "cli-table"; -import * as process from "process"; -import gr from "../../../services/global-resolver"; -/** - * Merge command parameters. Check the builder definition below for more details. - */ -type CLIArgs = { - id: string; -}; - -const services = [ - "storage", - "counter", - "message-queue", - "platform-services", - "applications", - "auth", - "realtime", - "websocket", - "user", - "search", - "database", - "webserver", - "statistics", -]; - -const command: yargs.CommandModule = { - command: "publish ", - describe: - "command that allow you to validate an application and make it available on the marketplace", - builder: { - force: { - default: false, - type: "boolean", - description: "Force update unrequested application", - }, - }, - handler: async argv => { - if (argv.id) { - let spinner = ora({ text: "Retrieving application" }).start(); - const platform = await twake.run(services); - await gr.doInit(platform); - let app = await gr.services.applications.marketplaceApps.get({ id: argv.id }, undefined); - spinner.stop(); - if (!app) { - console.error(`Application ${argv.id} not found`); - process.exit(1); - } - let table = new Table(); - table.push(app); - table.push({ name: app.identity.name }); - table.push({ description: app.identity.description }); - table.push({ website: app.identity.website }); - table.push({ requested: app.publication.requested }); - table.push({ published: app.publication.published }); - console.log(table.toString()); - - if (app.publication.published) { - console.error("Application already published"); - process.exit(1); - } - - if (!app.publication.requested && !argv.force) { - console.error( - "Application is not requested to be published. Use --force if you want to publish it anyway", - ); - process.exit(1); - } - - spinner = ora({ text: "Publishing application" }).start(); - await gr.services.applications.marketplaceApps.publish({ id: argv.id }, undefined); - app = await gr.services.applications.marketplaceApps.get({ id: argv.id }, undefined); - - spinner.stop(); - console.log("Application published"); - - table = new Table(); - table.push(app); - table.push({ published: app.publication.published }); - console.log(table.toString()); - } - process.exit(0); - }, -}; - -export default command; diff --git a/twake/backend/node/src/cli/cmds/applications_cmds/unpublish.ts b/twake/backend/node/src/cli/cmds/applications_cmds/unpublish.ts deleted file mode 100644 index 2e8b927c..00000000 --- a/twake/backend/node/src/cli/cmds/applications_cmds/unpublish.ts +++ /dev/null @@ -1,73 +0,0 @@ -import yargs from "yargs"; -import ora from "ora"; -import twake from "../../../twake"; -import Table from "cli-table"; -import * as process from "process"; -import gr from "../../../services/global-resolver"; -/** - * Merge command parameters. Check the builder definition below for more details. - */ -type CLIArgs = { - id: string; -}; - -const services = [ - "storage", - "counter", - "message-queue", - "platform-services", - "applications", - "auth", - "realtime", - "websocket", - "user", - "search", - "database", - "webserver", - "statistics", -]; - -const command: yargs.CommandModule = { - command: "unpublish ", - describe: "command that allow you to make application unavailable on the marketplace", - handler: async argv => { - if (argv.id) { - let spinner = ora({ text: "Retrieving application" }).start(); - const platform = await twake.run(services); - await gr.doInit(platform); - let app = await gr.services.applications.marketplaceApps.get({ id: argv.id }, undefined); - spinner.stop(); - if (!app) { - console.error(`Application ${argv.id} not found`); - process.exit(1); - } - let table = new Table(); - table.push(app); - table.push({ name: app.identity.name }); - table.push({ description: app.identity.description }); - table.push({ website: app.identity.website }); - table.push({ requested: app.publication.requested }); - table.push({ published: app.publication.published }); - console.log(table.toString()); - - if (!app.publication.published) { - console.error("Application is not published"); - process.exit(1); - } - - spinner = ora({ text: "Unpublishing application" }).start(); - await gr.services.applications.marketplaceApps.unpublish({ id: argv.id }, undefined); - app = await gr.services.applications.marketplaceApps.get({ id: argv.id }, undefined); - spinner.stop(); - console.log("Application unpublished"); - - table = new Table(); - table.push(app); - table.push({ published: app.publication.published }); - console.log(table.toString()); - } - process.exit(0); - }, -}; - -export default command; diff --git a/twake/backend/node/src/cli/cmds/export_cmds/export_company.ts b/twake/backend/node/src/cli/cmds/export_cmds/export_company.ts index 6d68b863..8c8b26b2 100644 --- a/twake/backend/node/src/cli/cmds/export_cmds/export_company.ts +++ b/twake/backend/node/src/cli/cmds/export_cmds/export_company.ts @@ -5,8 +5,6 @@ import { Pagination } from "../../../core/platform/framework/api/crud-service"; import WorkspaceUser from "../../../services/workspaces/entities/workspace_user"; import { ChannelVisibility } from "../../../services/channels/types"; import { Channel, ChannelMember } from "../../../services/channels/entities"; -import { Message } from "../../../services/messages/entities/messages"; -import { MessageWithReplies } from "../../../services/messages/types"; import { formatCompany } from "../../../services/user/utils"; import gr from "../../../services/global-resolver"; import { formatUser } from "../../../utils/users"; @@ -107,15 +105,6 @@ const command: yargs.CommandModule = { } writeFileSync(`${output}/users.json`, JSON.stringify(users)); - //Applications - console.log("- Create applications json file"); - const applications = await gr.services.applications.companyApps.list( - new Pagination(), - {}, - { company: { id: company.id }, user: { id: "", server_request: true } }, - ); - writeFileSync(`${output}/applications.json`, JSON.stringify(applications)); - //Channels console.log("- Create channels json file"); const directChannels: Channel[] = []; @@ -201,64 +190,6 @@ const command: yargs.CommandModule = { ); } - //Messages - console.log("- Create messages json file"); - //Note: direct channels content is private and not needed for R&D - for (const channel of [...allPublicChannels /*, ...directChannels*/]) { - let threads: MessageWithReplies[] = []; - let messages: Message[] = []; - let pagination = new Pagination(); - try { - do { - const page = await gr.services.messages.views.listChannel( - pagination, - { - include_users: false, - replies_per_thread: 10000, - emojis: false, - }, - { - user: { id: "", server_request: true }, - channel: { - company_id: channel.company_id, - workspace_id: channel.workspace_id, - id: channel.id, - }, - }, - ); - - for (const thread of page.getEntities()) { - messages = [...messages, ...thread.last_replies]; - } - - threads = [ - ...threads, - ...page.getEntities().map(thread => { - thread.last_replies = undefined; - return thread; - }), - ]; - - pagination = page.nextPage as Pagination; - } while (pagination.page_token); - } catch (err) { - console.log(`-- Error on the channel ${channel.id}`); - console.log(err); - } - - mkdirSync(`${output}/workspaces/${channel.workspace_id}/channels/${channel.id}`, { - recursive: true, - }); - writeFileSync( - `${output}/workspaces/${channel.workspace_id}/channels/${channel.id}/threads.json`, - JSON.stringify(threads), - ); - writeFileSync( - `${output}/workspaces/${channel.workspace_id}/channels/${channel.id}/messages.json`, - JSON.stringify(messages), - ); - } - await platform.stop(); }, }; diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/application.ts b/twake/backend/node/src/cli/cmds/migration_cmds/application.ts deleted file mode 100644 index 63c88264..00000000 --- a/twake/backend/node/src/cli/cmds/migration_cmds/application.ts +++ /dev/null @@ -1,264 +0,0 @@ -import yargs from "yargs"; -import twake from "../../../twake"; -import ora from "ora"; -import { TwakePlatform } from "../../../core/platform/platform"; -import { DatabaseServiceAPI } from "../../../core/platform/services/database/api"; -import PhpApplication, { - DepreciatedDisplayConfiguration, - TYPE as phpTYPE, -} from "./php-application/php-application-entity"; -import { Pagination } from "../../../core/platform/framework/api/crud-service"; -import Application, { TYPE } from "../../../services/applications/entities/application"; -import gr from "../../../services/global-resolver"; - -type Options = { - onlyApplication?: string; - replaceExisting?: boolean; -}; - -class ApplicationMigrator { - database: DatabaseServiceAPI; - - constructor(readonly platform: TwakePlatform) { - this.database = this.platform.getProvider("database"); - } - - public async run(options: Options = {}): Promise { - const phpRepository = await this.database.getRepository(phpTYPE, PhpApplication); - const repository = await this.database.getRepository(TYPE, Application); - - let page: Pagination = { limitStr: "100" }; - do { - const applicationListResult = await phpRepository.find({}, { pagination: page }, undefined); - page = applicationListResult.nextPage as Pagination; - - for (const application of applicationListResult.getEntities()) { - if ( - !(await repository.findOne( - { - id: application.id, - }, - {}, - undefined, - )) || - options.replaceExisting - ) { - const newApplication = importDepreciatedFields(application); - await repository.save(newApplication, undefined); - } - } - } while (page.page_token); - } -} - -const services = [ - "storage", - "counter", - "platform-services", - "user", - "search", - "channels", - "database", - "webserver", - "message-queue", - "applications", - "console", - "auth", - "statistics", - "realtime", -]; - -const command: yargs.CommandModule = { - command: "application", - describe: "command that allow you to migrate php applications to node", - builder: { - onlyApplication: { - default: null, - type: "string", - description: "Migrate only this application ID", - }, - replaceExisting: { - default: false, - type: "boolean", - description: "Replace already migrated applications", - }, - }, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - handler: async argv => { - const spinner = ora({ text: "Migrating php applications - " }).start(); - const platform = await twake.run(services); - await gr.doInit(platform); - const migrator = new ApplicationMigrator(platform); - - const onlyApplication = argv.onlyApplication as string | null; - const replaceExisting = (argv.replaceExisting || false) as boolean; - - await migrator.run({ - onlyApplication, - replaceExisting, - }); - - return spinner.stop(); - }, -}; - -export default command; - -export const importDepreciatedFields = (application: PhpApplication): Application => { - const newApplication = new Application(); - - newApplication.id = application.id; - newApplication.company_id = application.group_id; - newApplication.is_default = application.is_default; - - if (!newApplication.identity?.name) { - newApplication.identity = { - code: - application.depreciated_simple_name || - (application.depreciated_name || "").toLocaleLowerCase(), - name: application.depreciated_name, - icon: application.depreciated_icon_url, - description: application.depreciated_description, - website: "http://twake.app/", - categories: [], - compatibility: ["twake"], - }; - } - - if (newApplication.publication?.published === undefined) { - newApplication.publication = newApplication.publication || { - published: false, - requested: false, - }; - newApplication.publication.published = application.depreciated_is_available_to_public; - newApplication.publication.requested = - application.depreciated_public && !application.depreciated_twake_team_validation; - } - - if (!newApplication.stats?.version) { - newApplication.stats = newApplication.stats || { - created_at: null, - updated_at: null, - version: null, - }; - newApplication.stats.version = 1; - newApplication.stats.created_at = Date.now(); - newApplication.stats.updated_at = Date.now(); - } - - if (!newApplication.api?.private_key) { - newApplication.api = newApplication.api || { - hooks_url: null, - allowed_ips: null, - private_key: null, - }; - newApplication.api.hooks_url = application.depreciated_api_events_url; - newApplication.api.allowed_ips = application.depreciated_api_allowed_ip; - newApplication.api.private_key = application.depreciated_api_private_key; - } - - if (newApplication.access?.write === undefined) { - newApplication.access = newApplication.access || { - read: null, - write: null, - delete: null, - hooks: null, - }; - try { - newApplication.access.write = JSON.parse(application.depreciated_capabilities || "[]") || []; - newApplication.access.delete = JSON.parse(application.depreciated_capabilities || "[]") || []; - } catch (e) { - newApplication.access.write = []; - newApplication.access.delete = []; - } - try { - newApplication.access.read = JSON.parse(application.depreciated_privileges || "[]") || []; - } catch (e) { - newApplication.access.read = []; - } - try { - newApplication.access.hooks = JSON.parse(application.depreciated_hooks || "[]") || []; - } catch (e) { - newApplication.access.hooks = []; - } - } - - newApplication.display = importDepreciatedDisplayFields( - newApplication, - JSON.parse(application.depreciated_display_configuration), - ); - - return newApplication; -}; - -export const importDepreciatedDisplayFields = ( - application: Application, - depreciatedDisplay: DepreciatedDisplayConfiguration, -): Application["display"] => { - let display = application.display; - - if (!display?.twake) { - display = display || { twake: {} }; - display.twake = display.twake || {}; - } - - display.twake.tab = depreciatedDisplay?.channel_tab - ? { url: depreciatedDisplay?.channel_tab?.iframe } || true - : undefined; - - display.twake.standalone = depreciatedDisplay?.app - ? { url: depreciatedDisplay?.app?.iframe } || true - : undefined; - - display.twake.configuration = []; - if (depreciatedDisplay?.configuration?.can_configure_in_workspace) - display.twake.configuration.push("global"); - if (depreciatedDisplay?.configuration?.can_configure_in_channel) - display.twake.configuration.push("channel"); - - display.twake.direct = depreciatedDisplay?.member_app - ? { name: application.identity.name, icon: application.identity.icon } || true - : undefined; - - if (depreciatedDisplay?.drive_module) { - display.twake.files = { - editor: undefined, - actions: [], - }; - - display.twake.files.editor = { - preview_url: depreciatedDisplay?.drive_module?.can_open_files?.preview_url, - edition_url: depreciatedDisplay?.drive_module?.can_open_files?.url, - extensions: [ - ...(depreciatedDisplay?.drive_module?.can_open_files?.main_ext || []), - ...(depreciatedDisplay?.drive_module?.can_open_files?.other_ext || []), - ], - empty_files: (depreciatedDisplay?.drive_module?.can_create_files as any) || [], - }; - } - - if (depreciatedDisplay?.messages_module) { - display.twake.chat = { - input: - depreciatedDisplay?.messages_module?.in_plus || - depreciatedDisplay?.messages_module?.right_icon - ? { - icon: application.identity.icon, - } - : undefined, - commands: - (depreciatedDisplay?.messages_module - ?.commands as Application["display"]["twake"]["chat"]["commands"]) || undefined, - actions: depreciatedDisplay?.messages_module?.action - ? [ - { - name: depreciatedDisplay?.messages_module?.action.description, - id: "default", - }, - ] - : undefined, - }; - } - - return display; -}; diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/channel-member-read-cursors.ts b/twake/backend/node/src/cli/cmds/migration_cmds/channel-member-read-cursors.ts deleted file mode 100644 index aec9c6a7..00000000 --- a/twake/backend/node/src/cli/cmds/migration_cmds/channel-member-read-cursors.ts +++ /dev/null @@ -1,173 +0,0 @@ -import { DatabaseServiceAPI } from "../../../core/platform/services/database/api"; -import { TwakePlatform } from "../../../core/platform/platform"; -import Repository from "../../../core/platform/services/database/services/orm/repository/repository"; -import { ChannelMember, ChannelMemberReadCursors } from "../../../services/channels/entities"; -import { - ExecutionContext, - Paginable, - Pagination, -} from "../../../core/platform/framework/api/crud-service"; -import gr from "../../../services/global-resolver"; -import yargs from "yargs"; -import ora from "ora"; -import twake from "../../../twake"; - -class ChannelMemberReadCursorsMigrator { - readSectionRepository: Repository; - database: DatabaseServiceAPI; - - constructor(readonly _platform: TwakePlatform) { - this.database = this._platform.getProvider("database"); - } - - public async run(_options = {}, context?: ExecutionContext): Promise { - const companyPagination: Paginable = new Pagination(null, "100"); - const companies = await gr.services.companies.getCompanies(companyPagination); - - context.user.server_request = true; - - for (const company of companies.getEntities()) { - const { id: CompanyId } = company; - - const workspaces = await gr.services.workspaces.getAllForCompany(CompanyId); - const workspaceIds = [...workspaces.map(({ id }) => id), "direct"]; - - for (const workspaceId of workspaceIds) { - const channels = await gr.services.channels.channels.getAllChannelsInWorkspace( - CompanyId, - workspaceId, - ); - - for (const channel of channels) { - const { id: channelId } = channel; - const membersPagination: Pagination = new Pagination(null, "100"); - const threadsPagination: Pagination = new Pagination(null, "1"); - let members: ChannelMember[]; - - try { - const membersList = await gr.services.channels.members.list( - membersPagination, - {}, - { - ...context, - channel: { - company_id: CompanyId, - id: channelId, - workspace_id: workspaceId, - }, - }, - ); - members = membersList.getEntities(); - } catch (error) { - continue; - } - - let lastMessage = await gr.services.messages.views.listChannel( - threadsPagination, - {}, - { - ...context, - channel: { - company_id: CompanyId, - id: channelId, - workspace_id: workspaceId, - }, - user: { - id: null, - }, - }, - ); - - const firstMessage = await gr.services.messages.views.listChannel( - { - ...threadsPagination, - reversed: true, - }, - {}, - { - ...context, - channel: { - company_id: CompanyId, - id: channelId, - workspace_id: workspaceId, - }, - user: { - id: null, - }, - }, - ); - - if (!firstMessage.getEntities().length) { - continue; - } - - if (!lastMessage.getEntities().length) { - lastMessage = firstMessage; - } - - const firstMessageId = firstMessage.getEntities()[0].id; - const lastMessageId = lastMessage.getEntities()[0].id; - - if (!firstMessageId) { - continue; - } else { - for (const member of members) { - await gr.services.channels.members.setChannelMemberReadSections( - { - start: firstMessageId, - end: lastMessageId, - }, - { - ...context, - channel_id: channelId, - workspace_id: workspaceId, - company: { - id: CompanyId, - }, - user: { - id: member.user_id, - }, - }, - ); - } - } - } - } - } - } -} - -const services = [ - "search", - "database", - "webserver", - "auth", - "counter", - "cron", - "message-queue", - "push", - "realtime", - "storage", - "tracker", - "websocket", - "email-pusher", -]; - -const command: yargs.CommandModule = { - command: "channel-member-read-cursor-repair", - describe: "fixes the channel members read cursors for old messages", - builder: {}, - - handler: async () => { - const spinner = ora({ text: "Fixing channel members read cursors" }).start(); - const platform = await twake.run(services); - await gr.doInit(platform); - const migrator = new ChannelMemberReadCursorsMigrator(platform); - - await migrator.run({}); - - return spinner.stop(); - }, -}; - -export default command; diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/devices.ts b/twake/backend/node/src/cli/cmds/migration_cmds/devices.ts deleted file mode 100644 index 8027d3d1..00000000 --- a/twake/backend/node/src/cli/cmds/migration_cmds/devices.ts +++ /dev/null @@ -1,103 +0,0 @@ -import yargs from "yargs"; -import twake from "../../../twake"; -import ora from "ora"; -import { TwakePlatform } from "../../../core/platform/platform"; -import { DatabaseServiceAPI } from "../../../core/platform/services/database/api"; -import { PhpDevice, TYPE as phpTYPE } from "./php-device/php-device-entity"; -import { Pagination } from "../../../core/platform/framework/api/crud-service"; -import Device, { TYPE } from "../../../services/user/entities/device"; -import User, { TYPE as userTYPE } from "../../../services/user/entities/user"; -import _ from "lodash"; -import gr from "../../../services/global-resolver"; - -type Options = { - replaceExisting?: boolean; -}; - -class DeviceMigrator { - database: DatabaseServiceAPI; - - constructor(readonly platform: TwakePlatform) { - this.database = this.platform.getProvider("database"); - } - - public async run(options: Options = {}): Promise { - const phpRepository = await this.database.getRepository(phpTYPE, PhpDevice); - const userRepository = await this.database.getRepository(userTYPE, User); - const repository = await this.database.getRepository(TYPE, Device); - - // Get all companies - let page: Pagination = { limitStr: "100" }; - // For each devices - do { - const deviceListResult = await phpRepository.find({}, { pagination: page }, undefined); - page = deviceListResult.nextPage as Pagination; - - for (const device of deviceListResult.getEntities()) { - if ( - !(await repository.findOne({ id: device.value }, {}, undefined)) || - options.replaceExisting - ) { - if (device.type === "FCM" || device.type === "fcm") { - const newDevice = new Device(); - newDevice.id = device.value; - newDevice.type = "FCM"; - newDevice.user_id = device.user_id; - newDevice.version = device.version; - await repository.save(newDevice, undefined); - - const user = await userRepository.findOne({ id: device.user_id }, {}, undefined); - if (user) { - user.devices = _.uniq([...(user.devices || []), newDevice.id]); - await userRepository.save(user, undefined); - } - } - } - } - } while (page.page_token); - } -} - -const services = [ - "storage", - "counter", - "platform-services", - "user", - "search", - "channels", - "database", - "webserver", - "message-queue", - "workspaces", - "console", - "auth", -]; - -const command: yargs.CommandModule = { - command: "device", - describe: "command that allow you to migrate php devices to node", - builder: { - replaceExisting: { - default: false, - type: "boolean", - description: "Replace already migrated devices", - }, - }, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - handler: async argv => { - const spinner = ora({ text: "Migrating php devices - " }).start(); - const platform = await twake.run(services); - await gr.doInit(platform); - const migrator = new DeviceMigrator(platform); - - const replaceExisting = (argv.replaceExisting || false) as boolean; - - await migrator.run({ - replaceExisting, - }); - - return spinner.stop(); - }, -}; - -export default command; diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/drive.ts b/twake/backend/node/src/cli/cmds/migration_cmds/drive.ts deleted file mode 100644 index e5dba126..00000000 --- a/twake/backend/node/src/cli/cmds/migration_cmds/drive.ts +++ /dev/null @@ -1,40 +0,0 @@ -import ora from "ora"; -import globalResolver from "../../../services/global-resolver"; -import twake from "../../../twake"; -import yargs from "yargs"; -import DriveMigrator from "./php-drive-file/drive-migrator-service"; - -const services = [ - "search", - "database", - "webserver", - "auth", - "counter", - "cron", - "message-queue", - "push", - "realtime", - "storage", - "tracker", - "websocket", - "email-pusher", -]; - -const command: yargs.CommandModule = { - command: "drive", - describe: "migrate php drive items to node", - builder: {}, - handler: async _argv => { - console.log("test"); - const spinner = ora({ text: "Migrating php drive - " }).start(); - const platform = await twake.run(services); - await globalResolver.doInit(platform); - const migrator = new DriveMigrator(platform); - - await migrator.run(); - - return spinner.stop(); - }, -}; - -export default command; diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/message-channel-repair.ts b/twake/backend/node/src/cli/cmds/migration_cmds/message-channel-repair.ts deleted file mode 100644 index 80f85fbe..00000000 --- a/twake/backend/node/src/cli/cmds/migration_cmds/message-channel-repair.ts +++ /dev/null @@ -1,96 +0,0 @@ -import yargs from "yargs"; -import twake from "../../../twake"; -import ora from "ora"; -import { DatabaseServiceAPI } from "../../../core/platform/services/database/api"; -import { TwakePlatform } from "../../../core/platform/platform"; -import gr from "../../../services/global-resolver"; - -type Options = { - from?: string; - onlyCompany?: string; - onlyWorkspace?: string; - onlyChannel?: string; - dryRun?: boolean; -}; - -class MessageReferenceRepair { - database: DatabaseServiceAPI; - - constructor(readonly platform: TwakePlatform) { - this.database = this.platform.getProvider("database"); - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - public async run(_options: Options = {}): Promise { - //TODO repair messages - } -} - -const services = [ - "storage", - "counter", - "platform-services", - "user", - "search", - "channels", - "database", - "webserver", - "message-queue", - "messages", -]; - -const command: yargs.CommandModule = { - command: "message-channel-repair", - describe: "command that allow you to repair messages references in channels", - builder: { - from: { - default: null, - type: "string", - description: "Start migration from this company ID", - }, - onlyCompany: { - default: null, - type: "string", - description: "Migrate only this company ID", - }, - onlyWorkspace: { - default: null, - type: "string", - description: "Migrate only this workspace ID", - }, - onlyChannel: { - default: null, - type: "string", - description: "Migrate only this channel ID", - }, - dryRun: { - default: false, - type: "boolean", - description: "Do not save anything and show missing references", - }, - }, - handler: async argv => { - const spinner = ora({ text: "Fixing messages references - " }).start(); - const platform = await twake.run(services); - await gr.doInit(platform); - const migrator = new MessageReferenceRepair(platform); - - const from = argv.from as string | null; - const onlyCompany = argv.onlyCompany as string | null; - const onlyWorkspace = argv.onlyWorkspace as string | null; - const onlyChannel = argv.onlyChannel as string | null; - const dryRun = (argv.dryRun || false) as boolean; - - await migrator.run({ - from, - onlyCompany, - onlyWorkspace, - onlyChannel, - dryRun, - }); - - return spinner.stop(); - }, -}; - -export default command; diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/message-files-cache.ts b/twake/backend/node/src/cli/cmds/migration_cmds/message-files-cache.ts deleted file mode 100644 index 0e56413e..00000000 --- a/twake/backend/node/src/cli/cmds/migration_cmds/message-files-cache.ts +++ /dev/null @@ -1,190 +0,0 @@ -import yargs from "yargs"; -import twake from "../../../twake"; -import ora from "ora"; -import { TwakePlatform } from "../../../core/platform/platform"; -import { DatabaseServiceAPI } from "../../../core/platform/services/database/api"; -import { - ExecutionContext, - Paginable, - Pagination, -} from "../../../core/platform/framework/api/crud-service"; -import gr from "../../../services/global-resolver"; -import { getInstance, MessageFileRef } from "../../../services/messages/entities/message-file-refs"; -import Repository from "../../../core/platform/services/database/services/orm/repository/repository"; -import uuid from "node-uuid"; -import { MessageFile } from "../../../services/messages/entities/message-files"; -import { ThreadExecutionContext } from "../../../services/messages/types"; - -type Options = Record; - -class MessageFilesCacheMigrator { - database: DatabaseServiceAPI; - repository: Repository; - messageFileRepository: Repository; - - constructor(readonly platform: TwakePlatform) { - this.database = this.platform.getProvider("database"); - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - public async run(_options: Options = {}, context?: ExecutionContext): Promise { - this.repository = await gr.database.getRepository( - "message_file_refs", - MessageFileRef, - ); - this.messageFileRepository = await gr.database.getRepository( - "message_files", - MessageFile, - ); - - let count = 0; - - let companyPagination: Paginable = new Pagination(null, "100"); - do { - const companyList = await gr.services.companies.getCompanies(companyPagination); - companyPagination = companyList.nextPage; - - for (const company of companyList.getEntities()) { - const companyId: string = company.id; - const workspaceList = await gr.services.workspaces.getAllForCompany(companyId); - - for (const workspaceId of [...workspaceList.map(w => w.id), "direct"]) { - const channelsList = await gr.services.channels.channels.getAllChannelsInWorkspace( - companyId, - workspaceId, - context, - ); - - for (const channel of channelsList) { - const channelId = channel.id; - - let threadPagination: Paginable = new Pagination(null, "100"); - do { - const threadList = await gr.services.messages.views.listChannel( - threadPagination, - {}, - { - channel: { - company_id: companyId, - workspace_id: workspaceId, - id: channelId, - }, - user: { id: null }, - }, - ); - - for (const thread of threadList.getEntities()) { - let messagesPagination: Paginable = new Pagination(null, "100"); - do { - const messagesList = await gr.services.messages.messages.list( - messagesPagination, - {}, - { - thread: { id: thread.id }, - company: { id: companyId }, - workspace: { id: workspaceId }, - channel: { id: channelId }, - user: { id: null }, - } as ThreadExecutionContext, - ); - messagesPagination = messagesList.nextPage; - messagesPagination.page_token = messagesList.getEntities()[0]?.id; - - for (const message of messagesList.getEntities()) { - if (message.files && message.files.length > 0) { - for (const _messageFile of message.files) { - count++; - try { - const messageFile = await this.messageFileRepository.findOne( - { - message_id: message.id, - id: _messageFile.id, - }, - {}, - undefined, - ); - - if (messageFile) { - console.log(messageFile.metadata.name); - - //Update user uploads - const fileRef = getInstance({ - target_type: "user_upload", - target_id: message.user_id, - id: uuid.v1({ msecs: message.created_at }), - created_at: message.created_at, - company_id: companyId, - workspace_id: workspaceId, - channel_id: channelId, - thread_id: message.thread_id, - message_id: message.id, - message_file_id: messageFile.id, - file_id: messageFile.metadata.external_id, - }); - await this.repository.save(fileRef, undefined); - - //Update messageFileRepository - - messageFile.cache = { - company_id: companyId, - workspace_id: workspaceId, - channel_id: channelId, - user_id: message.user_id, - }; - messageFile.thread_id = message.thread_id; - - await this.messageFileRepository.save(messageFile, undefined); - } - } catch (e) {} - } - } - } - } while (messagesPagination.page_token); - } - - threadPagination = threadList.nextPage; - threadPagination.page_token = - threadPagination.page_token && - threadList.getEntities()?.[threadList.getEntities().length - 1]?.thread_id; - } while (false); // && threadPagination.page_token && threadList.getEntities().length > 50); - } - } - } - console.log("updated messages: ", count); - } while (companyPagination.page_token); - } -} - -const services = [ - "search", - "database", - "webserver", - "auth", - "counter", - "cron", - "message-queue", - "push", - "realtime", - "storage", - "tracker", - "websocket", -]; - -const command: yargs.CommandModule = { - command: "message-files-cache", - describe: "command that allow you to fix cache for each message-file refs", - builder: {}, - - handler: async () => { - const spinner = ora({ text: "Migrating messages - " }).start(); - const platform = await twake.run(services); - await gr.doInit(platform); - const migrator = new MessageFilesCacheMigrator(platform); - - await migrator.run({}); - - return spinner.stop(); - }, -}; - -export default command; diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/message.ts b/twake/backend/node/src/cli/cmds/migration_cmds/message.ts deleted file mode 100644 index b2814cf9..00000000 --- a/twake/backend/node/src/cli/cmds/migration_cmds/message.ts +++ /dev/null @@ -1,91 +0,0 @@ -import yargs from "yargs"; -import twake from "../../../twake"; -import ora from "ora"; -import MessageMigrator from "./php-message/message-migrator-service"; -import gr from "../../../services/global-resolver"; - -const services = [ - "storage", - "counter", - "platform-services", - "user", - "search", - "channels", - "database", - "webserver", - "message-queue", - "messages", - "statistics", -]; - -const command: yargs.CommandModule = { - command: "message", - describe: "command that allow you to migrate php messages to node", - builder: { - from: { - default: null, - type: "string", - description: "Start migration from this company ID", - }, - onlyCompany: { - default: null, - type: "string", - description: "Migrate only this company ID", - }, - onlyWorkspace: { - default: null, - type: "string", - description: "Migrate only this workspace ID", - }, - onlyChannel: { - default: null, - type: "string", - description: "Migrate only this channel ID", - }, - ignoreExisting: { - default: false, - type: "boolean", - description: "Skip existing message ids", - }, - backToPhp: { - default: false, - type: "boolean", - description: - "Run the migration back, put node messages into php table (only the one that are not already here)", - }, - dryRun: { - default: false, - type: "boolean", - description: "Do not save anything", - }, - }, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - handler: async argv => { - const spinner = ora({ text: "Migrating php messages - " }).start(); - const platform = await twake.run(services); - await gr.doInit(platform); - const migrator = new MessageMigrator(platform); - - const from = argv.from as string | null; - const onlyCompany = argv.onlyCompany as string | null; - const onlyWorkspace = argv.onlyWorkspace as string | null; - const onlyChannel = argv.onlyChannel as string | null; - const ignoreExisting = (argv.ignoreExisting || false) as boolean; - const backToPhp = (argv.backToPhp || false) as boolean; - const dryRun = (argv.dryRun || false) as boolean; - - await migrator.run({ - from, - onlyCompany, - onlyWorkspace, - onlyChannel, - ignoreExisting, - backToPhp, - dryRun, - }); - - return spinner.stop(); - }, -}; - -export default command; diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/messages-files-medias-separation.ts b/twake/backend/node/src/cli/cmds/migration_cmds/messages-files-medias-separation.ts deleted file mode 100644 index 35b23c09..00000000 --- a/twake/backend/node/src/cli/cmds/migration_cmds/messages-files-medias-separation.ts +++ /dev/null @@ -1,125 +0,0 @@ -import yargs from "yargs"; -import twake from "../../../twake"; -import ora from "ora"; -import { DatabaseServiceAPI } from "../../../core/platform/services/database/api"; -import { TwakePlatform } from "../../../core/platform/platform"; -import gr from "../../../services/global-resolver"; -import { MessageFileRef } from "../../../services/messages/entities/message-file-refs"; -import { Paginable } from "../../../core/platform/framework/api/crud-service"; -import { Pagination } from "../../../core/platform/framework/api/crud-service"; -import { fileIsMedia } from "../../../services/files/utils"; -import { MessageFile } from "../../../services/messages/entities/message-files"; -import _ from "lodash"; - -type Options = Record; - -class MessageReferenceRepair { - database: DatabaseServiceAPI; - - constructor(readonly platform: TwakePlatform) { - this.database = this.platform.getProvider("database"); - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - public async run(_options: Options = {}): Promise { - const repository = await gr.database.getRepository( - "message_file_refs", - MessageFileRef, - ); - const repositoryMessageFile = await gr.database.getRepository( - "message_files", - MessageFile, - ); - - let count = 0; - - let companyPagination: Paginable = new Pagination(null, "100"); - do { - const companyList = await gr.services.companies.getCompanies(companyPagination); - companyPagination = companyList.nextPage; - - for (const company of companyList.getEntities()) { - const companyId: string = company.id; - const workspaceList = await gr.services.workspaces.getAllForCompany(companyId); - - for (const workspaceId of [...workspaceList.map(w => w.id), "direct"]) { - const channelsList = await gr.services.channels.channels.getAllChannelsInWorkspace( - companyId, - workspaceId, - ); - - for (const channel of channelsList) { - const channelId = channel.id; - let filePagination: Pagination = new Pagination(null, "100"); - - do { - const items = await repository.find( - { - target_type: "channel", - target_id: channelId, - company_id: companyId, - }, - { pagination: filePagination }, - ); - - for (const item of items.getEntities()) { - try { - const msgFile = await repositoryMessageFile.findOne({ - message_id: item.message_id, - id: item.message_file_id, - }); - if (msgFile) { - count++; - const isMedia = fileIsMedia(msgFile); - const ref = _.cloneDeep(item); - ref.target_type = isMedia ? "channel_media" : "channel_file"; - await repository.save(ref); - } - } catch (e) { - console.log("Error", e); - } - } - - filePagination = new Pagination(items.nextPage.page_token, "100"); - } while (filePagination.page_token); - } - } - } - - console.log("updated files refs: ", count); - } while (companyPagination.page_token); - } -} - -const services = [ - "search", - "database", - "webserver", - "auth", - "counter", - "cron", - "message-queue", - "push", - "realtime", - "storage", - "tracker", - "websocket", -]; - -const command: yargs.CommandModule = { - command: "messages-files-medias-separation", - describe: "command to separate medias and files in messages-files channels refs", - builder: {}, - handler: async () => { - const spinner = ora({ text: "Fixing messages references - " }).start(); - const platform = await twake.run(services); - await gr.doInit(platform); - const migrator = new MessageReferenceRepair(platform); - - await migrator.run({}); - - return spinner.stop(); - }, -}; - -export default command; diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/php-application/php-application-entity.ts b/twake/backend/node/src/cli/cmds/migration_cmds/php-application/php-application-entity.ts deleted file mode 100644 index 5df778ab..00000000 --- a/twake/backend/node/src/cli/cmds/migration_cmds/php-application/php-application-entity.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { Type } from "class-transformer"; -import { - Column, - Entity, -} from "../../../../core/platform/services/database/services/orm/decorators"; - -export const TYPE = "application"; - -@Entity(TYPE, { - primaryKey: [["group_id"], "app_group_name", "id"], - type: TYPE, -}) -export default class PhpApplication { - @Type(() => String) - @Column("id", "timeuuid", { generator: "timeuuid" }) - id: string; - - @Type(() => String) - @Column("group_id", "timeuuid") - group_id: string; - - @Type(() => String) - @Column("app_group_name", "string") - app_group_name = ""; - - @Column("name", "encoded_string") - depreciated_name: string; - - @Column("simple_name", "encoded_string") - depreciated_simple_name: string; - - @Column("description", "encoded_string") - depreciated_description: string; - - @Column("icon_url", "encoded_string") - depreciated_icon_url: string; - - @Column("is_default", "twake_boolean") - is_default: boolean; - - @Column("public", "twake_boolean") - depreciated_public: boolean; - - @Column("twake_team_validation", "twake_boolean") - depreciated_twake_team_validation: boolean; - - @Column("is_available_to_public", "twake_boolean") - depreciated_is_available_to_public: boolean; //Vrai si $public ET $twake_team_validation - - @Column("api_events_url", "encoded_string") - depreciated_api_events_url: string; - - @Column("api_allowed_ip", "encoded_string") - depreciated_api_allowed_ip: string; - - @Column("api_private_key", "encoded_string") - depreciated_api_private_key: string; - - @Column("privileges", "encoded_string") - depreciated_privileges = "[]"; - - @Column("capabilities", "encoded_string") - depreciated_capabilities = "[]"; - - @Column("hooks", "encoded_string") - depreciated_hooks = "[]"; - - @Column("display_configuration", "encoded_string") - depreciated_display_configuration = "{}"; -} - -export type DepreciatedDisplayConfiguration = { - version?: 0; //Legacy - tasks_module?: { - can_connect_to_tasks?: boolean; - }; - calendar_module?: { - can_connect_to_calendar?: boolean; - }; - drive_module?: { - can_connect_to_directory?: boolean; - can_open_files?: { - url?: string; //Une url à appeler pour éditer le fichier (ouvert dans un onglet) - preview_url?: string; //Une url à appeler pour prévisualiser un fichier (iframe) - main_ext?: string[]; //Extensions principales - other_ext?: string[]; //Extensions secondaires - }; - can_create_files?: { - url?: string; - filename?: string; - name?: string; - }[]; - }; - member_app?: boolean; // Si défini, votre application génèrera un membre - // virtuel dans l'espace de travail avec lequel les - // utilisateurs pourront discuter. - messages_module?: { - in_plus?: { - should_wait_for_popup: boolean; - }; - right_icon?: { - icon_url?: string; //If defined replace original icon url of your app - should_wait_for_popup?: boolean; - type?: string; //"file" | "call" - }; - action?: { - should_wait_for_popup?: boolean; - description?: string; //Description de l'action, sinon remplacé par le nom de l'app - }; - commands?: { - command?: string; // my_app mycommand - description?: string; - }[]; - }; - channel?: { - can_connect_to_channel?: string; - }; - channel_tab?: { - iframe?: string; - }; - app?: { - iframe?: string; - plus_btn: { - should_wait_for_popup?: boolean; - }; - }; - configuration?: { - can_configure_in_workspace?: boolean; - can_configure_in_channel?: boolean; - can_configure_in_calendar?: boolean; - can_configure_in_tasks?: boolean; - }; -}; diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/php-device/php-device-entity.ts b/twake/backend/node/src/cli/cmds/migration_cmds/php-device/php-device-entity.ts deleted file mode 100644 index 9860b252..00000000 --- a/twake/backend/node/src/cli/cmds/migration_cmds/php-device/php-device-entity.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { - Column, - Entity, -} from "../../../../core/platform/services/database/services/orm/decorators"; - -export const TYPE = "device"; -@Entity(TYPE, { - primaryKey: [["id"]], - type: TYPE, -}) -export class PhpDevice { - @Column("id", "timeuuid") - id: string; - - @Column("user_id", "timeuuid") - user_id: string; - - @Column("type", "string") - type: string; - - @Column("version", "string") - version: string; - - @Column("value", "encoded_string") - value: string; -} diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/drive-migrator-service.ts b/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/drive-migrator-service.ts deleted file mode 100644 index 2c28b2a1..00000000 --- a/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/drive-migrator-service.ts +++ /dev/null @@ -1,535 +0,0 @@ -import { logger } from "../../../../core/platform/framework"; -import { ExecutionContext, Pagination } from "../../../../core/platform/framework/api/crud-service"; -import { TwakePlatform } from "../../../../core/platform/platform"; -import Repository from "../../../../core/platform/services/database/services/orm/repository/repository"; -import { DriveFile, AccessInformation } from "../../../../services/documents/entities/drive-file"; -import { - generateAccessToken, - getDefaultDriveItem, - getDefaultDriveItemVersion, -} from "../../../../services/documents/utils"; -import globalResolver from "../../../../services/global-resolver"; -import Company from "../../../../services/user/entities/company"; -import Workspace from "../../../../services/workspaces/entities/workspace"; -import { PhpDriveFile } from "./php-drive-file-entity"; -import { PhpDriveFileService } from "./php-drive-service"; -import mimes from "../../../../utils/mime"; -import WorkspaceUser from "../../../../services/workspaces/entities/workspace_user"; -import CompanyUser from "src/services/user/entities/company_user"; - -interface CompanyExecutionContext extends ExecutionContext { - company: { - id: string; - }; -} - -interface WorkspaceExecutionContext extends CompanyExecutionContext { - workspace_id: string; -} - -class DriveMigrator { - private phpDriveService: PhpDriveFileService; - private nodeRepository: Repository; - - constructor(readonly _platform: TwakePlatform) { - this.phpDriveService = new PhpDriveFileService(); - } - - /** - * Run - */ - public run = async (): Promise => { - await globalResolver.doInit(this._platform); - await this.phpDriveService.init(); - - this.nodeRepository = await globalResolver.database.getRepository( - "drive_files", - DriveFile, - ); - - let page: Pagination = { limitStr: "100" }; - const context: ExecutionContext = { - user: { - id: null, - server_request: true, - }, - }; - - do { - const companyListResult = await globalResolver.services.companies.getCompanies(page); - page = companyListResult.nextPage as Pagination; - - for (const company of companyListResult.getEntities()) { - await this.migrateCompany(company, { - ...context, - company: { id: company.id }, - }); - } - } while (page.page_token); - }; - - /** - * Migrate a company drive files. - * - * @param {Company} company - the company to migrate - */ - private migrateCompany = async ( - company: Company, - context: CompanyExecutionContext, - ): Promise => { - logger.info(`Migrating company ${company.id}`); - - const companyAdminOrOwnerId = await this.getCompanyOwnerOrAdminId(company.id, context); - const workspaceList = await globalResolver.services.workspaces.getAllForCompany(company.id); - - for (const workspace of workspaceList) { - const wsContext = { - ...context, - workspace_id: workspace.id, - user: { id: companyAdminOrOwnerId, server_request: true }, - }; - const access = await this.getWorkspaceAccess(workspace, company, wsContext); - - await this.migrateWorkspace(workspace, access, wsContext); - } - }; - - /** - * Migrates a workspace drive files. - * - * @param {Workspace} workspace - the workspace to migrate - */ - private migrateWorkspace = async ( - workspace: Workspace, - access: AccessInformation, - context: WorkspaceExecutionContext, - ): Promise => { - let page: Pagination = { limitStr: "100" }; - - console.debug(`Migrating workspace ${workspace.id} of company ${context.company.id}`); - logger.info(`Migrating workspace ${workspace.id} root folder`); - - const workspaceFolder = await this.createWorkspaceFolder(workspace, access, context); - // Migrate the root folder. - do { - const phpDriveFiles = await this.phpDriveService.listDirectory( - page, - "", - workspace.id, - context, - ); - page = phpDriveFiles.nextPage as Pagination; - - for (const phpDriveFile of phpDriveFiles.getEntities()) { - await this.migrateDriveFile(phpDriveFile, workspaceFolder.id, access, context); - } - } while (page.page_token); - - logger.info(`Migrating workspace ${workspace.id} trash`); - // Migrate the trash. - page = { limitStr: "100" }; - - do { - const phpDriveFiles = await this.phpDriveService.listDirectory(page, "trash", workspace.id); - page = phpDriveFiles.nextPage as Pagination; - - for (const phpDriveFile of phpDriveFiles.getEntities()) { - await this.migrateDriveFile(phpDriveFile, "trash", access, context); - } - } while (page.page_token); - }; - - /** - * Migrates a php drive item to a Node drive file - * - * @param {PhpDriveItem} item - the php drive file to migrate. - */ - private migrateDriveFile = async ( - item: PhpDriveFile, - parentId: string, - access: AccessInformation, - context: WorkspaceExecutionContext, - ): Promise => { - logger.info(`Migrating php drive item ${item.id} - parent: ${parentId ?? "root"}`); - - try { - const migrationRecord = await this.phpDriveService.getMigrationRecord( - item.id, - context.company.id, - ); - - const newDriveItem = getDefaultDriveItem( - { - name: item.name || item.id, - extension: item.extension, - added: item.added.toString(), - content_keywords: - item.content_keywords && item.content_keywords.length - ? item.content_keywords.join(",") - : "", - creator: item.creator || context.user.id, - is_directory: item.isdirectory, - is_in_trash: item.isintrash, - description: item.description, - tags: item.tags || [], - parent_id: parentId, - company_id: context.company.id, - access_info: access, - }, - context, - ); - - if (migrationRecord && migrationRecord.company_id === context.company.id) { - console.debug(`${item.id} is already migrated`); - } else { - await this.nodeRepository.save(newDriveItem); - } - - if (item.isdirectory) { - const newParentId = - migrationRecord && migrationRecord.company_id === context.company.id - ? migrationRecord.new_id - : newDriveItem.id; - - let page: Pagination = { limitStr: "100" }; - - do { - const directoryChildren = await this.phpDriveService.listDirectory( - page, - item.id, - context.workspace_id, - ); - page = directoryChildren.nextPage as Pagination; - - for (const child of directoryChildren.getEntities()) { - try { - await this.migrateDriveFile(child, newParentId, access, context); - } catch (error) { - logger.error(`Failed to migrate drive item ${child.id}`); - console.error(`Failed to migrate drive item ${child.id}`); - } - } - } while (page.page_token); - } else { - let versionPage: Pagination = { limitStr: "100" }; - if ( - migrationRecord && - migrationRecord.item_id === item.id && - migrationRecord.company_id === context.company.id - ) { - logger.info(`item is already migrated - ${item.id} - skipping`); - console.log(`item is already migrated - ${item.id} - skipping`); - return; - } - - const mime = mimes[item.extension]; - - let createdVersions = 0; - - do { - const itemVersions = await this.phpDriveService.listItemVersions( - versionPage, - item.id, - context, - ); - versionPage = itemVersions.nextPage as Pagination; - - for (const version of itemVersions.getEntities()) { - try { - const newVersion = getDefaultDriveItemVersion( - { - creator_id: version.creator_id || context.user.id, - data: version.data, - date_added: +version.date_added, - drive_item_id: newDriveItem.id, - file_size: version.file_size, - filename: version.filename, - key: version.key, - provider: version.provider, - realname: version.realname, - mode: version.mode, - }, - context, - ); - - const file = await this.phpDriveService.migrate( - version.file_id, - item.workspace_id, - version.id, - { - filename: version.filename, - userId: version.creator_id || context.user.id, - totalSize: version.file_size, - waitForThumbnail: true, - chunkNumber: 1, - totalChunks: 1, - type: mime, - }, - context, - ); - - if (!file) { - throw Error("cannot download file version"); - } - - newVersion.file_metadata = { - external_id: file.id, - mime: file.metadata.mime, - name: file.metadata.name || version.filename, - size: file.upload_data.size || version.file_size, - }; - - await globalResolver.services.documents.documents.createVersion( - newDriveItem.id, - newVersion, - context, - ); - - createdVersions++; - } catch (error) { - logger.error(`Failed to migrate version ${version.id} for drive item ${item.id}`); - console.error(`Failed to migrate version ${version.id} for drive item ${item.id}`); - } - } - } while (versionPage.page_token); - - if (createdVersions === 0) { - await this.nodeRepository.remove(newDriveItem); - return; - } - } - - if (!migrationRecord) { - await this.phpDriveService.markAsMigrated(item.id, newDriveItem.id, context.company.id); - } - } catch (error) { - logger.error( - `Failed to migrate Drive item ${item.id} / workspace ${item.workspace_id} / company_id: ${context.company.id}`, - error, - ); - console.error(`Failed to migrate Drive item ${item.id}`, error); - } - }; - - /** - * Fetches the first found company owner or admin identifier. - * - * @param {string} companyId - the companyId - * @param {ExecutionContext} context - the execution context - * @returns {Promise} - */ - private getCompanyOwnerOrAdminId = async ( - companyId: string, - context: ExecutionContext, - ): Promise => { - let pagination: Pagination = { limitStr: "100" }; - let companyOwnerOrAdminId = null; - - do { - const companyUsers = await globalResolver.services.companies.companyUserRepository.find( - { group_id: companyId }, - { pagination }, - context, - ); - - pagination = companyUsers.nextPage as Pagination; - - const companyAdminOrOwner = companyUsers - .getEntities() - .find(({ role }) => ["admin", "owner"].includes(role)); - - if (companyAdminOrOwner) { - companyOwnerOrAdminId = companyAdminOrOwner.id; - } - } while (pagination && !companyOwnerOrAdminId); - - return companyOwnerOrAdminId; - }; - - /** - * Compute the Access Information for the workspace folder to be created. - * - * @param {Workspace} workspace - the target workspace - * @param {Company} company - the target company - * @param {WorkspaceExecutionContext} context - the execution context - * @returns {Promise} - */ - private getWorkspaceAccess = async ( - workspace: Workspace, - company: Company, - context: WorkspaceExecutionContext, - ): Promise => { - const companyUsersCount = await globalResolver.services.companies.getUsersCount(company.id); - const workspaceUsersCount = await globalResolver.services.workspaces.getUsersCount( - workspace.id, - ); - - if (companyUsersCount === workspaceUsersCount) { - return { - entities: [ - { - id: "parent", - type: "folder", - level: "manage", - }, - { - id: company.id, - type: "company", - level: "none", - }, - { - id: context.user?.id, - type: "user", - level: "manage", - }, - ], - public: { - level: "none", - token: generateAccessToken(), - }, - }; - } - - let workspaceUsers: WorkspaceUser[] = []; - let wsUsersPagination: Pagination = { limitStr: "100" }; - - do { - const wsUsersQuery = await globalResolver.services.workspaces.getUsers( - { workspaceId: workspace.id }, - wsUsersPagination, - context, - ); - wsUsersPagination = wsUsersQuery.nextPage as Pagination; - - workspaceUsers = [...workspaceUsers, ...wsUsersQuery.getEntities()]; - } while (wsUsersPagination.page_token); - - if (companyUsersCount < 30 || workspaceUsersCount < 30) { - return { - entities: [ - { - id: "parent", - type: "folder", - level: "none", - }, - { - id: company.id, - type: "company", - level: "none", - }, - { - id: context.user?.id, - type: "user", - level: "manage", - }, - ...workspaceUsers.reduce((acc, curr) => { - acc = [ - ...acc, - { - id: curr.userId, - type: "user", - level: "manage", - }, - ]; - - return acc; - }, []), - ], - public: { - level: "none", - token: generateAccessToken(), - }, - }; - } - - let companyUsers: CompanyUser[] = []; - let companyUsersPaginations: Pagination = { limitStr: "100" }; - do { - const companyUsersQuery = await globalResolver.services.companies.getUsers( - { group_id: company.id }, - companyUsersPaginations, - {}, - context, - ); - companyUsersPaginations = companyUsersQuery.nextPage as Pagination; - companyUsers = [...companyUsers, ...companyUsersQuery.getEntities()]; - } while (companyUsersPaginations.page_token); - return { - entities: [ - { - id: "parent", - type: "folder", - level: "none", - }, - { - id: company.id, - type: "company", - level: "manage", - }, - { - id: context.user?.id, - type: "user", - level: "manage", - }, - ...companyUsers.reduce((acc, curr) => { - if (workspaceUsers.find(({ userId }) => curr.user_id === userId)) { - return acc; - } - - acc = [ - ...acc, - { - id: curr.user_id, - type: "user", - level: "none", - }, - ]; - - return acc; - }, []), - ], - public: { - level: "none", - token: generateAccessToken(), - }, - }; - }; - - /** - * Creates a folder for the workspace to migrate. - * - * @param {Workspace} workspace - the workspace to migrate. - * @param {AccessInformation} access - the access information. - * @param {WorkspaceExecutionContext} context - the execution context - * @returns {Promise} - */ - private createWorkspaceFolder = async ( - workspace: Workspace, - access: AccessInformation, - context: WorkspaceExecutionContext, - ): Promise => { - const workspaceFolder = getDefaultDriveItem( - { - name: workspace.name || workspace.id, - extension: "", - content_keywords: "", - creator: context.user.id, - is_directory: true, - is_in_trash: false, - description: "", - tags: [], - parent_id: "root", - company_id: context.company.id, - access_info: access, - }, - context, - ); - - await this.nodeRepository.save(workspaceFolder); - await this.phpDriveService.markAsMigrated(workspace.id, workspaceFolder.id, context.company.id); - - return workspaceFolder; - }; -} - -export default DriveMigrator; diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-file-entity.ts b/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-file-entity.ts deleted file mode 100644 index 05b32680..00000000 --- a/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-file-entity.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { Type } from "class-transformer"; -import { - Column, - Entity, -} from "../../../../core/platform/services/database/services/orm/decorators"; - -export const TYPE = "drive_file"; - -@Entity(TYPE, { - primaryKey: [["workspace_id"], "parent_id", "id"], - type: TYPE, -}) -export class PhpDriveFile { - @Type(() => String) - @Column("workspace_id", "timeuuid") - workspace_id: string; - - @Type(() => String) - @Column("parent_id", "string") - parent_id: string | ""; - - @Type(() => String) - @Column("id", "timeuuid", { generator: "timeuuid" }) - id: string; - - @Column("isintrash", "boolean") - isintrash: boolean; - - @Column("added", "string") - added: string; - - @Column("attachements", "encoded_json") - attachements: unknown; - - @Column("content_keywords", "encoded_json") - content_keywords: string[] | null; - - @Column("creator", "string") - creator: string; - - @Column("description", "string") - description: string; - - @Column("extension", "string") - extension: string; - - @Column("hidden_data", "encoded_json") - hidden_data: unknown; - - @Column("isdirectory", "boolean") - isdirectory: boolean; - - @Column("last_modified", "string") - last_modified: string; - - @Column("name", "encoded_string") - name: string; - - @Column("size", "number") - size: number; - - @Column("tags", "encoded_json") - tags: string[] | null; -} diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-file-version-entity.ts b/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-file-version-entity.ts deleted file mode 100644 index 69ecdd05..00000000 --- a/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-file-version-entity.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { Type } from "class-transformer"; -import { - Column, - Entity, -} from "../../../../core/platform/services/database/services/orm/decorators"; - -export const TYPE = "drive_file_version"; - -@Entity(TYPE, { - primaryKey: [["file_id"], "id"], - type: TYPE, -}) -export class PhpDriveFileVersion { - @Type(() => String) - @Column("id", "timeuuid", { generator: "timeuuid" }) - id: string; - - @Type(() => String) - @Column("file_id", "string") - file_id: string; - - @Type(() => String) - @Column("creator_id", "string") - creator_id: string; - - @Type(() => String) - @Column("realname", "encoded_string") - realname: string; - - @Type(() => String) - @Column("key", "string") - key: string; - - @Type(() => String) - @Column("mode", "string") - mode: string; - - @Type(() => String) - @Column("file_size", "number") - file_size: number; - - @Type(() => String) - @Column("date_added", "string") - date_added: string; - - @Type(() => String) - @Column("filename", "encoded_string") - filename: string; - - @Type(() => String) - @Column("provider", "string") - provider: string; - - @Column("data", "encoded_json") - data: unknown; -} - -export type PhpDriveFileVersionPrimaryKey = Pick; diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-migration-record-entity.ts b/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-migration-record-entity.ts deleted file mode 100644 index 0461d6b0..00000000 --- a/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-migration-record-entity.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { Type } from "class-transformer"; -import { - Column, - Entity, -} from "../../../../core/platform/services/database/services/orm/decorators"; - -export const TYPE = "php_drive_migration_record"; - -@Entity(TYPE, { - primaryKey: [["company_id"], "item_id"], - type: TYPE, -}) -export class phpDriveMigrationRecord { - @Type(() => String) - @Column("item_id", "timeuuid") - item_id: string; - - @Type(() => String) - @Column("company_id", "uuid") - company_id: string; - - @Type(() => String) - @Column("new_id", "string") - new_id: string; -} diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-service.ts b/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-service.ts deleted file mode 100644 index 9397a031..00000000 --- a/twake/backend/node/src/cli/cmds/migration_cmds/php-drive-file/php-drive-service.ts +++ /dev/null @@ -1,189 +0,0 @@ -import { - ExecutionContext, - ListResult, - Pagination, -} from "../../../../core/platform/framework/api/crud-service"; -import Repository from "../../../../core/platform/services/database/services/orm/repository/repository"; -import globalResolver from "../../../../services/global-resolver"; -import { PhpDriveFile, TYPE as DRIVE_FILE_TABLE } from "./php-drive-file-entity"; -import { Initializable, logger, TwakeServiceProvider } from "../../../../core/platform/framework"; -import { - PhpDriveFileVersion, - TYPE as DRIVE_FILE_VERSION_TABLE, -} from "./php-drive-file-version-entity"; -import axios from "axios"; -import { Multipart } from "fastify-multipart"; -import { CompanyExecutionContext } from "../../../../services/files/web/types"; -import { File } from "../../../../services/files/entities/file"; -import { UploadOptions } from "../../../../services/files/types"; -import { - phpDriveMigrationRecord, - TYPE as MIGRATION_RECORD_TABLE, -} from "./php-drive-migration-record-entity"; - -export interface MigrateOptions extends UploadOptions { - userId: string; -} - -export interface PhpDriveServiceAPI extends TwakeServiceProvider, Initializable {} - -export class PhpDriveFileService implements PhpDriveServiceAPI { - version: "1"; - public repository: Repository; - public versionRepository: Repository; - public migrationRepository: Repository; - - /** - * Init the service. - * - * @returns {PhpDriveFileService} - */ - async init(): Promise { - this.repository = await globalResolver.database.getRepository( - DRIVE_FILE_TABLE, - PhpDriveFile, - ); - - this.versionRepository = await globalResolver.database.getRepository( - DRIVE_FILE_VERSION_TABLE, - PhpDriveFileVersion, - ); - - this.migrationRepository = await globalResolver.database.getRepository( - MIGRATION_RECORD_TABLE, - phpDriveMigrationRecord, - ); - - return this; - } - - /** - * Lists the drive item directory children. - * - * @param {Pagination} pagination - the page. - * @param {string} directory - the drive item / directory id to search within. - * @param {string} workspaceId - the workspace id - * @param {ExecutionContext} context - the execution context. - * @returns {Promise>} - the drive item children. - */ - listDirectory = async ( - pagination: Pagination, - directory: string | "" | "trash", - workspaceId: string, - context?: ExecutionContext, - ): Promise> => - await this.repository.find( - { - workspace_id: workspaceId, - parent_id: directory, - }, - { pagination }, - context, - ); - - /** - * Lists the versions of a drive item. - * - * @param {Pagination} pagination - the page. - * @param {string} itemId - the drive item id. - * @param {ExecutionContext} context - the execution context. - * @returns {Promise>} - the list of the item versions. - */ - listItemVersions = async ( - pagination: Pagination, - itemId: string, - context?: ExecutionContext, - ): Promise> => - await this.versionRepository.find( - { - file_id: itemId, - }, - { pagination }, - context, - ); - - /** - * Downloads a file version from the old drive and uploads it to the new Drive. - * - * @param {string} fileId - the old file id - * @param {string} workspaceId - the workspace id - * @param {string} versionId - the version id - * @param {MigrateOptions} options - the file upload / migration options. - * @param {CompanyExecutionContext} context - the company execution context. - * @param {string} public_access_key - the file public access key. - * @returns {Promise} - the uploaded file information. - */ - migrate = async ( - fileId: string, - workspaceId: string, - versionId: string, - options: MigrateOptions, - context: CompanyExecutionContext, - public_access_key?: string, - ): Promise => { - try { - const url = `https://web.twake.app/ajax/drive/download?workspace_id=${workspaceId}&element_id=${fileId}&version_id=${versionId}&download=1${ - public_access_key ? `&public_access_key=${public_access_key}` : "" - }`; - - const response = await axios.get(url, { - responseType: "stream", - }); - - if (!response.data) { - throw Error("invalid download response"); - } - - const file = { - file: response.data, - }; - - return await globalResolver.services.files.save(null, file as Multipart, options, { - ...context, - user: { - id: options.userId, - }, - }); - } catch (error) { - logger.error(`Failed to migrate file ${fileId} on workspace ${workspaceId}`, error); - throw Error(error); - } - }; - - /** - * Saves a drive item. - * - * @param {PhpDriveFile} item - the php drive item. - * @returns {Promise} - */ - save = async (item: PhpDriveFile): Promise => await this.repository.save(item); - - /** - * Marks a drive item as migrated. - * - * @param {string} itemId - the drive item. - * @param {string} newId - the new drive item id. - * @param {string} companyId - the company id. - */ - markAsMigrated = async (itemId: string, newId: string, companyId: string): Promise => { - const migrationRecord = new phpDriveMigrationRecord(); - migrationRecord.item_id = itemId; - migrationRecord.new_id = newId; - migrationRecord.company_id = companyId; - - await this.migrationRepository.save(migrationRecord); - }; - - /** - * Fetches the drive item migration record. - * - * @param {string} itemId - the drive item id. - * @param {string} companyId - the company id. - * @returns {Promise} - */ - getMigrationRecord = async ( - itemId: string, - companyId: string, - ): Promise => - await this.migrationRepository.findOne({ item_id: itemId, company_id: companyId }); -} diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/php-message/message-migrator-service.ts b/twake/backend/node/src/cli/cmds/migration_cmds/php-message/message-migrator-service.ts deleted file mode 100644 index fa4199a0..00000000 --- a/twake/backend/node/src/cli/cmds/migration_cmds/php-message/message-migrator-service.ts +++ /dev/null @@ -1,560 +0,0 @@ -import { PhpMessagesService } from "./php-message-service"; -import { convertUuidV4ToV1 } from "./utils"; -import Company from "../../../../services/user/entities/company"; -import { Pagination } from "../../../../core/platform/framework/api/crud-service"; -import { PhpMessage } from "./php-message-entity"; -import { TwakePlatform } from "../../../../core/platform/platform"; -import { - Message, - MessageEdited, - MessageOverride, - MessagePinnedInfo, - MessageReaction, -} from "../../../../services/messages/entities/messages"; -import { MessageChannelRef } from "../../../../services/messages/entities/message-channel-refs"; -import { ParticipantObject, Thread } from "../../../../services/messages/entities/threads"; -import { Block } from "../../../../services/messages/blocks-types"; -import { WorkspaceExecutionContext } from "../../../../services/workspaces/types"; -import gr from "../../../../services/global-resolver"; - -type MigratedChannel = { - id: string; - workspace_id: string; - company_id: string; - owner?: string; -}; - -type Options = { - from?: string; - onlyCompany?: string; - onlyWorkspace?: string; - onlyChannel?: string; - ignoreExisting?: boolean; - backToPhp?: boolean; - dryRun?: boolean; -}; -class MessageMigrator { - private phpMessageService: PhpMessagesService; - private migratedMessages = 0; - private options: Options = {}; - - constructor(readonly _platform: TwakePlatform) { - this.phpMessageService = new PhpMessagesService(); - } - - public async run(options: Options = {}): Promise { - await gr.doInit(this._platform); - this.options = options; - - await this.phpMessageService.init(); - - if (this.options.onlyCompany) { - const company = await gr.services.companies.getCompany({ id: options.onlyCompany }); - await this.migrateCompanyMessages(company); - } else { - let waitForCompany = false; - if (this.options.from) { - waitForCompany = true; - } - - // Get all companies - let page: Pagination = { limitStr: "100" }; - // For each companies find workspaces - do { - const companyListResult = await gr.services.companies.getCompanies(page); - page = companyListResult.nextPage as Pagination; - - for (const company of companyListResult.getEntities()) { - if (waitForCompany && this.options.from == `${company.id}`) { - waitForCompany = false; - } - - if (!waitForCompany) { - await this.migrateCompanyMessages(company); - } - } - } while (page.page_token); - } - - console.log( - `Php Messages successfully migrated to node: (${this.migratedMessages} messages) !`, - ); - } - - private async migrateCompanyMessages(company: Company) { - console.log(`Start migration for ${company.id}...`); - - if (!this.options.onlyWorkspace || this.options.onlyWorkspace === "direct") { - await this.migrateCompanyDirectMessages(company); - console.log( - `${company.id} - (1/2) migrated direct messages (total: ${this.migratedMessages} messages) ✅`, - ); - } - - if (!this.options.onlyWorkspace || this.options.onlyWorkspace !== "direct") { - await this.migrateCompanyChannelsMessages(company); - console.log(`${company.id} - (2/2) completed (total: ${this.migratedMessages} messages) ✅`); - } - } - - /** - * Set all direct messages in company and set them to channelPhpMessages - */ - private async migrateCompanyDirectMessages(company: Company) { - await gr.doInit(this._platform); - let pageDirectChannels: Pagination = { limitStr: "100" }; - // For each directChannels find messages - do { - const directChannelsInCompanyResult = await gr.services.channels.channels.list( - pageDirectChannels, - {}, - { - workspace: { - workspace_id: "direct", - company_id: company.id, - }, - user: { id: null, server_request: true }, - }, - ); - - pageDirectChannels = directChannelsInCompanyResult.nextPage as Pagination; - - for (const directChannel of directChannelsInCompanyResult.getEntities()) { - await this.migrateChannelsMessages(company, { - id: directChannel.id, - workspace_id: "direct", - company_id: directChannel.company_id, - }); - } - } while (pageDirectChannels.page_token); - } - - /** - * Set all messages in company and set them to channelPhpMessages - */ - private async migrateCompanyChannelsMessages(company: Company) { - await gr.doInit(this._platform); - - // Get all workspaces in company - const workspacesInCompany = ( - await gr.services.workspaces.list({ limitStr: "" }, {}, { - user: { - id: null, - server_request: true, - }, - company_id: company.id, - } as WorkspaceExecutionContext) - ).getEntities(); - - // For each workspaces find channels - for (const workspace of workspacesInCompany) { - if (this.options.onlyWorkspace && `${workspace.id}` !== this.options.onlyWorkspace) { - continue; - } - - // Get all channels in workspace - let pageChannels: Pagination = { limitStr: "1" }; - do { - const channelsInWorkspace = await gr.services.channels.channels.list( - pageChannels, - {}, - { - workspace: { - workspace_id: workspace.id, - company_id: workspace.company_id, - }, - user: { id: null, server_request: true }, - }, - ); - pageChannels = channelsInWorkspace.nextPage as Pagination; - - // For each channels find messages - for (const channel of channelsInWorkspace.getEntities()) { - await this.migrateChannelsMessages(company, { - id: channel.id, - workspace_id: channel.workspace_id, - company_id: channel.company_id, - owner: channel.owner, - }); - } - } while (pageChannels.page_token); - } - } - - //Params: company, channel - private async migrateChannelsMessages(company: Company, channel: MigratedChannel) { - if (this.options.onlyChannel && `${channel.id}` !== this.options.onlyChannel) { - return; - } - - if (this.options.backToPhp) { - await this.migrateChannelsMessagesBackToPhp(company, channel); - return; - } - - //This function will migrate all messages in a channel - let pagePhpMessages: Pagination = { limitStr: "100" }; - do { - const messages = await this.phpMessageService.list( - pagePhpMessages, - {}, - { - channel_id: convertUuidV4ToV1(channel.id), - user: { id: null, server_request: true }, - }, - ); - - for (const message of messages.getEntities()) { - try { - await this.migrateMessage(company, channel, message); - } catch (err) { - console.log(err); - } - } - - pagePhpMessages = messages.nextPage as Pagination; - } while (pagePhpMessages.page_token); - } - - private migratedThreads: string[] = []; - - /** - * Migrate php message to node - * @param company - * @param channel - * @param message - */ - private async migrateMessage(company: Company, channel: MigratedChannel, message: PhpMessage) { - if (!message.id) { - return; - } - await gr.doInit(this._platform); - //Create thread first if not exists - const threadId = message.parent_message_id || message.id; - - if (this.options.ignoreExisting) { - const msg = await gr.services.messages.messages.get({ - thread_id: threadId, - id: message.id, - }); - if (msg) { - return; - } - } - - const threadDoesNotExists = !this.migratedThreads.includes(threadId); - if (threadDoesNotExists) { - await this.migratePhpMessageToNodeThread(message, channel, company); - this.migratedThreads.push(threadId); - } - - //Migrate message itself - await this.migratePhpMessageToNodeMessage(threadId, message, company); - - this.migratedMessages++; - - if (this.migratedMessages % 100 == 0) { - console.log(`${company.id} - ... (total: ${this.migratedMessages} messages)`); - } - - //Force delay between channels - await new Promise(r => { - setTimeout(r, 40); - }); - } - - private async migrateChannelsMessagesBackToPhp(company: Company, channel: MigratedChannel) { - await gr.doInit(this._platform); - const channelRefRepository = await gr.database.getRepository( - "message_channel_refs", - MessageChannelRef, - ); - const messageRepository = await gr.database.getRepository("messages", Message); - - //This function will migrate all messages in a channel - let pageMessages: Pagination = { limitStr: "100" }; - do { - const messages = await channelRefRepository.find( - { - company_id: company.id, - workspace_id: channel.workspace_id, - channel_id: channel.id, - }, - { pagination: pageMessages }, - undefined, - ); - - for (const messageRef of messages.getEntities()) { - const messages = await messageRepository.find( - { - thread_id: messageRef.thread_id, - }, - {}, - undefined, - ); - - for (const message of messages.getEntities()) { - const uuidv1_channel_id = - channel.id.substring(0, 14) + "1" + channel.id.substring(14 + 1); - - let phpMessage = await this.phpMessageService.get({ - id: message.id, - }); - - if (!phpMessage) { - phpMessage = await this.phpMessageService.get({ - channel_id: uuidv1_channel_id, - parent_message_id: message.thread_id, - id: message.id, - }); - } - - if (!phpMessage && message.subtype !== "deleted") { - //This message doesn't exists in php, move it to php - - const newPhpMessage = new PhpMessage(); - newPhpMessage.id = message.id; - newPhpMessage.channel_id = uuidv1_channel_id; - newPhpMessage.parent_message_id = message.thread_id; - newPhpMessage.application_id = message.application_id || null; - newPhpMessage.modification_date = Math.floor( - message.edited?.edited_at || message.created_at, - ); - newPhpMessage.creation_date = Math.floor(message.created_at); - newPhpMessage.sender = message.user_id; - newPhpMessage.pinned = !!message.pinned_info?.pinned_at; - newPhpMessage.edited = !!message.edited?.edited_at; - newPhpMessage.message_type = - message.subtype === "application" ? 1 : message.subtype === "system" ? 2 : 0; - newPhpMessage.hidden_data = message.context; - newPhpMessage.reactions = "{}"; - newPhpMessage.responses_count = 0; - - let prepared: any[] = []; - (message.blocks || []).map(block => { - if (block.type === "twacode") { - prepared = block["elements"]; - } - if (block.type === "section" && prepared.length > 0) { - prepared = [ - { - type: "twacode", - content: block?.text?.text || "", - }, - ]; - } - }); - newPhpMessage.content = { - fallback_string: message.text, - original_str: message.text, - files: (message.files || []).map(f => { - return { - content: f, - mode: "mini", - type: "file", - }; - }), - prepared: prepared, - }; - - if (!this.options.dryRun) { - await this.phpMessageService.repository.save(newPhpMessage, undefined); - } - - this.migratedMessages++; - if (this.migratedMessages % 100 == 0) { - console.log(`${company.id} - ... (total: ${this.migratedMessages} messages)`); - } - } - } - } - - pageMessages = messages.nextPage as Pagination; - } while (pageMessages.page_token); - } - - /** - * Migrate php message to node thread - * @param message PhpMessage - * @param channel MigratedChannel - */ - private async migratePhpMessageToNodeThread( - message: PhpMessage, - channel: MigratedChannel, - company: Company, - ) { - const thread = new Thread(); - - // Set nodeThread values - thread.id = message.parent_message_id || message.id; - thread.created_at = message.creation_date; - thread.last_activity = message.modification_date; - thread.answers = 0; - thread.participants = [ - { - type: "channel", - id: channel.id, - company_id: channel.company_id, - workspace_id: channel.workspace_id, - created_at: message.creation_date, - created_by: message.sender, - } as ParticipantObject, - ]; - - if (this.options.dryRun) { - return; - } - - // Create nodeThread - return await gr.services.messages.threads.save( - thread, - {}, - { user: { id: null, server_request: true }, company }, - ); - } - - /** - * Set message string type - * @param integer 0 = null | 1 = system | 2 = application - */ - private setMessageType(integer: number): { type: Message["type"]; subtype: Message["subtype"] } { - switch (integer) { - case 1: - return { - type: "message", - subtype: "application", - }; - case 2: - return { - type: "message", - subtype: "system", - }; - case 0: - default: - return { - type: "message", - subtype: null, - }; - } - } - - /** - * Set blocks array - * @param content { [key: string]: any } - */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private setBlocks(content: { [key: string]: any }): Block[] { - const blocks: Block[] = []; - - if (!content) return blocks; - - const new_block_format: Block = { - type: "twacode", - elements: content.formatted || content.prepared || content || [], - }; - - blocks.push(new_block_format); - - return blocks; - } - - /** - * Set edited message Object - * @param modification_date timestamp - */ - private setMessageEditedObject(message: PhpMessage): MessageEdited { - return message.edited ? { edited_at: message.modification_date } : null; - } - - /** - * Set pinned message Object - * @param message PhpMessage - */ - private setMessagePinnedObject(message: PhpMessage): MessagePinnedInfo { - if (message.pinned) { - return { pinned_at: 0, pinned_by: message.sender }; - } - return null; - } - - /** - * Set reactions message object - * @param reactions JSON - */ - private setMessageReactionsObject(reactions: string): MessageReaction[] { - const parsed_reactions = JSON.parse(reactions); - - if (!parsed_reactions) return []; - - const new_reactions_array: MessageReaction[] = []; - - for (const reaction_name in parsed_reactions) { - const new_reaction_object: MessageReaction = { - name: reaction_name, - users: parsed_reactions[reaction_name].users, - count: parsed_reactions[reaction_name].count, - }; - - new_reactions_array.push(new_reaction_object); - } - - return new_reactions_array; - } - - /** - * Set override message Object - * @param message PhpMessage - */ - private setMessageOverrideObject(_message: PhpMessage): MessageOverride { - return null; - } - - /** - * Migrate php message to node message - * @param message PhpMessage - */ - private async migratePhpMessageToNodeMessage( - threadId: string, - message: PhpMessage, - company: Company, - ) { - const nodeMessage = new Message(); - - // Set nodeMessage values - nodeMessage.id = message.id; - nodeMessage.thread_id = threadId; - nodeMessage.type = this.setMessageType(message.message_type).type; - nodeMessage.subtype = this.setMessageType(message.message_type).subtype; - nodeMessage.created_at = message.creation_date; - nodeMessage.user_id = message.sender; - nodeMessage.application_id = message.application_id; - nodeMessage.text = message.content?.original_str || ""; - nodeMessage.blocks = this.setBlocks(message.content); - nodeMessage.context = message.hidden_data || {}; - nodeMessage.edited = this.setMessageEditedObject(message); - nodeMessage.pinned_info = this.setMessagePinnedObject(message); - nodeMessage.reactions = this.setMessageReactionsObject(message.reactions); - nodeMessage.override = this.setMessageOverrideObject(message); - - nodeMessage.context._front_id = message.id; - - if (this.options.dryRun) { - return; - } - - // Create nodeMessage then add it to thread - return await gr.services.messages.messages.save( - nodeMessage, - { - enforceViewPropagation: true, - }, - { - user: { id: null, server_request: true }, - thread: { id: threadId }, - company: { id: company.id }, - }, - ); - } -} - -export default MessageMigrator; diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/php-message/php-message-entity.ts b/twake/backend/node/src/cli/cmds/migration_cmds/php-message/php-message-entity.ts deleted file mode 100644 index 0623db4b..00000000 --- a/twake/backend/node/src/cli/cmds/migration_cmds/php-message/php-message-entity.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { Type } from "class-transformer"; -import { merge } from "lodash"; -import { - Column, - Entity, -} from "../../../../core/platform/services/database/services/orm/decorators"; - -export const TYPE = "message"; -@Entity(TYPE, { - primaryKey: [["channel_id"], "parent_message_id", "id"], - type: TYPE, -}) -export class PhpMessage { - @Type(() => String) - @Column("id", "timeuuid", { generator: "timeuuid" }) - id: string; - - @Type(() => String) - @Column("channel_id", "timeuuid") - channel_id: string; - - @Type(() => String) - @Column("parent_message_id", "encoded_string") - parent_message_id: string; - - @Type(() => String) - @Column("application_id", "encoded_string") - application_id: string | null; - - @Column("modification_date", "twake_datetime") - modification_date: number | null; - - @Column("creation_date", "twake_datetime") - creation_date: number; - - @Column("message_type", "number") - message_type: number; - - @Column("sender_id", "timeuuid") - sender: string; - - @Column("content", "encoded_json") - // eslint-disable-next-line @typescript-eslint/no-explicit-any - content: { [key: string]: any } | null; - - @Column("hidden_data", "encoded_json") - // eslint-disable-next-line @typescript-eslint/no-explicit-any - hidden_data: { [key: string]: any } | null; - - @Column("reactions", "encoded_string") - reactions: string | null; - - @Column("pinned", "twake_boolean") - pinned: boolean; - - @Column("edited", "twake_boolean") - edited: boolean; - - @Column("responses_count", "twake_int") - responses_count: number | null; -} - -export type PhpMessagePrimaryKey = Pick; - -export function getInstance(message: PhpMessage): PhpMessage { - return merge(new PhpMessage(), message); -} diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/php-message/php-message-service.ts b/twake/backend/node/src/cli/cmds/migration_cmds/php-message/php-message-service.ts deleted file mode 100644 index 5995a24f..00000000 --- a/twake/backend/node/src/cli/cmds/migration_cmds/php-message/php-message-service.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { - DeleteResult, - ExecutionContext, - ListResult, - Pagination, -} from "../../../../core/platform/framework/api/crud-service"; -import Repository, { - FindFilter, -} from "../../../../core/platform/services/database/services/orm/repository/repository"; -import { PhpMessagesServiceAPI } from "./types"; -import { PhpMessage, PhpMessagePrimaryKey } from "./php-message-entity"; -import gr from "../../../../services/global-resolver"; - -export interface PhpMessageExecutionContext extends ExecutionContext { - channel_id: string; - parent_message_id?: string; - id?: string; -} - -export class PhpMessagesService implements PhpMessagesServiceAPI { - version: "1"; - public repository: Repository; - - async init(): Promise { - this.repository = await gr.database.getRepository("message", PhpMessage); - return this; - } - - get(pk: { parent_message_id?: string; channel_id?: string; id: string }): Promise { - if (pk.channel_id) { - pk.channel_id = `${pk.channel_id}`; - pk.channel_id.substring(0, 14) + "1" + pk.channel_id.substring(14 + 1); - } - return this.repository.findOne(pk, {}, undefined); - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - async delete(pk: PhpMessagePrimaryKey): Promise> { - throw Error("not implemented"); - } - - async list( - // eslint-disable-next-line @typescript-eslint/no-unused-vars - pagination: Pagination, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - options?: ListOptions, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - context?: PhpMessageExecutionContext, - ): Promise> { - const findFilter: FindFilter = { - channel_id: context.channel_id, - parent_message_id: context.parent_message_id, - //id: context.id, - }; - - const list = await this.repository.find(findFilter, { pagination }, undefined); - return list; - } -} diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/php-message/types.ts b/twake/backend/node/src/cli/cmds/migration_cmds/php-message/types.ts deleted file mode 100644 index 067e7abc..00000000 --- a/twake/backend/node/src/cli/cmds/migration_cmds/php-message/types.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { Initializable, TwakeServiceProvider } from "../../../../core/platform/framework"; - -export interface PhpMessagesServiceAPI extends TwakeServiceProvider, Initializable {} diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/php-message/utils.ts b/twake/backend/node/src/cli/cmds/migration_cmds/php-message/utils.ts deleted file mode 100644 index 61334010..00000000 --- a/twake/backend/node/src/cli/cmds/migration_cmds/php-message/utils.ts +++ /dev/null @@ -1,11 +0,0 @@ -export const convertUuidV4ToV1 = (str: string): string => { - const uuid = [...str]; - uuid[14] = "1"; - return uuid.join(""); -}; - -export const convertUuidV1ToV4 = (str: string): string => { - const uuid = [...str]; - uuid[14] = "4"; - return uuid.join(""); -}; diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/php-workspace/php-workspace-entity.ts b/twake/backend/node/src/cli/cmds/migration_cmds/php-workspace/php-workspace-entity.ts deleted file mode 100644 index 9ad04b4f..00000000 --- a/twake/backend/node/src/cli/cmds/migration_cmds/php-workspace/php-workspace-entity.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { merge } from "lodash"; -import { - Column, - Entity, -} from "../../../../core/platform/services/database/services/orm/decorators"; - -export const TYPE = "workspace"; -@Entity(TYPE, { - primaryKey: [["id"]], - type: TYPE, -}) -export class PhpWorkspace { - @Column("id", "timeuuid") - id: string; - - @Column("group_id", "timeuuid") - group_id: string; - - @Column("name", "encoded_string") - name: string; - - @Column("logo", "encoded_string") - logo: string; - - @Column("stats", "encoded_string") - stats: string; - - @Column("is_deleted", "boolean") - isDeleted: boolean; - - @Column("is_archived", "boolean") - isArchived: boolean; - - @Column("is_default", "boolean") - isDefault: boolean; - - @Column("date_added", "number") - dateAdded: number; -} - -export type PhpMessagePrimaryKey = Pick; - -export function getInstance(workspace: PhpWorkspace): PhpWorkspace { - return merge(new PhpWorkspace(), workspace); -} diff --git a/twake/backend/node/src/cli/cmds/migration_cmds/workspace.ts b/twake/backend/node/src/cli/cmds/migration_cmds/workspace.ts deleted file mode 100644 index 96a17ef7..00000000 --- a/twake/backend/node/src/cli/cmds/migration_cmds/workspace.ts +++ /dev/null @@ -1,148 +0,0 @@ -import yargs from "yargs"; -import twake from "../../../twake"; -import ora from "ora"; -import { TwakePlatform } from "../../../core/platform/platform"; -import { DatabaseServiceAPI } from "../../../core/platform/services/database/api"; -import { PhpWorkspace, TYPE as phpTYPE } from "./php-workspace/php-workspace-entity"; -import { Pagination } from "../../../core/platform/framework/api/crud-service"; -import Workspace, { TYPE, getInstance } from "../../../services/workspaces/entities/workspace"; -import _ from "lodash"; -import gr from "../../../services/global-resolver"; - -type Options = { - from?: string; - onlyCompany?: string; - onlyWorkspace?: string; - replaceExisting?: boolean; -}; - -class WorkspaceMigrator { - database: DatabaseServiceAPI; - - constructor(readonly platform: TwakePlatform) { - this.database = this.platform.getProvider("database"); - } - - public async run(options: Options = {}): Promise { - const phpRepository = await this.database.getRepository(phpTYPE, PhpWorkspace); - const repository = await this.database.getRepository(TYPE, Workspace); - - let waitForCompany = false; - if (options.from) { - waitForCompany = true; - } - - // Get all companies - let page: Pagination = { limitStr: "100" }; - // For each companies find workspaces - do { - const workspaceListResult = await phpRepository.find({}, { pagination: page }, undefined); - page = workspaceListResult.nextPage as Pagination; - - for (const workspace of workspaceListResult.getEntities()) { - if (waitForCompany && options.from == `${workspace.group_id}`) { - waitForCompany = false; - } - - if (!waitForCompany) { - if ( - (!options.onlyCompany && !options.onlyWorkspace) || - options.onlyCompany == `${workspace.group_id}` - ) { - if ( - !(await repository.findOne( - { company_id: workspace.group_id, id: workspace.id }, - {}, - undefined, - )) || - options.replaceExisting - ) { - const newWorkspace = getInstance( - _.pick( - workspace, - "id", - "company_id", - "name", - "logo", - "stats", - "is_deleted", - "is_archived", - "is_default", - "date_added", - ), - ); - newWorkspace.company_id = workspace.group_id; - await repository.save(newWorkspace, undefined); - } - } - } - } - } while (page.page_token); - } -} - -const services = [ - "storage", - "counter", - "platform-services", - "applications", - "user", - "search", - "channels", - "database", - "webserver", - "message-queue", - "workspaces", - "console", - "auth", -]; - -const command: yargs.CommandModule = { - command: "workspace", - describe: "command that allow you to migrate php workspaces to node", - builder: { - from: { - default: null, - type: "string", - description: "Start migration from this workspace ID", - }, - onlyCompany: { - default: null, - type: "string", - description: "Migrate only this workspace ID", - }, - onlyWorkspace: { - default: null, - type: "string", - description: "Migrate only this workspace ID", - }, - replaceExisting: { - default: false, - type: "boolean", - description: "Replace already migrated workspaces", - }, - }, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - handler: async argv => { - const spinner = ora({ text: "Migrating php worskpaces - " }).start(); - const platform = await twake.run(services); - await gr.doInit(platform); - const migrator = new WorkspaceMigrator(platform); - - const from = argv.from as string | null; - const onlyCompany = argv.onlyCompany as string | null; - const onlyWorkspace = argv.onlyWorkspace as string | null; - const replaceExisting = (argv.replaceExisting || false) as boolean; - - await migrator.run({ - from, - onlyCompany, - onlyWorkspace, - replaceExisting, - }); - - return spinner.stop(); - }, -}; - -export default command; diff --git a/twake/backend/node/src/cli/cmds/search_cmds/index_all.ts b/twake/backend/node/src/cli/cmds/search_cmds/index_all.ts index a6872830..54ee745c 100644 --- a/twake/backend/node/src/cli/cmds/search_cmds/index_all.ts +++ b/twake/backend/node/src/cli/cmds/search_cmds/index_all.ts @@ -7,18 +7,10 @@ import { Pagination } from "../../../core/platform/framework/api/crud-service"; import User, { TYPE as UserTYPE } from "../../../services/user/entities/user"; import { Channel } from "../../../services/channels/entities"; -import Application, { - TYPE as ApplicationTYPE, -} from "../../../services/applications/entities/application"; import Repository from "../../../core/platform/services/database/services/orm/repository/repository"; import { SearchServiceAPI } from "../../../core/platform/services/search/api"; import CompanyUser, { TYPE as CompanyUserTYPE } from "../../../services/user/entities/company_user"; -import { Message, TYPE as MessageTYPE } from "../../../services/messages/entities/messages"; import gr from "../../../services/global-resolver"; -import { - MessageFile, - TYPE as MessageFileTYPE, -} from "../../../services/messages/entities/message-files"; type Options = { repository?: string; @@ -36,17 +28,8 @@ class SearchIndexAll { public async run(options: Options = {}): Promise { const repositories: Map> = new Map(); - repositories.set("messages", await this.database.getRepository(MessageTYPE, Message)); - repositories.set( - "message_files", - await this.database.getRepository(MessageFileTYPE, MessageFile), - ); repositories.set("users", await this.database.getRepository(UserTYPE, User)); repositories.set("channels", await this.database.getRepository("channels", Channel)); - repositories.set( - "applications", - await this.database.getRepository(ApplicationTYPE, Application), - ); const repository = repositories.get(options.repository); if (!repository) { diff --git a/twake/backend/node/src/core/platform/services/email-pusher/types.ts b/twake/backend/node/src/core/platform/services/email-pusher/types.ts index 16765ac1..af17e49e 100644 --- a/twake/backend/node/src/core/platform/services/email-pusher/types.ts +++ b/twake/backend/node/src/core/platform/services/email-pusher/types.ts @@ -1,5 +1,4 @@ import { Channel } from "../../../../services/channels/entities/channel"; -import { Message } from "../../../../services/messages/entities/messages"; import Company from "../../../../services/user/entities/company"; import Workspace from "../../../../services/workspaces/entities/workspace"; import User from "../../../../services/user/entities/user"; @@ -11,7 +10,6 @@ export type EmailBuilderDataPayload = { notifications: { channel: Channel; workspace: Workspace; - message: Message & { user: UserObject }; }[]; }; diff --git a/twake/backend/node/src/core/platform/services/knowledge-graph/api-client.ts b/twake/backend/node/src/core/platform/services/knowledge-graph/api-client.ts index 4da68de8..f958655c 100644 --- a/twake/backend/node/src/core/platform/services/knowledge-graph/api-client.ts +++ b/twake/backend/node/src/core/platform/services/knowledge-graph/api-client.ts @@ -5,7 +5,6 @@ import { KnowledgeGraphCreateBodyRequest, KnowledgeGraphCreateMessageObjectData import { md5 } from "../../../../core/crypto"; import { Channel } from "../../../../services/channels/entities"; import gr from "../../../../services/global-resolver"; -import { Message } from "../../../../services/messages/entities/messages"; import Company from "../../../../services/user/entities/company"; import User from "../../../../services/user/entities/user"; import Workspace from "../../../../services/workspaces/entities/workspace"; @@ -139,41 +138,6 @@ export default class KnowledgeGraphAPIClient { } } - public async onMessageUpsert( - channelId: string, - message: Partial, - sensitiveData: boolean, - ): Promise { - const response = await this.send({ - records: [ - { - key: "null", - value: { - id: "Message", - properties: { - _kg_user_id: await this.getUserKGId(message.user_id), - _kg_email_id: await this.getUserKGMailId(message.user_id), - _kg_company_id: await this.getCompanyKGId(message.cache?.company_id), - message_thread_id: message.thread_id, - message_content: sensitiveData ? message.text : "", - type_message: message.type, - message_created_at: message.created_at, - message_updated_at: message.updated_at, - user_id: message.user_id, - channel_id: message.cache?.channel_id || channelId, - workspace_id: message.cache?.workspace_id, - company_id: message.cache?.company_id, - }, - }, - }, - ], - }); - - if (response.statusText === "OK") { - this.logger.info("onMessageUpsert %o", response.config.data); - } - } - private async send(data: any) { return await this.axiosInstance.post< KnowledgeGraphCreateBodyRequest diff --git a/twake/backend/node/src/core/platform/services/knowledge-graph/index.ts b/twake/backend/node/src/core/platform/services/knowledge-graph/index.ts index 4e4af125..154a7611 100644 --- a/twake/backend/node/src/core/platform/services/knowledge-graph/index.ts +++ b/twake/backend/node/src/core/platform/services/knowledge-graph/index.ts @@ -5,7 +5,6 @@ import Workspace from "../../../../services/workspaces/entities/workspace"; import Company from "../../../../services/user/entities/company"; import User from "../../../../services/user/entities/user"; import { Channel } from "../../../../services/channels/entities"; -import { Message } from "../../../../services/messages/entities/messages"; import { KnowledgeGraphGenericEventPayload, KnowledgeGraphEvents, @@ -48,11 +47,6 @@ export default class KnowledgeGraphService this.onChannelCreated.bind(this), ); - localEventBus.subscribe>( - KnowledgeGraphEvents.MESSAGE_UPSERT, - this.onMessageUpsert.bind(this), - ); - localEventBus.subscribe>( KnowledgeGraphEvents.USER_UPSERT, this.onUserCreated.bind(this), @@ -94,23 +88,6 @@ export default class KnowledgeGraphService } } - async onMessageUpsert(data: KnowledgeGraphGenericEventPayload): Promise { - this.logger.debug(`${KnowledgeGraphEvents.MESSAGE_UPSERT} %o`, data); - - const allowedToShare = await this.shouldForwardEvent( - [data.resource.cache.company_id], - data.resource.user_id, - ); - - if (this.kgAPIClient && allowedToShare) { - this.kgAPIClient.onMessageUpsert( - data.resource.cache.company_id, - data.resource, - allowedToShare === "all", - ); - } - } - async onUserCreated(data: KnowledgeGraphGenericEventPayload): Promise { this.logger.info(`${KnowledgeGraphEvents.USER_UPSERT} %o`, data); diff --git a/twake/backend/node/src/core/platform/services/knowledge-graph/provider.ts b/twake/backend/node/src/core/platform/services/knowledge-graph/provider.ts index fc5a1f59..46eecd0c 100644 --- a/twake/backend/node/src/core/platform/services/knowledge-graph/provider.ts +++ b/twake/backend/node/src/core/platform/services/knowledge-graph/provider.ts @@ -3,13 +3,11 @@ import { Channel } from "../../../../services/channels/entities"; import { TwakeServiceProvider } from "../../framework"; import { KnowledgeGraphGenericEventPayload } from "./types"; import Workspace from "../../../../services/workspaces/entities/workspace"; -import { Message } from "../../../../services/messages/entities/messages"; import User from "../../../../services/user/entities/user"; export default interface KnowledgeGraphAPI extends TwakeServiceProvider { onCompanyCreated(data: KnowledgeGraphGenericEventPayload): void; onWorkspaceCreated(data: KnowledgeGraphGenericEventPayload): void; onChannelCreated(data: KnowledgeGraphGenericEventPayload): void; - onMessageUpsert(data: KnowledgeGraphGenericEventPayload): void; onUserCreated(data: KnowledgeGraphGenericEventPayload): void; } diff --git a/twake/backend/node/src/core/platform/services/tracker/index.ts b/twake/backend/node/src/core/platform/services/tracker/index.ts index c89c71d9..ea8852cf 100644 --- a/twake/backend/node/src/core/platform/services/tracker/index.ts +++ b/twake/backend/node/src/core/platform/services/tracker/index.ts @@ -43,23 +43,6 @@ export default class Tracker extends TwakeService implements Tracker ); }); - const messageSentEvent = "channel:message_sent"; - localEventBus.subscribe(messageSentEvent, data => { - logger.debug(`Tracker - New ${messageSentEvent} event`); - this.track( - { - user: data.user, - event: messageSentEvent, - properties: { - is_direct: data.message.workspace_id === "direct" ? true : false, - is_thread_reply: data.message.thread_id ? true : false, - }, - }, - (err: Error) => - err ? logger.error({ err }, "Tracker - Error while tracking", messageSentEvent) : false, - ); - }); - const channelCreatedEvent = "channel:created"; localEventBus.subscribe(channelCreatedEvent, data => { logger.debug(`Tracker - New ${channelCreatedEvent} event`); diff --git a/twake/backend/node/src/services/applications-api/index.ts b/twake/backend/node/src/services/applications-api/index.ts deleted file mode 100644 index 5343ad5a..00000000 --- a/twake/backend/node/src/services/applications-api/index.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { Prefix, TwakeService } from "../../core/platform/framework"; -import WebServerAPI from "../../core/platform/services/webserver/provider"; -import web from "./web/index"; -import FastProxy from "fast-proxy"; - -@Prefix("/api") -export default class ApplicationsApiService extends TwakeService { - version = "1"; - name = "applicationsapi"; - - public async doInit(): Promise { - const fastify = this.context.getProvider("webserver").getServer(); - fastify.register((instance, _opts, next) => { - web(instance, { prefix: this.prefix }); - next(); - }); - - //Redirect requests from /plugins/* to the plugin server (if installed) - const { proxy, close } = FastProxy({ - base: this.configuration.get("plugins.server"), - }); - fastify.addHook("onClose", close); - fastify.all("/plugins/*", (req, rep) => { - proxy(req.raw, rep.raw, req.url, {}); - }); - - return this; - } - - // TODO: remove - api(): undefined { - return undefined; - } -} diff --git a/twake/backend/node/src/services/applications-api/types.ts b/twake/backend/node/src/services/applications-api/types.ts deleted file mode 100644 index 9919e15d..00000000 --- a/twake/backend/node/src/services/applications-api/types.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { Channel } from "../channels/entities"; -import { Message } from "../messages/entities/messages"; -import { Thread } from "../messages/entities/threads"; - -export type HookType = { - type: "message"; - application_id: string; - company_id: string; - - channel?: Channel; - thread: Thread; - message: Message; -}; diff --git a/twake/backend/node/src/services/applications-api/web/controllers/index.ts b/twake/backend/node/src/services/applications-api/web/controllers/index.ts deleted file mode 100644 index afbf0f85..00000000 --- a/twake/backend/node/src/services/applications-api/web/controllers/index.ts +++ /dev/null @@ -1,181 +0,0 @@ -import { FastifyInstance, FastifyReply, FastifyRequest, HTTPMethods } from "fastify"; -import { ApplicationObject } from "../../../applications/entities/application"; -import { - ApplicationApiExecutionContext, - ApplicationLoginRequest, - ApplicationLoginResponse, - ConfigureRequest, -} from "../types"; -import { ResourceGetResponse } from "../../../../utils/types"; -import { CrudException } from "../../../../core/platform/framework/api/crud-service"; -import { localEventBus } from "../../../../core/platform/framework/event-bus"; -import { - RealtimeApplicationEvent, - RealtimeBaseBusEvent, -} from "../../../../core/platform/services/realtime/types"; -import gr from "../../../global-resolver"; -import _ from "lodash"; -import { v4 } from "uuid"; - -export class ApplicationsApiController { - async token( - request: FastifyRequest<{ Body: ApplicationLoginRequest }>, - ): Promise> { - const context = getExecutionContext(request); - - if (!request.body.id || !request.body.secret) { - throw CrudException.forbidden("Application not found"); - } - - const app = await gr.services.applications.marketplaceApps.get( - { - id: request.body.id, - }, - context, - ); - - if (!app) { - throw CrudException.forbidden("Application not found"); - } - - if (!app.api.private_key || app.api.private_key !== request.body.secret) { - throw CrudException.forbidden("Secret key is not valid"); - } - - return { - resource: { - access_token: gr.platformServices.auth.generateJWT(request.body.id, null, { - track: false, - provider_id: "", - application_id: request.body.id, - }), - }, - }; - } - - async me( - request: FastifyRequest, - _reply: FastifyReply, - ): Promise> { - const context = getExecutionContext(request); - - const entity = await gr.services.applications.marketplaceApps.get( - { - id: context.application_id, - }, - context, - ); - if (!entity) { - throw CrudException.notFound("Application not found"); - } - - return { resource: entity.getApplicationObject() }; - } - - async configure( - request: FastifyRequest<{ Body: ConfigureRequest }>, - _reply: FastifyReply, - ): Promise> { - const app_id = request.currentUser.application_id; - const context = getExecutionContext(request); - const application = await gr.services.applications.marketplaceApps.get({ id: app_id }, context); - - if (!application) { - throw CrudException.forbidden("Application not found"); - } - - const body = request.body; - - const data = { - action: "configure", - application: { - id: app_id, - identity: application.identity, - }, - form: body.form, - connection_id: body.connection_id, - hidden_data: {}, - configurator_id: v4(), - }; - - localEventBus.publish("realtime:event", { - room: "/me/" + body.user_id, - type: "application", - data, - } as RealtimeBaseBusEvent); - - return { status: "ok" }; - } - - async proxy( - request: FastifyRequest<{ Params: { company_id: string; service: string; version: string } }>, - reply: FastifyReply, - fastify: FastifyInstance, - ): Promise { - // Check the application has access to this company - const company_id = request.params.company_id; - const companyApplication = gr.services.applications.companyApps.get({ - company_id, - application_id: request.currentUser.application_id, - id: undefined, - }); - if (!companyApplication) { - throw CrudException.forbidden("This application is not installed in the requested company"); - } - - const context = getExecutionContext(request); - const app = await gr.services.applications.marketplaceApps.get( - { - id: request.currentUser.application_id, - }, - context, - ); - - // Check call can be done from this IP - if ( - app.api.allowed_ips.trim() && - app.api.allowed_ips !== "*" && - !_.includes( - app.api.allowed_ips - .split(",") - .map(a => a.trim()) - .filter(a => a), - request.ip, - ) - ) { - throw CrudException.forbidden( - `This application is not allowed to access from this IP (${request.ip})`, - ); - } - - //TODO Check application access rights (write, read, remove for each micro services) - const _access = app.access; - - //TODO save some statistics about API usage for application and per companies - - const route = request.url.replace("/api/", "/internal/services/"); - - fastify.inject( - { - method: request.method as HTTPMethods, - url: route, - payload: request.body as any, - headers: _.pick(request.headers, "authorization"), - }, - (err, response) => { - reply.headers(response.headers); - reply.send(response.payload); - }, - ); - } -} - -function getExecutionContext(request: FastifyRequest): ApplicationApiExecutionContext { - return { - application_id: request.currentUser?.application_id, - user: request.currentUser, - url: request.url, - method: request.routerMethod, - transport: "http", - }; -} diff --git a/twake/backend/node/src/services/applications-api/web/index.ts b/twake/backend/node/src/services/applications-api/web/index.ts deleted file mode 100644 index de7176c9..00000000 --- a/twake/backend/node/src/services/applications-api/web/index.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { FastifyInstance, FastifyRegisterOptions } from "fastify"; -import routes from "./routes"; - -export default ( - fastify: FastifyInstance, - options: FastifyRegisterOptions<{ prefix: string }>, -): void => { - fastify.log.debug("Configuring /internal/services/applications/v1 routes"); - fastify.register(routes, options); -}; diff --git a/twake/backend/node/src/services/applications-api/web/routes.ts b/twake/backend/node/src/services/applications-api/web/routes.ts deleted file mode 100644 index 3af81dbb..00000000 --- a/twake/backend/node/src/services/applications-api/web/routes.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { FastifyInstance, FastifyPluginCallback, FastifyRequest } from "fastify"; - -import { ApplicationsApiController } from "./controllers"; -import { ApplicationApiBaseRequest } from "./types"; -import { logger as log } from "../../../core/platform/framework"; -import { configureRequestSchema } from "./schemas"; - -const routes: FastifyPluginCallback = (fastify: FastifyInstance, options, next) => { - const controller = new ApplicationsApiController(); - - const checkApplication = async (request: FastifyRequest<{ Body: ApplicationApiBaseRequest }>) => { - if (!request.currentUser.application_id) { - log.debug(request.currentUser); - throw fastify.httpErrors.forbidden("You should log in as application"); - } - }; - - //Authenticate the application - fastify.route({ - method: "POST", - url: "/console/v1/login", - handler: controller.token.bind(controller), - }); - - //Get myself as an application - fastify.route({ - method: "GET", - url: "/console/v1/me", - preValidation: [fastify.authenticate], - preHandler: [checkApplication], - handler: controller.me.bind(controller), - }); - - //Open a configuration popup on the client side - fastify.route({ - method: "POST", - url: "/console/v1/configure", - preValidation: [fastify.authenticate], - schema: configureRequestSchema, - handler: controller.configure.bind(controller), - }); - - //Get myself as an application - fastify.route({ - method: ["POST", "GET", "DELETE", "PUT"], - url: "/:service/:version/companies/:company_id/*", - preValidation: [fastify.authenticate], - handler: (request, reply) => controller.proxy.bind(controller)(request, reply, fastify), - }); - - next(); -}; - -export default routes; diff --git a/twake/backend/node/src/services/applications-api/web/schemas.ts b/twake/backend/node/src/services/applications-api/web/schemas.ts deleted file mode 100644 index 1ee1b370..00000000 --- a/twake/backend/node/src/services/applications-api/web/schemas.ts +++ /dev/null @@ -1,16 +0,0 @@ -export const applicationsSchema = { - type: "object", - properties: {}, -}; - -export const configureRequestSchema = { - body: { - type: "object", - properties: { - user_id: { type: "string" }, - connection_id: { type: "string" }, - form: {}, - }, - required: ["user_id", "connection_id"], - }, -}; diff --git a/twake/backend/node/src/services/applications-api/web/types.ts b/twake/backend/node/src/services/applications-api/web/types.ts deleted file mode 100644 index 04556d9e..00000000 --- a/twake/backend/node/src/services/applications-api/web/types.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { AccessToken } from "../../../utils/types"; -import { ExecutionContext } from "../../../core/platform/framework/api/crud-service"; - -export interface ApplicationApiBaseRequest { - id: string; - secret: string; -} - -export type ApplicationLoginRequest = ApplicationApiBaseRequest; - -export interface ApplicationLoginResponse { - access_token: AccessToken; -} - -export interface ApplicationApiExecutionContext extends ExecutionContext { - application_id: string; -} - -export interface ConfigureRequest { - user_id: string; - connection_id: string; - form?: any; -} diff --git a/twake/backend/node/src/services/applications/entities/application.search.ts b/twake/backend/node/src/services/applications/entities/application.search.ts deleted file mode 100644 index 669e0245..00000000 --- a/twake/backend/node/src/services/applications/entities/application.search.ts +++ /dev/null @@ -1,33 +0,0 @@ -import Application, { TYPE } from "./application"; - -export default { - index: TYPE, - source: (entity: Application) => { - return { - company_id: entity.company_id, - name: entity.identity.name, - description: entity.identity.description, - categories: entity.identity.categories, - compatibility: entity.identity.compatibility, - published: entity.publication.published, - created_at: entity.stats.created_at, - }; - }, - mongoMapping: { - text: { - name: "text", - description: "text", - }, - }, - esMapping: { - properties: { - company_id: { type: "keyword" }, - name: { type: "text", index_prefixes: { min_chars: 1 } }, - description: { type: "text" }, - categories: { type: "keyword" }, - compatibility: { type: "keyword" }, - published: { type: "boolean" }, - created_at: { type: "number" }, - }, - }, -}; diff --git a/twake/backend/node/src/services/applications/entities/application.ts b/twake/backend/node/src/services/applications/entities/application.ts deleted file mode 100644 index 17cf4f54..00000000 --- a/twake/backend/node/src/services/applications/entities/application.ts +++ /dev/null @@ -1,211 +0,0 @@ -import { Type } from "class-transformer"; -import _, { merge } from "lodash"; -import { Column, Entity } from "../../../core/platform/services/database/services/orm/decorators"; -import search from "./application.search"; - -export const TYPE = "applications"; - -@Entity(TYPE, { - primaryKey: ["id"], - type: TYPE, - search, -}) -export default class Application { - @Type(() => String) - @Column("id", "timeuuid", { generator: "timeuuid" }) - id: string; - - @Type(() => String) - @Column("group_id", "timeuuid") - company_id: string; - - @Column("is_default", "boolean") - is_default: boolean; - - @Column("identity", "json") - identity: ApplicationIdentity; - - //This information is private to the application, make sure not to disclose it - @Column("api", "encoded_json") - api: ApplicationApi; - - @Column("access", "json") - access: ApplicationAccess; - - @Column("display", "json") - display: ApplicationDisplay; - - @Column("publication", "json") - publication: ApplicationPublication; - - @Column("stats", "json") - stats: ApplicationStatistics; - - getPublicObject(): PublicApplicationObject { - const i = _.pick( - this, - "id", - "company_id", - "is_default", - "identity", - "access", - "display", - "publication", - "stats", - ); - - i.is_default = !!i.is_default; - return i; - } - - getApplicationObject(): ApplicationObject { - const i = _.pick( - this, - "id", - "company_id", - "is_default", - "identity", - "access", - "display", - "publication", - "stats", - "api", - ); - - i.is_default = !!i.is_default; - return i; - } -} - -export type PublicApplicationObject = Pick< - Application, - "id" | "company_id" | "is_default" | "identity" | "access" | "display" | "publication" | "stats" ->; - -export type ApplicationObject = Pick< - Application, - | "id" - | "company_id" - | "is_default" - | "identity" - | "access" - | "display" - | "publication" - | "stats" - | "api" ->; - -export type ApplicationPrimaryKey = { id: string }; - -export function getInstance(message: Application): Application { - return merge(new Application(), message); -} - -export type ApplicationIdentity = { - code: string; - name: string; - icon: string; - description: string; - website: string; - categories: string[]; - compatibility: "twake"[]; -}; - -export type ApplicationPublication = { - published: boolean; //Publication accepted // RO - requested: boolean; //Publication requested -}; - -export type ApplicationStatistics = { - created_at: number; // RO - updated_at: number; // RO - version: number; // RO -}; - -export type ApplicationApi = { - hooks_url: string; - allowed_ips: string; - private_key: string; // RO -}; - -type ApplicationScopes = - | "files" - | "applications" - | "workspaces" - | "users" - | "messages" - | "channels"; - -export type ApplicationAccess = { - read: ApplicationScopes[]; - write: ApplicationScopes[]; - delete: ApplicationScopes[]; - hooks: ApplicationScopes[]; -}; - -export type ApplicationDisplay = { - twake: { - files?: { - editor?: { - preview_url: string; //Open a preview inline (iframe) - edition_url: string; //Url to edit the file (full screen) - extensions?: string[]; //Main extensions app can read - // if file was created by the app, then the app is able to edit with or without extension - empty_files?: { - url: string; // "https://[...]/empty.docx"; - filename: string; // "Untitled.docx"; - name: string; // "Word Document"; - }[]; - }; - actions?: //List of action that can apply on a file - { - name: string; - id: string; - }[]; - }; - - //Chat plugin - chat?: { - input?: - | true - | { - icon?: string; //If defined replace original icon url of your app - type?: "file" | "call"; //To add in existing apps folder / default icon - }; - commands?: { - command: string; // my_app mycommand - description: string; - }[]; - actions?: //List of action that can apply on a message - { - name: string; - id: string; - }[]; - }; - - //Allow app to appear as a bot user in direct chat - direct?: - | true - | { - name?: string; - icon?: string; //If defined replace original icon url of your app - }; - - //Display app as a standalone application in a tab - tab?: - | { - url: string; - } - | true; - - //Display app as a standalone application on the left bar - standalone?: - | { - url: string; - } - | true; - - //Define where the app can be configured from - configuration?: ("global" | "channel")[]; - }; -}; diff --git a/twake/backend/node/src/services/applications/entities/company-application.ts b/twake/backend/node/src/services/applications/entities/company-application.ts deleted file mode 100644 index 7598e05f..00000000 --- a/twake/backend/node/src/services/applications/entities/company-application.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { Type } from "class-transformer"; -import { Column, Entity } from "../../../core/platform/services/database/services/orm/decorators"; -import { PublicApplicationObject } from "./application"; - -export const TYPE = "group_app"; - -@Entity(TYPE, { - primaryKey: [["group_id"], "app_id", "id"], - type: TYPE, -}) -export default class CompanyApplication { - @Type(() => String) - @Column("group_id", "timeuuid") - company_id: string; - - @Type(() => String) - @Column("app_id", "timeuuid") - application_id: string; - - @Type(() => String) - @Column("id", "timeuuid", { generator: "timeuuid" }) - id: string; - - @Column("created_at", "number") - created_at: number; - - @Type(() => String) - @Column("created_by", "string") - created_by: string; //Will be the default delegated user when doing actions on Twake -} - -export type CompanyApplicationPrimaryKey = Pick< - CompanyApplication, - "company_id" | "application_id" | "id" ->; - -export class CompanyApplicationWithApplication extends CompanyApplication { - //Not in database but attached to this object - application?: PublicApplicationObject; -} diff --git a/twake/backend/node/src/services/applications/index.ts b/twake/backend/node/src/services/applications/index.ts deleted file mode 100644 index 7c73c6a8..00000000 --- a/twake/backend/node/src/services/applications/index.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { Prefix, TwakeService } from "../../core/platform/framework"; -import WebServerAPI from "../../core/platform/services/webserver/provider"; -import web from "./web"; - -@Prefix("/internal/services/applications/v1") -export default class ApplicationsService extends TwakeService { - version = "1"; - name = "applications"; - - public async doInit(): Promise { - const fastify = this.context.getProvider("webserver").getServer(); - fastify.register((instance, _opts, next) => { - web(instance, { prefix: this.prefix }); - next(); - }); - - return this; - } - - // TODO: remove - api(): undefined { - return undefined; - } -} diff --git a/twake/backend/node/src/services/applications/realtime.ts b/twake/backend/node/src/services/applications/realtime.ts deleted file mode 100644 index 074f8a3e..00000000 --- a/twake/backend/node/src/services/applications/realtime.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { WebsocketMetadata } from "../../utils/types"; - -export function getCompanyApplicationRooms(companyId: string): WebsocketMetadata[] { - return [ - { - room: getCompanyApplicationRoom(companyId), - }, - ]; -} - -export function getCompanyApplicationRoom(companyApplicationId: string): string { - return `/company-application/${companyApplicationId}`; -} diff --git a/twake/backend/node/src/services/applications/services/applications.ts b/twake/backend/node/src/services/applications/services/applications.ts deleted file mode 100644 index 4eecdd8f..00000000 --- a/twake/backend/node/src/services/applications/services/applications.ts +++ /dev/null @@ -1,143 +0,0 @@ -import Application, { - ApplicationPrimaryKey, - getInstance as getApplicationInstance, - PublicApplicationObject, - TYPE, -} from "../entities/application"; -import Repository from "../../../core/platform/services/database/services/orm/repository/repository"; -import { Initializable, logger, TwakeServiceProvider } from "../../../core/platform/framework"; -import { - DeleteResult, - ExecutionContext, - ListResult, - OperationType, - Pagination, - SaveResult, -} from "../../../core/platform/framework/api/crud-service"; -import SearchRepository from "../../../core/platform/services/search/repository"; -import assert from "assert"; - -import gr from "../../global-resolver"; -import { InternalToHooksProcessor } from "./internal-event-to-hooks"; - -export class ApplicationServiceImpl implements TwakeServiceProvider, Initializable { - version: "1"; - repository: Repository; - searchRepository: SearchRepository; - - async init(): Promise { - try { - this.searchRepository = gr.platformServices.search.getRepository( - TYPE, - Application, - ); - this.repository = await gr.database.getRepository(TYPE, Application); - } catch (err) { - console.log(err); - logger.error("Error while initializing applications service"); - } - - gr.platformServices.messageQueue.processor.addHandler(new InternalToHooksProcessor()); - - return this; - } - - async get(pk: ApplicationPrimaryKey, context: ExecutionContext): Promise { - return await this.repository.findOne(pk, {}, context); - } - - async list( - pagination: Pagination, - options?: { search?: string }, - context?: ExecutionContext, - ): Promise> { - let entities: ListResult; - if (options.search) { - entities = await this.searchRepository.search( - {}, - { - pagination, - $text: { - $search: options.search, - }, - }, - context, - ); - } else { - entities = await this.repository.find({}, { pagination }, context); - } - entities.filterEntities(app => app.publication.published); - - const applications = entities - .getEntities() - .filter(app => app) - .map(app => app.getPublicObject()); - return new ListResult(entities.type, applications, entities.nextPage); - } - - async listUnpublished(context: ExecutionContext): Promise { - const entities = await this.repository.find({}, {}, context); - entities.filterEntities(app => !app.publication.published); - return entities.getEntities(); - } - - async listDefaults(context: ExecutionContext): Promise> { - const entities = []; - - let page: Pagination = { limitStr: "100" }; - do { - const applicationListResult = await this.repository.find({}, { pagination: page }, context); - page = applicationListResult.nextPage as Pagination; - applicationListResult.filterEntities(app => app.publication.published && app.is_default); - - for (const application of applicationListResult.getEntities()) { - if (application) entities.push(application.getPublicObject()); - } - } while (page.page_token); - - return new ListResult(TYPE, entities); - } - - async save( - item: Application, - options?: SaveOptions, - context?: ExecutionContext, - ): Promise> { - assert(item.company_id, "company_id is not defined"); - - try { - const entity = getApplicationInstance(item); - await this.repository.save(entity, context); - return new SaveResult("application", entity, OperationType.UPDATE); - } catch (e) { - throw e; - } - } - - async delete( - pk: ApplicationPrimaryKey, - context?: ExecutionContext, - ): Promise> { - const entity = await this.get(pk, context); - await this.repository.remove(entity, context); - return new DeleteResult("application", entity, true); - } - - async publish(pk: ApplicationPrimaryKey, context: ExecutionContext): Promise { - const entity = await this.get(pk, context); - if (!entity) { - throw new Error("Entity not found"); - } - entity.publication.published = true; - await this.repository.save(entity, context); - } - - async unpublish(pk: ApplicationPrimaryKey, context: ExecutionContext): Promise { - const entity = await this.get(pk, context); - if (!entity) { - throw new Error("Entity not found"); - } - entity.publication.published = false; - await this.repository.save(entity, context); - } -} diff --git a/twake/backend/node/src/services/applications/services/company-applications.ts b/twake/backend/node/src/services/applications/services/company-applications.ts deleted file mode 100644 index 5ae82621..00000000 --- a/twake/backend/node/src/services/applications/services/company-applications.ts +++ /dev/null @@ -1,188 +0,0 @@ -import CompanyApplication, { - CompanyApplicationPrimaryKey, - CompanyApplicationWithApplication, - TYPE, -} from "../entities/company-application"; -import Repository from "../../../core/platform/services/database/services/orm/repository/repository"; -import { - Initializable, - logger, - RealtimeDeleted, - RealtimeSaved, - TwakeServiceProvider, -} from "../../../core/platform/framework"; -import { - DeleteResult, - ListResult, - OperationType, - Paginable, - Pagination, - SaveResult, -} from "../../../core/platform/framework/api/crud-service"; -import { CompanyExecutionContext } from "../web/types"; -import { getCompanyApplicationRoom } from "../realtime"; -import gr from "../../global-resolver"; - -export class CompanyApplicationServiceImpl implements TwakeServiceProvider, Initializable { - version: "1"; - repository: Repository; - - async init(): Promise { - try { - this.repository = await gr.database.getRepository( - TYPE, - CompanyApplication, - ); - } catch (err) { - console.log(err); - logger.error("Error while initializing applications service"); - } - return this; - } - - // TODO: remove logic from context - async get( - pk: Pick & { id?: string }, - context?: CompanyExecutionContext, - ): Promise { - const companyApplication = await this.repository.findOne( - { - group_id: context ? context.company.id : pk.company_id, - app_id: pk.application_id, - }, - {}, - context, - ); - - const application = await gr.services.applications.marketplaceApps.get( - { - id: pk.application_id, - }, - context, - ); - - return { - ...companyApplication, - application: application, - }; - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - @RealtimeSaved((companyApplication, _context) => { - return [ - { - room: getCompanyApplicationRoom(companyApplication.id), - resource: companyApplication, - }, - ]; - }) - async save( - item: Pick, - _?: SaveOptions, - context?: CompanyExecutionContext, - ): Promise> { - if (!context?.user?.id && !context?.user?.server_request) { - throw new Error("Only an user of a company can add an application to a company."); - } - - let operation = OperationType.UPDATE; - let companyApplication = await this.repository.findOne( - { - group_id: context?.company.id, - app_id: item.application_id, - }, - {}, - context, - ); - if (!companyApplication) { - operation = OperationType.CREATE; - - companyApplication = new CompanyApplication(); - companyApplication.company_id = context.company.id; - companyApplication.application_id = item.application_id; - companyApplication.created_at = new Date().getTime(); - companyApplication.created_by = context?.user?.id || ""; - - await this.repository.save(companyApplication, context); - } - - return new SaveResult(TYPE, companyApplication, operation); - } - - async initWithDefaultApplications( - companyId: string, - context: CompanyExecutionContext, - ): Promise { - const defaultApps = await gr.services.applications.marketplaceApps.listDefaults(context); - for (const defaultApp of defaultApps.getEntities()) { - await this.save({ company_id: companyId, application_id: defaultApp.id }, {}, context); - } - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - @RealtimeDeleted((companyApplication, _context) => { - return [ - { - room: getCompanyApplicationRoom(companyApplication.id), - resource: companyApplication, - }, - ]; - }) - async delete( - pk: CompanyApplicationPrimaryKey, - context?: CompanyExecutionContext, - ): Promise> { - const companyApplication = await this.repository.findOne( - { - group_id: context.company.id, - app_id: pk.application_id, - }, - {}, - context, - ); - - let deleted = false; - if (companyApplication) { - this.repository.remove(companyApplication, context); - deleted = true; - } - - return new DeleteResult(TYPE, companyApplication, deleted); - } - - async list( - pagination: Paginable, - options?: ListOptions, - context?: CompanyExecutionContext, - ): Promise> { - const companyApplications = await this.repository.find( - { - group_id: context.company.id, - }, - { pagination: Pagination.fromPaginable(pagination) }, - context, - ); - - const applications = []; - - for (const companyApplication of companyApplications.getEntities()) { - const application = await gr.services.applications.marketplaceApps.get( - { - id: companyApplication.application_id, - }, - context, - ); - if (application) - applications.push({ - ...companyApplication, - application: application, - }); - } - - return new ListResult( - TYPE, - applications, - companyApplications.nextPage, - ); - } -} diff --git a/twake/backend/node/src/services/applications/services/hooks.ts b/twake/backend/node/src/services/applications/services/hooks.ts deleted file mode 100644 index d34a5f15..00000000 --- a/twake/backend/node/src/services/applications/services/hooks.ts +++ /dev/null @@ -1,96 +0,0 @@ -import Application from "../entities/application"; -import Repository from "../../../core/platform/services/database/services/orm/repository/repository"; -import { - Initializable, - logger as log, - TwakeServiceProvider, -} from "../../../core/platform/framework"; -import { CrudException, ExecutionContext } from "../../../core/platform/framework/api/crud-service"; -import SearchRepository from "../../../core/platform/services/search/repository"; -import axios from "axios"; -import * as crypto from "crypto"; -import { isObject } from "lodash"; -import gr from "../../global-resolver"; - -export class ApplicationHooksService implements TwakeServiceProvider, Initializable { - version: "1"; - repository: Repository; - searchRepository: SearchRepository; - - async init() { - return this; - } - - async notifyApp( - application_id: string, - connection_id: string, - user_id: string, - type: string, - name: string, - content: any, - company_id: string, - workspace_id: string, - context: ExecutionContext, - ): Promise { - const app = await gr.services.applications.marketplaceApps.get({ id: application_id }, context); - if (!app) { - throw CrudException.notFound("Application not found"); - } - - if (!app.api.hooks_url) { - throw CrudException.badRequest("Application hooks_url is not defined"); - } - - const payload = { - type, - name, - content, - connection_id: connection_id, - user_id: user_id, - company_id, - workspace_id, - }; - - const signature = crypto - .createHmac("sha256", app.api.private_key) - .update(JSON.stringify(payload)) - .digest("hex"); - - return await axios - .post(app.api.hooks_url, payload, { - headers: { - "Content-Type": "application/json", - "X-Twake-Signature": signature, - }, - }) - - .then(({ data }) => data) - .catch(e => { - log.error(e.message); - const r = e.response; - - if (!r) { - throw CrudException.badGateway("Can't connect remote application"); - } - - let msg = r.data; - - if (isObject(msg)) { - // parse typical responses - if (r.data.message) { - msg = r.data.message; - } else if (r.data.error) { - msg = r.data.error; - } else { - msg = JSON.stringify(r.data); - } - } - - if (r.status == 403) { - throw CrudException.forbidden(msg); - } else { - throw CrudException.badRequest(msg); - } - }); - } -} diff --git a/twake/backend/node/src/services/applications/services/internal-event-to-hooks.ts b/twake/backend/node/src/services/applications/services/internal-event-to-hooks.ts deleted file mode 100644 index 77fe8d74..00000000 --- a/twake/backend/node/src/services/applications/services/internal-event-to-hooks.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { logger } from "../../../core/platform/framework"; -import { HookType } from "../../applications-api/types"; -import { MessageQueueHandler } from "../../../core/platform/services/message-queue/api"; -import { MessageHook } from "../../messages/types"; -import gr from "../../global-resolver"; -import { ExecutionContext } from "../../../core/platform/framework/api/crud-service"; - -export class InternalToHooksProcessor implements MessageQueueHandler { - readonly topics = { - in: "application:hook:message", - }; - - readonly options = { - unique: true, - ack: true, - queue: "application:hook:message:consumer1", - }; - - readonly name = "Application::InternalToHooksProcessor"; - - validate(_: HookType): boolean { - return true; - } - - async process(message: HookType, context?: ExecutionContext): Promise { - logger.debug(`${this.name} - Receive hook of type ${message.type}`); - - const application = await gr.services.applications.marketplaceApps.get( - { - id: message.application_id, - }, - context, - ); - - //TODO Check application access rights (hooks) - const _access = application.access; - - // Check application still exists in the company - if ( - !(await gr.services.applications.companyApps.get({ - company_id: message.company_id, - application_id: message.application_id, - id: undefined, - })) - ) { - logger.error( - `${this.name} - Application ${message.application_id} not found in company ${message.company_id}`, - ); - return; - } - - await gr.services.applications.hooks.notifyApp( - message.application_id, - null, - null, - "hook", - null, - { message }, - null, - null, - context, - ); - } -} diff --git a/twake/backend/node/src/services/applications/web/controllers/applications.ts b/twake/backend/node/src/services/applications/web/controllers/applications.ts deleted file mode 100644 index 418e9a92..00000000 --- a/twake/backend/node/src/services/applications/web/controllers/applications.ts +++ /dev/null @@ -1,265 +0,0 @@ -import { FastifyReply, FastifyRequest } from "fastify"; -import { CrudController } from "../../../../core/platform/services/webserver/types"; -import { - PaginationQueryParameters, - ResourceCreateResponse, - ResourceDeleteResponse, - ResourceGetResponse, - ResourceListResponse, - ResourceUpdateResponse, -} from "../../../../utils/types"; -import Application, { - ApplicationObject, - PublicApplicationObject, -} from "../../entities/application"; -import { - CrudException, - ExecutionContext, - Pagination, -} from "../../../../core/platform/framework/api/crud-service"; -import _ from "lodash"; -import { randomBytes } from "crypto"; -import { ApplicationEventRequestBody } from "../types"; -import { logger as log } from "../../../../core/platform/framework"; -import { hasCompanyAdminLevel } from "../../../../utils/company"; -import gr from "../../../global-resolver"; - -export class ApplicationController - implements - CrudController< - ResourceGetResponse, - ResourceUpdateResponse, - ResourceListResponse, - ResourceDeleteResponse - > -{ - async get( - request: FastifyRequest<{ Params: { application_id: string } }>, - ): Promise> { - const context = getExecutionContext(request); - - const entity = await gr.services.applications.marketplaceApps.get( - { - id: request.params.application_id, - }, - context, - ); - - const companyUser = await gr.services.companies.getCompanyUser( - { id: entity.company_id }, - { id: context.user.id }, - ); - - const isAdmin = companyUser && companyUser.role == "admin"; - - return { - resource: isAdmin ? entity.getApplicationObject() : entity.getPublicObject(), - }; - } - - async list( - request: FastifyRequest<{ - Querystring: PaginationQueryParameters & { search: string }; - }>, - ): Promise> { - const entities = await gr.services.applications.marketplaceApps.list(new Pagination(), { - search: request.query.search, - }); - return { - resources: entities.getEntities(), - next_page_token: entities.nextPage.page_token, - }; - } - - async save( - request: FastifyRequest<{ - Params: { application_id: string }; - Body: { resource: Application }; - }>, - _reply: FastifyReply, - ): Promise> { - const context = getExecutionContext(request); - - try { - const app = request.body.resource; - const now = new Date().getTime(); - - let entity: Application; - - if (request.params.application_id) { - entity = await gr.services.applications.marketplaceApps.get( - { - id: request.params.application_id, - }, - context, - ); - - if (!entity) { - throw CrudException.notFound("Application not found"); - } - - entity.publication.requested = app.publication.requested; - if (app.publication.requested === false) { - entity.publication.published = false; - } - - if (entity.publication.published) { - if ( - !_.isEqual( - _.pick(entity, "identity", "api", "access", "display"), - _.pick(app, "identity", "api", "access", "display"), - ) - ) { - throw CrudException.badRequest( - "You can't update applications details while it published", - ); - } - } - - entity.identity = app.identity; - entity.api.hooks_url = app.api.hooks_url; - entity.api.allowed_ips = app.api.allowed_ips; - entity.access = app.access; - entity.display = app.display; - - entity.stats.updated_at = now; - entity.stats.version++; - - const res = await gr.services.applications.marketplaceApps.save(entity); - entity = res.entity; - } else { - // INSERT - - app.is_default = false; - app.publication.published = false; - app.api.private_key = randomBytes(32).toString("base64"); - - app.stats = { - created_at: now, - updated_at: now, - version: 0, - }; - - const res = await gr.services.applications.marketplaceApps.save(app); - entity = res.entity; - } - - return { - resource: entity.getApplicationObject(), - }; - } catch (e) { - log.error(e); - throw e; - } - } - - async delete( - request: FastifyRequest<{ Params: { application_id: string } }>, - reply: FastifyReply, - ): Promise { - const context = getExecutionContext(request); - - const application = await gr.services.applications.marketplaceApps.get( - { - id: request.params.application_id, - }, - context, - ); - - const compUser = await gr.services.companies.getCompanyUser( - { id: application.company_id }, - { id: context.user.id }, - ); - if (!compUser || !hasCompanyAdminLevel(compUser.role)) { - throw CrudException.forbidden("You don't have the rights to delete this application"); - } - - const deleteResult = await gr.services.applications.marketplaceApps.delete( - { - id: request.params.application_id, - }, - context, - ); - - if (deleteResult.deleted) { - reply.code(204); - - return { - status: "success", - }; - } - - return { - status: "error", - }; - } - - async event( - request: FastifyRequest<{ - Body: ApplicationEventRequestBody; - Params: { application_id: string }; - }>, - _reply: FastifyReply, - ): Promise> { - const context = getExecutionContext(request); - - const content = request.body.data; - - const applicationEntity = await gr.services.applications.marketplaceApps.get( - { - id: request.params.application_id, - }, - context, - ); - - if (!applicationEntity) { - throw CrudException.notFound("Application not found"); - } - - const companyUser = gr.services.companies.getCompanyUser( - { id: request.body.company_id }, - { id: context.user.id }, - ); - - if (!companyUser) { - throw CrudException.badRequest( - "You cannot send event to an application from another company", - ); - } - - const applicationInCompany = await gr.services.applications.companyApps.get({ - company_id: request.body.company_id, - application_id: request.params.application_id, - id: undefined, - }); - - if (!applicationInCompany) { - throw CrudException.badRequest("Application isn't installed in this company"); - } - - const hookResponse = await gr.services.applications.hooks.notifyApp( - request.params.application_id, - request.body.connection_id, - context.user.id, - request.body.type, - request.body.name, - content, - request.body.company_id, - request.body.workspace_id, - context, - ); - - return { - resource: hookResponse, - }; - } -} - -function getExecutionContext(request: FastifyRequest): ExecutionContext { - return { - user: request.currentUser, - url: request.url, - method: request.routerMethod, - transport: "http", - }; -} diff --git a/twake/backend/node/src/services/applications/web/controllers/company-applications.ts b/twake/backend/node/src/services/applications/web/controllers/company-applications.ts deleted file mode 100644 index 26542e39..00000000 --- a/twake/backend/node/src/services/applications/web/controllers/company-applications.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { FastifyReply, FastifyRequest } from "fastify"; - -import { - PaginationQueryParameters, - ResourceDeleteResponse, - ResourceGetResponse, - ResourceListResponse, - ResourceUpdateResponse, -} from "../../../../utils/types"; -import { PublicApplicationObject } from "../../entities/application"; -import { CompanyExecutionContext } from "../types"; -import { CrudController } from "../../../../core/platform/services/webserver/types"; -import { getCompanyApplicationRooms } from "../../realtime"; -import gr from "../../../global-resolver"; - -export class CompanyApplicationController - implements - CrudController< - ResourceGetResponse, - ResourceUpdateResponse, - ResourceListResponse, - ResourceDeleteResponse - > -{ - async get( - request: FastifyRequest<{ Params: { company_id: string; application_id: string } }>, - ): Promise> { - const context = getCompanyExecutionContext(request); - const resource = await gr.services.applications.companyApps.get( - { - application_id: request.params.application_id, - company_id: context.company.id, - id: undefined, - }, - context, - ); - return { - resource: resource?.application, - }; - } - - async list( - request: FastifyRequest<{ - Params: { company_id: string }; - Querystring: PaginationQueryParameters & { search: string }; - }>, - ): Promise> { - const context = getCompanyExecutionContext(request); - const resources = await gr.services.applications.companyApps.list( - request.query, - { search: request.query.search }, - context, - ); - - return { - resources: resources.getEntities().map(ca => ca.application), - next_page_token: resources.nextPage.page_token, - websockets: - gr.platformServices.realtime.sign( - getCompanyApplicationRooms(request.params.company_id), - context.user.id, - ) || [], - }; - } - - async save( - request: FastifyRequest<{ - Params: { company_id: string; application_id: string }; - Body: PublicApplicationObject; - }>, - ): Promise> { - const context = getCompanyExecutionContext(request); - - const resource = await gr.services.applications.companyApps.save( - { application_id: request.params.application_id, company_id: context.company.id }, - {}, - context, - ); - - const app = await gr.services.applications.companyApps.get(resource.entity); - - return { - resource: app.application, - }; - } - - async delete( - request: FastifyRequest<{ Params: { company_id: string; application_id: string } }>, - _reply: FastifyReply, - ): Promise { - const context = getCompanyExecutionContext(request); - const resource = await gr.services.applications.companyApps.delete( - { - application_id: request.params.application_id, - company_id: context.company.id, - id: undefined, - }, - context, - ); - return { - status: resource.deleted ? "success" : "error", - }; - } -} - -function getCompanyExecutionContext( - request: FastifyRequest<{ - Params: { company_id: string }; - }>, -): CompanyExecutionContext { - return { - user: request.currentUser, - company: { id: request.params.company_id }, - url: request.url, - method: request.routerMethod, - reqId: request.id, - transport: "http", - }; -} diff --git a/twake/backend/node/src/services/applications/web/controllers/index.ts b/twake/backend/node/src/services/applications/web/controllers/index.ts deleted file mode 100644 index ef52722d..00000000 --- a/twake/backend/node/src/services/applications/web/controllers/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./applications"; diff --git a/twake/backend/node/src/services/applications/web/index.ts b/twake/backend/node/src/services/applications/web/index.ts deleted file mode 100644 index 3c2df979..00000000 --- a/twake/backend/node/src/services/applications/web/index.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { FastifyInstance, FastifyRegisterOptions } from "fastify"; -import routes from "./routes"; - -export default ( - fastify: FastifyInstance, - options: FastifyRegisterOptions<{ - prefix: string; - }>, -): void => { - fastify.log.debug("Configuring /internal/services/applications/v1 routes"); - fastify.register(routes, options); -}; diff --git a/twake/backend/node/src/services/applications/web/routes.ts b/twake/backend/node/src/services/applications/web/routes.ts deleted file mode 100644 index 3d5835c7..00000000 --- a/twake/backend/node/src/services/applications/web/routes.ts +++ /dev/null @@ -1,161 +0,0 @@ -import { FastifyInstance, FastifyPluginCallback, FastifyRequest } from "fastify"; -import { ApplicationController } from "./controllers/applications"; -import { CompanyApplicationController } from "./controllers/company-applications"; - -import Application from "../entities/application"; -import { applicationEventHookSchema, applicationPostSchema } from "./schemas"; -import { logger as log } from "../../../core/platform/framework"; -import { checkUserBelongsToCompany, hasCompanyAdminLevel } from "../../../utils/company"; -import gr from "../../global-resolver"; - -const applicationsUrl = "/applications"; -const companyApplicationsUrl = "/companies/:company_id/applications"; - -const routes: FastifyPluginCallback = (fastify: FastifyInstance, options, next) => { - const applicationController = new ApplicationController(); - const companyApplicationController = new CompanyApplicationController(); - - const adminCheck = async ( - request: FastifyRequest<{ - Body: { resource: Application }; - Params: { application_id: string }; - }>, - ) => { - try { - let companyId: string = request.body?.resource?.company_id; - - if (request.params.application_id) { - const application = await gr.services.applications.marketplaceApps.get( - { - id: request.params.application_id, - }, - undefined, - ); - - if (!application) { - throw fastify.httpErrors.notFound("Application is not defined"); - } - - companyId = application.company_id; - } - - const userId = request.currentUser.id; - - if (!companyId) { - throw fastify.httpErrors.forbidden(`Company ${companyId} not found`); - } - - const companyUser = await checkUserBelongsToCompany(userId, companyId); - - if (!hasCompanyAdminLevel(companyUser.role)) { - throw fastify.httpErrors.forbidden("You must be an admin of this company"); - } - } catch (e) { - log.error(e); - throw e; - } - }; - - /** - * Applications collection - * Marketplace of applications - */ - - //Get and search list of applications in the marketplace - fastify.route({ - method: "GET", - url: `${applicationsUrl}`, - preValidation: [fastify.authenticate], - // schema: applicationGetSchema, - handler: applicationController.list.bind(applicationController), - }); - - //Get a single application in the marketplace - fastify.route({ - method: "GET", - url: `${applicationsUrl}/:application_id`, - preValidation: [fastify.authenticate], - // schema: applicationGetSchema, - handler: applicationController.get.bind(applicationController), - }); - - //Create application (must be my company application and I must be company admin) - fastify.route({ - method: "POST", - url: `${applicationsUrl}`, - preHandler: [adminCheck], - preValidation: [fastify.authenticate], - schema: applicationPostSchema, - handler: applicationController.save.bind(applicationController), - }); - - //Edit application (must be my company application and I must be company admin) - fastify.route({ - method: "POST", - url: `${applicationsUrl}/:application_id`, - preHandler: [adminCheck], - preValidation: [fastify.authenticate], - schema: applicationPostSchema, - handler: applicationController.save.bind(applicationController), - }); - - // Delete application (must be my company application and I must be company admin) - fastify.route({ - method: "DELETE", - url: `${applicationsUrl}/:application_id`, - preHandler: [adminCheck], - preValidation: [fastify.authenticate], - handler: applicationController.delete.bind(applicationController), - }); - - /** - * Company applications collection - * Company-wide available applications - * (must be my company application and I must be company admin) - */ - - //Get list of applications for a company - fastify.route({ - method: "GET", - url: `${companyApplicationsUrl}`, - preValidation: [fastify.authenticate], - handler: companyApplicationController.list.bind(companyApplicationController), - }); - - //Get one application of a company - fastify.route({ - method: "GET", - url: `${companyApplicationsUrl}/:application_id`, - preValidation: [fastify.authenticate], - handler: companyApplicationController.get.bind(companyApplicationController), - }); - - //Remove an application from a company - fastify.route({ - method: "DELETE", - url: `${companyApplicationsUrl}/:application_id`, - preValidation: [fastify.authenticate], - handler: companyApplicationController.delete.bind(companyApplicationController), - }); - - //Add an application to the company - fastify.route({ - method: "POST", - url: `${companyApplicationsUrl}/:application_id`, - preValidation: [fastify.authenticate], - handler: companyApplicationController.save.bind(companyApplicationController), - }); - - //Application event triggered by a user - fastify.route({ - method: "POST", - url: `${applicationsUrl}/:application_id/event`, - preValidation: [fastify.authenticate], - schema: applicationEventHookSchema, - handler: applicationController.event.bind(applicationController), - }); - - next(); -}; - -export default routes; diff --git a/twake/backend/node/src/services/applications/web/schemas.ts b/twake/backend/node/src/services/applications/web/schemas.ts deleted file mode 100644 index 996cea70..00000000 --- a/twake/backend/node/src/services/applications/web/schemas.ts +++ /dev/null @@ -1,129 +0,0 @@ -export const applicationsSchema = { - type: "object", - properties: {}, -}; - -const applicationIdentity = { - type: "object", - properties: { - code: { type: "string" }, - name: { type: "string" }, - icon: { type: "string" }, - description: { type: "string" }, - website: { type: "string" }, - categories: { type: "array", items: { type: "string" } }, - compatibility: { type: "array", items: { type: "string" } }, - }, - required: ["code", "name", "icon", "description", "website", "categories", "compatibility"], -}; - -const applicationAccess = { - type: "object", - properties: { - read: { type: "array", items: { type: "string" } }, - write: { type: "array", items: { type: "string" } }, - delete: { type: "array", items: { type: "string" } }, - hooks: { type: "array", items: { type: "string" } }, - }, - required: ["read", "write", "delete", "hooks"], -}; - -const requestApplicationPublication = { - type: "object", - properties: { - requested: { type: "boolean" }, - }, - required: ["requested"], -}; - -const responseApplicationPublication = { - type: "object", - properties: { - published: { type: "boolean" }, - requested: { type: "boolean" }, - }, - required: ["requested", "published"], -}; - -const applicationStats = { - type: "object", - properties: { - created_at: { type: "number" }, - updated_at: { type: "number" }, - version: { type: "number" }, - }, - required: ["created_at", "updated_at", "version"], -}; - -const apiObject = { - type: "object", - properties: { - hooks_url: { type: "string" }, - allowed_ips: { type: "string" }, - private_key: { type: "string" }, - }, - required: ["hooks_url", "allowed_ips"], -}; - -const requestApplicationObject = { - type: "object", - properties: { - company_id: { type: "string" }, - identity: applicationIdentity, - access: applicationAccess, - display: {}, - api: apiObject, - publication: requestApplicationPublication, - }, - required: ["company_id", "identity", "access", "display", "api", "publication"], - additionalProperties: false, -}; - -const responseApplicationObject = { - type: "object", - properties: { - id: { type: "string" }, - is_default: { type: "boolean" }, - company_id: { type: "string" }, - identity: applicationIdentity, - access: applicationAccess, - display: {}, - publication: responseApplicationPublication, - api: apiObject, - stats: applicationStats, - }, - required: [ - "id", - "is_default", - "company_id", - "identity", - "access", - "display", - "publication", - "stats", - ], - additionalProperties: false, -}; - -export const applicationPostSchema = { - body: { type: "object", properties: { resource: requestApplicationObject } }, - response: { - "2xx": { - resource: responseApplicationObject, - }, - }, -}; - -export const applicationEventHookSchema = { - body: { - type: "object", - properties: { - company_id: { type: "string" }, - workspace_id: { type: "string" }, - type: { type: "string" }, - name: { type: "string" }, - content: {}, - }, - required: ["company_id", "workspace_id", "type", "content"], - }, -}; diff --git a/twake/backend/node/src/services/applications/web/types.ts b/twake/backend/node/src/services/applications/web/types.ts deleted file mode 100644 index 5a60f3b3..00000000 --- a/twake/backend/node/src/services/applications/web/types.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { ExecutionContext } from "../../../core/platform/framework/api/crud-service"; - -export interface CompanyExecutionContext extends ExecutionContext { - company: { id: string }; -} - -export interface ApplicationEventRequestBody { - company_id: string; - workspace_id: string; - connection_id: string; - type: string; - name?: string; - content: any; - data: any; -} diff --git a/twake/backend/node/src/services/channels/services/member/service.ts b/twake/backend/node/src/services/channels/services/member/service.ts index e344771d..014f19ce 100644 --- a/twake/backend/node/src/services/channels/services/member/service.ts +++ b/twake/backend/node/src/services/channels/services/member/service.ts @@ -26,6 +26,7 @@ import { ChannelMemberType, ChannelVisibility, WorkspaceExecutionContext, + CompanyExecutionContext, } from "../../types"; import { Channel, ResourceEventsPayload, User } from "../../../../utils/types"; import { cloneDeep, isNil, omitBy } from "lodash"; @@ -51,7 +52,6 @@ import { WorkspacePrimaryKey } from "../../../workspaces/entities/workspace"; import gr from "../../../global-resolver"; import uuidTime from "uuid-time"; -import { CompanyExecutionContext } from "../../../../services/messages/types"; import { ChannelObject } from "../channel/types"; const USER_CHANNEL_KEYS = [ diff --git a/twake/backend/node/src/services/channels/services/pubsub/index.ts b/twake/backend/node/src/services/channels/services/pubsub/index.ts index e8d452e0..383e0cba 100644 --- a/twake/backend/node/src/services/channels/services/pubsub/index.ts +++ b/twake/backend/node/src/services/channels/services/pubsub/index.ts @@ -1,6 +1,5 @@ import { Initializable } from "../../../../core/platform/framework"; import { NewChannelActivityProcessor } from "./new-channel-activity"; -import { NewDirectChannelMessageProcessor } from "./new-direct-channel-message"; import { NewUserInWorkspaceJoinDefaultChannelsProcessor } from "./new-user-in-workspace-join-default-channels"; import { NewPendingEmailsInWorkspaceJoinChannelsProcessor } from "./new-pending-emails-in-workspace-join-channels"; import { NewWorkspaceProcessor } from "./new-workspace"; @@ -10,7 +9,6 @@ export class ChannelsMessageQueueListener implements Initializable { async init(): Promise { const channelActivityProcessor = await new NewChannelActivityProcessor().init(); gr.platformServices.messageQueue.processor.addHandler(channelActivityProcessor); - gr.platformServices.messageQueue.processor.addHandler(new NewDirectChannelMessageProcessor()); gr.platformServices.messageQueue.processor.addHandler( new NewUserInWorkspaceJoinDefaultChannelsProcessor(), ); diff --git a/twake/backend/node/src/services/channels/services/pubsub/new-direct-channel-message.ts b/twake/backend/node/src/services/channels/services/pubsub/new-direct-channel-message.ts deleted file mode 100644 index 9aee6a36..00000000 --- a/twake/backend/node/src/services/channels/services/pubsub/new-direct-channel-message.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { without } from "lodash"; -import { Channel, ChannelMember } from "../../entities"; -import { getLogger } from "../../../../core/platform/framework"; -import { MessageQueueHandler } from "../../../../core/platform/services/message-queue/api"; -import { MessageNotification } from "../../../messages/types"; -import gr from "../../../global-resolver"; -import { ExecutionContext } from "../../../../core/platform/framework/api/crud-service"; - -const logger = getLogger("channel.message-queue.new-direct-channel-message"); - -export class NewDirectChannelMessageProcessor - implements MessageQueueHandler -{ - readonly topics = { - in: "message:created", - }; - - readonly options = { - unique: true, - ack: true, - queue: "message:created:consumer1", - }; - - readonly name = "Channel::NewDirectChannelMessageProcessor"; - - validate(message: MessageNotification): boolean { - return !!(message && message.channel_id && message.company_id && message.workspace_id); - } - - async process(message: MessageNotification, context?: ExecutionContext): Promise { - logger.info( - `${this.name} - Processing notification for message ${message.thread_id}/${message.id} in channel ${message.channel_id}`, - ); - logger.debug(`${this.name} - Notification message ${JSON.stringify(message)}`); - - try { - const channel = await gr.services.channels.channels.get( - { - company_id: message.company_id, - id: message.channel_id, - workspace_id: message.workspace_id, - }, - context, - ); - - if (!channel || !Channel.isDirectChannel(channel)) { - logger.debug(`${this.name} - Not a direct channel`); - return; - } - - const memberIds = without(channel.members || [], String(message.sender)); - if (!memberIds.length) { - logger.debug(`${this.name} - No members to notify. Original array was %o`, channel.members); - } - - const members = memberIds.map(user_id => { - return { - user_id, - channel_id: channel.id, - workspace_id: channel.workspace_id, - company_id: channel.company_id, - } as ChannelMember; - }); - - await Promise.all( - members.map(async member => { - try { - logger.info(`${this.name} - Adding ${member.user_id} to channel ${message.channel_id}`); - const memberSaved = await gr.services.channels.members.save(member, { - channel, - user: { id: member.user_id }, - }); - logger.info( - `${this.name} - Member added to channel ${message.channel_id} - ${JSON.stringify( - memberSaved, - )}`, - ); - } catch (err) { - logger.info( - { err }, - `${this.name} - Error while adding user ${member.user_id} to direct channel ${member.channel_id}`, - ); - } - }), - ); - } catch (err) { - logger.error( - { err }, - `${this.name} - Error while processing direct channel members for message ${message.thread_id}/${message.id} in channel ${message.channel_id}`, - ); - } - } -} diff --git a/twake/backend/node/src/services/channels/types.ts b/twake/backend/node/src/services/channels/types.ts index 20894579..a703d724 100644 --- a/twake/backend/node/src/services/channels/types.ts +++ b/twake/backend/node/src/services/channels/types.ts @@ -52,3 +52,7 @@ export type ChannelActivityNotification = { sender_name: string; body: string; }; + +export interface CompanyExecutionContext extends ExecutionContext { + company: { id: string }; +} \ No newline at end of file diff --git a/twake/backend/node/src/services/documents/services/engine/extract-keywords.ts b/twake/backend/node/src/services/documents/services/engine/extract-keywords.ts index 7a9fa87c..37be4a8d 100644 --- a/twake/backend/node/src/services/documents/services/engine/extract-keywords.ts +++ b/twake/backend/node/src/services/documents/services/engine/extract-keywords.ts @@ -2,8 +2,7 @@ import globalResolver from "../../../../services/global-resolver"; import { logger } from "../../../../core/platform/framework"; import { MessageQueueHandler } from "../../../../core/platform/services/message-queue/api"; import { DocumentsMessageQueueCallback, DocumentsMessageQueueRequest } from "../../types"; -import { extractKeywords, officeFileToString, pdfFileToString } from "../../utils"; -import { isFileType } from "../../../../services/previews/utils"; +import { extractKeywords, officeFileToString, pdfFileToString, isFileType } from "../../utils"; import { officeExtensions, textExtensions, pdfExtensions } from "../../../../utils/mime"; import { readableToString } from "../../../../utils/files"; diff --git a/twake/backend/node/src/services/documents/utils.ts b/twake/backend/node/src/services/documents/utils.ts index 5edf1180..c12527f4 100644 --- a/twake/backend/node/src/services/documents/utils.ts +++ b/twake/backend/node/src/services/documents/utils.ts @@ -1,3 +1,4 @@ +import mimes from "../../utils/mime"; import { merge } from "lodash"; import { DriveFile } from "./entities/drive-file"; import { @@ -347,17 +348,6 @@ export const getAccessLevel = async ( repository: Repository, context: CompanyExecutionContext & { public_token?: string; twake_tab_token?: string }, ): Promise => { - if ( - context.user?.application_id && - ( - await globalResolver.services.applications.companyApps.get({ - company_id: context.company.id, - application_id: context.user.application_id, - }) - )?.application?.access //TODO check precise access right for applications - ) { - return "manage"; - } if (!id || id === "root") return !context?.user?.id ? "none" : (await isCompanyGuest(context)) ? "read" : "manage"; @@ -804,3 +794,10 @@ export const canMoveItem = async ( return true; }; + +export function isFileType(fileMime: string, fileName: string, requiredExtensions: string[]): any { + const extension = fileName.split(".").pop(); + const secondaryExtensions = Object.keys(mimes).filter(k => mimes[k] === fileMime); + const fileExtensions = [extension, ...secondaryExtensions]; + return fileExtensions.some(e => requiredExtensions.includes(e)); +} \ No newline at end of file diff --git a/twake/backend/node/src/services/files/services/index.ts b/twake/backend/node/src/services/files/services/index.ts index 3cb1c354..e8e20bc2 100644 --- a/twake/backend/node/src/services/files/services/index.ts +++ b/twake/backend/node/src/services/files/services/index.ts @@ -6,8 +6,6 @@ import { File } from "../entities/file"; import Repository from "../../../../src/core/platform/services/database/services/orm/repository/repository"; import { CompanyExecutionContext } from "../web/types"; import { logger } from "../../../core/platform/framework"; -import { PreviewClearMessageQueueRequest, PreviewMessageQueueRequest } from "../../previews/types"; -import { PreviewFinishedProcessor } from "./preview"; import _ from "lodash"; import { getDownloadRoute, getThumbnailRoute } from "../web/routes"; import { @@ -27,9 +25,6 @@ export class FileServiceImpl { try { await Promise.all([ (this.repository = await gr.database.getRepository("files", File)), - gr.platformServices.messageQueue.processor.addHandler( - new PreviewFinishedProcessor(this, this.repository), - ), ]); } catch (err) { logger.error("Error while initializing files service", err); @@ -116,61 +111,6 @@ export class FileServiceImpl { await this.repository.save(entity, context); } - //Fixme: detect in multichunk when all chunks are uploaded to trigger this. For now we do only single chunks for preview - if (entity.upload_data.chunks === 1 && totalUploadedSize) { - /** Send preview generation task */ - if (entity.upload_data.size < this.max_preview_file_size) { - const document: PreviewMessageQueueRequest["document"] = { - id: JSON.stringify(_.pick(entity, "id", "company_id")), - provider: gr.platformServices.storage.getConnectorType(), - - path: getFilePath(entity), - encryption_algo: this.algorithm, - encryption_key: entity.encryption_key, - chunks: entity.upload_data.chunks, - - filename: entity.metadata.name, - mime: entity.metadata.mime, - }; - const output = { - provider: gr.platformServices.storage.getConnectorType(), - path: `${getFilePath(entity)}/thumbnails/`, - encryption_algo: this.algorithm, - encryption_key: entity.encryption_key, - pages: 10, - }; - - entity.metadata.thumbnails_status = "waiting"; - await this.repository.save(entity, context); - - try { - await gr.platformServices.messageQueue.publish( - "services:preview", - { - data: { document, output }, - }, - ); - - if (options.waitForThumbnail) { - entity = await gr.services.files.getFile( - { - id: entity.id, - company_id: context.company.id, - }, - context, - { waitForThumbnail: true }, - ); - } - } catch (err) { - entity.metadata.thumbnails_status = "error"; - await this.repository.save(entity, context); - - logger.warn({ err }, "Previewing - Error while sending "); - } - } - - /** End preview generation task generation */ - } } return entity; @@ -286,22 +226,6 @@ export class FileServiceImpl { totalChunks: fileToDelete.upload_data.chunks, }); - if (fileToDelete.thumbnails.length > 0) { - await gr.platformServices.messageQueue.publish( - "services:preview:clear", - { - data: { - document: { - id: JSON.stringify(_.pick(fileToDelete, "id", "company_id")), - provider: gr.platformServices.storage.getConnectorType(), - path: `${path}/thumbnails/`, - thumbnails_number: fileToDelete.thumbnails.length, - }, - }, - }, - ); - } - return new DeleteResult("files", fileToDelete, true); } } diff --git a/twake/backend/node/src/services/files/services/preview.ts b/twake/backend/node/src/services/files/services/preview.ts deleted file mode 100644 index ec808eae..00000000 --- a/twake/backend/node/src/services/files/services/preview.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { logger, TwakeContext } from "../../../core/platform/framework"; -import { PreviewMessageQueueCallback } from "../../../../src/services/previews/types"; -import Repository from "../../../../src/core/platform/services/database/services/orm/repository/repository"; -import { File } from "../entities/file"; -import { FileServiceImpl } from "./index"; -import { MessageQueueHandler } from "../../../core/platform/services/message-queue/api"; -import { ExecutionContext } from "../../../core/platform/framework/api/crud-service"; - -/** - * Update the file metadata and upload the thumbnails in storage - */ -export class PreviewFinishedProcessor - implements MessageQueueHandler -{ - constructor(readonly service: FileServiceImpl, private repository: Repository) {} - - async init(_context?: TwakeContext): Promise { - return this; - } - - readonly topics = { - in: "services:preview:callback", - }; - - readonly options = { - unique: true, - ack: true, - }; - - name = "FilePreviewProcessor"; - - validate(message: PreviewMessageQueueCallback): boolean { - return !!(message && message.document && message.thumbnails); - } - - async process(message: PreviewMessageQueueCallback, context?: ExecutionContext): Promise { - logger.info( - `${this.name} - Updating file metadata with preview generation ${message.thumbnails.length}`, - ); - - const pk: { company_id: string; id: string } = JSON.parse(message.document.id); - const entity = await this.repository.findOne(pk, {}, context); - - if (!entity) { - logger.info(`This file ${message.document.id} does not exists anymore.`); - return; - } - - entity.thumbnails = message.thumbnails.map((thumb, index) => { - return { - index, - id: thumb.path.split("/").pop(), - size: thumb.size, - type: thumb.type, - width: thumb.width, - height: thumb.height, - url: `/internal/services/files/v1/companies/${entity.company_id}/files/${entity.id}/thumbnails/${index}`, - }; - }); - - if (!entity.metadata) entity.metadata = {}; - entity.metadata.thumbnails_status = "done"; - - await this.repository.save(entity, context); - return "done"; - } -} diff --git a/twake/backend/node/src/services/files/utils.ts b/twake/backend/node/src/services/files/utils.ts index 946c53c6..e0584396 100644 --- a/twake/backend/node/src/services/files/utils.ts +++ b/twake/backend/node/src/services/files/utils.ts @@ -1,4 +1,3 @@ -import { MessageFile } from "../messages/entities/message-files"; import { File, PublicFile } from "./entities/file"; export const formatPublicFile = (file: Partial): PublicFile => { @@ -16,7 +15,7 @@ export const formatPublicFile = (file: Partial): PublicFile = } as PublicFile; }; -export const fileIsMedia = (file: Partial): boolean => { +export const fileIsMedia = (file: Partial): boolean => { return ( file.metadata?.mime?.startsWith("video/") || file.metadata?.mime?.startsWith("audio/") || diff --git a/twake/backend/node/src/services/global-resolver.ts b/twake/backend/node/src/services/global-resolver.ts index 4d477f62..a7f111ed 100644 --- a/twake/backend/node/src/services/global-resolver.ts +++ b/twake/backend/node/src/services/global-resolver.ts @@ -24,15 +24,7 @@ import { WorkspaceServiceImpl } from "./workspaces/services/workspace"; import { UserExternalLinksServiceImpl } from "./user/services/external_links"; import { UserNotificationBadgeService } from "./notifications/services/bages"; import { NotificationPreferencesService } from "./notifications/services/preferences"; -import { ThreadMessagesService } from "./messages/services/messages"; -import { MessagesFilesService } from "./messages/services/messages-files"; -import { ThreadsService } from "./messages/services/threads"; -import { UserBookmarksService } from "./messages/services/user-bookmarks"; import { UserServiceImpl } from "./user/services/users/service"; -import { CompanyApplicationServiceImpl } from "./applications/services/company-applications"; -import { ApplicationServiceImpl } from "./applications/services/applications"; -import { ViewsServiceImpl } from "./messages/services/views"; -import { MessagesEngine } from "./messages/services/engine"; import { FileServiceImpl } from "./files/services"; import { ChannelServiceImpl } from "./channels/services/channel/service"; import { MemberServiceImpl } from "./channels/services/member/service"; @@ -44,13 +36,8 @@ import { NotificationEngine } from "./notifications/services/engine"; import { MobilePushService } from "./notifications/services/mobile-push"; import { ChannelMemberPreferencesServiceImpl } from "./notifications/services/channel-preferences"; import { ChannelThreadUsersServiceImpl } from "./notifications/services/channel-thread-users"; -import { PreviewProcessService } from "./previews/services/files/processing/service"; -import { ApplicationHooksService } from "./applications/services/hooks"; import OnlineServiceImpl from "./online/service"; -import { PreviewEngine } from "./previews/services/files/engine"; import { ChannelsMessageQueueListener } from "./channels/services/pubsub"; -import { LinkPreviewProcessService } from "./previews/services/links/processing/service"; -import { LinkPreviewEngine } from "./previews/services/links/engine"; import { UserNotificationDigestService } from "./notifications/services/digest"; import { DocumentsService } from "./documents/services"; import { DocumentsEngine } from "./documents/services/engine"; @@ -88,23 +75,6 @@ type TwakeServices = { mobilePush: MobilePushService; digest: UserNotificationDigestService; }; - preview: { - files: PreviewProcessService; - links: LinkPreviewProcessService; - }; - messages: { - messages: ThreadMessagesService; - messagesFiles: MessagesFilesService; - threads: ThreadsService; - userBookmarks: UserBookmarksService; - views: ViewsServiceImpl; - engine: MessagesEngine; - }; - applications: { - marketplaceApps: ApplicationServiceImpl; - companyApps: CompanyApplicationServiceImpl; - hooks: ApplicationHooksService; - }; files: FileServiceImpl; channels: { channels: ChannelServiceImpl; @@ -159,9 +129,6 @@ class GlobalResolver { assert(service, `Platform service ${key} was not initialized`); }); - await new PreviewEngine().init(); - await new LinkPreviewEngine().init(); - this.services = { workspaces: await new WorkspaceServiceImpl().init(), companies: await new CompanyServiceImpl().init(), @@ -178,23 +145,6 @@ class GlobalResolver { mobilePush: await new MobilePushService().init(), digest: await new UserNotificationDigestService().init(), }, - preview: { - files: await new PreviewProcessService().init(), - links: await new LinkPreviewProcessService().init(), - }, - messages: { - messages: await new ThreadMessagesService().init(platform), - messagesFiles: await new MessagesFilesService().init(), - threads: await new ThreadsService().init(), - userBookmarks: await new UserBookmarksService().init(), - views: await new ViewsServiceImpl().init(), - engine: await new MessagesEngine().init(), - }, - applications: { - marketplaceApps: await new ApplicationServiceImpl().init(), - companyApps: await new CompanyApplicationServiceImpl().init(), - hooks: await new ApplicationHooksService().init(), - }, files: await new FileServiceImpl().init(), channels: { channels: await new ChannelServiceImpl().init(), diff --git a/twake/backend/node/src/services/notifications/services/bages.ts b/twake/backend/node/src/services/notifications/services/bages.ts index bf278211..41676467 100644 --- a/twake/backend/node/src/services/notifications/services/bages.ts +++ b/twake/backend/node/src/services/notifications/services/bages.ts @@ -1,302 +1,261 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ import { - Initializable, - RealtimeDeleted, - RealtimeSaved, - TwakeContext, - TwakeServiceProvider, -} from "../../../core/platform/framework"; -import { ResourcePath } from "../../../core/platform/services/realtime/types"; -import { - CrudException, - DeleteResult, - ExecutionContext, - ListResult, - OperationType, - Pagination, - SaveResult, -} from "../../../core/platform/framework/api/crud-service"; -import { - getUserNotificationBadgeInstance, - UserNotificationBadge, - UserNotificationBadgePrimaryKey, - UserNotificationBadgeType, -} from "../entities"; -import { NotificationExecutionContext } from "../types"; -import { getNotificationRoomName } from "./realtime"; -import Repository from "../../../core/platform/services/database/services/orm/repository/repository"; -import gr from "../../global-resolver"; -import _, { pick, uniq } from "lodash"; - -export class UserNotificationBadgeService implements TwakeServiceProvider, Initializable { - version: "1"; - repository: Repository; - - async init(context: TwakeContext): Promise { - this.repository = await gr.database.getRepository( - UserNotificationBadgeType, - UserNotificationBadge, - ); - - return this; - } - - async get( - pk: UserNotificationBadgePrimaryKey, - context: ExecutionContext, - ): Promise { - return await this.repository.findOne(pk, {}, context); - } - - @RealtimeSaved((badge, context) => { - return [ - { - room: ResourcePath.get(getNotificationRoomName(badge.user_id)), - }, - ]; - }) - async save( - badge: UserNotificationBadge, - context: ExecutionContext, - ): Promise> { - //Initiate the digest - await gr.services.notifications.digest.putBadge(badge); - - await this.repository.save(getUserNotificationBadgeInstance(badge), context); - return new SaveResult(UserNotificationBadgeType, badge, OperationType.CREATE); - } - - @RealtimeDeleted((badge, context) => { - return [ - { - room: ResourcePath.get(getNotificationRoomName(badge.user_id)), - }, - ]; - }) - async delete( - pk: UserNotificationBadgePrimaryKey, - context?: NotificationExecutionContext, - ): Promise> { - //Cancel the current digest as we just read the badges - await gr.services.notifications.digest.cancelDigest(pk.company_id, pk.user_id); - - await this.repository.remove(pk as UserNotificationBadge, context); - return new DeleteResult(UserNotificationBadgeType, pk as UserNotificationBadge, true); - } - - list(): Promise> { - throw new Error("Not implemented"); - } - - async listForUserPerCompanies( - user_id: string, - context: ExecutionContext, - ): Promise> { - //We remove all badge from current company as next block will create dupicates - const companies_ids = (await gr.services.companies.getAllForUser(user_id)).map( - gu => gu.group_id, - ); - - let result: UserNotificationBadge[] = []; - let type = ""; - for (const company_id of companies_ids) { - const find = await this.repository.find( - { - company_id, - user_id, - }, - { - pagination: new Pagination("", "1"), - }, - context, + Initializable, + RealtimeDeleted, + RealtimeSaved, + TwakeContext, + TwakeServiceProvider, + } from "../../../core/platform/framework"; + import { ResourcePath } from "../../../core/platform/services/realtime/types"; + import { + CrudException, + DeleteResult, + ExecutionContext, + ListResult, + OperationType, + Pagination, + SaveResult, + } from "../../../core/platform/framework/api/crud-service"; + import { + getUserNotificationBadgeInstance, + UserNotificationBadge, + UserNotificationBadgePrimaryKey, + UserNotificationBadgeType, + } from "../entities"; + import { NotificationExecutionContext } from "../types"; + import { getNotificationRoomName } from "./realtime"; + import Repository from "../../../core/platform/services/database/services/orm/repository/repository"; + import gr from "../../global-resolver"; + import _, { pick, uniq } from "lodash"; + + export class UserNotificationBadgeService implements TwakeServiceProvider, Initializable { + version: "1"; + repository: Repository; + + async init(context: TwakeContext): Promise { + this.repository = await gr.database.getRepository( + UserNotificationBadgeType, + UserNotificationBadge, ); - type = find.type; - result = result.concat(find.getEntities()); + + return this; } - - const badges = new ListResult(type, result); - await this.ensureBadgesAreReachable(badges, context); - - return badges; - } - - async listForUser( - company_id: string, - user_id: string, - filter: Pick, - context?: ExecutionContext, - ): Promise> { - if (!company_id || !user_id) { - throw CrudException.badRequest("company_id and user_id are required"); + + async get( + pk: UserNotificationBadgePrimaryKey, + context: ExecutionContext, + ): Promise { + return await this.repository.findOne(pk, {}, context); } - - //Cancel the current digest as we just read the badges - await gr.services.notifications.digest.cancelDigest(company_id, user_id); - - const badges = await this.repository.find( - { - ...{ - company_id, - user_id, + + @RealtimeSaved((badge, context) => { + return [ + { + room: ResourcePath.get(getNotificationRoomName(badge.user_id)), }, - ...pick(filter, ["workspace_id", "channel_id", "thread_id"]), - }, - {}, - context, - ); - - await this.ensureBadgesAreReachable(badges, context); - - return badges; - } - - // This will ensure we are still in the channels and if not, we'll remove the badge - // We need to also ensure more than that - // - Are we in the workspace? - // - Are we in the company? - async ensureBadgesAreReachable( - badges: ListResult, - context?: ExecutionContext, - ): Promise> { - if (badges.getEntities().length === 0) { - return badges; + ]; + }) + async save( + badge: UserNotificationBadge, + context: ExecutionContext, + ): Promise> { + //Initiate the digest + await gr.services.notifications.digest.putBadge(badge); + + await this.repository.save(getUserNotificationBadgeInstance(badge), context); + return new SaveResult(UserNotificationBadgeType, badge, OperationType.CREATE); } - - const userId = badges.getEntities()[0].user_id; - - const channels = uniq(badges.getEntities().map(r => r.channel_id)); - for (const channelId of channels) { - const someBadge = badges.getEntities().find(b => b.channel_id === channelId); - const channelMemberPk = { - company_id: someBadge.company_id, - workspace_id: someBadge.workspace_id, - channel_id: channelId, - user_id: userId, - }; - const context = { - user: { id: channelMemberPk.user_id, server_request: true }, - channel: { id: channelId, ...channelMemberPk }, - }; - const exists = - (await gr.services.channels.channels.get( + + @RealtimeDeleted((badge, context) => { + return [ + { + room: ResourcePath.get(getNotificationRoomName(badge.user_id)), + }, + ]; + }) + async delete( + pk: UserNotificationBadgePrimaryKey, + context?: NotificationExecutionContext, + ): Promise> { + //Cancel the current digest as we just read the badges + await gr.services.notifications.digest.cancelDigest(pk.company_id, pk.user_id); + + await this.repository.remove(pk as UserNotificationBadge, context); + return new DeleteResult(UserNotificationBadgeType, pk as UserNotificationBadge, true); + } + + list(): Promise> { + throw new Error("Not implemented"); + } + + async listForUserPerCompanies( + user_id: string, + context: ExecutionContext, + ): Promise> { + //We remove all badge from current company as next block will create dupicates + const companies_ids = (await gr.services.companies.getAllForUser(user_id)).map( + gu => gu.group_id, + ); + + let result: UserNotificationBadge[] = []; + let type = ""; + for (const company_id of companies_ids) { + const find = await this.repository.find( { - id: channelId, - ..._.pick(channelMemberPk, "company_id", "workspace_id"), + company_id, + user_id, + }, + { + pagination: new Pagination("", "1"), }, context, - )) && (await gr.services.channels.members.get(channelMemberPk, context)); - if (!exists) { - for (const badge of badges.getEntities()) { - if (badge.channel_id === channelId) this.removeUserChannelBadges(badge, context); - } - badges.filterEntities(b => b.channel_id !== channelId); + ); + type = find.type; + result = result.concat(find.getEntities()); } + + const badges = new ListResult(type, result); + await this.ensureBadgesAreReachable(badges, context); + + return badges; } - - const badgePerWorkspace = _.uniqBy(badges.getEntities(), r => r.workspace_id); - for (const badge of badgePerWorkspace) { - const workspaceId = badge.workspace_id; - const companyId = badge.company_id; - if (!workspaceId || workspaceId === "direct") { - continue; + + async listForUser( + company_id: string, + user_id: string, + filter: Pick, + context?: ExecutionContext, + ): Promise> { + if (!company_id || !user_id) { + throw CrudException.badRequest("company_id and user_id are required"); } - try { - const exists = - (await gr.services.workspaces.get({ - id: workspaceId, - company_id: companyId, - })) && - (await gr.services.workspaces.getUser({ - workspaceId, - userId, - })); - if (!exists) { - await gr.services.channels.members.ensureUserNotInWorkspaceIsNotInChannel( - { id: userId }, - { id: workspaceId, company_id: companyId }, - context, - ); - for (const badge of badges.getEntities()) { - if (badge.workspace_id === workspaceId) this.removeUserChannelBadges(badge, context); - } - badges.filterEntities(b => b.workspace_id !== workspaceId); - } - } catch (e) {} - } - - return badges; - } - - /** - * FIXME: This is a temporary implementation which is sending as many websocket notifications as there are badges to remove - * A better implementation will be to do a bulk delete and have a single websocket notification event - * @param filter - * @param context - */ - async removeUserChannelBadges( - filter: Pick< - UserNotificationBadgePrimaryKey, - "workspace_id" | "company_id" | "channel_id" | "user_id" - >, - context?: ExecutionContext, - ): Promise { - const badges = ( - await this.repository.find( - _.pick(filter, ["workspace_id", "company_id", "channel_id", "user_id"]), + + //Cancel the current digest as we just read the badges + await gr.services.notifications.digest.cancelDigest(company_id, user_id); + + const badges = await this.repository.find( + { + ...{ + company_id, + user_id, + }, + ...pick(filter, ["workspace_id", "channel_id", "thread_id"]), + }, {}, context, - ) - ).getEntities(); - - return ( - await Promise.all( - badges.map(async badge => { - try { - return (await this.delete(badge)).deleted; - } catch (err) {} - }), - ) - ).filter(Boolean).length; - } - - /** - * acknowledge a notification and set the message status to delivered. - * - * @param {UserNotificationBadgePrimaryKey} pk - The primary key of the badge to acknowledge - * @param {ExecutionContext} context - The context of the acknowledge - * @returns {Promise} - The result of the acknowledge - */ - async acknowledge( - notification: UserNotificationBadgePrimaryKey & { message_id: string }, - context: ExecutionContext, - ): Promise { - const { message_id, ...pk } = notification; - const badge = await this.repository.findOne(pk, {}, context); - const payload = badge || notification; - - const ThreadExecutionContext = { - company: { - id: payload.company_id, - }, - thread: { - id: payload.thread_id, - }, - message_id, - ...context, - }; - - const result = await gr.services.messages.messages.updateDeliveryStatus( - { - ...payload, - status: "delivered", - }, - ThreadExecutionContext, - ); - - if (result) { - return true; + ); + + await this.ensureBadgesAreReachable(badges, context); + + return badges; } - - return false; - } -} + + // This will ensure we are still in the channels and if not, we'll remove the badge + // We need to also ensure more than that + // - Are we in the workspace? + // - Are we in the company? + async ensureBadgesAreReachable( + badges: ListResult, + context?: ExecutionContext, + ): Promise> { + if (badges.getEntities().length === 0) { + return badges; + } + + const userId = badges.getEntities()[0].user_id; + + const channels = uniq(badges.getEntities().map(r => r.channel_id)); + for (const channelId of channels) { + const someBadge = badges.getEntities().find(b => b.channel_id === channelId); + const channelMemberPk = { + company_id: someBadge.company_id, + workspace_id: someBadge.workspace_id, + channel_id: channelId, + user_id: userId, + }; + const context = { + user: { id: channelMemberPk.user_id, server_request: true }, + channel: { id: channelId, ...channelMemberPk }, + }; + const exists = + (await gr.services.channels.channels.get( + { + id: channelId, + ..._.pick(channelMemberPk, "company_id", "workspace_id"), + }, + context, + )) && (await gr.services.channels.members.get(channelMemberPk, context)); + if (!exists) { + for (const badge of badges.getEntities()) { + if (badge.channel_id === channelId) this.removeUserChannelBadges(badge, context); + } + badges.filterEntities(b => b.channel_id !== channelId); + } + } + + const badgePerWorkspace = _.uniqBy(badges.getEntities(), r => r.workspace_id); + for (const badge of badgePerWorkspace) { + const workspaceId = badge.workspace_id; + const companyId = badge.company_id; + if (!workspaceId || workspaceId === "direct") { + continue; + } + try { + const exists = + (await gr.services.workspaces.get({ + id: workspaceId, + company_id: companyId, + })) && + (await gr.services.workspaces.getUser({ + workspaceId, + userId, + })); + if (!exists) { + await gr.services.channels.members.ensureUserNotInWorkspaceIsNotInChannel( + { id: userId }, + { id: workspaceId, company_id: companyId }, + context, + ); + for (const badge of badges.getEntities()) { + if (badge.workspace_id === workspaceId) this.removeUserChannelBadges(badge, context); + } + badges.filterEntities(b => b.workspace_id !== workspaceId); + } + } catch (e) {} + } + + return badges; + } + + /** + * FIXME: This is a temporary implementation which is sending as many websocket notifications as there are badges to remove + * A better implementation will be to do a bulk delete and have a single websocket notification event + * @param filter + * @param context + */ + async removeUserChannelBadges( + filter: Pick< + UserNotificationBadgePrimaryKey, + "workspace_id" | "company_id" | "channel_id" | "user_id" + >, + context?: ExecutionContext, + ): Promise { + const badges = ( + await this.repository.find( + _.pick(filter, ["workspace_id", "company_id", "channel_id", "user_id"]), + {}, + context, + ) + ).getEntities(); + + return ( + await Promise.all( + badges.map(async badge => { + try { + return (await this.delete(badge)).deleted; + } catch (err) {} + }), + ) + ).filter(Boolean).length; + } + } \ No newline at end of file diff --git a/twake/backend/node/src/services/notifications/services/digest.ts b/twake/backend/node/src/services/notifications/services/digest.ts index a9cf3c23..e410cbd0 100644 --- a/twake/backend/node/src/services/notifications/services/digest.ts +++ b/twake/backend/node/src/services/notifications/services/digest.ts @@ -1,10 +1,8 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ -import { MessageWithReplies } from "../../../services/messages/types"; import { Initializable, TwakeServiceProvider } from "../../../core/platform/framework"; import { Paginable, Pagination } from "../../../core/platform/framework/api/crud-service"; import Repository from "../../../core/platform/services/database/services/orm/repository/repository"; import { Channel } from "../../../services/channels/entities"; -import { Message } from "../../../services/messages/entities/messages"; import { UserObject } from "../../../services/user/web/types"; import Workspace from "../../../services/workspaces/entities/workspace"; import gr from "../../global-resolver"; @@ -104,7 +102,6 @@ export class UserNotificationDigestService implements TwakeServiceProvider, Init const notifications: { channel: Channel; workspace: Workspace; - message: Message & { user: UserObject }; }[] = []; const workspaces: { [key: string]: Workspace } = {}; const channels: { [key: string]: Channel } = {}; @@ -112,15 +109,7 @@ export class UserNotificationDigestService implements TwakeServiceProvider, Init for (const badge of badges.getEntities()) { if (!badge.thread_id) continue; try { - const message = await gr.services.messages.messages.includeUsersInMessageWithReplies( - (await gr.services.messages.messages.get({ - id: badge.thread_id, - thread_id: badge.thread_id, - })) as MessageWithReplies, - ); - - if (message.created_at < digest.created_at - 60 * 1000) continue; - + channels[badge.channel_id] = channels[badge.channel_id] || (await gr.services.channels.channels.get({ @@ -141,7 +130,6 @@ export class UserNotificationDigestService implements TwakeServiceProvider, Init notifications.push({ channel: channels[badge.channel_id], workspace: workspaces[badge.workspace_id], - message: { ...message, user: (message.users || []).find(u => u.id === message.user_id) }, }); } catch (e) {} } diff --git a/twake/backend/node/src/services/notifications/services/engine/index.ts b/twake/backend/node/src/services/notifications/services/engine/index.ts index 96b6fe4e..53a7ceb9 100644 --- a/twake/backend/node/src/services/notifications/services/engine/index.ts +++ b/twake/backend/node/src/services/notifications/services/engine/index.ts @@ -1,7 +1,6 @@ import { Initializable } from "../../../../core/platform/framework"; import { MarkChannelAsReadMessageProcessor } from "./processors/mark-channel-as-read"; import { MarkChannelAsUnreadMessageProcessor } from "./processors/mark-channel-as-unread"; -import { NewChannelMessageProcessor } from "./processors/new-channel-message"; import { PushNotificationMessageProcessor } from "./processors/mobile-push-notifications"; import { PushNotificationToUsersMessageProcessor } from "./processors/push-to-users"; import { LeaveChannelMessageProcessor } from "./processors/channel-member-deleted"; @@ -24,7 +23,6 @@ export class NotificationEngine implements Initializable { gr.platformServices.messageQueue.processor.addHandler( new MarkChannelAsUnreadMessageProcessor(), ); - gr.platformServices.messageQueue.processor.addHandler(new NewChannelMessageProcessor()); gr.platformServices.messageQueue.processor.addHandler(new PushNotificationMessageProcessor()); gr.platformServices.messageQueue.processor.addHandler( new PushNotificationToUsersMessageProcessor(), diff --git a/twake/backend/node/src/services/notifications/services/engine/processors/new-channel-message.ts b/twake/backend/node/src/services/notifications/services/engine/processors/new-channel-message.ts deleted file mode 100644 index 175bdff1..00000000 --- a/twake/backend/node/src/services/notifications/services/engine/processors/new-channel-message.ts +++ /dev/null @@ -1,259 +0,0 @@ -import { isDirectChannel } from "../../../../channels/utils"; -import { logger } from "../../../../../core/platform/framework"; -import { MessageNotification } from "../../../../messages/types"; -import { - ChannelMemberNotificationPreference, - ChannelThreadUsers, - getChannelThreadUsersInstance, -} from "../../../entities"; -import { ChannelMemberNotificationLevel } from "../../../../channels/types"; -import { MentionNotification, NotificationMessageQueueHandler } from "../../../types"; -import { ChannelType } from "../../../../../utils/types"; -import gr from "../../../../global-resolver"; -import { ExecutionContext } from "../../../../../core/platform/framework/api/crud-service"; - -export class NewChannelMessageProcessor - implements NotificationMessageQueueHandler -{ - readonly topics = { - in: "message:created", - out: "notification:mentions", - }; - - readonly options = { - unique: true, - ack: true, - queue: "message:created:consumer2", - }; - - readonly name = "NewChannelMessageProcessor"; - - validate(message: MessageNotification): boolean { - return !!( - message && - message.channel_id && - message.company_id && - message.workspace_id && - message.creation_date > Date.now() - 5 * 60 * 1000 - ); - } - - async process(message: MessageNotification): Promise { - logger.info( - `${this.name} - Processing notification for message ${message.thread_id}/${message.id} in channel ${message.channel_id}`, - ); - logger.debug(`${this.name} - Notification message ${JSON.stringify(message)}`); - - try { - if (message.workspace_id === ChannelType.DIRECT) { - //Fixme: Monkey fix until we find a way to add user to channel BEFORE to add the badge to this channel - await new Promise(r => setTimeout(r, 1000)); - } - - const usersToNotify = await this.getUsersToNotify(message); - - if (!usersToNotify?.length) { - logger.info( - `${this.name} - No users to notify for message ${message.thread_id}/${message.id} in channel ${message.channel_id}`, - ); - return; - } - - logger.info( - `${this.name} - Users to notify for message ${message.thread_id}/${message.id} in channel ${ - message.channel_id - } : ['${usersToNotify.join("', '")}']`, - ); - - //Fixme Add user full names if known, need to import microservice - const users_names = {}; - //Fixme add channel full names, need to import microservice - const channels_names = {}; - - return { - channel_id: message.channel_id, - company_id: message.company_id, - message_id: message.id, - thread_id: message.thread_id, - workspace_id: message.workspace_id, - creation_date: message.creation_date, - mentions: { - users: usersToNotify || [], - }, - object_names: { - users: users_names, - channels: channels_names, - }, - - //Temp: should not be used like this when migrating messages to node - //But we don't remember why, so keeping this like this as it works well - text: message.text, - title: message.title, - } as MentionNotification; - } catch (err) { - logger.error( - { err }, - `${this.name} - Error while gettings users to notify for message ${message.thread_id}/${message.id} in channel ${message.channel_id}`, - ); - } - } - - async getUsersToNotify( - message: MessageNotification, - context?: ExecutionContext, - ): Promise { - let channelPreferencesForUsers: ChannelMemberNotificationPreference[]; - const threadId = message.thread_id || message.id; - const isNewThread = !message.thread_id || `${message.thread_id}` === `${message.id}`; - const isDirect = isDirectChannel({ workspace_id: message.workspace_id }); - const isAllOrHereMention = this.isAllOrHereMention(message); - - const users: ChannelThreadUsers[] = [ - // message sender is a user in the thread - ...[ - getChannelThreadUsersInstance({ - company_id: message.company_id, - channel_id: message.channel_id, - thread_id: threadId, - user_id: message.sender, - }), - ], - - // mentionned users are users in the thread - ...(message?.mentions?.users?.length - ? message.mentions.users.map(user_id => - getChannelThreadUsersInstance({ - company_id: message.company_id, - channel_id: message.channel_id, - thread_id: threadId, - user_id, - }), - ) - : []), - ]; - - await gr.services.notifications.channelThreads.bulkSave(users, context); - - if (isNewThread || isDirect || isAllOrHereMention) { - //get the channel level preferences - channelPreferencesForUsers = ( - await gr.services.notifications.channelPreferences.getChannelPreferencesForUsers({ - company_id: message.company_id, - channel_id: message.channel_id, - }) - ).getEntities(); - - return this.filterMembersToNotify(message, channelPreferencesForUsers).map(m => m.user_id); - } - - // get the preferences of the users involved in the thread - channelPreferencesForUsers = await this.getAllInvolvedUsersPreferences( - { - channel_id: message.channel_id, - company_id: message.company_id, - thread_id: threadId, - }, - context, - ); - - return this.filterThreadMembersToNotify(message, channelPreferencesForUsers).map( - m => m.user_id, - ); - } - - protected filterMembersToNotify( - message: MessageNotification, - membersPreferences: ChannelMemberNotificationPreference[], - ): ChannelMemberNotificationPreference[] { - logger.debug(`${this.name} - Filter members ${JSON.stringify(membersPreferences)}`); - const isAllOrHere = this.isAllOrHereMention(message); - const isDirect = isDirectChannel({ workspace_id: message.workspace_id }); - return ( - membersPreferences - // 1. Remove the ones which does not want any notification (preference === NONE) - .filter(preference => preference.preferences !== ChannelMemberNotificationLevel.NONE) - // 2. Remove the sender - .filter(preference => String(preference.user_id) !== String(message.sender)) - // 3. Filter based on truth table based on user preferences and current message - .filter(memberPreference => { - const userIsMentionned = this.userIsMentionned(memberPreference.user_id, message); - - const truthTable = [ - isDirect, - // all - memberPreference.preferences === ChannelMemberNotificationLevel.ALL, - // mentions - memberPreference.preferences === ChannelMemberNotificationLevel.MENTIONS && - (isAllOrHere || userIsMentionned), - // me - memberPreference.preferences === ChannelMemberNotificationLevel.ME && userIsMentionned, - ]; - - logger.debug( - `${this.name} - ${ - memberPreference.user_id - } truth table [direct, all, mentions, me] : ${JSON.stringify(truthTable)}`, - ); - - return truthTable.includes(true); - }) - ); - } - - protected filterThreadMembersToNotify( - message: MessageNotification, - membersPreferences: ChannelMemberNotificationPreference[], - ): ChannelMemberNotificationPreference[] { - logger.debug(`${this.name} - Filter thread members ${JSON.stringify(membersPreferences)}`); - return ( - membersPreferences - // 1. Remove the ones which does not want any notification (preference === NONE) - .filter(preference => preference.preferences !== ChannelMemberNotificationLevel.NONE) - //2. Remove the sender - .filter(preference => String(preference.user_id) !== String(message.sender)) - ); - } - - /** - * When message is a response in a thread, get all the users involved in the thread - * ie the ones who where initially mentionned, mentionned in children messages, and the ones who replied - */ - protected async getAllInvolvedUsersPreferences( - thread: { - company_id: string; - channel_id: string; - thread_id: string; - }, - context: ExecutionContext, - ): Promise { - const usersIds: string[] = ( - await gr.services.notifications.channelThreads.getUsersInThread(thread, context) - ) - .getEntities() - .map(thread => thread.user_id); - - return ( - await gr.services.notifications.channelPreferences.getChannelPreferencesForUsers( - { company_id: thread.company_id, channel_id: thread.channel_id }, - usersIds, - undefined, - context, - ) - ).getEntities(); - } - - private isAllOrHereMention(message: MessageNotification): boolean { - return ( - message.mentions && - message.mentions.specials && - (message.mentions.specials.includes("all") || - message.mentions.specials.includes("here") || - message.mentions.specials.includes("channel") || - message.mentions.specials.includes("everyone")) - ); - } - - private userIsMentionned(user: string, message: MessageNotification) { - return message?.mentions?.users?.includes(String(user)); - } -} diff --git a/twake/backend/node/src/services/notifications/services/engine/processors/push-to-users.ts b/twake/backend/node/src/services/notifications/services/engine/processors/push-to-users.ts index 44e1445f..d016df61 100644 --- a/twake/backend/node/src/services/notifications/services/engine/processors/push-to-users.ts +++ b/twake/backend/node/src/services/notifications/services/engine/processors/push-to-users.ts @@ -168,12 +168,6 @@ export class PushNotificationToUsersMessageProcessor message_id: badge.message_id, user_id: user, mention_type: mentions.users.includes(user) - ? "me" - : mentions.specials.length > 0 - ? "global" - : badge.thread_id !== badge.message_id - ? "reply" - : null, }); return this.saveBadge(badgeEntity, context); }), diff --git a/twake/backend/node/src/services/notifications/services/engine/processors/reaction-notification.ts b/twake/backend/node/src/services/notifications/services/engine/processors/reaction-notification.ts index 044cd01d..c06848ba 100644 --- a/twake/backend/node/src/services/notifications/services/engine/processors/reaction-notification.ts +++ b/twake/backend/node/src/services/notifications/services/engine/processors/reaction-notification.ts @@ -102,12 +102,7 @@ export class PushReactionNotification }), gr.services.users.get({ id: reaction_user_id }), ]); - - const msg = await gr.services.messages.messages.get({ - thread_id: thread_id, - id: message.message_id, - }); - + const companyName = company?.name || ""; const workspaceName = workspace_id === "direct" ? "Direct" : workspace?.name || ""; const userName = this.getUserName(user) || "Twake"; @@ -120,7 +115,7 @@ export class PushReactionNotification return { title, - text: `${userName}: ${reaction} to ${msg?.text || "your message"}`, + text: `${userName}: ${reaction}`, }; } diff --git a/twake/backend/node/src/services/notifications/types.ts b/twake/backend/node/src/services/notifications/types.ts index dc62345e..2d7bd4ec 100644 --- a/twake/backend/node/src/services/notifications/types.ts +++ b/twake/backend/node/src/services/notifications/types.ts @@ -1,7 +1,6 @@ import { ExecutionContext } from "../../core/platform/framework/api/crud-service"; import { Channel, ChannelMember } from "../channels/entities"; import { PaginationQueryParameters } from "../channels/web/types"; -import { SpecialMention } from "../messages/types"; import { uuid } from "../../utils/types"; import { MessageQueueHandler } from "../../core/platform/services/message-queue/api"; @@ -26,7 +25,6 @@ export type MentionNotification = { creation_date: number; mentions?: { users: uuid[]; - specials?: SpecialMention[]; }; object_names?: { users: { [id: string]: string }; diff --git a/twake/backend/node/src/services/notifications/web/controllers/notification.ts b/twake/backend/node/src/services/notifications/web/controllers/notification.ts index 0519552b..2d8a5de4 100644 --- a/twake/backend/node/src/services/notifications/web/controllers/notification.ts +++ b/twake/backend/node/src/services/notifications/web/controllers/notification.ts @@ -66,45 +66,6 @@ export class NotificationController }), }; } - - /** - * Acknowledge a notification - * - * @param {FastifyRequest} request - The request object - * @param {FastifyReply} reply - The reply object - * @returns {Promise} - The response object - */ - async acknowledge( - request: FastifyRequest<{ - Params: { - company_id: string; - }; - Body: NotificationAcknowledgeBody; - }>, - reply: FastifyReply, - ): Promise { - const context = getExecutionContext(request); - const { company_id } = request.params; - const { workspace_id, channel_id, thread_id, message_id } = request.body; - - try { - await gr.services.notifications.badges.acknowledge( - { - channel_id, - company_id, - thread_id, - user_id: context.user.id, - workspace_id, - message_id, - }, - context, - ); - - return reply.send(true); - } catch (err) { - return reply.send(false); - } - } } function getExecutionContext(request: FastifyRequest): WorkspaceExecutionContext { diff --git a/twake/backend/node/src/services/notifications/web/routes.ts b/twake/backend/node/src/services/notifications/web/routes.ts index 2749c367..29585943 100644 --- a/twake/backend/node/src/services/notifications/web/routes.ts +++ b/twake/backend/node/src/services/notifications/web/routes.ts @@ -31,13 +31,6 @@ const routes: FastifyPluginCallback = (fastify: FastifyInstance, options, next) handler: notificationPreferencesController.save.bind(notificationPreferencesController), }); - fastify.route({ - method: "POST", - url: `${badgesUrl}/:company_id/acknowledge`, - preValidation: [fastify.authenticate], - handler: notificationController.acknowledge.bind(notificationController), - }); - next(); }; diff --git a/twake/backend/node/src/services/previews/index.ts b/twake/backend/node/src/services/previews/index.ts deleted file mode 100644 index 222ae336..00000000 --- a/twake/backend/node/src/services/previews/index.ts +++ /dev/null @@ -1,23 +0,0 @@ -import WebServerAPI from "../../core/platform/services/webserver/provider"; -import { Prefix, TwakeService } from "../../core/platform/framework"; -import web from "./web"; - -@Prefix("/internal/services/previews/v1") -export default class PreviewsService extends TwakeService { - version = "1"; - name = "previews"; - - public async doInit(): Promise { - const fastify = this.context.getProvider("webserver").getServer(); - fastify.register((instance, _opts, next) => { - web(instance, { prefix: this.prefix }); - next(); - }); - return this; - } - - // TODO: remove - api(): undefined { - return undefined; - } -} diff --git a/twake/backend/node/src/services/previews/services/files/engine/clear.ts b/twake/backend/node/src/services/previews/services/files/engine/clear.ts deleted file mode 100644 index 06749c49..00000000 --- a/twake/backend/node/src/services/previews/services/files/engine/clear.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { logger, TwakeContext } from "../../../../../core/platform/framework"; -import { PreviewClearMessageQueueRequest, PreviewMessageQueueCallback } from "../../../types"; -import gr from "../../../../global-resolver"; -import { MessageQueueHandler } from "../../../../../core/platform/services/message-queue/api"; - -/** - * Clear thumbnails when the delete task is called - */ -export class ClearProcessor - implements MessageQueueHandler -{ - readonly name = "ClearProcessor"; - - readonly topics = { - in: "services:preview:clear", - out: "services:preview:callback", - }; - - readonly options = { - unique: true, - ack: true, - }; - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - init?(context?: TwakeContext): Promise { - throw new Error("Method not implemented."); - } - - validate(message: PreviewClearMessageQueueRequest): boolean { - return !!(message && message.document); - } - - async process(message: PreviewClearMessageQueueRequest): Promise { - logger.info(`${this.name} - Processing preview generation ${message.document.id}`); - - if (!this.validate(message)) { - throw new Error("Missing required fields"); - } - - for (let i = 0; i < message.document.thumbnails_number; i++) { - await gr.platformServices.storage.remove( - `${message.document.path.replace(/\/$/, "")}/${i}.png`, - ); - } - - return { document: message.document, thumbnails: [] }; - } -} diff --git a/twake/backend/node/src/services/previews/services/files/engine/index.ts b/twake/backend/node/src/services/previews/services/files/engine/index.ts deleted file mode 100644 index 0d04a2b5..00000000 --- a/twake/backend/node/src/services/previews/services/files/engine/index.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { Initializable } from "../../../../../core/platform/framework"; -import { ClearProcessor } from "./clear"; -import { PreviewProcessor } from "./service"; -import gr from "../../../../global-resolver"; - -/** - * The notification engine is in charge of processing data and delivering user notifications on the right place - */ -export class PreviewEngine implements Initializable { - async init(): Promise { - gr.platformServices.messageQueue.processor.addHandler(new PreviewProcessor()); - gr.platformServices.messageQueue.processor.addHandler(new ClearProcessor()); - return this; - } -} diff --git a/twake/backend/node/src/services/previews/services/files/engine/service.ts b/twake/backend/node/src/services/previews/services/files/engine/service.ts deleted file mode 100644 index 46f85930..00000000 --- a/twake/backend/node/src/services/previews/services/files/engine/service.ts +++ /dev/null @@ -1,118 +0,0 @@ -import fs, { promises as fsPromise } from "fs"; -import { logger } from "../../../../../core/platform/framework"; -import { - PreviewMessageQueueCallback, - PreviewMessageQueueRequest, - ThumbnailResult, -} from "../../../types"; -import gr from "../../../../global-resolver"; -import { MessageQueueHandler } from "../../../../../core/platform/services/message-queue/api"; -import { getTmpFile } from "../../../../../utils/files"; - -const { unlink } = fsPromise; -/** - * Generate thumbnails when the upload is finished - */ -export class PreviewProcessor - implements MessageQueueHandler -{ - readonly name = "PreviewProcessor"; - - init?(): Promise { - throw new Error("Method not implemented."); - } - - readonly topics = { - in: "services:preview", - out: "services:preview:callback", - }; - - readonly options = { - unique: true, - ack: true, - }; - - validate(message: PreviewMessageQueueRequest): boolean { - return !!(message && message.document && message.output); - } - - async process(message: PreviewMessageQueueRequest): Promise { - logger.info(`${this.name} - Processing preview generation ${message.document.id}`); - - let res: PreviewMessageQueueCallback = { document: message.document, thumbnails: [] }; - try { - res = await this.generate(message); - } catch (err) { - logger.error(`${this.name} - Can't generate thumbnails ${err}`); - } - - logger.info( - `${this.name} - Generated ${res.thumbnails.length} thumbnails from ${ - message.document.filename || message.document.id - }`, - ); - - return res; - } - - async generate(message: PreviewMessageQueueRequest): Promise { - //Download original file - const readable = await gr.platformServices.storage.read(message.document.path, { - totalChunks: message.document.chunks, - encryptionAlgo: message.document.encryption_algo, - encryptionKey: message.document.encryption_key, - }); - if (!readable) { - return { document: message.document, thumbnails: [] }; - } - - const inputPath = getTmpFile(); - const writable = fs.createWriteStream(inputPath); - - readable.pipe(writable); - - await new Promise(r => { - writable.on("finish", r); - }); - - writable.end(); - - //Generate previews - let localThumbnails: ThumbnailResult[] = []; - - try { - localThumbnails = await gr.services.preview.files.generateThumbnails( - { path: inputPath, mime: message.document.mime, filename: message.document.filename }, - message.output, - ); - } catch (err) { - logger.error(`${this.name} - Can't generate thumbnails ${err}`); - localThumbnails = []; - throw Error("Can't generate thumbnails."); - } - - const thumbnails: PreviewMessageQueueCallback["thumbnails"] = []; - - for (let i = 0; i < localThumbnails.length; i++) { - const uploadThumbnailPath = `${message.output.path.replace(/\/$/, "")}/${i}.png`; - const uploadThumbnail = fs.createReadStream(localThumbnails[i].path); - - await gr.platformServices.storage.write(uploadThumbnailPath, uploadThumbnail, { - encryptionAlgo: message.output.encryption_algo, - encryptionKey: message.output.encryption_key, - }); - - thumbnails.push({ - path: uploadThumbnailPath, - size: localThumbnails[i].size, - type: localThumbnails[i].type, - width: localThumbnails[i].width, - height: localThumbnails[i].height, - }); - - await unlink(localThumbnails[i].path); - } - - return { document: message.document, thumbnails }; - } -} diff --git a/twake/backend/node/src/services/previews/services/files/processing/image.ts b/twake/backend/node/src/services/previews/services/files/processing/image.ts deleted file mode 100644 index 349f5b22..00000000 --- a/twake/backend/node/src/services/previews/services/files/processing/image.ts +++ /dev/null @@ -1,54 +0,0 @@ -import sharp from "sharp"; -import { cleanFiles, getTmpFile } from "../../../../../utils/files"; -import { PreviewMessageQueueRequest, ThumbnailResult } from "../../../types"; -import { logger } from "../../../../../core/platform/framework/logger"; - -export async function generatePreview( - inputPaths: string[], - options: PreviewMessageQueueRequest["output"], -): Promise<{ - output: ThumbnailResult[]; - done: boolean; - error?: string; -}> { - const output: ThumbnailResult[] = []; - - for (const inputPath of inputPaths) { - let result: sharp.OutputInfo; - const outputPath = getTmpFile(); - try { - const inputMetadata = await sharp(inputPath).metadata(); - const outputFormat = computeNewFormat(inputMetadata, options); - - result = await sharp(inputPath).rotate().resize(outputFormat).toFile(outputPath); - output.push({ - path: outputPath, - width: result.width, - height: result.height, - type: "image/png", - size: result.size, - }); - } catch (error) { - logger.info(`sharp cant process ${error}`); - cleanFiles([outputPath]); - throw Error("Can't resize thumnail with Sharp"); - } - } - - return { - output, - done: true, - }; -} - -function computeNewFormat( - inputMetadata: sharp.Metadata, - options?: PreviewMessageQueueRequest["output"], -): { width: number; height: number } { - const maxOutputWidth = options?.width || 600; - const maxOutputHeight = options?.height || 400; - const inputWidth = inputMetadata.width; - const inputHeight = inputMetadata.height; - const scale = Math.max(inputWidth / maxOutputWidth, inputHeight / maxOutputHeight); - return { width: Math.round(inputWidth / scale), height: Math.round(inputHeight / scale) }; -} diff --git a/twake/backend/node/src/services/previews/services/files/processing/office.ts b/twake/backend/node/src/services/previews/services/files/processing/office.ts deleted file mode 100644 index 2cd99a3c..00000000 --- a/twake/backend/node/src/services/previews/services/files/processing/office.ts +++ /dev/null @@ -1,29 +0,0 @@ -// eslint-disable-next-line @typescript-eslint/no-var-requires -const unoconv = require("unoconv-promise"); -import { cleanFiles } from "../../../../../utils/files"; -import { logger } from "../../../../../core/platform/framework/logger"; - -export async function convertFromOffice( - path: string, - options: { - numberOfPages?: number; - }, -): Promise<{ output: string; done: boolean }> { - if (options.numberOfPages >= 1) { - const outputPath = `${path}.pdf`; - try { - await unoconv.run({ - file: path, - output: outputPath, - export: `PageRange=1-${options.numberOfPages}`, - }); - } catch (err) { - logger.error(`unoconv cant process ${err}`); - cleanFiles([outputPath]); - throw Error("Can't convert file with unoconv"); - } - return { output: outputPath, done: true }; - } else { - logger.error("Unoconv can't processe, number of pages lower than 1"); - } -} diff --git a/twake/backend/node/src/services/previews/services/files/processing/pdf.ts b/twake/backend/node/src/services/previews/services/files/processing/pdf.ts deleted file mode 100644 index 4ce052af..00000000 --- a/twake/backend/node/src/services/previews/services/files/processing/pdf.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { fromPath } from "pdf2pic"; -import { mkdirSync } from "fs"; -import { cleanFiles, getTmpFile } from "../../../../../utils/files"; - -export async function convertFromPdf( - inputPath: string, - options: { - numberOfPages: number; - }, -): Promise<{ output: string[]; done: boolean }> { - const pages: string[] = []; - - try { - const pdfOptions = { - density: 100, - saveFilename: "output", - savePath: getTmpFile(), - format: "png", - }; - mkdirSync(pdfOptions.savePath, { recursive: true }); - const storeAsImage = fromPath(inputPath, pdfOptions); - try { - for (let i = 1; i <= options.numberOfPages; i++) { - const image = await storeAsImage(i); - pages.push( - `${pdfOptions.savePath}/${pdfOptions.saveFilename}.${image.page}.${pdfOptions.format}`, - ); - } - } catch (err) { - if (!pages.length) { - throw err; - } - //Just no more page to convert - } - } catch (error) { - console.error(error); - for (const file of pages) { - cleanFiles([file]); - } - throw Error("Can't convert file with pdf-image."); - } - return { output: pages, done: true }; -} diff --git a/twake/backend/node/src/services/previews/services/files/processing/service.ts b/twake/backend/node/src/services/previews/services/files/processing/service.ts deleted file mode 100644 index 139a5316..00000000 --- a/twake/backend/node/src/services/previews/services/files/processing/service.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { generatePreview as thumbnailsFromImages } from "./image"; -import { convertFromOffice } from "./office"; -import { convertFromPdf } from "./pdf"; -import { isFileType } from "../../../utils"; -import { - imageExtensions, - officeExtensions, - pdfExtensions, - videoExtensions, -} from "../../../../../utils/mime"; -import { PreviewMessageQueueRequest, ThumbnailResult } from "../../../types"; -import { generateVideoPreview } from "./video"; -import { Initializable, TwakeServiceProvider } from "../../../../../core/platform/framework"; -import { cleanFiles } from "../../../../../utils/files"; - -export class PreviewProcessService implements TwakeServiceProvider, Initializable { - name: "PreviewProcessService"; - version: "1"; - - async init(): Promise { - return this; - } - - async generateThumbnails( - document: Pick, - options: PreviewMessageQueueRequest["output"], - ): Promise { - if (isFileType(document.mime, document.filename, officeExtensions)) { - const pdfPath = await convertFromOffice(document.path, { - numberOfPages: options.pages, - }); - await cleanFiles([document.path]); - const thumbnailPath = await convertFromPdf(pdfPath.output, { - numberOfPages: options.pages, - }); - await cleanFiles([pdfPath.output]); - const images = (await thumbnailsFromImages(thumbnailPath.output, options)).output; - await cleanFiles(thumbnailPath.output); - return images; - } - - if (isFileType(document.mime, document.filename, pdfExtensions)) { - const thumbnailPath = await convertFromPdf(document.path, { - numberOfPages: options.pages, - }); - await cleanFiles([document.path]); - const images = (await thumbnailsFromImages(thumbnailPath.output, options)).output; - await cleanFiles(thumbnailPath.output); - return images; - } - - if (isFileType(document.mime, document.filename, imageExtensions)) { - const images = (await thumbnailsFromImages([document.path], options)).output; - await cleanFiles([document.path]); - return images; - } - - if (isFileType(document.mime, document.filename, videoExtensions)) { - try { - const images = await generateVideoPreview([document.path]); - await cleanFiles([document.path]); - - return images; - } catch (error) { - throw Error("failed to generate video preview"); - } - } - - throw "Can not proccess, file type can't be defined"; - } -} diff --git a/twake/backend/node/src/services/previews/services/files/processing/video.ts b/twake/backend/node/src/services/previews/services/files/processing/video.ts deleted file mode 100644 index e64041dc..00000000 --- a/twake/backend/node/src/services/previews/services/files/processing/video.ts +++ /dev/null @@ -1,167 +0,0 @@ -import ffmpeg from "fluent-ffmpeg"; -import { temporaryThumbnailFile, ThumbnailResult } from "../../../types"; -import fs from "fs"; -import { cleanFiles, getTmpFile } from "../../../../../utils/files"; - -/** - * Generate thumbnails for given video files. - * - * @param {String[]} videoPaths - the input video paths - * @returns {Promise} - resolves when the thumbnails are generated - */ -export async function generateVideoPreview(videoPaths: string[]): Promise { - const output: ThumbnailResult[] = []; - - for (const videoPath of videoPaths) { - const { fileName, folder, filePath } = getTemporaryThumbnailFile(); - - try { - const { width, height } = await takeVideoScreenshot(videoPath, folder, fileName); - - output.push({ - ...getThumbnailInformation(filePath), - width, - height, - }); - } catch (error) { - cleanFiles([filePath]); - throw Error(`failed to generate video preview: ${error}`); - } - } - - return output; -} - -/** - * Generate one thumbnail from a single video file. - * the thumbnail is the first frame from the video - * - * @param {String} inputPath - the input video Path - * @param {String} outputFolder - the output folder - * @param {String} outputFile - the output file name - * @returns {Primise} - resolves when the thumbnail is generated - */ -const takeVideoScreenshot = async ( - inputPath: string, - outputFolder: string, - outputFile: string, -): Promise<{ width: number; height: number }> => { - return new Promise(async (resolve, reject) => { - try { - const { width, height } = await getVideoDimensions(inputPath); - const { width: outputWidth, height: outputHeight } = calculateThumbnailDimensions( - width, - height, - ); - - ffmpeg(inputPath) - .screenshot({ - count: 1, - filename: outputFile, - folder: outputFolder, - timemarks: ["0"], - size: `${outputWidth}x${outputHeight}`, - }) - .on("end", () => { - resolve({ width: outputWidth, height: outputHeight }); - }) - .on("error", error => { - reject(error); - }); - } catch (error) { - reject(error); - } - }); -}; - -/** - * Get the generated thumbnail information. - * - * @param {String} path - the path to the thumbnail - * @param {PreviewMessageQueueRequest["output"]} options - the options for the thumbnails - * @returns { { path: string, type: string, size: number } } - the thumbnail information - */ -const getThumbnailInformation = ( - path: string, -): { - path: string; - type: string; - size: number; -} => { - const stats = fs.statSync(path); - - return { - size: stats.size, - type: "image/jpg", - path, - }; -}; - -/** - * generate a temporary thumbnail file. - * - * @returns {temporaryThumbnailFile} - the temporary thumbnail file information - */ -const getTemporaryThumbnailFile = (): temporaryThumbnailFile => { - const filePath = `${getTmpFile()}.jpg`; - const fileName = filePath.split("/").pop(); - const folder = filePath.substring(0, filePath.lastIndexOf("/")); - - return { - folder, - fileName, - filePath, - }; -}; - -/** - * Detect video dimensions - * - * @param {String} videoPath - the video path - * @returns {Promise<{ width: number, height: number }>} - the video dimensions - */ -async function getVideoDimensions(videoPath: string): Promise<{ width: number; height: number }> { - return new Promise((resolve, reject) => { - ffmpeg.ffprobe(videoPath, (err, metadata) => { - if (err) { - reject(err); - } - - const { width, height } = metadata.streams[0]; - resolve({ width, height }); - }); - }); -} - -/** - * Calculate the thumbnail dimensions. - * maximum dimension is 1080p and the minimum dimension is 320x240. - * The aspect ratio is preserved. - * - * @param {Number} width - the video width - * @param {Number} height - the video height - * @returns { { width: number, height: number } } - the thumbnail dimensions - */ -function calculateThumbnailDimensions( - width: number, - height: number, -): { - width: number; - height: number; -} { - let newWidth = Math.min(width, 1920); - let newHeight = Math.min(height, 1080); - const ratio = width / height; - - if (width > 1920 || height > 1080) { - newWidth = Math.min(1920, width); - newHeight = Math.min(1080, newWidth / ratio); - } - - if (width < 320 || height < 240) { - newWidth = Math.max(320, width); - newHeight = Math.max(240, newWidth / ratio); - } - - return { width: newWidth, height: newHeight }; -} diff --git a/twake/backend/node/src/services/previews/services/links/engine/index.ts b/twake/backend/node/src/services/previews/services/links/engine/index.ts deleted file mode 100644 index 8f857fa3..00000000 --- a/twake/backend/node/src/services/previews/services/links/engine/index.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Initializable } from "../../../../../core/platform/framework"; -import { LinkPreviewProcessor } from "./service"; -import gr from "../../../../global-resolver"; - -export class LinkPreviewEngine implements Initializable { - async init(): Promise { - gr.platformServices.messageQueue.processor.addHandler(new LinkPreviewProcessor()); - - return this; - } -} diff --git a/twake/backend/node/src/services/previews/services/links/engine/service.ts b/twake/backend/node/src/services/previews/services/links/engine/service.ts deleted file mode 100644 index 8cd7d39e..00000000 --- a/twake/backend/node/src/services/previews/services/links/engine/service.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { - LinkPreview, - LinkPreviewMessageQueueCallback, - LinkPreviewMessageQueueRequest, -} from "../../../types"; -import { logger } from "../../../../../core/platform/framework"; -import gr from "../../../../global-resolver"; -import { MessageQueueHandler } from "../../../../../core/platform/services/message-queue/api"; - -export class LinkPreviewProcessor - implements MessageQueueHandler -{ - readonly name = "LinkPreviewProcessor"; - - readonly topics = { - in: "services:preview:links", - out: "services:preview:links:callback", - }; - - readonly options = { - unique: true, - ack: true, - }; - - init?(): Promise { - throw new Error("Method not implemented."); - } - - /** - * Checks if the message is valid - * - * @param {LinkPreviewMessageQueueRequest} message - The message to check - * @returns {Boolean} - true if the message is valid - */ - validate(message: LinkPreviewMessageQueueRequest): boolean { - return !!(message && message.links && message.links.length); - } - - /** - * process the links preview generation message - * - * @param {LinkPreviewMessageQueueRequest} message - The message to process - * @returns {Promise} - links preview callback - */ - async process(message: LinkPreviewMessageQueueRequest): Promise { - logger.info(`${this.name} - Processing preview generation for ${message.links.length} links`); - - let res: LinkPreviewMessageQueueCallback = { previews: [], message: message.message }; - - try { - res = await this.generate(message); - } catch (err) { - logger.error(`${this.name} - Can't generate link previews ${err}`); - } - - logger.info(`${this.name} - Generated ${res.previews.length} link previews`); - - return res; - } - - /** - * Generate previews for links - * - * @param {LinkPreviewMessageQueueRequest} message - The message to process - * @returns {Promise} - links preview callback - */ - async generate( - message: LinkPreviewMessageQueueRequest, - ): Promise { - let previews: LinkPreview[] = []; - try { - previews = await gr.services.preview.links.generatePreviews(message.links); - } catch (err) { - logger.error(`${this.name} - Can't generate link previews ${err}`); - throw Error(`cannot generate link previews: ${err}`); - } - - return { previews, message: message.message }; - } -} diff --git a/twake/backend/node/src/services/previews/services/links/processing/image.ts b/twake/backend/node/src/services/previews/services/links/processing/image.ts deleted file mode 100644 index 77a6c3f3..00000000 --- a/twake/backend/node/src/services/previews/services/links/processing/image.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { LinkPreview } from "../../../types"; -import { logger } from "../../../../../core/platform/framework"; -import imageProbe from "probe-image-size"; -import { getUrlFavicon, getDomain } from "../../../utils"; - -/** - * Generate a preview for a given image url. - * - * @param {String} url - the input url - * @returns {Promise} - resolves when the preview is generated - */ -export const generateImageUrlPreview = async (url: string): Promise => { - try { - const favicon = (await getUrlFavicon(getWebsiteUrl(url))) || null; - const domain = getDomain(url); - const title = url.split("/").pop(); - const { height: img_height, width: img_width } = await imageProbe(url); - - return { - title, - domain, - favicon, - url, - img_height, - img_width, - description: null, - img: url, - }; - } catch (error) { - logger.error(`failed to generate image url preview: ${error}`); - } -}; - -const getWebsiteUrl = (url: string) => { - const urlObj = new URL(url); - return `${urlObj.protocol}//${urlObj.hostname}`; -}; diff --git a/twake/backend/node/src/services/previews/services/links/processing/link.ts b/twake/backend/node/src/services/previews/services/links/processing/link.ts deleted file mode 100644 index 292699fb..00000000 --- a/twake/backend/node/src/services/previews/services/links/processing/link.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { parser } from "html-metadata-parser"; -import { LinkPreview } from "../../../types"; -import { logger } from "../../../../../core/platform/framework"; -import imageProbe from "probe-image-size"; -import { getUrlFavicon, getDomain, TIMEOUT, MAX_SIZE } from "../../../utils"; -import ua from "random-useragent"; - -type HtmlImage = { - src: string; -}; - -/** - * Generate a thumbnail for a given url. - * - * @param {String[]} urls - the input urls - * @returns {Promise} - resolves when the preview is generated - */ -export async function generateLinkPreview(url: string): Promise { - try { - return await getUrlInformation(url); - } catch (error) { - logger.error(`failed to generate link preview: ${error}`); - } -} - -/** - * get url information - * - * @param {String} url - the input url - * @returns {Promise} - resolves when the information is retrieved - */ -const getUrlInformation = async (url: string): Promise => { - try { - const parsedPage = await parser(url, { - timeout: TIMEOUT, - maxContentLength: MAX_SIZE, - headers: { - "User-Agent": ua.getRandom(), - }, - }); - const title = parsedPage.og?.title || parsedPage.meta?.title || null; - const description = parsedPage.og?.description || parsedPage.meta?.description || null; - let img = parsedPage.og?.image || parsedPage.meta?.image || parsedPage.images?.[0] || null; - - if (!title && !description && !img) { - throw new Error("not enough information to generate link preview"); - } - - const favicon = (await getUrlFavicon(url)) || null; - const domain = getDomain(url); - let img_height: number | null = null, - img_width: number | null = null; - - if (img) { - if (typeof img === "object") { - img = (img as HtmlImage).src; - } - - const dimensions = await imageProbe(img); - img_height = dimensions.height; - img_width = dimensions.width; - } - - return { - title, - domain, - description, - favicon, - img, - img_height, - img_width, - url, - }; - } catch (error) { - throw Error(`failed to get url information: ${error}`); - } -}; diff --git a/twake/backend/node/src/services/previews/services/links/processing/service.ts b/twake/backend/node/src/services/previews/services/links/processing/service.ts deleted file mode 100644 index 7a38236d..00000000 --- a/twake/backend/node/src/services/previews/services/links/processing/service.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { LinkPreview, LinkPreviewMessageQueueRequest } from "../../../types"; -import { generateLinkPreview } from "./link"; -import { checkUrlContents } from "../../../utils"; -import { generateImageUrlPreview } from "./image"; -import { - Initializable, - logger, - TwakeServiceProvider, -} from "../../../../../core/platform/framework"; - -export class LinkPreviewProcessService implements TwakeServiceProvider, Initializable { - name: "LinkPreviewProcessService"; - version: "1"; - - async init(): Promise { - return this; - } - - /** - * Generate previews for links - * - * @param {LinkPreviewMessageQueueRequest["links"]} links - input urls - * @returns {Promise} - The generated url previews - */ - async generatePreviews(links: LinkPreviewMessageQueueRequest["links"]): Promise { - const result: LinkPreview[] = []; - - for (const link of links) { - try { - const contentType = await checkUrlContents(link); - - if (!contentType) { - continue; - } - - if (contentType.includes("text/html")) { - result.push(await generateLinkPreview(link)); - } - - if (contentType.startsWith("image")) { - result.push(await generateImageUrlPreview(link)); - } - } catch (error) { - logger.error(`failed to generate link preview: ${error}`); - } - } - - return result.filter( - preview => preview && (preview.title || preview.description || preview.img), - ); - } -} diff --git a/twake/backend/node/src/services/previews/types.ts b/twake/backend/node/src/services/previews/types.ts deleted file mode 100644 index 5232ef37..00000000 --- a/twake/backend/node/src/services/previews/types.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { MessageLocalEvent } from "../messages/types"; - -export type PreviewMessageQueueRequest = { - document: { - id: string; - provider: string; - path: string; - encryption_algo?: string; - encryption_key?: string; - chunks?: number; - mime?: string; - filename?: string; - }; - output: { - provider: string; - path: string; - encryption_algo?: string; - encryption_key?: string; - pages?: number; //Max number of pages for the document - width?: number; //Max width for the thumbnails - height?: number; //Max height for the thumbnails - }; -}; - -export type PreviewClearMessageQueueRequest = { - document: { - id: string; - provider: string; - path: string; - thumbnails_number: number; - }; -}; - -export type PreviewMessageQueueCallback = { - document: { - id: string; - path: string; - provider: string; - }; - thumbnails: { - path: string; - size: number; - type: string; - width: number; - height: number; - provider?: string; - index?: number; - }[]; -}; - -export type ThumbnailResult = { - path: string; - width: number; - height: number; - size: number; - type: string; -}; - -export type temporaryThumbnailFile = { - filePath: string; - fileName: string; - folder: string; -}; - -export type LinkPreview = { - title: string; - description: string | null; - domain: string; - favicon: string | null; - img: string | null; - img_height: number | null; - img_width: number | null; - url: string; -}; - -export type LinkPreviewMessageQueueRequest = { - links: string[]; - message: MessageLocalEvent; -}; - -export type LinkPreviewMessageQueueCallback = { - message: MessageLocalEvent; - previews: LinkPreview[]; -}; diff --git a/twake/backend/node/src/services/previews/utils.ts b/twake/backend/node/src/services/previews/utils.ts deleted file mode 100644 index 5d829358..00000000 --- a/twake/backend/node/src/services/previews/utils.ts +++ /dev/null @@ -1,67 +0,0 @@ -import mimes from "../../utils/mime"; -import getFavicons from "get-website-favicon"; -import { logger } from "../../core/platform/framework"; -import axios from "axios"; - -export const TIMEOUT = 5 * 1000; -export const MAX_SIZE = 5 * 1024 * 1024; - -export function isFileType(fileMime: string, fileName: string, requiredExtensions: string[]): any { - const extension = fileName.split(".").pop(); - const secondaryExtensions = Object.keys(mimes).filter(k => mimes[k] === fileMime); - const fileExtensions = [extension, ...secondaryExtensions]; - return fileExtensions.some(e => requiredExtensions.includes(e)); -} - -/** - * get url favicon - * - * @param {String} url - the input url - * @returns {Promise} - resolves when the favicon is retrieved - */ -export const getUrlFavicon = async (url: string): Promise => { - try { - const result = await getFavicons(url); - if (!result.icons || !result.icons.length) { - return; - } - - return result.icons[0].src; - } catch (error) { - logger.error(`failed to get url favicon: ${error}`); - } -}; - -/** - * Get domain from a given url. - * - * @param {String} url - the input url - * @returns {String} - resolves when the domain is retrieved - */ -export const getDomain = (url: string): string => { - try { - const domain = new URL(url).hostname; - return domain.replace(/^www\./, ""); - } catch (error) { - throw Error(`failed to get domain: ${error}`); - } -}; - -/** - * Get url content type headers - * - * @param {string} url - the input url - * @returns {Promise} - the request headers - */ -export const checkUrlContents = async (url: string): Promise => { - try { - const response = await axios(url, { - maxContentLength: MAX_SIZE, - timeout: TIMEOUT, - }); - - return response.headers["content-type"]; - } catch (error) { - throw Error(`failed to check url contents: ${error}`); - } -}; diff --git a/twake/backend/node/src/services/previews/web/controllers/index.ts b/twake/backend/node/src/services/previews/web/controllers/index.ts deleted file mode 100644 index 8581813f..00000000 --- a/twake/backend/node/src/services/previews/web/controllers/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./previews"; diff --git a/twake/backend/node/src/services/previews/web/controllers/previews.ts b/twake/backend/node/src/services/previews/web/controllers/previews.ts deleted file mode 100644 index 21fa83a0..00000000 --- a/twake/backend/node/src/services/previews/web/controllers/previews.ts +++ /dev/null @@ -1 +0,0 @@ -export class PreviewController {} diff --git a/twake/backend/node/src/services/previews/web/index.ts b/twake/backend/node/src/services/previews/web/index.ts deleted file mode 100644 index e5dd872c..00000000 --- a/twake/backend/node/src/services/previews/web/index.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { FastifyInstance, FastifyRegisterOptions } from "fastify"; -import routes from "./routes"; - -export default ( - fastify: FastifyInstance, - options: FastifyRegisterOptions<{ - prefix: string; - }>, -): void => { - fastify.log.debug("Configuring /internal/services/previews/v1 routes"); - fastify.register(routes, options); -}; diff --git a/twake/backend/node/src/services/previews/web/routes.ts b/twake/backend/node/src/services/previews/web/routes.ts deleted file mode 100644 index f10656d3..00000000 --- a/twake/backend/node/src/services/previews/web/routes.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { FastifyInstance, FastifyPluginCallback } from "fastify"; - -const routes: FastifyPluginCallback = (fastify: FastifyInstance, options, next) => { - next(); -}; - -export default routes; diff --git a/twake/backend/node/src/services/statistics/pubsub/messages.ts b/twake/backend/node/src/services/statistics/pubsub/messages.ts deleted file mode 100644 index 1a489e5f..00000000 --- a/twake/backend/node/src/services/statistics/pubsub/messages.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { MessageQueueHandler } from "../../../core/platform/services/message-queue/api"; -import { getLogger } from "../../../core/platform/framework"; -import { MessageNotification } from "../../messages/types"; -import gr from "../../global-resolver"; - -const logger = getLogger("statistics.messages"); - -export class StatisticsMessageProcessor implements MessageQueueHandler { - readonly topics = { - in: "message:created", - }; - - readonly options = { - unique: true, - ack: true, - queue: "message:created:statistics-consumer", - }; - - readonly name = "Channel::StatisticsMessageProcessor"; - - validate(message: MessageNotification): boolean { - return !!(message && message.channel_id && message.company_id && message.workspace_id); - } - - async process(message: MessageNotification): Promise { - logger.info(`${this.name} - Processing increasing messages counter for ${message.company_id}`); - try { - await gr.services.statistics.increase(message.company_id, "messages"); - } catch (err) { - logger.error( - { err }, - `${this.name} - Error while processing direct channel members for message ${message.thread_id}/${message.id} in channel ${message.channel_id}`, - ); - } - } -} diff --git a/twake/backend/node/src/services/workspaces/services/workspace.ts b/twake/backend/node/src/services/workspaces/services/workspace.ts index 281f9527..87f6ff6e 100644 --- a/twake/backend/node/src/services/workspaces/services/workspace.ts +++ b/twake/backend/node/src/services/workspaces/services/workspace.ts @@ -150,14 +150,6 @@ export class WorkspaceServiceImpl implements TwakeServiceProvider, Initializable { company_id: workspace.company_id, user: { id: userId, server_request: true } }, ); - await gr.services.applications.companyApps.initWithDefaultApplications( - created.entity.company_id, - { - company: { id: created.entity.company_id }, - user: { id: userId, server_request: true }, - }, - ); - return new CreateResult(TYPE, created.entity); } diff --git a/twake/backend/node/src/utils/types.ts b/twake/backend/node/src/utils/types.ts index f81d0bb5..550086d3 100644 --- a/twake/backend/node/src/utils/types.ts +++ b/twake/backend/node/src/utils/types.ts @@ -10,7 +10,6 @@ import { ChannelTab, } from "../services/channels/entities"; import { ChannelParameters } from "../services/channels/web/types"; -import { MessageNotification } from "../services/messages/types"; export type uuid = string; @@ -107,7 +106,6 @@ export interface ResourceEventsPayload { channelParameters?: ChannelParameters; guest?: ChannelPendingEmails; member?: ChannelMember; - message?: Pick; actor?: User; resourcesBefore?: (User | ChannelEntity | ChannelTab | ChannelMember)[]; resourcesAfter?: (User | ChannelEntity | ChannelTab | ChannelMember)[];