📁 Changed TDrive root folder (#16)

📁 Changed TDrive root folder
This commit is contained in:
Montassar Ghanmy
2023-04-11 10:04:29 +01:00
committed by GitHub
parent 3b3f67af07
commit e0615fa867
10548 changed files with 48 additions and 48 deletions
@@ -0,0 +1,171 @@
export const stopWords = [
"about",
"above",
"after",
"again",
"all",
"also",
"am",
"an",
"and",
"another",
"any",
"are",
"as",
"at",
"be",
"because",
"been",
"before",
"being",
"below",
"between",
"both",
"but",
"by",
"came",
"can",
"cannot",
"come",
"could",
"did",
"do",
"does",
"doing",
"during",
"each",
"few",
"for",
"from",
"further",
"get",
"got",
"has",
"had",
"he",
"have",
"her",
"here",
"him",
"himself",
"his",
"how",
"if",
"in",
"into",
"is",
"it",
"its",
"itself",
"like",
"make",
"many",
"me",
"might",
"more",
"most",
"much",
"must",
"my",
"myself",
"never",
"now",
"of",
"on",
"only",
"or",
"other",
"our",
"ours",
"ourselves",
"out",
"over",
"own",
"said",
"same",
"see",
"should",
"since",
"so",
"some",
"still",
"such",
"take",
"than",
"that",
"the",
"their",
"theirs",
"them",
"themselves",
"then",
"there",
"these",
"they",
"this",
"those",
"through",
"to",
"too",
"under",
"until",
"up",
"very",
"was",
"way",
"we",
"well",
"were",
"what",
"where",
"when",
"which",
"while",
"who",
"whom",
"with",
"would",
"why",
"you",
"your",
"yours",
"yourself",
"a",
"b",
"c",
"d",
"e",
"f",
"g",
"h",
"i",
"j",
"k",
"l",
"m",
"n",
"o",
"p",
"q",
"r",
"s",
"t",
"u",
"v",
"w",
"x",
"y",
"z",
"$",
"1",
"2",
"3",
"4",
"5",
"6",
"7",
"8",
"9",
"0",
"_",
];
@@ -0,0 +1,33 @@
import { DriveFile, TYPE } from "./drive-file";
export default {
index: TYPE,
source: (entity: DriveFile) => ({
content_keywords: entity.content_keywords,
tags: (entity.tags || []).join(" "),
creator: entity.creator,
added: entity.added,
name: entity.name,
company_id: entity.company_id,
}),
mongoMapping: {
text: {
content_keywords: "text",
tags: "text",
creator: "text",
added: "text",
name: "text",
company_id: "text",
},
},
esMapping: {
properties: {
name: { type: "text", index_prefixes: { min_chars: 1 } },
content_keywords: { type: "text", index_prefixes: { min_chars: 1 } },
tags: { type: "keyword" },
creator: { type: "keyword" },
added: { type: "keyword" },
company_id: { type: "keyword" },
},
},
};
@@ -0,0 +1,90 @@
import { Type } from "class-transformer";
import { Column, Entity } from "../../../core/platform/services/database/services/orm/decorators";
import { DriveFileAccessLevel, publicAccessLevel } from "../types";
import { FileVersion } from "./file-version";
import search from "./drive-file.search";
export const TYPE = "drive_files";
@Entity(TYPE, {
globalIndexes: [["company_id", "parent_id"]],
primaryKey: [["company_id"], "id"],
type: TYPE,
search,
})
export class DriveFile {
@Type(() => String)
@Column("company_id", "uuid")
company_id: string;
@Type(() => String)
@Column("id", "uuid", { generator: "uuid" })
id: string;
@Type(() => String)
@Column("parent_id", "string")
parent_id: string;
@Type(() => Boolean)
@Column("is_in_trash", "boolean")
is_in_trash: boolean;
@Type(() => Boolean)
@Column("is_directory", "boolean")
is_directory: boolean;
@Type(() => String)
@Column("name", "string")
name: string;
@Type(() => String)
@Column("extension", "string")
extension: string;
@Type(() => String)
@Column("description", "string")
description: string;
@Column("tags", "encoded_json")
tags: string[];
@Type(() => String)
@Column("added", "string")
added: string;
@Type(() => String)
@Column("last_modified", "string")
last_modified: string;
@Column("access_info", "encoded_json")
access_info: AccessInformation;
@Type(() => String)
@Column("content_keywords", "string")
content_keywords: string;
@Type(() => String)
@Column("creator", "uuid")
creator: string;
@Type(() => Number)
@Column("size", "number")
size: number;
@Column("last_version_cache", "encoded_json")
last_version_cache: Partial<FileVersion>;
}
export type AccessInformation = {
public?: {
token: string;
level: publicAccessLevel;
};
entities: AuthEntity[];
};
type AuthEntity = {
type: "user" | "channel" | "company" | "folder";
id: string | "parent";
level: publicAccessLevel | DriveFileAccessLevel;
};
@@ -0,0 +1,30 @@
import { Type } from "class-transformer";
import { Column, Entity } from "../../../core/platform/services/database/services/orm/decorators";
export const TYPE = "drive_tdrive_tab";
@Entity(TYPE, {
primaryKey: [["company_id"], "tab_id"],
type: TYPE,
})
export class DriveTdriveTab {
@Type(() => String)
@Column("company_id", "string")
company_id: string;
@Type(() => String)
@Column("tab_id", "string")
tab_id: string;
@Type(() => String)
@Column("channel__id", "string")
channel_id: string;
@Type(() => String)
@Column("item_id", "string")
item_id: string;
@Type(() => String)
@Column("level", "string")
level: "read" | "write";
}
@@ -0,0 +1,83 @@
import { Type } from "class-transformer";
import { Column, Entity } from "../../../core/platform/services/database/services/orm/decorators";
export const TYPE = "drive_file_versions";
@Entity(TYPE, {
primaryKey: [["drive_item_id"], "id"],
type: TYPE,
})
export class FileVersion {
@Type(() => String)
@Column("drive_item_id", "uuid")
drive_item_id: string;
@Type(() => String)
@Column("id", "uuid", { generator: "uuid" })
id: string;
@Type(() => String)
@Column("provider", "string")
provider: "internal" | "drive" | string;
@Column("file_metadata", "encoded_json")
file_metadata: DriveFileMetadata;
@Type(() => Number)
@Column("date_added", "number")
date_added: number;
@Type(() => String)
@Column("creator_id", "uuid")
creator_id: string;
@Type(() => String)
@Column("application_id", "string")
application_id: string;
@Type(() => String)
@Column("realname", "string")
realname: string;
@Type(() => String)
@Column("key", "string")
key: string;
@Type(() => String)
@Column("mode", "string")
mode: string | "OpenSSL-2";
@Type(() => Number)
@Column("file_size", "number")
file_size: number;
@Type(() => String)
@Column("filename", "string")
filename: string;
@Column("data", "encoded_json")
data: unknown;
}
export type DriveFileMetadata = {
source?: "internal" | "drive" | string;
external_id: string;
name?: string;
mime?: string;
size?: number;
thumbnails?: DriveFileThumbnail[];
};
type DriveFileThumbnail = {
index: number;
id: string;
type: string;
size: number;
width: number;
height: number;
url: string;
full_url?: string;
};
@@ -0,0 +1,23 @@
import { Prefix, TdriveService } from "../../core/platform/framework";
import WebServerAPI from "../../core/platform/services/webserver/provider";
import web from "./web";
@Prefix("/internal/services/documents/v1")
export default class DocumentsService extends TdriveService<undefined> {
version = "1";
name = "documents";
public doInit: () => Promise<this> = async () => {
const fastify = this.context.getProvider<WebServerAPI>("webserver").getServer();
fastify.register((instance, _options, next) => {
web(instance, { prefix: this.prefix });
next();
});
return this;
};
api = (): undefined => {
return undefined;
};
}
@@ -0,0 +1,78 @@
import globalResolver from "../../../../services/global-resolver";
import { logger } from "../../../../core/platform/framework";
import { MessageQueueHandler } from "../../../../core/platform/services/message-queue/api";
import { DocumentsMessageQueueCallback, DocumentsMessageQueueRequest } from "../../types";
import { extractKeywords, officeFileToString, pdfFileToString, isFileType } from "../../utils";
import { officeExtensions, textExtensions, pdfExtensions } from "../../../../utils/mime";
import { readableToString } from "../../../../utils/files";
export class DocumentsProcessor
implements MessageQueueHandler<DocumentsMessageQueueRequest, DocumentsMessageQueueCallback>
{
readonly name = "DocumentsProcessor";
readonly topics = {
in: "services:documents:process",
out: "services:documents:process:callback",
};
readonly options = {
unique: true,
ack: true,
};
validate(message: DocumentsMessageQueueRequest): boolean {
return !!(
message &&
message.context &&
message.item &&
message.item.id &&
message.version &&
message.version.file_metadata &&
message.version.file_metadata.external_id &&
!message.item.is_directory
);
}
async process(message: DocumentsMessageQueueRequest): Promise<DocumentsMessageQueueCallback> {
logger.info(`${this.name} - process document content keywords for ${message.item.id}`);
return await this.generate(message);
}
async generate(message: DocumentsMessageQueueRequest): Promise<DocumentsMessageQueueCallback> {
let content_keywords = "";
let content_strings = "";
try {
const storedFile = await globalResolver.services.files.download(
message.version.file_metadata.external_id,
message.context,
);
const extension = storedFile.name.split(".").pop();
if (isFileType(storedFile.mime, storedFile.name, textExtensions)) {
logger.info("Processing text file");
content_strings = await readableToString(storedFile.file);
}
if (isFileType(storedFile.mime, storedFile.name, pdfExtensions)) {
logger.info("Processing PDF file");
content_strings = await pdfFileToString(storedFile.file);
}
if (isFileType(storedFile.mime, storedFile.name, officeExtensions)) {
logger.info("Processing office file");
content_strings = await officeFileToString(storedFile.file, extension);
}
content_keywords = extractKeywords(content_strings);
} catch (error) {
console.debug(error);
logger.error("Failed to generate content keywords", error);
}
return { content_keywords, item: message.item };
}
}
@@ -0,0 +1,18 @@
import globalResolver from "../../../global-resolver";
import { Initializable } from "../../../../core/platform/framework";
import { DocumentsProcessor } from "./extract-keywords";
import { DriveFile, TYPE } from "../../entities/drive-file";
import { DocumentsFinishedProcess } from "./save-keywords";
export class DocumentsEngine implements Initializable {
async init(): Promise<this> {
const repository = await globalResolver.database.getRepository<DriveFile>(TYPE, DriveFile);
globalResolver.platformServices.messageQueue.processor.addHandler(new DocumentsProcessor());
globalResolver.platformServices.messageQueue.processor.addHandler(
new DocumentsFinishedProcess(repository),
);
return this;
}
}
@@ -0,0 +1,65 @@
import { logger } from "../../../../core/platform/framework";
import { ExecutionContext } from "../../../../core/platform/framework/api/crud-service";
import Repository from "../../../../core/platform/services/database/services/orm/repository/repository";
import { MessageQueueHandler } from "../../../../core/platform/services/message-queue/api";
import { DriveFile } from "../../entities/drive-file";
import { DocumentsMessageQueueCallback } from "../../types";
export class DocumentsFinishedProcess
implements MessageQueueHandler<DocumentsMessageQueueCallback, void>
{
constructor(private repository: Repository<DriveFile>) {}
readonly name = "DocumentsFinishedProcess";
readonly topics = {
in: "services:documents:process:callback",
};
readonly options = {
unique: true,
ack: true,
};
init? = (): Promise<this> => {
throw Error("Method not implemented.");
};
validate = (message: DocumentsMessageQueueCallback): boolean => {
return !!(
message &&
message.content_keywords &&
message.content_keywords.length &&
message.item &&
message.item.id
);
};
process = async (
message: DocumentsMessageQueueCallback,
context?: ExecutionContext,
): Promise<void> => {
logger.info(`${this.name} - updating drive item content keywords`);
try {
const entity = await this.repository.findOne(
{
id: message.item.id,
company_id: message.item.company_id,
},
{},
context,
);
if (!entity) {
throw Error("Drive item not found");
}
entity.content_keywords = message.content_keywords;
return await this.repository.save(entity, context);
} catch (error) {
logger.error(`${this.name} - Failed to set content keywords`, error);
return;
}
};
}
@@ -0,0 +1,840 @@
import SearchRepository from "../../../core/platform/services/search/repository";
import { getLogger, logger, TdriveLogger } from "../../../core/platform/framework";
import { CrudException, ListResult } from "../../../core/platform/framework/api/crud-service";
import Repository from "../../../core/platform/services/database/services/orm/repository/repository";
import { PublicFile } from "../../../services/files/entities/file";
import globalResolver from "../../../services/global-resolver";
import { hasCompanyAdminLevel } from "../../../utils/company";
import gr from "../../global-resolver";
import { DriveFile, TYPE } from "../entities/drive-file";
import { FileVersion, TYPE as FileVersionType } from "../entities/file-version";
import {
DriveTdriveTab as DriveTdriveTabEntity,
TYPE as DriveTdriveTabRepoType,
} from "../entities/drive-tdrive-tab";
import {
DriveExecutionContext,
DocumentsMessageQueueRequest,
DriveItemDetails,
RootType,
SearchDocumentsOptions,
TrashType,
CompanyExecutionContext,
DriveTdriveTab,
} from "../types";
import {
addDriveItemToArchive,
calculateItemSize,
canMoveItem,
checkAccess,
getAccessLevel,
getDefaultDriveItem,
getDefaultDriveItemVersion,
getFileMetadata,
getItemName,
getPath,
hasAccessLevel,
makeStandaloneAccessLevel,
updateItemSize,
} from "../utils";
import { websocketEventBus } from "../../../core/platform/services/realtime/bus";
import archiver from "archiver";
import internal from "stream";
import {
RealtimeEntityActionType,
ResourcePath,
} from "../../../core/platform/services/realtime/types";
export class DocumentsService {
version: "1";
repository: Repository<DriveFile>;
searchRepository: SearchRepository<DriveFile>;
fileVersionRepository: Repository<FileVersion>;
driveTdriveTabRepository: Repository<DriveTdriveTabEntity>;
ROOT: RootType = "root";
TRASH: TrashType = "trash";
logger: TdriveLogger = getLogger("Documents Service");
async init(): Promise<this> {
try {
this.repository = await globalResolver.database.getRepository<DriveFile>(TYPE, DriveFile);
this.searchRepository = globalResolver.platformServices.search.getRepository<DriveFile>(
TYPE,
DriveFile,
);
this.fileVersionRepository = await globalResolver.database.getRepository<FileVersion>(
FileVersionType,
FileVersion,
);
this.driveTdriveTabRepository =
await globalResolver.database.getRepository<DriveTdriveTabEntity>(
DriveTdriveTabRepoType,
DriveTdriveTabEntity,
);
} catch (error) {
logger.error("Error while initializing Documents Service", error);
}
return this;
}
/**
* Fetches a drive element
*
* @param {string} id - the id of the DriveFile to fetch or "trash" or an empty string for root folder.
* @param {DriveExecutionContext} context
* @returns {Promise<DriveItemDetails>}
*/
get = async (id: string, context: DriveExecutionContext): Promise<DriveItemDetails> => {
if (!context) {
this.logger.error("invalid context");
return null;
}
id = id || this.ROOT;
//Get requested entity
const entity =
id === this.ROOT || id === this.TRASH
? null
: await this.repository.findOne(
{
company_id: context.company.id,
id,
},
{},
context,
);
if (!entity && !(id === this.ROOT || id === this.TRASH)) {
this.logger.error("Drive item not found");
throw new CrudException("Item not found", 404);
}
//Check access to entity
try {
const hasAccess = await checkAccess(id, entity, "read", this.repository, context);
if (!hasAccess) {
this.logger.error("user does not have access drive item ", id);
throw Error("user does not have access to this item");
}
} catch (error) {
this.logger.error("Failed to grant access to the drive item", error);
throw new CrudException("User does not have access to this item or its children", 401);
}
const isDirectory = entity ? entity.is_directory : true;
//Get entity version in case of a file
const versions = isDirectory
? []
: (
await this.fileVersionRepository.find(
{
drive_item_id: entity.id,
},
{},
context,
)
).getEntities();
//Get children if it is a directory
let children = isDirectory
? (
await this.repository.find(
{
company_id: context.company.id,
parent_id: id,
},
{},
context,
)
).getEntities()
: [];
//Check each children for access
const accessMap: { [key: string]: boolean } = {};
await Promise.all(
children.map(async child => {
accessMap[child.id] = await checkAccess(child.id, child, "read", this.repository, context);
}),
);
children = children.filter(child => accessMap[child.id]);
//Return complete object
return {
path: await getPath(id, this.repository, false, context),
item:
entity ||
({
id,
parent_id: null,
name: id === this.ROOT ? "root" : id === this.TRASH ? "trash" : "unknown",
size: await calculateItemSize(
id === this.ROOT ? this.ROOT : "trash",
this.repository,
context,
),
} as DriveFile),
versions: versions,
children: children,
access: await getAccessLevel(id, entity, this.repository, context),
};
};
/**
* Creates a DriveFile item.
*
* @param {PublicFile} file - the multipart file
* @param {Partial<DriveFile>} content - the DriveFile item to create
* @param {Partial<FileVersion>} version - the DriveFile version.
* @param {DriveExecutionContext} context - the company execution context.
* @returns {Promise<DriveFile>} - the created DriveFile
*/
create = async (
file: PublicFile | null,
content: Partial<DriveFile>,
version: Partial<FileVersion>,
context: DriveExecutionContext,
): Promise<DriveFile> => {
try {
const driveItem = getDefaultDriveItem(content, context);
const driveItemVersion = getDefaultDriveItemVersion(version, context);
const hasAccess = await checkAccess(
driveItem.parent_id,
null,
"write",
this.repository,
context,
);
if (!hasAccess) {
this.logger.error("User does not have access to parent drive item", driveItem.parent_id);
throw Error("User does not have access to this item parent");
}
if (file || driveItem.is_directory === false) {
let fileToProcess;
if (file) {
fileToProcess = file;
} else if (driveItemVersion.file_metadata.external_id) {
fileToProcess = await globalResolver.services.files.getFile(
{
id: driveItemVersion.file_metadata.external_id,
company_id: driveItem.company_id,
},
context,
{ waitForThumbnail: true },
);
}
if (fileToProcess) {
driveItem.size = fileToProcess.upload_data.size;
driveItem.is_directory = false;
driveItem.extension = fileToProcess.metadata.name.split(".").pop();
driveItemVersion.filename = driveItemVersion.filename || fileToProcess.metadata.name;
driveItemVersion.file_size = fileToProcess.upload_data.size;
driveItemVersion.file_metadata.external_id = fileToProcess.id;
driveItemVersion.file_metadata.mime = fileToProcess.metadata.mime;
driveItemVersion.file_metadata.size = fileToProcess.upload_data.size;
driveItemVersion.file_metadata.name = fileToProcess.metadata.name;
driveItemVersion.file_metadata.thumbnails = fileToProcess.thumbnails;
if (context.user.application_id) {
driveItemVersion.application_id = context.user.application_id;
}
}
}
driveItem.name = await getItemName(
driveItem.parent_id,
driveItem.id,
driveItem.name,
driveItem.is_directory,
this.repository,
context,
);
await this.repository.save(driveItem);
driveItemVersion.drive_item_id = driveItem.id;
await this.fileVersionRepository.save(driveItemVersion);
driveItem.last_version_cache = driveItemVersion;
await this.repository.save(driveItem);
await updateItemSize(driveItem.parent_id, this.repository, context);
this.notifyWebsocket(driveItem.parent_id, context);
globalResolver.platformServices.messageQueue.publish<DocumentsMessageQueueRequest>(
"services:documents:process",
{
data: {
item: driveItem,
version: driveItemVersion,
context,
},
},
);
return driveItem;
} catch (error) {
this.logger.error("Failed to create drive item", error);
throw new CrudException("Failed to create item", 500);
}
};
/**
* Updates a DriveFile item
*
* @param {string} id - the id of the DriveFile to update.
* @param {Partial<DriveFile>} content - the updated content
* @param {DriveExecutionContext} context - the company execution context
* @returns {Promise<DriveFile>} - the updated DriveFile
*/
update = async (
id: string,
content: Partial<DriveFile>,
context: CompanyExecutionContext,
): Promise<DriveFile> => {
if (!context) {
this.logger.error("invalid execution context");
return null;
}
try {
let oldParent = null;
const level = await getAccessLevel(id, null, this.repository, context);
const hasAccess = hasAccessLevel("write", level);
if (!hasAccess) {
this.logger.error("user does not have access drive item ", id);
throw Error("user does not have access to this item");
}
const item = await this.repository.findOne({
company_id: context.company.id,
id,
});
if (!item) {
this.logger.error("Drive item not found");
throw Error("Item not found");
}
if (content.id && content.id !== id) {
this.logger.error("content mismatch");
throw Error("content mismatch");
}
const updatable = ["access_info", "name", "tags", "parent_id", "description"];
for (const key of updatable) {
if ((content as any)[key]) {
if (
key === "parent_id" &&
!(await canMoveItem(item.id, content.parent_id, this.repository, context))
) {
throw Error("Move operation not permitted");
} else {
oldParent = item.parent_id;
}
if (key === "name") {
item.name = await getItemName(
content.parent_id || item.parent_id,
item.id,
content.name,
item.is_directory,
this.repository,
context,
);
} else {
(item as any)[key] = (content as any)[key];
}
}
}
//We cannot do a change that would make the item unreachable
if (
level === "manage" &&
!(await checkAccess(item.id, item, "manage", this.repository, context))
) {
throw new Error("Cannot change access level to make the item unreachable");
}
await this.repository.save(item);
await updateItemSize(item.parent_id, this.repository, context);
if (oldParent) {
await updateItemSize(oldParent, this.repository, context);
this.notifyWebsocket(oldParent, context);
}
this.notifyWebsocket(item.parent_id, context);
if (item.parent_id === this.TRASH) {
//When moving to trash we recompute the access level to make them flat
item.access_info = await makeStandaloneAccessLevel(
item.company_id,
item.id,
item,
this.repository,
);
}
return item;
} catch (error) {
this.logger.error("Failed to update drive item", error);
throw new CrudException("Failed to update item", 500);
}
};
/**
* deletes or moves to Trash a Drive Document and its children
*
* @param {string} id - the item id
* @param {DriveExecutionContext} context - the execution context
* @returns {Promise<void>}
*/
delete = async (
id: string | RootType | TrashType,
item?: DriveFile,
context?: DriveExecutionContext,
): Promise<void> => {
if (!id) {
//We can't remove the root folder
return;
}
//In the case of the trash we definitively delete the items
if (id === "trash") {
//Only administrators can execute this action
const role = await gr.services.companies.getUserRole(context.company.id, context.user?.id);
if (hasCompanyAdminLevel(role) === false) {
throw new CrudException("Only administrators can empty the trash", 403);
}
try {
const itemsInTrash = await this.repository.find(
{
company_id: context.company.id,
parent_id: "trash",
},
{},
context,
);
await Promise.all(
itemsInTrash.getEntities().map(async item => {
await this.delete(item.id, item, context);
}),
);
} catch (error) {
this.logger.error("Failed to empty trash", error);
throw new CrudException("Failed to empty trash", 500);
}
return;
} else {
item =
item ||
(await this.repository.findOne({
company_id: context.company.id,
id,
}));
if (!item) {
this.logger.error("item to delete not found");
throw new CrudException("Drive item not found", 404);
}
try {
if (!(await checkAccess(item.id, item, "manage", this.repository, context))) {
this.logger.error("user does not have access drive item ", id);
throw Error("user does not have access to this item");
}
} catch (error) {
this.logger.error("Failed to grant access to the drive item", error);
throw new CrudException("User does not have access to this item or its children", 401);
}
const previousParentId = item.parent_id;
if (
item.parent_id === this.TRASH ||
(await getPath(item.parent_id, this.repository, true, context))[0].id === this.TRASH
) {
//This item is already in trash, we can delete it definitively
if (item.is_directory) {
//We delete the children
const children = await this.repository.find(
{
company_id: context.company.id,
parent_id: item.id,
},
{},
context,
);
await Promise.all(
children.getEntities().map(async child => {
await this.delete(child.id, child, context);
}),
);
} else {
//Delete the version and stored file
const itemVersions = await this.fileVersionRepository.find(
{
drive_item_id: item.id,
},
{},
context,
);
await Promise.all(
itemVersions.getEntities().map(async version => {
await this.fileVersionRepository.remove(version);
await gr.services.files.delete(version.file_metadata.external_id, context);
}),
);
}
await this.repository.remove(item);
} else {
//This item is not in trash, we move it to trash
item.parent_id = this.TRASH;
await this.update(item.id, item, context);
}
await updateItemSize(previousParentId, this.repository, context);
this.notifyWebsocket(previousParentId, context);
}
this.notifyWebsocket("trash", context);
};
/**
* Create a Drive item version
*
* @param {string} id - the Drive item id to create a version for.
* @param {Partial<FileVersion>} version - the version item.
* @param {DriveExecutionContext} context - the company execution context
* @returns {Promise<FileVersion>} - the created FileVersion
*/
createVersion = async (
id: string,
version: Partial<FileVersion>,
context: DriveExecutionContext,
): Promise<FileVersion> => {
if (!context) {
this.logger.error("invalid execution context");
return null;
}
try {
const hasAccess = await checkAccess(id, null, "write", this.repository, context);
if (!hasAccess) {
this.logger.error("user does not have access drive item ", id);
throw Error("user does not have access to this item");
}
const item = await this.repository.findOne(
{
id,
company_id: context.company.id,
},
{},
context,
);
if (!item) {
throw Error("Drive item not found");
}
if (item.is_directory) {
throw Error("cannot create version for a directory");
}
const driveItemVersion = getDefaultDriveItemVersion(version, context);
const metadata = await getFileMetadata(driveItemVersion.file_metadata.external_id, context);
driveItemVersion.file_size = metadata.size;
driveItemVersion.file_metadata.size = metadata.size;
driveItemVersion.file_metadata.name = metadata.name;
driveItemVersion.file_metadata.mime = metadata.mime;
driveItemVersion.file_metadata.thumbnails = metadata.thumbnails;
driveItemVersion.drive_item_id = item.id;
if (context.user.application_id) {
driveItemVersion.application_id = context.user.application_id;
}
await this.fileVersionRepository.save(driveItemVersion);
item.last_version_cache = driveItemVersion;
item.size = driveItemVersion.file_size;
await this.repository.save(item);
this.notifyWebsocket(item.parent_id, context);
await updateItemSize(item.parent_id, this.repository, context);
globalResolver.platformServices.messageQueue.publish<DocumentsMessageQueueRequest>(
"services:documents:process",
{
data: {
item,
version: driveItemVersion,
context,
},
},
);
return driveItemVersion;
} catch (error) {
this.logger.error("Failed to create Drive item version", error);
throw new CrudException("Failed to create Drive item version", 500);
}
};
downloadGetToken = async (
ids: string[],
versionId: string | null,
context: DriveExecutionContext,
): Promise<string> => {
for (const id of ids) {
const item = await this.get(id, context);
if (!item) {
throw new CrudException("Drive item not found", 404);
}
}
return globalResolver.platformServices.auth.sign({
ids,
version_id: versionId,
company_id: context.company.id,
user_id: context.user?.id,
});
};
applyDownloadTokenToContext = async (
ids: string[],
versionId: string | null,
token: string,
context: DriveExecutionContext,
): Promise<void> => {
try {
const v = globalResolver.platformServices.auth.verifyTokenObject<{
ids: string[];
version_id: string;
company_id: string;
user_id: string;
}>(token);
if (
ids.some(a => !(v?.ids || [])?.includes(a)) ||
(v?.version_id && versionId && v?.version_id !== versionId)
) {
return;
}
context.company = { id: v.company_id };
context.user = { id: v.user_id };
} catch (e) {
if (token) throw new CrudException("Invalid token", 401);
}
};
download = async (
id: string,
versionId: string | null,
context: DriveExecutionContext,
): Promise<{
archive?: archiver.Archiver;
file?: {
file: internal.Readable;
name: string;
mime: string;
size: number;
};
}> => {
const item = await this.get(id, context);
if (item.item.is_directory) {
return { archive: await this.createZip([id], context) };
}
let version = item.item.last_version_cache;
if (versionId) version = item.versions.find(version => version.id === versionId);
if (!version) {
throw new CrudException("Version not found", 404);
}
const fileId = version.file_metadata.external_id;
const file = await globalResolver.services.files.download(fileId, context);
return { file };
};
/**
* Creates a zip archive containing the drive items.
*
* @param {string[]} ids - the drive item list
* @param {DriveExecutionContext} context - the execution context
* @returns {Promise<archiver.Archiver>} the created archive.
*/
createZip = async (
ids: string[] = [],
context: DriveExecutionContext,
): Promise<archiver.Archiver> => {
if (!context) {
this.logger.error("invalid execution context");
return null;
}
const archive = archiver("zip", {
zlib: { level: 9 },
});
for (const id of ids) {
if (!(await checkAccess(id, null, "read", this.repository, context))) {
this.logger.warn(`not enough permissions to download ${id}, skipping`);
return;
}
try {
await addDriveItemToArchive(id, null, archive, this.repository, context);
} catch (error) {
console.error(error);
this.logger.warn("failed to add item to archive", error);
}
}
archive.finalize();
return archive;
};
notifyWebsocket = async (id: string, context: DriveExecutionContext) => {
websocketEventBus.publish(RealtimeEntityActionType.Event, {
type: "documents:updated",
room: ResourcePath.get(`/companies/${context.company.id}/documents/item/${id}`),
entity: {
companyId: context.company.id,
id: id,
},
resourcePath: null,
result: null,
});
};
/**
* Search for Drive items.
*
* @param {SearchDocumentsOptions} options - the search optins.
* @param {DriveExecutionContext} context - the execution context.
* @returns {Promise<ListResult<DriveFile>>} - the search result.
*/
search = async (
options: SearchDocumentsOptions,
context?: DriveExecutionContext,
): Promise<ListResult<DriveFile>> => {
const result = await this.searchRepository.search(
{},
{
pagination: {
limitStr: "100",
},
...(options.company_id ? { $in: [["company_id", [options.company_id]]] } : {}),
...(options.creator ? { $in: [["creator", [options.creator]]] } : {}),
...(options.added ? { $in: [["added", [options.added]]] } : {}),
$text: {
$search: options.search,
},
},
context,
);
// Use Promise.all to check access on each item in parallel
const filteredResult = await Promise.all(
result.getEntities().filter(async item => {
try {
// Check access for each item
const hasAccess = await checkAccess(item.id, null, "read", this.repository, context);
// Return true if the user has access
return hasAccess;
} catch (error) {
this.logger.warn("failed to check item access", error);
return false;
}
}),
);
return new ListResult(result.type, filteredResult, result.nextPage);
};
getTab = async (tabId: string, context: CompanyExecutionContext): Promise<DriveTdriveTab> => {
const tab = await this.driveTdriveTabRepository.findOne(
{ company_id: context.company.id, tab_id: tabId },
{},
context,
);
return tab;
};
setTab = async (
tabId: string,
channelId: string,
itemId: string,
level: "read" | "write",
context: CompanyExecutionContext,
): Promise<DriveTdriveTab> => {
const hasAccess = await checkAccess(itemId, null, "manage", this.repository, context);
if (!hasAccess) {
throw new CrudException("Not enough permissions", 403);
}
const previousTabConfiguration = await this.getTab(tabId, context);
const item = await this.repository.findOne(
{
company_id: context.company.id,
id: itemId,
},
{},
context,
);
await this.driveTdriveTabRepository.save(
Object.assign(new DriveTdriveTabEntity(), {
company_id: context.company.id,
tab_id: tabId,
channel_id: channelId,
item_id: itemId,
level,
}),
context,
);
await this.update(
item.id,
{
...item,
access_info: {
...item.access_info,
entities: [
...(item.access_info?.entities || []).filter(
e =>
!previousTabConfiguration ||
!(e.type === "channel" && e.id !== previousTabConfiguration.channel_id),
),
{
type: "channel",
id: channelId,
level: level === "write" ? "write" : "read",
},
],
},
},
context,
);
return await this.getTab(tabId, context);
};
}
@@ -0,0 +1,75 @@
import { ExecutionContext } from "../../core/platform/framework/api/crud-service";
import { DriveFile } from "./entities/drive-file";
import { FileVersion } from "./entities/file-version";
export interface CompanyExecutionContext extends ExecutionContext {
company: { id: string };
}
export type DriveExecutionContext = CompanyExecutionContext & {
public_token?: string;
};
export type RequestParams = {
company_id: string;
};
export type ItemRequestParams = RequestParams & {
id: string;
};
export type DriveItemDetails = {
path: DriveFile[];
item?: DriveFile;
versions?: FileVersion[];
children: DriveFile[];
access: DriveFileAccessLevel | "none";
};
export type DriveFileAccessLevel = "read" | "write" | "manage";
export type publicAccessLevel = "write" | "read" | "none";
export type RootType = "root";
export type TrashType = "trash";
export type DownloadZipBodyRequest = {
items: string[];
};
export type SearchDocumentsOptions = {
search?: string;
company_id?: string;
creator?: string;
added?: string;
};
export type SearchDocumentsBody = {
search?: string;
company_id?: string;
creator?: string;
added?: string;
};
export type DocumentsMessageQueueRequest = {
item: DriveFile;
version: FileVersion;
context: CompanyExecutionContext;
};
export type DocumentsMessageQueueCallback = {
item: DriveFile;
content_keywords: string;
};
export type exportKeywordPayload = {
file_id: string;
company_id: string;
};
export type DriveTdriveTab = {
company_id: string;
tab_id: string;
channel_id: string;
item_id: string;
level: "read" | "write";
};
@@ -0,0 +1,802 @@
import mimes from "../../utils/mime";
import { merge } from "lodash";
import { DriveFile } from "./entities/drive-file";
import {
CompanyExecutionContext,
DriveExecutionContext,
DriveFileAccessLevel,
RootType,
TrashType,
} from "./types";
import crypto from "crypto";
import { FileVersion, DriveFileMetadata } from "./entities/file-version";
import globalResolver from "../global-resolver";
import Repository from "../../core/platform/services/database/services/orm/repository/repository";
import archiver from "archiver";
import { Readable } from "stream";
import { stopWords } from "./const";
import unoconv from "unoconv-promise";
import {
writeToTemporaryFile,
cleanFiles,
getTmpFile,
readFromTemporaryFile,
readableToBuffer,
} from "../../utils/files";
import PdfParse from "pdf-parse";
import _ from "lodash";
import { logger } from "../../core/platform/framework";
/**
* Returns the default DriveFile object using existing data
*
* @param {Partial<DriveFile>} item - the partial drive file item.
* @param {CompanyExecutionContext} context - the company execution context
* @returns {DriveFile} - the Default DriveFile
*/
export const getDefaultDriveItem = (
item: Partial<DriveFile>,
context: CompanyExecutionContext,
): DriveFile => {
const defaultDriveItem = merge<DriveFile, Partial<DriveFile>>(new DriveFile(), {
company_id: context.company.id,
added: item.added || new Date().getTime().toString(),
creator: item.creator || context.user?.id,
is_directory: item.is_directory || false,
is_in_trash: item.is_in_trash || false,
last_modified: new Date().getTime().toString(),
parent_id: item.parent_id || "root",
content_keywords: item.content_keywords || "",
description: item.description || "",
access_info: item.access_info || {
entities: [
{
id: "parent",
type: "folder",
level: "manage",
},
{
id: item.company_id,
type: "company",
level: "none",
},
{
id: context.user?.id,
type: "user",
level: "manage",
},
],
public: {
level: "none",
token: generateAccessToken(),
},
},
extension: item.extension || "",
last_version_cache: item.last_version_cache,
name: item.name || "",
size: item.size || 0,
tags: item.tags || [],
});
if (item.id) {
defaultDriveItem.id = item.id;
}
return defaultDriveItem;
};
/**
* Returns the default FileVersion item.
*
* @param {Partial<FileVersion>} version - the partial version item
* @param {CompanyExecutionContext} context - the execution context
* @returns
*/
export const getDefaultDriveItemVersion = (
version: Partial<FileVersion>,
context: CompanyExecutionContext,
): FileVersion => {
const defaultVersion = merge(new FileVersion(), {
application_id: version.application_id || "",
creator_id: version.creator_id || context.user?.id,
data: version.data || {},
date_added: version.date_added || new Date().getTime(),
drive_item_id: version.drive_item_id || "",
file_metadata: version.file_metadata || {},
file_size: version.file_size || 0,
filename: version.filename || "",
key: version.key || "",
mode: version.mode || "OpenSSL-2",
provider: version.provider,
realname: version.realname,
});
if (version.id) {
defaultVersion.id = version.id;
}
return defaultVersion;
};
/**
* Generates a random sha1 access token
*
* @returns {String} - the random access token ( sha1 hex digest ).
*/
export const generateAccessToken = (): string => {
const randomBytes = crypto.randomBytes(64);
return crypto.createHash("sha1").update(randomBytes).digest("hex");
};
/**
* Checks if the level meets the required level.
*
* @param {publicAccessLevel | DriveFileAccessLevel} requiredLevel
* @param {publicAccessLevel} level
* @returns {boolean}
*/
export const hasAccessLevel = (
requiredLevel: DriveFileAccessLevel | "none",
level: DriveFileAccessLevel | "none",
): boolean => {
if (requiredLevel === level) return true;
if (requiredLevel === "write") {
return level === "manage";
}
if (requiredLevel === "read") {
return level === "manage" || level === "write";
}
return requiredLevel === "none";
};
/**
* checks the current user is a guest
*
* @param {CompanyExecutionContext} context
* @returns {Promise<boolean>}
*/
export const isCompanyGuest = async (context: CompanyExecutionContext): Promise<boolean> => {
const userRole = await globalResolver.services.companies.getUserRole(
context.company.id,
context.user?.id,
);
return userRole === "guest" || !userRole;
};
/**
* checks the current user is a admin
*
* @param {CompanyExecutionContext} context
* @returns {Promise<boolean>}
*/
export const isCompanyAdmin = async (context: CompanyExecutionContext): Promise<boolean> => {
const userRole = await globalResolver.services.companies.getUserRole(
context.company.id,
context.user?.id,
);
return userRole === "admin";
};
/**
* Calculates the size of the Drive Item
*
* @param {DriveFile} item - The file or directory
* @param {Repository<DriveFile>} repository - the database repository
* @param {CompanyExecutionContext} context - the execution context
* @returns {Promise<number>} - the size of the Drive Item
*/
export const calculateItemSize = async (
item: DriveFile | TrashType | RootType,
repository: Repository<DriveFile>,
context: CompanyExecutionContext,
): Promise<number> => {
if (item === "trash") {
const trashedItems = await repository.find(
{ company_id: context.company.id, parent_id: "trash" },
{},
context,
);
return trashedItems.getEntities().reduce((acc, curr) => acc + curr.size, 0);
}
if (item === "root" || !item) {
const rootFolderItems = await repository.find(
{ company_id: context.company.id, parent_id: "root" },
{},
context,
);
return rootFolderItems.getEntities().reduce((acc, curr) => acc + curr.size, 0);
}
if (item.is_directory) {
const children = await repository.find(
{
company_id: context.company.id,
parent_id: item.id,
},
{},
context,
);
return children.getEntities().reduce((acc, curr) => acc + curr.size, 0);
}
return item.size;
};
/**
* Recalculates and updates the Drive item size
*
* @param {string} id - the item id
* @param {Repository<DriveFile>} repository
* @param {CompanyExecutionContext} context - the execution context
* @returns {Promise<void>}
*/
export const updateItemSize = async (
id: string,
repository: Repository<DriveFile>,
context: CompanyExecutionContext,
): Promise<void> => {
if (!id || id === "root" || id === "trash") return;
const item = await repository.findOne({ id, company_id: context.company.id });
if (!item) {
throw Error("Drive item doesn't exist");
}
item.size = await calculateItemSize(item, repository, context);
await repository.save(item);
if (item.parent_id === "root" || item.parent_id === "trash") {
return;
}
await updateItemSize(item.parent_id, repository, context);
};
/**
* gets the path for the driveitem
*
* @param {string} id
* @param {Repository<DriveFile>} repository
* @param {boolean} ignoreAccess
* @param {CompanyExecutionContext} context
* @returns
*/
export const getPath = async (
id: string,
repository: Repository<DriveFile>,
ignoreAccess?: boolean,
context?: DriveExecutionContext,
): Promise<DriveFile[]> => {
id = id || "root";
if (id === "root" || id === "trash")
return !context.public_token || ignoreAccess
? [
{
id,
name: id === "root" ? "Home" : "Trash",
} as DriveFile,
]
: [];
const item = await repository.findOne({
id,
company_id: context.company.id,
});
if (!item || (!(await checkAccess(id, item, "read", repository, context)) && !ignoreAccess)) {
return [];
}
return [...(await getPath(item.parent_id, repository, ignoreAccess, context)), item];
};
/**
* checks if access can be granted for the drive item
*
* @param {string} id
* @param {DriveFile | null} item
* @param {DriveFileAccessLevel} level
* @param {Repository<DriveFile>} repository
* @param {CompanyExecutionContext} context
* @param {string} token
* @returns {Promise<boolean>}
*/
export const checkAccess = async (
id: string,
item: DriveFile | null,
level: DriveFileAccessLevel,
repository: Repository<DriveFile>,
context: CompanyExecutionContext & { public_token?: string; tdrive_tab_token?: string },
): Promise<boolean> => {
if (context.user?.server_request) {
return true;
}
const grantedLevel = await getAccessLevel(id, item, repository, context);
const hasAccess = hasAccessLevel(level, grantedLevel);
logger.info(
`Got level ${grantedLevel} for drive item ${id} and required ${level} - returning ${hasAccess}`,
);
return hasAccess;
};
/**
* get maximum level for the drive item
*
* @param {string} id
* @param {DriveFile | null} item
* @param {Repository<DriveFile>} repository
* @param {CompanyExecutionContext} context
* @param {string} token
* @returns {Promise<boolean>}
*/
export const getAccessLevel = async (
id: string,
item: DriveFile | null,
repository: Repository<DriveFile>,
context: CompanyExecutionContext & { public_token?: string; tdrive_tab_token?: string },
): Promise<DriveFileAccessLevel | "none"> => {
if (!id || id === "root")
return !context?.user?.id ? "none" : (await isCompanyGuest(context)) ? "read" : "manage";
if (id === "trash")
return (await isCompanyGuest(context)) || !context?.user?.id
? "none"
: (await isCompanyAdmin(context))
? "manage"
: "write";
const publicToken = context.public_token;
try {
item =
item ||
(await repository.findOne({
id,
company_id: context.company.id,
}));
if (!item) {
throw Error("Drive item doesn't exist");
}
/*
* Specific user or channel rule is applied first. Then less restrictive level will be chosen
* between the parent folder and company accesses.
*/
//Public access
if (publicToken) {
if (!item.access_info.public.token) return "none";
const { token: itemToken, level: itemLevel } = item.access_info.public;
if (itemToken === publicToken) return itemLevel;
}
const accessEntities = item.access_info.entities || [];
const otherLevels = [];
//From there a user must be logged in
if (context?.user?.id) {
//Users
const matchingUser = accessEntities.find(a => a.type === "user" && a.id === context.user?.id);
if (matchingUser) return matchingUser.level;
//Channels
if (context.tdrive_tab_token) {
try {
const [channelId] = context.tdrive_tab_token.split("+"); //First item will be the channel id
const matchingChannel = accessEntities.find(
a => a.type === "channel" && a.id === channelId,
);
if (matchingChannel) return matchingChannel.level;
} catch (e) {
console.log(e);
}
}
//Companies
const matchingCompany = accessEntities.find(
a => a.type === "company" && a.id === context.company.id,
);
if (matchingCompany) otherLevels.push(matchingCompany.level);
}
//Parent folder
const maxParentFolderLevel =
accessEntities.find(a => a.type === "folder" && a.id === "parent")?.level || "none";
if (maxParentFolderLevel === "none") {
otherLevels.push(maxParentFolderLevel);
} else {
const parentFolderLevel = await getAccessLevel(item.parent_id, null, repository, context);
otherLevels.push(parentFolderLevel);
}
//Return least restrictive level of otherLevels
return otherLevels.reduce(
(previousValue, b) =>
hasAccessLevel(b as DriveFileAccessLevel, previousValue as DriveFileAccessLevel)
? previousValue
: b,
"none",
) as DriveFileAccessLevel | "none";
} catch (error) {
throw Error(error);
}
};
/**
* Isolate access level information from parent folder logic
* Used when putting folder in the trash
* @param id
* @param item
* @param repository
*/
export const makeStandaloneAccessLevel = async (
companyId: string,
id: string,
item: DriveFile | null,
repository: Repository<DriveFile>,
options: { removePublicAccess?: boolean } = { removePublicAccess: true },
): Promise<DriveFile["access_info"]> => {
item =
item ||
(await repository.findOne({
id,
company_id: companyId,
}));
if (!item) {
throw Error("Drive item doesn't exist");
}
const accessInfo = _.cloneDeep(item.access_info);
if (options?.removePublicAccess && accessInfo?.public?.level) accessInfo.public.level = "none";
const parentFolderAccess = accessInfo.entities.find(
a => a.type === "folder" && a.id === "parent",
);
if (!parentFolderAccess || parentFolderAccess.level === "none") {
return accessInfo;
} else if (item.parent_id !== "root" && item.parent_id !== "trash") {
// Get limitations from parent folder
const accessEntitiesFromParent = await makeStandaloneAccessLevel(
companyId,
item.parent_id,
null,
repository,
options,
);
let mostRestrictiveFolderLevel = parentFolderAccess.level as DriveFileAccessLevel | "none";
const keptEntities = accessEntitiesFromParent.entities.filter(a => {
if (["user", "channel"].includes(a.type)) {
return !accessInfo.entities.find(b => b.type === a.type && b.id === a.id);
} else {
if (a.type === "folder" && a.id === "parent") {
mostRestrictiveFolderLevel = hasAccessLevel(a.level, mostRestrictiveFolderLevel)
? a.level
: mostRestrictiveFolderLevel;
}
return false;
}
});
accessInfo.entities = accessInfo.entities.map(a => {
if (a.type === "folder" && a.id === "parent") {
a.level = mostRestrictiveFolderLevel;
}
return a;
}) as DriveFile["access_info"]["entities"];
accessInfo.entities = [...accessInfo.entities, ...keptEntities];
}
return accessInfo;
};
/**
* Adds drive items to an archive recursively
*
* @param {string} id - the drive item id
* @param {DriveFile | null } entity - the drive item entity
* @param {archiver.Archiver} archive - the archive
* @param {Repository<DriveFile>} repository - the repository
* @param {CompanyExecutionContext} context - the execution context
* @param {string} prefix - folder prefix
* @returns {Promise<void>}
*/
export const addDriveItemToArchive = async (
id: string,
entity: DriveFile | null,
archive: archiver.Archiver,
repository: Repository<DriveFile>,
context: CompanyExecutionContext,
prefix?: string,
): Promise<void> => {
const item = entity || (await repository.findOne({ id, company_id: context.company.id }));
if (!item) {
throw Error("item not found");
}
if (!item.is_directory) {
const file_id = item.last_version_cache.file_metadata.external_id;
const file = await globalResolver.services.files.download(file_id, context);
if (!file) {
throw Error("file not found");
}
archive.append(file.file, { name: file.name, prefix: prefix ?? "" });
return;
} else {
const items = await repository.find({
parent_id: item.id,
company_id: context.company.id,
});
for (const child of items.getEntities()) {
await addDriveItemToArchive(
child.id,
child,
archive,
repository,
context,
`${prefix || ""}${item.name}/`,
);
}
return;
}
};
/**
* Extracts the most popular 250 keywords from a text.
*
* @param {string} data - file data string.
* @returns {string}
*/
export const extractKeywords = (data: string): string => {
const words = data.toLowerCase().split(/[^a-zA-Z']+/);
const filteredWords = words.filter(word => !stopWords.includes(word) && word.length > 3);
const wordFrequency = filteredWords.reduce((acc: Record<string, number>, word: string) => {
acc[word] = (acc[word] || 0) + 1;
return acc;
}, {});
const sortedFrequency = Object.entries(wordFrequency)
.sort((a, b) => b[1] - a[1])
.reduce((acc: Record<string, number>, [key, val]) => {
acc[key] = val;
return acc;
}, {});
return Object.keys(sortedFrequency).slice(0, 250).join(" ");
};
/**
* Converts an office file stream into a human readable string.
*
* @param {Readable} file - the input file stream.
* @param {string} extension - the file extension.
* @returns {Promise<string>}
*/
export const officeFileToString = async (file: Readable, extension: string): Promise<string> => {
const officeFilePath = await writeToTemporaryFile(file, extension);
const outputPath = getTmpFile(".pdf");
try {
await unoconv.run({
file: officeFilePath,
output: outputPath,
});
cleanFiles([officeFilePath]);
return await pdfFileToString(outputPath);
} catch (error) {
cleanFiles([officeFilePath]);
throw Error(error);
}
};
/**
* Converts a PDF file stream into a human readable string.
*
* @param {Readable | string} file - the input file stream or path.
* @returns {Promise<string>}
*/
export const pdfFileToString = async (file: Readable | string): Promise<string> => {
let inputBuffer: Buffer;
try {
if (typeof file === "string") {
inputBuffer = await readFromTemporaryFile(file);
cleanFiles([file]);
} else {
inputBuffer = await readableToBuffer(file);
}
const result = await PdfParse(inputBuffer);
return result.text;
} catch (error) {
if (typeof file === "string") {
cleanFiles([file]);
}
throw Error(error);
}
};
/**
* returns the file metadata.
*
* @param {string} fileId - the file id
* @param {CompanyExecutionContext} context - the execution context
* @returns {DriveFileMetadata}
*/
export const getFileMetadata = async (
fileId: string,
context: CompanyExecutionContext,
): Promise<DriveFileMetadata> => {
const file = await globalResolver.services.files.getFile(
{
id: fileId,
company_id: context.company.id,
},
context,
{ ...(context.user.server_request ? {} : { waitForThumbnail: true }) },
);
if (!file) {
throw Error("File doesn't exist");
}
return {
source: "internal",
external_id: fileId,
mime: file.metadata.mime,
name: file.metadata.name,
size: file.upload_data.size,
thumbnails: file.thumbnails,
} as DriveFileMetadata;
};
/**
* Finds a suitable name for an item based on items inside the same folder.
*
* @param {string} parent_id - the parent id.
* @param {string} name - the item name.
* @param {Repository<DriveFile>} repository - the drive repository.
* @param {CompanyExecutionContext} context - the execution context.
* @returns {Promise<string>} - the drive item name.
*/
export const getItemName = async (
parent_id: string,
id: string,
name: string,
is_directory: boolean,
repository: Repository<DriveFile>,
context: CompanyExecutionContext,
): Promise<string> => {
try {
let newName = name.substring(0, 255);
let exists = true;
const children = await repository.find(
{
parent_id,
company_id: context.company.id,
},
{},
context,
);
while (exists) {
exists = !!children
.getEntities()
.find(
child => child.name === newName && child.is_directory === is_directory && child.id !== id,
);
if (exists) {
const ext = newName.split(".").pop();
newName =
ext && ext !== newName ? `${newName.slice(0, -ext.length - 1)}-2.${ext}` : `${newName}-2`;
}
}
return newName;
} catch (error) {
throw Error("Failed to get item name");
}
};
/**
* Checks if an item can be moved to its destination
* An item cannot be moved to itself or any of its derived chilren.
*
* @param {string} source - the to be moved item id.
* @param {string} target - the to be moved to item id.
* @param {string} repository - the Drive item repository.
* @param {CompanyExecutionContex} context - the execution context.
* @returns {Promise<boolean>} - whether the move is possible or not.
*/
export const canMoveItem = async (
source: string,
target: string,
repository: Repository<DriveFile>,
context: CompanyExecutionContext,
): Promise<boolean> => {
if (source === target) return false;
if (target === "root" || target === "trash") return true;
const item = await repository.findOne({
id: source,
company_id: context.company.id,
});
if (!item.is_directory) {
return true;
}
const targetItem = await repository.findOne({
id: target,
company_id: context.company.id,
});
if (!targetItem || !targetItem.is_directory) {
throw Error("target item doesn't exist or not a directory");
}
if (!(await checkAccess(target, targetItem, "write", repository, context))) {
return false;
}
if (!item) {
throw Error("Item not found");
}
const children = (
await repository.find({
parent_id: source,
company_id: context.company.id,
})
).getEntities();
if (children.some(child => child.id === target)) {
return false;
}
for (const child of children) {
if (child.is_directory && !(await canMoveItem(child.id, target, repository, context))) {
return false;
}
}
return true;
};
export function isFileType(fileMime: string, fileName: string, requiredExtensions: string[]): any {
const extension = fileName.split(".").pop();
const secondaryExtensions = Object.keys(mimes).filter(k => mimes[k] === fileMime);
const fileExtensions = [extension, ...secondaryExtensions];
return fileExtensions.some(e => requiredExtensions.includes(e));
}
@@ -0,0 +1,401 @@
import { FastifyReply, FastifyRequest } from "fastify";
import { logger } from "../../../../core/platform/framework";
import { CrudException, ListResult } from "../../../../core/platform/framework/api/crud-service";
import { File } from "../../../../services/files/entities/file";
import { UploadOptions } from "../../../../services/files/types";
import globalResolver from "../../../../services/global-resolver";
import { PaginationQueryParameters, ResourceWebsocket } from "../../../../utils/types";
import { DriveFile } from "../../entities/drive-file";
import { FileVersion } from "../../entities/file-version";
import {
CompanyExecutionContext,
DriveExecutionContext,
DriveItemDetails,
DriveTdriveTab,
ItemRequestParams,
RequestParams,
SearchDocumentsBody,
SearchDocumentsOptions,
} from "../../types";
export class DocumentsController {
/**
* Creates a DriveFile item
*
* @param {FastifyRequest} request
* @returns
*/
create = async (
request: FastifyRequest<{
Params: RequestParams;
Querystring: Record<string, string>;
Body: {
item: Partial<DriveFile>;
version: Partial<FileVersion>;
};
}>,
): Promise<DriveFile> => {
try {
const context = getDriveExecutionContext(request);
let createdFile: File = null;
if (request.isMultipart()) {
const file = await request.file();
const q = request.query;
const options: UploadOptions = {
totalChunks: parseInt(q.resumableTotalChunks || q.total_chunks) || 1,
totalSize: parseInt(q.resumableTotalSize || q.total_size) || 0,
chunkNumber: parseInt(q.resumableChunkNumber || q.chunk_number) || 1,
filename: q.resumableFilename || q.filename || file?.filename || undefined,
type: q.resumableType || q.type || file?.mimetype || undefined,
waitForThumbnail: !!q.thumbnail_sync,
};
createdFile = await globalResolver.services.files.save(null, file, options, context);
}
const { item, version } = request.body;
return await globalResolver.services.documents.documents.create(
createdFile,
item,
version,
context,
);
} catch (error) {
logger.error("Failed to create Drive item", error);
throw new CrudException("Failed to create Drive item", 500);
}
};
/**
* Deletes a DriveFile item or empty the trash or delete root folder contents
*
* @param {FastifyRequest} request
* @returns {Promise<void>}
*/
delete = async (
request: FastifyRequest<{ Params: ItemRequestParams; Querystring: { public_token?: string } }>,
reply: FastifyReply,
): Promise<void> => {
try {
const context = getDriveExecutionContext(request);
await globalResolver.services.documents.documents.delete(request.params.id, null, context);
reply.status(200).send();
} catch (error) {
logger.error("Failed to delete drive item", error);
throw new CrudException("Failed to delete drive item", 500);
}
};
/**
* Lists the drive root folder.
*
* @param {FastifyRequest} request
* @returns {Promise<DriveItemDetails>}
*/
listRootFolder = async (
request: FastifyRequest<{
Params: RequestParams;
Querystring: PaginationQueryParameters & { public_token?: string };
}>,
): Promise<DriveItemDetails> => {
const context = getDriveExecutionContext(request);
return await globalResolver.services.documents.documents.get(null, context);
};
/**
* Fetches a DriveFile item.
*
* @param {FastifyRequest} request
* @returns {Promise<DriveItemDetails>}
*/
get = async (
request: FastifyRequest<{
Params: ItemRequestParams;
Querystring: PaginationQueryParameters & { public_token?: string };
}>,
): Promise<DriveItemDetails & { websockets: ResourceWebsocket[] }> => {
const context = getDriveExecutionContext(request);
const { id } = request.params;
return {
...(await globalResolver.services.documents.documents.get(id, context)),
websockets: request.currentUser?.id
? globalResolver.platformServices.realtime.sign(
[{ room: `/companies/${context.company.id}/documents/item/${id}` }],
request.currentUser?.id,
)
: [],
};
};
/**
* Update drive item
*
* @param {FastifyRequest} request
* @returns {Promise<DriveFile>}
*/
update = async (
request: FastifyRequest<{
Params: ItemRequestParams;
Body: Partial<DriveFile>;
Querystring: { public_token?: string };
}>,
): Promise<DriveFile> => {
const context = getDriveExecutionContext(request);
const { id } = request.params;
const update = request.body;
if (!id) throw new CrudException("Missing id", 400);
return await globalResolver.services.documents.documents.update(id, update, context);
};
/**
* Create a drive file version.
*
* @param {FastifyRequest} request
* @returns {Promise<FileVersion>}
*/
createVersion = async (
request: FastifyRequest<{
Params: ItemRequestParams;
Body: Partial<FileVersion>;
Querystring: { public_token?: string };
}>,
): Promise<FileVersion> => {
const context = getDriveExecutionContext(request);
const { id } = request.params;
const version = request.body;
if (!id) throw new CrudException("Missing id", 400);
return await globalResolver.services.documents.documents.createVersion(id, version, context);
};
downloadGetToken = async (
request: FastifyRequest<{
Params: ItemRequestParams;
Querystring: { version_id?: string; items?: string; public_token?: string };
}>,
): Promise<{ token: string }> => {
const ids = (request.query.items || "").split(",");
const context = getDriveExecutionContext(request);
return {
token: await globalResolver.services.documents.documents.downloadGetToken(
ids,
request.query.version_id,
context,
),
};
};
/**
* Shortcut to download a file (you can also use the file-service directly).
* If the item is a folder, a zip will be automatically generated.
*
* @param {FastifyRequest} request
* @param {FastifyReply} reply
*/
download = async (
request: FastifyRequest<{
Params: ItemRequestParams;
Querystring: { version_id?: string; token?: string; public_token?: string };
}>,
response: FastifyReply,
): Promise<void> => {
const context = getDriveExecutionContext(request);
const id = request.params.id || "";
const versionId = request.query.version_id || null;
const token = request.query.token;
await globalResolver.services.documents.documents.applyDownloadTokenToContext(
[id],
versionId,
token,
context,
);
try {
const archiveOrFile = await globalResolver.services.documents.documents.download(
id,
versionId,
context,
);
if (archiveOrFile.archive) {
const archive = archiveOrFile.archive;
archive.on("finish", () => {
response.status(200);
});
archive.on("error", () => {
response.internalServerError();
});
archive.pipe(response.raw);
} else if (archiveOrFile.file) {
const data = archiveOrFile.file;
const filename = data.name.replace(/[^a-zA-Z0-9 -_.]/g, "");
response.header("Content-disposition", `attachment; filename="${filename}"`);
if (data.size) response.header("Content-Length", data.size);
response.type(data.mime);
response.send(data.file);
}
} catch (error) {
logger.error("failed to download file", error);
throw new CrudException("Failed to download file", 500);
}
};
/**
* Downloads a zip archive containing the drive items.
*
* @param {FastifyRequest} request
* @param {FastifyReply} reply
*/
downloadZip = async (
request: FastifyRequest<{
Params: RequestParams;
Querystring: { token?: string; items: string; public_token?: string };
}>,
reply: FastifyReply,
): Promise<void> => {
const context = getDriveExecutionContext(request);
let ids = (request.query.items || "").split(",");
const token = request.query.token;
await globalResolver.services.documents.documents.applyDownloadTokenToContext(
ids,
null,
token,
context,
);
if (ids[0] === "root") {
const items = await globalResolver.services.documents.documents.get(ids[0], context);
ids = items.children.map(item => item.id);
}
try {
const archive = await globalResolver.services.documents.documents.createZip(ids, context);
archive.on("finish", () => {
reply.status(200);
});
archive.on("error", () => {
reply.internalServerError();
});
archive.pipe(reply.raw);
} catch (error) {
logger.error("failed to send zip file", error);
throw new CrudException("Failed to create zip file", 500);
}
};
/**
* Search for documents.
*
* @param {FastifyRequest} request
* @returns {Promise<ListResult<DriveFile>>}
*/
search = async (
request: FastifyRequest<{
Params: RequestParams;
Body: SearchDocumentsBody;
Querystring: { public_token?: string };
}>,
): Promise<ListResult<DriveFile>> => {
try {
const context = getDriveExecutionContext(request);
const { search = "", added = "", company_id = "", creator = "" } = request.body;
const options: SearchDocumentsOptions = {
company_id: company_id || context.company.id,
...(search ? { search } : {}),
...(added ? { added } : {}),
...(creator ? { creator } : {}),
};
if (!Object.keys(options).length) {
throw Error("Search options are empty");
}
return await globalResolver.services.documents.documents.search(options, context);
} catch (error) {
logger.error("error while searching for document", error);
throw new CrudException("Failed to search for documents", 500);
}
};
getTab = async (
request: FastifyRequest<{
Params: { tab_id: string; company_id: string };
}>,
): Promise<DriveTdriveTab> => {
const context = getCompanyExecutionContext(request);
const { tab_id } = request.params;
return await globalResolver.services.documents.documents.getTab(tab_id, context);
};
setTab = async (
request: FastifyRequest<{
Params: { tab_id: string; company_id: string };
Body: DriveTdriveTab;
}>,
): Promise<DriveTdriveTab> => {
const context = getCompanyExecutionContext(request);
const { tab_id } = request.params;
if (!request.body.channel_id || !request.body.item_id)
throw new Error("Missing parameters (channel_id, item_id)");
return await globalResolver.services.documents.documents.setTab(
tab_id,
request.body.channel_id,
request.body.item_id,
request.body.level,
context,
);
};
}
/**
* Gets the company execution context
*
* @param { FastifyRequest<{ Params: { company_id: string } }>} req
* @returns {CompanyExecutionContext}
*/
const getDriveExecutionContext = (
req: FastifyRequest<{ Params: { company_id: string }; Querystring: { public_token?: string } }>,
): DriveExecutionContext => ({
public_token: req.query.public_token,
user: req.currentUser,
company: { id: req.params.company_id },
url: req.url,
method: req.routerMethod,
reqId: req.id,
transport: "http",
});
function getCompanyExecutionContext(
request: FastifyRequest<{
Params: { company_id: string };
}>,
): CompanyExecutionContext {
return {
user: request.currentUser,
company: { id: request.params.company_id },
url: request.url,
method: request.routerMethod,
reqId: request.id,
transport: "http",
};
}
@@ -0,0 +1 @@
export * from "./documents";
@@ -0,0 +1,12 @@
import fastifyCaching from "@fastify/caching";
import { FastifyInstance, FastifyRegisterOptions } from "fastify";
import routes from "./routes";
export default (
fastify: FastifyInstance,
options: FastifyRegisterOptions<{ prefix: string }>,
): void => {
fastify.log.debug("configuring /internal/services/documents/v1 routes");
fastify.register(fastifyCaching, { expiresIn: 31536000, privacy: fastifyCaching.privacy.PUBLIC });
fastify.register(routes, options);
};
@@ -0,0 +1,100 @@
import { FastifyInstance, FastifyPluginCallback } from "fastify";
import { DocumentsController } from "./controllers";
import { createDocumentSchema, createVersionSchema } from "./schemas";
const baseUrl = "/companies/:company_id";
const serviceUrl = `${baseUrl}/item`;
const routes: FastifyPluginCallback = (fastify: FastifyInstance, _options, next) => {
const documentsController = new DocumentsController();
fastify.route({
method: "GET",
url: `${serviceUrl}`,
preValidation: [fastify.authenticateOptional],
handler: documentsController.listRootFolder.bind(documentsController),
});
fastify.route({
method: "GET",
url: `${serviceUrl}/:id`,
preValidation: [fastify.authenticateOptional],
handler: documentsController.get.bind(documentsController),
});
fastify.route({
method: "POST",
url: serviceUrl,
preValidation: [fastify.authenticateOptional],
schema: createDocumentSchema,
handler: documentsController.create.bind(documentsController),
});
fastify.route({
method: "POST",
url: `${serviceUrl}/:id`,
preValidation: [fastify.authenticateOptional],
handler: documentsController.update.bind(documentsController),
});
fastify.route({
method: "DELETE",
url: `${serviceUrl}/:id`,
preValidation: [fastify.authenticateOptional],
handler: documentsController.delete.bind(documentsController),
});
fastify.route({
method: "POST",
url: `${serviceUrl}/:id/version`,
preValidation: [fastify.authenticateOptional],
schema: createVersionSchema,
handler: documentsController.createVersion.bind(documentsController),
});
fastify.route({
method: "GET",
url: `${serviceUrl}/download/token`,
preValidation: [fastify.authenticateOptional],
handler: documentsController.downloadGetToken.bind(documentsController),
});
fastify.route({
method: "GET",
url: `${serviceUrl}/:id/download`,
preValidation: [fastify.authenticateOptional],
handler: documentsController.download.bind(documentsController),
});
fastify.route({
method: "GET",
url: `${serviceUrl}/download/zip`,
preValidation: [fastify.authenticateOptional],
handler: documentsController.downloadZip.bind(documentsController),
});
fastify.route({
method: "POST",
url: `${baseUrl}/search`,
preValidation: [fastify.authenticate],
handler: documentsController.search.bind(documentsController),
});
fastify.route({
method: "GET",
url: `${baseUrl}/tabs/:tab_id`,
preValidation: [fastify.authenticate],
handler: documentsController.getTab.bind(documentsController),
});
fastify.route({
method: "POST",
url: `${baseUrl}/tabs/:tab_id`,
preValidation: [fastify.authenticate],
handler: documentsController.setTab.bind(documentsController),
});
return next();
};
export default routes;
@@ -0,0 +1,108 @@
const fileVersionSchema = {
type: "object",
properties: {
id: { type: "string" },
provider: { type: "string" },
drive_item_id: { type: "string" },
file_metadata: {
type: "object",
properties: {
source: { type: "string" },
external_id: { type: "string" },
name: { type: "string" },
mime: { type: "string" },
size: { type: "string" },
thumbnails: {
type: "object",
properties: {
index: { type: "number" },
id: { type: "string" },
type: { type: "string" },
size: { type: "number" },
width: { type: "number" },
height: { type: "number" },
url: { type: "string" },
full_url: { type: "string" },
},
},
},
},
date_added: { type: "string" },
creator_id: { type: "string" },
application_id: { type: "string" },
realname: { type: "string" },
key: { type: "string" },
mode: { type: "string" },
file_size: { type: "number" },
filename: { type: "string" },
data: {},
},
};
const documentSchema = {
type: "object",
properties: {
id: { type: "string" },
company_id: { type: "string" },
parent_id: { type: "string" },
is_in_trash: { type: "boolean" },
is_directory: { type: "boolean" },
name: { type: "string" },
extension: { type: "string" },
description: { type: "string" },
tags: { type: "array" },
added: { type: "string" },
last_modified: { type: "string" },
access_info: {
type: "object",
properties: {
public: {
type: "object",
properties: {
token: { type: "string" },
level: { type: "string" },
},
},
entities: { type: "array" },
},
},
content_keywords: { type: "string" },
hidden_data: {},
root_group_folder: { type: "string" },
creator: { type: "string" },
size: { type: "number" },
detached_file: { type: "boolean" },
has_preview: { type: "boolean" },
shared: { type: "boolean" },
url: { type: "string" },
preview_link: { type: "string" },
object_link_cache: { type: "string" },
external_storage: { type: "boolean" },
last_user: { type: "string" },
attachements: { type: "array" },
last_version_cache: fileVersionSchema,
},
};
export const createDocumentSchema = {
body: {
type: "object",
properties: {
item: { type: "object" },
version: { type: "object" },
},
required: ["item", "version"],
},
response: {
"2xx": documentSchema,
},
};
export const createVersionSchema = {
body: {
type: "object",
},
response: {
"2xx": fileVersionSchema,
},
};