eduvia/workavia-drive
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

♻️ Refactor keyword extraction into utils (#438)

Browse Source
This commit is contained in:
Eric Doughty-Papassideris
2024-03-26 12:06:22 +01:00
committed by ericlinagora
parent e2c5d3016b
commit 723fab4ef6
2 changed files with 35 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tdrive/backend/node/src/services/documents/services/engine/extract-keywords.ts
+2 -25
View File
@@ -2,9 +2,7 @@ import globalResolver from "../../../../services/global-resolver";
import { logger } from "../../../../core/platform/framework";
import { MessageQueueHandler } from "../../../../core/platform/services/message-queue/api";
import { DocumentsMessageQueueCallback, DocumentsMessageQueueRequest } from "../../types";
import { extractKeywords, officeFileToString, pdfFileToString, isFileType } from "../../utils";
import { officeExtensions, textExtensions, pdfExtensions } from "../../../../utils/mime";
import { readableToString } from "../../../../utils/files";
import { getKeywordsOfFile } from "../../utils";
export class DocumentsProcessor
implements MessageQueueHandler<DocumentsMessageQueueRequest, DocumentsMessageQueueCallback>
@@ -42,37 +40,16 @@ export class DocumentsProcessor
async generate(message: DocumentsMessageQueueRequest): Promise<DocumentsMessageQueueCallback> {
let content_keywords = "";
let content_strings = "";
try {
const storedFile = await globalResolver.services.files.download(
message.version.file_metadata.external_id,
message.context,
);
const extension = storedFile.name.split(".").pop();
if (isFileType(storedFile.mime, storedFile.name, textExtensions)) {
logger.info("Processing text file");
content_strings = await readableToString(storedFile.file);
}
if (isFileType(storedFile.mime, storedFile.name, pdfExtensions)) {
logger.info("Processing PDF file");
content_strings = await pdfFileToString(storedFile.file);
}
if (isFileType(storedFile.mime, storedFile.name, officeExtensions)) {
logger.info("Processing office file");
content_strings = await officeFileToString(storedFile.file, extension);
}
content_keywords = extractKeywords(content_strings);
content_keywords = await getKeywordsOfFile(storedFile.mime, storedFile.name, storedFile.file);
} catch (error) {
console.debug(error);
logger.error({ error: `${error}` }, "Failed to generate content keywords");
}
return { content_keywords, item: message.item };
}
}
tdrive/backend/node/src/services/documents/utils.ts
+33
View File
@@ -610,3 +610,36 @@ export const isInTrash = async (
// Recursively check the parent item
return isInTrash(parentItem, repository, context);
};
import { logger } from "../../core/platform/framework";
import { officeExtensions, textExtensions, pdfExtensions } from "../../utils/mime";
import { readableToString } from "../../utils/files";
/** Return true if the given filename and mime type could be analysed by `getKeywordsOfFile` */
export const couldGetKeywordsOfFile = async (mime: string, filename: string): Promise<boolean> =>
isFileType(mime, filename, textExtensions) ||
isFileType(mime, filename, pdfExtensions) ||
isFileType(mime, filename, officeExtensions);
/** Extract keyword text from the provided readable */
export const getKeywordsOfFile = async (
mime: string,
filename: string,
file: Readable,
): Promise<string> => {
let content_strings = "";
const extension = filename.split(".").pop();
if (isFileType(mime, filename, textExtensions)) {
logger.info(`Processing text file: ${filename}`);
content_strings = await readableToString(file);
}
if (isFileType(mime, filename, pdfExtensions)) {
logger.info(`Processing PDF file: ${filename}`);
content_strings = await pdfFileToString(file);
}
if (isFileType(mime, filename, officeExtensions)) {
logger.info(`Processing office file: ${filename}`);
content_strings = await officeFileToString(file, extension);
}
return extractKeywords(content_strings);
};