♻️ Refactor keyword extraction into utils (#438)

tdrive/backend/node/src/services/documents/services/engine/extract-keywords.ts

@@ -2,9 +2,7 @@ import globalResolver from "../../../../services/global-resolver";
 import { logger } from "../../../../core/platform/framework";
 import { MessageQueueHandler } from "../../../../core/platform/services/message-queue/api";
 import { DocumentsMessageQueueCallback, DocumentsMessageQueueRequest } from "../../types";
-import { extractKeywords, officeFileToString, pdfFileToString, isFileType } from "../../utils";
+import { getKeywordsOfFile } from "../../utils";
-import { officeExtensions, textExtensions, pdfExtensions } from "../../../../utils/mime";
-import { readableToString } from "../../../../utils/files";
 
 export class DocumentsProcessor
   implements MessageQueueHandler<DocumentsMessageQueueRequest, DocumentsMessageQueueCallback>
@@ -42,37 +40,16 @@ export class DocumentsProcessor
 
   async generate(message: DocumentsMessageQueueRequest): Promise<DocumentsMessageQueueCallback> {
     let content_keywords = "";
-    let content_strings = "";
-
     try {
       const storedFile = await globalResolver.services.files.download(
         message.version.file_metadata.external_id,
         message.context,
       );
-
+      content_keywords = await getKeywordsOfFile(storedFile.mime, storedFile.name, storedFile.file);
-      const extension = storedFile.name.split(".").pop();
-
-      if (isFileType(storedFile.mime, storedFile.name, textExtensions)) {
-        logger.info("Processing text file");
-        content_strings = await readableToString(storedFile.file);
-      }
-
-      if (isFileType(storedFile.mime, storedFile.name, pdfExtensions)) {
-        logger.info("Processing PDF file");
-        content_strings = await pdfFileToString(storedFile.file);
-      }
-
-      if (isFileType(storedFile.mime, storedFile.name, officeExtensions)) {
-        logger.info("Processing office file");
-        content_strings = await officeFileToString(storedFile.file, extension);
-      }
-
-      content_keywords = extractKeywords(content_strings);
     } catch (error) {
       console.debug(error);
       logger.error({ error: `${error}` }, "Failed to generate content keywords");
     }
-
     return { content_keywords, item: message.item };
   }
 }

tdrive/backend/node/src/services/documents/utils.ts

@@ -610,3 +610,36 @@ export const isInTrash = async (
   // Recursively check the parent item
   return isInTrash(parentItem, repository, context);
 };
+
+import { logger } from "../../core/platform/framework";
+import { officeExtensions, textExtensions, pdfExtensions } from "../../utils/mime";
+import { readableToString } from "../../utils/files";
+
+/** Return true if the given filename and mime type could be analysed by `getKeywordsOfFile` */
+export const couldGetKeywordsOfFile = async (mime: string, filename: string): Promise<boolean> =>
+  isFileType(mime, filename, textExtensions) ||
+  isFileType(mime, filename, pdfExtensions) ||
+  isFileType(mime, filename, officeExtensions);
+
+/** Extract keyword text from the provided readable */
+export const getKeywordsOfFile = async (
+  mime: string,
+  filename: string,
+  file: Readable,
+): Promise<string> => {
+  let content_strings = "";
+  const extension = filename.split(".").pop();
+  if (isFileType(mime, filename, textExtensions)) {
+    logger.info(`Processing text file:   ${filename}`);
+    content_strings = await readableToString(file);
+  }
+  if (isFileType(mime, filename, pdfExtensions)) {
+    logger.info(`Processing PDF file:    ${filename}`);
+    content_strings = await pdfFileToString(file);
+  }
+  if (isFileType(mime, filename, officeExtensions)) {
+    logger.info(`Processing office file: ${filename}`);
+    content_strings = await officeFileToString(file, extension);
+  }
+  return extractKeywords(content_strings);
+};