✨ oo-connector: OO forgotten files batch processor, and testing, but WIP (#525)

tdrive/connectors/onlyoffice-connector/src/app.ts

@@ -9,6 +9,8 @@ import logger from './lib/logger';
 import errorMiddleware from './middlewares/error.middleware';
 import { mountRoutes } from './routes';
 
+import forgottenProcessorService from './services/forgotten-processor.service';
+
 class App {
   public app: express.Application;
   public env: string;
@@ -22,6 +24,8 @@ class App {
     this.initMiddlewares();
     this.initRoutes();
     this.initErrorHandling();
+
+    forgottenProcessorService.makeSureItsLoaded();
   }
 
   public listen = () => {

tdrive/connectors/onlyoffice-connector/src/controllers/health-status.controller.ts

@@ -1,19 +1,27 @@
-import { NextFunction, Request, Response } from 'express';
+import { Request, Response } from 'express';
 import onlyofficeService from '@/services/onlyoffice.service';
 import apiService from '@/services/api.service';
 
+import forgottenProcessorService from '@/services/forgotten-processor.service';
+
 /**
  * Health response for devops operational purposes. Should not be exposed.
  */
 export const HealthStatusController = {
-  async get(req: Request<{}, {}, {}, {}>, res: Response, next: NextFunction): Promise<void> {
+  async get(req: Request<{}, {}, {}, {}>, res: Response): Promise<void> {
-    Promise.all([onlyofficeService.getLatestLicence(), apiService.hasToken(), onlyofficeService.getForgottenList()]).then(
+    Promise.all([
-      ([onlyOfficeLicense, twakeDriveToken, forgottenKeys]) =>
+      onlyofficeService.getLatestLicence(),
+      apiService.hasToken(),
+      onlyofficeService.getForgottenList(),
+      forgottenProcessorService.getLastStartTimeAgoS(),
+    ]).then(
+      ([onlyOfficeLicense, twakeDriveToken, forgottenKeys, forgottenLastProcessAgoS]) =>
         res.status(onlyOfficeLicense && twakeDriveToken ? 200 : 500).send({
           uptimeS: Math.floor(process.uptime()),
           onlyOfficeLicense,
           twakeDriveToken,
           forgottenCount: forgottenKeys?.length ?? 0,
+          forgottenLastProcessAgoS,
         }),
       err => res.status(500).send(err),
     );

tdrive/connectors/onlyoffice-connector/src/services/api.service.ts

@@ -5,17 +5,10 @@ import {
   IApiServiceApplicationTokenResponse,
 } from '@/interfaces/api.interface';
 import axios, { Axios, AxiosRequestConfig, AxiosResponse } from 'axios';
-import {
+import { CREDENTIALS_ENDPOINT, CREDENTIALS_ID, CREDENTIALS_SECRET, twakeDriveTokenRefrehPeriodMS } from '@config';
-  CREDENTIALS_ENDPOINT,
-  CREDENTIALS_ID,
-  CREDENTIALS_SECRET,
-  twakeDriveTokenRefrehPeriodMS,
-  onlyOfficeForgottenFilesCheckPeriodMS,
-} from '@config';
 import logger from '../lib/logger';
 import * as Utils from '@/utils';
 import { PolledThingieValue } from '@/lib/polled-thingie-value';
-import onlyofficeService from './onlyoffice.service';
 
 /**
  * Client for the Twake Drive backend API on behalf of the plugin (or provided token in parameters).
@@ -23,15 +16,9 @@ import onlyofficeService from './onlyoffice.service';
  */
 class ApiService implements IApiService {
   private readonly tokenPoller: PolledThingieValue<Axios>;
-  private readonly forgottenFilesPoller: PolledThingieValue<number>;
 
   constructor() {
     this.tokenPoller = new PolledThingieValue('Refresh Twake Drive token', async () => await this.refreshToken(), twakeDriveTokenRefrehPeriodMS);
-    this.forgottenFilesPoller = new PolledThingieValue(
-      'Process forgotten files in OO',
-      async () => await this.processForgottenFiles(),
-      onlyOfficeForgottenFilesCheckPeriodMS,
-    );
   }
 
   public async hasToken() {
@@ -42,14 +29,6 @@ class ApiService implements IApiService {
     return this.tokenPoller.requireLatestValueWithTry('No Twake Drive app token.');
   }
 
-  private async processForgottenFiles() {
-    if (!this.tokenPoller.hasValue()) return -1;
-    return await onlyofficeService.processForgotten(async (/* key, url */) => {
-      //TODO: when endpoint decided, call here. See if accept HTTP 202 for ex. to avoid deleting.
-      return false;
-    });
-  }
-
   public get = async <T>(params: IApiServiceRequestParams<T>): Promise<T> => {
     const { url, token, responseType, headers } = params;
 
@@ -95,6 +74,7 @@ class ApiService implements IApiService {
 
     const axiosWithToken = await this.requireAxios();
 
+    logger.info(`POST to Twake Drive ${url} - payload: ${payload}`);
     try {
       return await axiosWithToken.post(url, payload, {
         headers: {
@@ -103,7 +83,7 @@ class ApiService implements IApiService {
         },
       });
     } catch (error) {
-      logger.error('Failed to post to Twake drive: ', error.stack);
+      logger.error('Failed to post to Twake Drive: ', { error });
       this.refreshToken();
     }
   };

tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts

@@ -96,11 +96,20 @@ class DriveService implements IDriveService {
       if (!url) {
         throw Error('no url found');
       }
-
+      //TODO: It would be better to avoid two requests for this operation
       const originalFile = await this.getByEditingSessionKey({ editing_session_key });
 
       if (!originalFile) {
-        throw Error('original file not found');
+        // TODO: Make a single request and don't require the filename at all
+        //   then in POST /editing_session/... if the key is not found, just
+        //   put it in a users or company "lost and found" folder.
+        //   and accept without error. Because really, if backend doesn't know
+        //   the key anymore, there's not much we can do, and we should get OO
+        //   to clean up and stop trying to upload it.
+        //   but for today:
+        logger.fatal("Losing OO document because Twake Drive doesn't know that Key.", { editing_session_key, url });
+        throw new Error(`Unknown key ${JSON.stringify(editing_session_key)}`);
+        // TODO: Distinguish this case from a long disconnected browser tab waking up
       }
 
       const newFile = await apiService.get<Stream>({
@@ -137,6 +146,7 @@ class DriveService implements IDriveService {
    * Get the document information by the editing session key. Just simple call to the drive API
    * /item/editing_session/${editing_session_key}
    * @param params
+   * @returns null if the key was not found, or the api response body
    */
   public getByEditingSessionKey = async (params: { editing_session_key: string; user_token?: string }): Promise<DriveFileType['item']> => {
     try {
@@ -146,7 +156,7 @@ class DriveService implements IDriveService {
         token: params.user_token,
       });
     } catch (error) {
-      logger.error('Failed to fetch file metadata by editing session key: ', error.stack);
+      if (error?.response?.status === 404) return null;
       throw error;
     }
   };

tdrive/connectors/onlyoffice-connector/src/services/forgotten-processor.service.ts

@@ -0,0 +1,56 @@
+import { onlyOfficeForgottenFilesCheckPeriodMS } from '@/config';
+import { PolledThingieValue } from '@/lib/polled-thingie-value';
+
+import apiService from './api.service';
+import onlyofficeService from './onlyoffice.service';
+import driveService from './drive.service';
+import logger from '@/lib/logger';
+
+/**
+ * Periodically poll the Only Office document server for forgotten
+ * files and try to upload to Twake Drive or get rid if unknown.
+ */
+class ForgottenProcessor {
+  private readonly forgottenFilesPoller: PolledThingieValue<number>;
+  private lastStart = 0;
+
+  constructor() {
+    let skippedFirst = false;
+    this.forgottenFilesPoller = new PolledThingieValue(
+      'Process forgotten files in OO',
+      async () => (skippedFirst ? await this.processForgottenFiles() : ((skippedFirst = true), -1)),
+      onlyOfficeForgottenFilesCheckPeriodMS,
+    );
+  }
+
+  /** Get the number of seconds since the last time this process started */
+  public getLastStartTimeAgoS() {
+    return ~~((new Date().getTime() - this.lastStart) / 1000);
+  }
+
+  public makeSureItsLoaded() {
+    // The point of this is to ensure this file is imported,
+    // which is needed for side-effect of starting this timer
+    return true;
+  }
+
+  private async processForgottenFiles() {
+    this.lastStart = new Date().getTime();
+    if (!(await apiService.hasToken())) return -1;
+    return await onlyofficeService.processForgotten(async (key, url) => {
+      try {
+        await driveService.endEditing(key, url);
+        return true;
+      } catch (error) {
+        logger.error(`Error processing forgotten file by key ${JSON.stringify(key)}: ${error}`, { key, url, error });
+        // Can't do much about it here, hope it goes in retry, but don't
+        // throw to keep processing
+        //TODO: Maybe make a date string, compare to key, if old enough, delete...
+        // this logic should probably in Twake Drive backend though
+      }
+      return false;
+    });
+  }
+}
+
+export default new ForgottenProcessor();

tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts

@@ -249,8 +249,13 @@ class OnlyOfficeService {
    * @returns The number of files processed and deleted
    */
   public async processForgotten(processor: (key: string, url: string) => Promise<boolean>): Promise<number> {
+    logger.info(`Begin to process forgotten files in OnlyOffice`);
+    //TODO: filter by instance id of the key
     const forgottenFiles = await this.getForgottenList();
-    if (forgottenFiles.length === 0) return 0;
+    if (forgottenFiles.length === 0) {
+      logger.info(`No forgotten files in OnlyOffice`);
+      return 0;
+    }
     Utils.fisherYattesShuffleInPlace(forgottenFiles);
     logger.info(`Forgotten files found: ${forgottenFiles.length}`);
     let deleted = 0;