eduvia/workavia-drive
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ oo-connector: OO forgotten files batch processor, and testing, but WIP (#525)

Browse Source
This commit is contained in:
Eric Doughty-Papassideris
2024-09-18 01:55:18 +02:00
parent 15ca46b656
commit 18f9208a43
6 changed files with 94 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tdrive/connectors/onlyoffice-connector/src/app.ts
+4
View File
@@ -9,6 +9,8 @@ import logger from './lib/logger';
import errorMiddleware from './middlewares/error.middleware';
import { mountRoutes } from './routes';
import forgottenProcessorService from './services/forgotten-processor.service';
class App {
public app: express.Application;
public env: string;
@@ -22,6 +24,8 @@ class App {
this.initMiddlewares();
this.initRoutes();
this.initErrorHandling();
forgottenProcessorService.makeSureItsLoaded();
}
public listen = () => {
tdrive/connectors/onlyoffice-connector/src/controllers/health-status.controller.ts
+12 -4
View File
@@ -1,19 +1,27 @@
import { NextFunction, Request, Response } from 'express';
import { Request, Response } from 'express';
import onlyofficeService from '@/services/onlyoffice.service';
import apiService from '@/services/api.service';
import forgottenProcessorService from '@/services/forgotten-processor.service';
/**
* Health response for devops operational purposes. Should not be exposed.
*/
export const HealthStatusController = {
async get(req: Request<{}, {}, {}, {}>, res: Response, next: NextFunction): Promise<void> {
Promise.all([onlyofficeService.getLatestLicence(), apiService.hasToken(), onlyofficeService.getForgottenList()]).then(
([onlyOfficeLicense, twakeDriveToken, forgottenKeys]) =>
async get(req: Request<{}, {}, {}, {}>, res: Response): Promise<void> {
Promise.all([
onlyofficeService.getLatestLicence(),
apiService.hasToken(),
onlyofficeService.getForgottenList(),
forgottenProcessorService.getLastStartTimeAgoS(),
]).then(
([onlyOfficeLicense, twakeDriveToken, forgottenKeys, forgottenLastProcessAgoS]) =>
res.status(onlyOfficeLicense && twakeDriveToken ? 200 : 500).send({
uptimeS: Math.floor(process.uptime()),
onlyOfficeLicense,
twakeDriveToken,
forgottenCount: forgottenKeys?.length ?? 0,
forgottenLastProcessAgoS,
}),
err => res.status(500).send(err),
);
tdrive/connectors/onlyoffice-connector/src/services/api.service.ts
+3 -23
View File
@@ -5,17 +5,10 @@ import {
IApiServiceApplicationTokenResponse,
} from '@/interfaces/api.interface';
import axios, { Axios, AxiosRequestConfig, AxiosResponse } from 'axios';
import {
CREDENTIALS_ENDPOINT,
CREDENTIALS_ID,
CREDENTIALS_SECRET,
twakeDriveTokenRefrehPeriodMS,
onlyOfficeForgottenFilesCheckPeriodMS,
} from '@config';
import { CREDENTIALS_ENDPOINT, CREDENTIALS_ID, CREDENTIALS_SECRET, twakeDriveTokenRefrehPeriodMS } from '@config';
import logger from '../lib/logger';
import * as Utils from '@/utils';
import { PolledThingieValue } from '@/lib/polled-thingie-value';
import onlyofficeService from './onlyoffice.service';
/**
* Client for the Twake Drive backend API on behalf of the plugin (or provided token in parameters).
@@ -23,15 +16,9 @@ import onlyofficeService from './onlyoffice.service';
*/
class ApiService implements IApiService {
private readonly tokenPoller: PolledThingieValue<Axios>;
private readonly forgottenFilesPoller: PolledThingieValue<number>;
constructor() {
this.tokenPoller = new PolledThingieValue('Refresh Twake Drive token', async () => await this.refreshToken(), twakeDriveTokenRefrehPeriodMS);
this.forgottenFilesPoller = new PolledThingieValue(
'Process forgotten files in OO',
async () => await this.processForgottenFiles(),
onlyOfficeForgottenFilesCheckPeriodMS,
);
}
public async hasToken() {
@@ -42,14 +29,6 @@ class ApiService implements IApiService {
return this.tokenPoller.requireLatestValueWithTry('No Twake Drive app token.');
}
private async processForgottenFiles() {
if (!this.tokenPoller.hasValue()) return -1;
return await onlyofficeService.processForgotten(async (/* key, url */) => {
//TODO: when endpoint decided, call here. See if accept HTTP 202 for ex. to avoid deleting.
return false;
});
}
public get = async <T>(params: IApiServiceRequestParams<T>): Promise<T> => {
const { url, token, responseType, headers } = params;
@@ -95,6 +74,7 @@ class ApiService implements IApiService {
const axiosWithToken = await this.requireAxios();
logger.info(`POST to Twake Drive ${url} - payload: ${payload}`);
try {
return await axiosWithToken.post(url, payload, {
headers: {
@@ -103,7 +83,7 @@ class ApiService implements IApiService {
},
});
} catch (error) {
logger.error('Failed to post to Twake drive: ', error.stack);
logger.error('Failed to post to Twake Drive: ', { error });
this.refreshToken();
}
};
tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts
+13 -3
View File
@@ -96,11 +96,20 @@ class DriveService implements IDriveService {
if (!url) {
throw Error('no url found');
}
//TODO: It would be better to avoid two requests for this operation
const originalFile = await this.getByEditingSessionKey({ editing_session_key });
if (!originalFile) {
throw Error('original file not found');
// TODO: Make a single request and don't require the filename at all
// then in POST /editing_session/... if the key is not found, just
// put it in a users or company "lost and found" folder.
// and accept without error. Because really, if backend doesn't know
// the key anymore, there's not much we can do, and we should get OO
// to clean up and stop trying to upload it.
// but for today:
logger.fatal("Losing OO document because Twake Drive doesn't know that Key.", { editing_session_key, url });
throw new Error(`Unknown key ${JSON.stringify(editing_session_key)}`);
// TODO: Distinguish this case from a long disconnected browser tab waking up
}
const newFile = await apiService.get<Stream>({
@@ -137,6 +146,7 @@ class DriveService implements IDriveService {
* Get the document information by the editing session key. Just simple call to the drive API
* /item/editing_session/${editing_session_key}
* @param params
* @returns null if the key was not found, or the api response body
*/
public getByEditingSessionKey = async (params: { editing_session_key: string; user_token?: string }): Promise<DriveFileType['item']> => {
try {
@@ -146,7 +156,7 @@ class DriveService implements IDriveService {
token: params.user_token,
});
} catch (error) {
logger.error('Failed to fetch file metadata by editing session key: ', error.stack);
if (error?.response?.status === 404) return null;
throw error;
}
};
tdrive/connectors/onlyoffice-connector/src/services/forgotten-processor.service.ts
+56
View File
@@ -0,0 +1,56 @@
import { onlyOfficeForgottenFilesCheckPeriodMS } from '@/config';
import { PolledThingieValue } from '@/lib/polled-thingie-value';
import apiService from './api.service';
import onlyofficeService from './onlyoffice.service';
import driveService from './drive.service';
import logger from '@/lib/logger';
/**
* Periodically poll the Only Office document server for forgotten
* files and try to upload to Twake Drive or get rid if unknown.
*/
class ForgottenProcessor {
private readonly forgottenFilesPoller: PolledThingieValue<number>;
private lastStart = 0;
constructor() {
let skippedFirst = false;
this.forgottenFilesPoller = new PolledThingieValue(
'Process forgotten files in OO',
async () => (skippedFirst ? await this.processForgottenFiles() : ((skippedFirst = true), -1)),
onlyOfficeForgottenFilesCheckPeriodMS,
);
}
/** Get the number of seconds since the last time this process started */
public getLastStartTimeAgoS() {
return ~~((new Date().getTime() - this.lastStart) / 1000);
}
public makeSureItsLoaded() {
// The point of this is to ensure this file is imported,
// which is needed for side-effect of starting this timer
return true;
}
private async processForgottenFiles() {
this.lastStart = new Date().getTime();
if (!(await apiService.hasToken())) return -1;
return await onlyofficeService.processForgotten(async (key, url) => {
try {
await driveService.endEditing(key, url);
return true;
} catch (error) {
logger.error(`Error processing forgotten file by key ${JSON.stringify(key)}: ${error}`, { key, url, error });
// Can't do much about it here, hope it goes in retry, but don't
// throw to keep processing
//TODO: Maybe make a date string, compare to key, if old enough, delete...
// this logic should probably in Twake Drive backend though
}
return false;
});
}
}
export default new ForgottenProcessor();
tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts
+6 -1
View File
@@ -249,8 +249,13 @@ class OnlyOfficeService {
* @returns The number of files processed and deleted
*/
public async processForgotten(processor: (key: string, url: string) => Promise<boolean>): Promise<number> {
logger.info(`Begin to process forgotten files in OnlyOffice`);
//TODO: filter by instance id of the key
const forgottenFiles = await this.getForgottenList();
if (forgottenFiles.length === 0) return 0;
if (forgottenFiles.length === 0) {
logger.info(`No forgotten files in OnlyOffice`);
return 0;
}
Utils.fisherYattesShuffleInPlace(forgottenFiles);
logger.info(`Forgotten files found: ${forgottenFiles.length}`);
let deleted = 0;