import { spawnSync } from 'child_process'; // @ts-ignore import fs from 'fs'; import { ShellLdapUserProvider } from './shell_ldap_user'; import { TwakeDriveClient, TwakeDriveUser } from './twake_client'; import path from 'path'; import { logger } from "./logger" import { User, UserProvider, UserProviderFactory, UserProviderType } from "./user_privider"; import { FunctionExecutor } from "./executor"; export interface NextcloudMigrationConfiguration { shell: { baseDn: string, url: string, }, lemon: { url: string, auth: string, } drive: { url: string, credentials: { appId: string, secret: string, } }, tmpDir: string, nextcloudUrl: string userProvider: UserProviderType } export class NextcloudMigration { private config: NextcloudMigrationConfiguration; private userProvider: UserProvider; driveClient: TwakeDriveClient; executor = new FunctionExecutor(); constructor(config: NextcloudMigrationConfiguration) { this.config = config; this.userProvider = (new UserProviderFactory()).get(config.userProvider, config[config.userProvider]); this.driveClient = new TwakeDriveClient(this.config.drive); } async migrate(username: string, password: string, dir?: string) { const dirTmp = dir ? dir : this.createTmpDir(username); if (dir) console.log(`Using dir: ${dir}`); // const dir = "/tmp/to_upload" try { const user = await this.getLDAPUser(username); //create user if needed Twake Drive const driveUser = await this.driveClient.createUser(user); console.log(`Drive user ${driveUser.id} created`); //download all files from nextcloud to tmp dir if(!dir) await this.download(username, password, dirTmp); //upload files to the Twake Drive await this.upload(driveUser, dirTmp); this.executor.printStatistics(); this.executor.printFailedExecutions(); return this.executor.getStats(); } catch (e) { console.error('Error downloading files from next cloud', e); throw e; } finally { if(!dir) this.deleteDir(dirTmp); } } async download(username: string, password: string, dir: string) { return new Promise((resolve, reject) => { let args = [ '-s', '--non-interactive', '-u', username, '-p', password, dir, this.config.nextcloudUrl]; console.log('Start downloading data from Nextcloud'); const ret = spawnSync('nextcloudcmd', args); if (ret.stderr) { console.log('ERROR:', ret.stderr.toString()); } if (ret.stdout) { console.log('OUT: ', ret.stdout.toString()); } if (ret.error) { console.log(`ERROR running sync for the user: ${ret.error.message}`); reject(ret.error.message); } else { console.log('Download finished'); resolve(''); } }); } async getLDAPUser(username: string): Promise { const user = await this.userProvider.find(username); if (!user.email) { throw new Error(`User ${username} not found`); } return user; } createTmpDir(username: string) { console.log('Creating tmp directory for the user data'); const dir = this.config.tmpDir + '/' + username + '_' + new Date().getTime(); if (!fs.existsSync(dir)) { console.log(`Creating directory ${dir} ...`); fs.mkdirSync(dir); console.log(`Directory ${dir} created`); } else { this.deleteDir(dir); } return dir; } deleteDir(dir: string) { console.log(`Deleting directory ${dir} ...`); fs.rmSync(dir, { recursive: true, force: true }); console.log(`Directory ${dir} deleted`); } async upload(user: TwakeDriveUser, sourceDirPath: string, parentDirId = "user_" + user.id) { const dirsToUpload: Map = new Map(); const filesToUpload: string[] = []; const existingFiles: string[] = []; const parent = await this.driveClient.getDocument(parentDirId); const exists = (filename: string) => { let parsedPath = path.parse(filename); let name = `${parsedPath.name}${parsedPath.ext > '' ? parsedPath.ext : ''}`; return parent.children.filter(i => i.name == name).length > 0; } logger.debug(`Reading content of the directory ${sourceDirPath} ...`) fs.readdirSync(sourceDirPath).forEach(function (name) { const filePath = path.join(sourceDirPath, name); const stat = fs.statSync(filePath); if (exists(name)) { logger.info(`File ${name} already exists`); //find document that we need to replace existingFiles.push(filePath); //skip } else { if (stat.isFile()) { logger.info(`Add file for future upload ${filePath}`); filesToUpload.push(filePath) } else if (stat.isDirectory()) { logger.info(`Add directory for the future upload ${filePath}`); dirsToUpload.set(name, filePath); } } }); //check existing files for (const fPath of existingFiles) { logger.debug(`Check existing file ${fPath}`) let parsedPath = path.parse(fPath); let name = parsedPath.name + (parsedPath.ext > '' ? parsedPath.ext : ''); let candidatesWithTheSameName = parent.children.filter(i => i.name === name); if (candidatesWithTheSameName.length > 1) { logger.warn("WE HAVE MORE MORE THAN ONE FILE WITH NAME: " + name); } else { const doc = candidatesWithTheSameName[0]; if (doc.is_directory) { logger.info(`Directory ${name} exists, try to upload inside`); await this.upload(user, fPath, doc.id); } else { //check that it exists in S3 if (await this.driveClient.existsInS3(doc.id)) { logger.info(`File ${name}, docId = ${doc.id} exists in S3`); } else { //if it doesn't exists upload just one file logger.info(`File ${name}, is missing, uploading files`); const response = await this.driveClient.uploadToS3(fPath, doc.id); if (!response && !response.success) { console.log(`Error creating file '${name}' in S3`) } } } } } //upload all files logger.debug(`UPLOAD FILES FOR ${sourceDirPath}`) for (const file of filesToUpload) { logger.debug(`Upload file ${file}`) await this.executor.executeWithRetries(this.driveClient.createFile.bind(this.driveClient), [file, parentDirId], 3) // await this.driveClient.createFile(file, parentDirId); } logger.debug(`UPLOAD DIRS FOR ${sourceDirPath}`) for (const [name, path] of dirsToUpload) { logger.info(`Create directory ${name}`); const dir = await this.executor.executeWithRetries(this.driveClient.createDirectory.bind(this.driveClient), [name, parentDirId], 3) // const dir = await this.driveClient.createDirectory(name, parentDirId) if (dir) { await this.upload(user, path, dir.id); } } } }