[teraslice-cli] add earl jobs export command (#3717)

This PR makes the following changes:
- Add `jobs export` command to teraslice-cli
- Creates a `Jobs` class using the provided jobIds, extracts the
jobConfig for each job, and saves each config to the local file system.
- Each job is saved to `./<job.name>.json` by default. If a file name
already exists `-N` will be appended to the file name.
- `--outdir` flag will set a custom directory where job files are saved.
- A job-id of `all` will export all jobs. This can be combined with
`--status` to export all jobs with executions of a specific status.

Ref: #3695

docs/packages/teraslice-cli/overview.md

@@ -398,6 +398,18 @@ teraslice-cli jobs view <cluster> <job_id>
 teraslice-cli jobs view local 99999999-9999-9999-9999-999999999999
 ```
 
+### jobs export
+
+Export job on a cluster to a json file. By default the file is saved to the current working directory as <job.name>.json
+
+```sh
+teraslice-cli jobs export <cluster> <job_id>
+# export job to current directory
+teraslice-cli jobs export local 99999999-9999-9999-9999-999999999999
+# export job to custom directory
+teraslice-cli jobs export local 99999999-9999-9999-9999-999999999999 --outdir ~/my_jobs
+```
+
 ### jobs recover
 
 Recover a crashed jobs

packages/teraslice-cli/package.json

@@ -1,7 +1,7 @@
 {
     "name": "teraslice-cli",
     "displayName": "Teraslice CLI",
-    "version": "2.3.0",
+    "version": "2.4.0",
     "description": "Command line manager for teraslice jobs, assets, and cluster references.",
     "keywords": [
         "teraslice"

packages/teraslice-cli/src/cmds/jobs/export.ts

@@ -0,0 +1,37 @@
+import { CMD } from '../../interfaces.js';
+import Config from '../../helpers/config.js';
+import YargsOptions from '../../helpers/yargs-options.js';
+import Jobs from '../../helpers/jobs.js';
+import reply from '../../helpers/reply.js';
+
+const yargsOptions = new YargsOptions();
+
+export default {
+    command: 'export <cluster-alias> <job-id...>',
+    describe: 'Export job on a cluster to a json file. By default the file is saved to the current working directory as <job.name>.json\n',
+    builder(yargs: any) {
+        yargs.positional('job-id', yargsOptions.buildPositional('job-id'));
+        yargs.options('config-dir', yargsOptions.buildOption('config-dir'));
+        yargs.options('outdir', yargsOptions.buildOption('outdir'));
+        yargs.options('status', yargsOptions.buildOption('jobs-status'));
+        yargs.options('yes', yargsOptions.buildOption('yes'));
+        yargs.strict()
+            .example('$0 jobs export CLUSTER_ALIAS JOB1', 'exports job config as a tjm compatible JSON file')
+            .example('$0 jobs export CLUSTER_ALIAS JOB1 JOB2', 'exports job configs for two jobs')
+            .example('$0 jobs export CLUSTER_ALIAS JOB1 --outdir ~/my_jobs', 'exports a job to ~/my_jobs/<job.name>.json')
+            .example('$0 jobs export CLUSTER_ALIAS all --status failing', 'exports all failing jobs on a cluster (maximum 100)')
+            .example('$0 jobs export CLUSTER_ALIAS all -y', 'exports all jobs on a cluster (maximum 100) and bypasses any prompts');
+        return yargs;
+    },
+    async handler(argv: any) {
+        const cliConfig = new Config(argv);
+        const jobs = new Jobs(cliConfig);
+
+        try {
+            await jobs.initialize();
+            await jobs.export();
+        } catch (e) {
+            reply.fatal(e);
+        }
+    }
+} as CMD;

packages/teraslice-cli/src/cmds/jobs/index.ts

@@ -2,6 +2,7 @@ import { CMD } from '../../interfaces.js';
 import awaitCmd from './await.js';
 import deleteJob from './delete.js';
 import errors from './errors.js';
+import exportJob from './export.js';
 import list from './list.js';
 import pause from './pause.js';
 import recover from './recover.js';
@@ -19,6 +20,7 @@ const commandList = [
     awaitCmd,
     deleteJob,
     errors,
+    exportJob,
     list,
     pause,
     recover,

packages/teraslice-cli/src/helpers/config.ts

@@ -70,6 +70,18 @@ export default class Config {
         return `${this.configDir}/aliases.yaml`;
     }
 
+    /**
+     * Returns the user provided output directory or
+     * the current directory as default
+     */
+    get outdir(): string {
+        if (this.args.outdir) {
+            return this.args.outdir;
+        } else {
+            return process.cwd();
+        }
+    }
+
     get jobStateDir(): string {
         return `${this.configDir}/job_state_files`;
     }
@@ -85,7 +97,8 @@ export default class Config {
     get allSubDirs(): string[] {
         return [
             this.jobStateDir,
-            this.assetDir
+            this.assetDir,
+            this.outdir,
         ];
     }
 
@@ -96,7 +109,7 @@ export default class Config {
 
         this.allSubDirs.forEach((dir) => {
             if (!fs.existsSync(dir)) {
-                fs.mkdirSync(dir);
+                fs.mkdirSync(dir, { recursive: true });
             }
         });
     }

packages/teraslice-cli/src/helpers/jobs.ts

@@ -1,6 +1,6 @@
 import fs from 'fs-extra';
 import {
-    has, toString, pDelay, pMap,
+    has, toString, pDelay, pMap, pRetry,
 } from '@terascope/utils';
 import { Teraslice } from '@terascope/types';
 import chalk from 'chalk';
@@ -10,7 +10,7 @@ import { Job } from 'teraslice-client-js';
 import TerasliceUtil from './teraslice-util.js';
 import Display from './display.js';
 import reply from './reply.js';
-import { getJobConfigFromFile } from './tjm-util.js';
+import { getJobConfigFromFile, saveJobConfigToFile } from './tjm-util.js';
 import Config from './config.js';
 import {
     JobMetadata,
@@ -598,9 +598,9 @@ export default class Jobs {
                 return this.getJobIdsFromSavedState();
             }
 
-            // if action is delete we need to get inactive
-            // as well as active jobs
-            if (action === 'delete') {
+            // if action is delete or export  we need to
+            // get inactive as well as active jobs
+            if (action === 'delete' || action === 'export') {
                 return this.getActiveAndInactiveJobIds();
             }
 
@@ -681,11 +681,11 @@ export default class Jobs {
     }
 
     noJobsWithStatus() {
-        const cluster = `cluster: ${this.config.args.clusterUrl}`;
+        const cluster = `cluster: ${this.config.clusterUrl}`;
         const targetedStatus = `${this.config.args.status.join(' or ')}`;
 
         if (this.config.args.jobId.includes('all')) {
-            reply.fatal(`No jobs on ${cluster} with status ${targetedStatus}`);
+            reply.fatal(`No jobs on ${cluster} with status ${targetedStatus || '"any"'}`);
         }
 
         reply.fatal(`Jobs: ${this.config.args.jobId.join(', ')} on ${cluster} do not have status ${targetedStatus}`);
@@ -806,6 +806,53 @@ export default class Jobs {
         this.printDiff(diffObject, showUpdateField);
     }
 
+    async export() {
+        const jobIds = this.jobs.map((job) => job.id);
+
+        reply.yellow(`Saving jobFile(s) for ${jobIds.join(', ')} on ${this.config.args.clusterAlias}`);
+
+        await pMap(
+            this.jobs,
+            (job) => this.exportOne(job.config),
+            { concurrency: this.concurrency }
+        );
+
+        reply.green(`Saved jobFile(s) to ${this.config.outdir}`);
+    }
+
+    async exportOne(jobConfig: Teraslice.JobConfig) {
+        await pRetry(() => {
+            const filePath = this.createUniqueFilePath(jobConfig.name);
+            return saveJobConfigToFile(jobConfig, filePath, this.config.clusterUrl);
+        })
+    }
+
+    /**
+     * @param { string } jobConfigName
+     * @returns {string} A unique file path
+     *
+     * Using the name from a jobConfig and the outdir,
+     * creates a unique file path where a job can be exported.
+     * Spaces in the job name are replaced with underscores.
+     * If the file name exists a '-N' suffix will be added to the name.
+     *     ex: First export: '~/my_current_directory/my_job_name.json'
+     *        Second export: '~/my_current_directory/my_job_name-1.json'
+     */
+    private createUniqueFilePath(jobConfigName: string) {
+        const dirName = this.config.outdir;
+        const fileName = `${jobConfigName.replaceAll(' ', '_')}.json`;
+        const filePath = path.join(dirName, fileName);
+        let uniquePath = filePath;
+        let i = 1;
+
+        while (fs.existsSync(uniquePath)) {
+            uniquePath = `${filePath.slice(0, -5)}-${i}.json`;
+            i++;
+        }
+
+        return uniquePath;
+    }
+
     /**
      * @param args action and final property, final indicates if it is part of a series of commands
      * @param job job metadata

packages/teraslice-cli/src/helpers/tjm-util.ts

@@ -4,8 +4,10 @@ import {
     has,
     set,
     unset,
-    get
+    get,
+    cloneDeep
 } from '@terascope/utils';
+import { Teraslice } from '@terascope/types';
 import Config from './config.js';
 import Jobs from './jobs.js';
 import { getPackage } from './utils.js';
@@ -208,3 +210,26 @@ export function saveConfig(
 function hasMetadata(jobConfig: Record<string, any>): boolean {
     return has(jobConfig, '__metadata');
 }
+
+export async function saveJobConfigToFile(
+    jobConfig: Teraslice.JobConfig,
+    filePath: string,
+    clusterUrl: string
+) {
+    const jobConfigCopy = {};
+    const keysToSkip = ['job_id', '_created', '_context', '_updated', '_deleted', '_deleted_on']
+
+    for (const key of Object.keys(jobConfig)) {
+        if (!keysToSkip.includes(key)) {
+            jobConfigCopy[key] = cloneDeep(jobConfig[key]);
+        }
+    }
+
+    addMetaData(jobConfigCopy, jobConfig.job_id, clusterUrl);
+
+    if (!fs.existsSync(filePath)) {
+        await fs.writeJSON(filePath, jobConfigCopy);
+    } else {
+        throw new Error(`File already exists at ${filePath}`);
+    }
+}

packages/teraslice-cli/src/helpers/yargs-options.ts

@@ -233,10 +233,15 @@ export default class Options {
             default: false,
             type: 'boolean'
         }),
+        'outdir': () => ({
+            alias: 'o',
+            describe: 'Directory where exported job file will be saved',
+            type: 'string',
+            nargs: 1,
+        }),
         'active-job': () => ({
             describe: 'List active jobs',
             type: 'boolean'
-
         }),
         'show-deleted': () => ({
             describe: 'List deleted records',

packages/teraslice-cli/test/cmds/jobs/export-spec.ts

@@ -0,0 +1,35 @@
+import yargs from 'yargs';
+import exportJob from '../../../src/cmds/jobs/export.js';
+
+describe('jobs export', () => {
+    describe('-> parse', () => {
+        it('should parse properly', () => {
+            const yargsCmd = yargs().command(
+                // @ts-expect-error
+                exportJob.command,
+                exportJob.describe,
+                exportJob.builder,
+                () => true
+            );
+            const yargsResult = yargsCmd.parseSync(
+                'export ts-test1 all', {}
+            );
+            expect(yargsResult.clusterAlias).toEqual('ts-test1');
+        });
+
+        it('should parse properly with an id specifed', () => {
+            const yargsCmd = yargs().command(
+                // @ts-expect-error
+                exportJob.command,
+                exportJob.describe,
+                exportJob.builder,
+                () => true
+            );
+            const yargsResult = yargsCmd.parseSync(
+                'export ts-test1 99999999-9999-9999-9999-999999999999', {}
+            );
+            expect(yargsResult.clusterAlias).toEqual('ts-test1');
+            expect(yargsResult.jobId).toEqual(['99999999-9999-9999-9999-999999999999']);
+        });
+    });
+});

packages/teraslice-cli/test/fixtures/job_saves/aliases.yaml

@@ -5,3 +5,5 @@ clusters:
     host: http://test-host
   save_jobs2:
     host: http://test-host
+  export_jobs1:
+    host: http://test-host

packages/teraslice-cli/test/helpers/config-spec.ts

@@ -64,6 +64,27 @@ describe('config', () => {
         });
     });
 
+    describe('-> outdir', () => {
+        test('default directory should be defined', () => {
+            const odir = process.cwd();
+            expect(testConfig.outdir).toBe(odir);
+        });
+
+        test('custom directory should be defined', () => {
+            cliArgs = {
+                'cluster-manager-type': 'native',
+                'output-style': 'txt',
+                'config-dir': path.join(dirname, '../fixtures/config_dir'),
+                'cluster-alias': 'localhost',
+                'outdir': '/tmp/exportTest/test1'
+            };
+            testConfig = new Config(cliArgs);
+
+            const edir = path.join('/','tmp', 'exportTest', 'test1');
+            expect(testConfig.outdir).toBe(edir);
+        });
+    });
+
     describe('-> jobStateDir', () => {
         test('should be defined', () => {
             const adir = path.join(cliArgs['config-dir'], 'job_state_files');
@@ -83,9 +104,11 @@ describe('config', () => {
         test('should be defined', () => {
             const jdir = path.join(cliArgs['config-dir'], 'job_state_files');
             const adir = path.join(cliArgs['config-dir'], 'assets');
+            const odir = process.cwd();
             expect(testConfig.allSubDirs).toBeDefined();
             expect(testConfig.allSubDirs[0]).toBe(jdir);
             expect(testConfig.allSubDirs[1]).toBe(adir);
+            expect(testConfig.allSubDirs[2]).toBe(odir);
         });
     });