Skip to content

Commit

Permalink
more tests, fix race condition in file name
Browse files Browse the repository at this point in the history
  • Loading branch information
@busma13
busma13 committed Sep 9, 2024
1 parent 67e2ffd commit 439c093
Show file tree
Hide file tree
Showing 4 changed files with 197 additions and 13 deletions.
20 changes: 8 additions & 12 deletions packages/teraslice-cli/src/helpers/jobs.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import fs from 'fs-extra';
import {
has, toString, pDelay, pMap,
has, toString, pDelay, pMap, pRetry,
} from '@terascope/utils';
import { Teraslice } from '@terascope/types';
import chalk from 'chalk';
Expand Down Expand Up @@ -598,15 +598,9 @@ export default class Jobs {
return this.getJobIdsFromSavedState();
}

// if action is delete we need to get inactive
// as well as active jobs
if (action === 'delete') {
return this.getActiveAndInactiveJobIds();
}

// if action is export we need to get inactive
// as well as active jobs
if (this.config.args._action === 'export') {
// if action is delete or export we need to
// get inactive as well as active jobs
if (action === 'delete' || action === 'export') {
return this.getActiveAndInactiveJobIds();
}

Expand Down Expand Up @@ -827,8 +821,10 @@ export default class Jobs {
}

async exportOne(jobConfig: Teraslice.JobConfig) {
const filePath = this.createUniqueFilePath(jobConfig.name);
await saveJobConfigToFile(jobConfig, filePath, this.config.clusterUrl);
await pRetry(() => {
const filePath = this.createUniqueFilePath(jobConfig.name);
return saveJobConfigToFile(jobConfig, filePath, this.config.clusterUrl);
})
}

/**
Expand Down
6 changes: 5 additions & 1 deletion packages/teraslice-cli/src/helpers/tjm-util.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,5 +227,9 @@ export async function saveJobConfigToFile(

addMetaData(jobConfigCopy, jobConfig.job_id, clusterUrl);

await fs.writeJSON(filePath, jobConfigCopy);
if (!fs.existsSync(filePath)) {
await fs.writeJSON(filePath, jobConfigCopy);
} else {
throw new Error(`File already exists at ${filePath}`);
}
}
2 changes: 2 additions & 0 deletions packages/teraslice-cli/test/fixtures/job_saves/aliases.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,5 @@ clusters:
host: http://test-host
save_jobs2:
host: http://test-host
export_jobs1:
host: http://test-host
182 changes: 182 additions & 0 deletions packages/teraslice-cli/test/helpers/jobs-spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -944,4 +944,186 @@ describe('Job helper class', () => {
expect(reply.error).toHaveBeenCalledWith(expect.stringContaining('Job is in non-terminal status running, cannot delete. Skipping'))
});
});

describe('export', () => {
const action = 'export';

const exportPath = path.join(dirname, '..', 'fixtures', 'job_exports');

beforeEach(() => {
fs.mkdirSync(exportPath);
})

afterEach(async () => {
fs.removeSync(exportPath);
});

it('should export a job to the default directory', async () => {
const [jobId] = makeJobIds(1);

const jobController = clusterControllers([jobId]);
const jobExecution = getJobExecution(jobId);

tsClient
.get(`/v1/jobs/${jobId}/ex`)
.reply(200, { _status: 'running' })
.get(`/v1/jobs/${jobId}`)
.reply(200, testJobConfig(jobId))
.get(`/v1/jobs/${jobId}/controller`)
.reply(200, () => Promise.resolve(jobController))
.get(`/v1/jobs/${jobId}/ex`)
.reply(200, () => Promise.resolve(jobExecution));

const alias = 'export_jobs1';

const config = buildCLIConfig(
action,
{
'job-id': [jobId],
jobId: [jobId],
clusterAlias: alias
}
);

const job = new Jobs(config);

await job.initialize();

const originalDirectory = process.cwd();
fs.mkdirSync(path.join(exportPath, 'current'));
process.chdir(path.join(exportPath, 'current'));

await job.export();

const jobFile = fs.readJsonSync(path.join(process.cwd(), 'test-job.json'));

expect(jobFile.__metadata.cli.job_id).toBe(jobId);
expect(jobFile.assets).toEqual(['asset1:2.7.4', 'asset2:0.14.1']);
process.chdir(originalDirectory);
});

it('should export a job to a custom directory', async () => {
const customDir = path.join(exportPath, 'custom1');
const [jobId] = makeJobIds(1);

const jobController = clusterControllers([jobId]);
const jobExecution = getJobExecution(jobId);

tsClient
.get(`/v1/jobs/${jobId}/ex`)
.reply(200, { _status: 'running' })
.get(`/v1/jobs/${jobId}`)
.reply(200, testJobConfig(jobId))
.get(`/v1/jobs/${jobId}/controller`)
.reply(200, () => Promise.resolve(jobController))
.get(`/v1/jobs/${jobId}/ex`)
.reply(200, () => Promise.resolve(jobExecution));

const alias = 'export_jobs1';

const config = buildCLIConfig(
action,
{
'job-id': [jobId],
jobId: [jobId],
clusterAlias: alias,
outdir: customDir
}
);

const job = new Jobs(config);

await job.initialize();

await job.export();

const jobFile = fs.readJsonSync(path.join(customDir, 'test-job.json'));

expect(jobFile.__metadata.cli.job_id).toBe(jobId);
expect(jobFile.assets).toEqual(['asset1:2.7.4', 'asset2:0.14.1']);
});

it('should export all jobs to a custom directory', async () => {
const customDir = path.join(exportPath, 'custom2');

const jobIds = makeJobIds(3);
console.log('@@@@ jobIds: ', jobIds);
const [job1, job2, job3] = jobIds;
const jobsList = [{ job_id: job1}, { job_id: job2}, { job_id: job3}]
const jobControllers = clusterControllers(jobIds);

const [jobC1, jobC2, jobC3] = jobControllers;

const jobEx1 = getJobExecution(job1);
const jobEx2 = getJobExecution(job2);
const jobEx3 = getJobExecution(job3);

tsClient
.get('/v1/jobs')
.reply(200, () => Promise.resolve(jobsList))
.get(`/v1/jobs/${job1}/ex`)
.reply(200, { _status: 'running' })
.get(`/v1/jobs/${job2}/ex`)
.reply(200, { _status: 'running' })
.get(`/v1/jobs/${job3}/ex`)
.reply(200, { _status: 'running' })
.get(`/v1/jobs/${job1}`)
.reply(200, testJobConfig(job1))
.get(`/v1/jobs/${job2}`)
.reply(200, testJobConfig(job2))
.get(`/v1/jobs/${job3}`)
.reply(200, testJobConfig(job3))
.get(`/v1/jobs/${job1}/controller`)
.reply(200, () => Promise.resolve([jobC1]))
.get(`/v1/jobs/${job1}/ex`)
.reply(200, () => Promise.resolve(jobEx1))
.get(`/v1/jobs/${job2}/controller`)
.reply(200, () => Promise.resolve([jobC2]))
.get(`/v1/jobs/${job2}/ex`)
.reply(200, () => Promise.resolve(jobEx2))
.get(`/v1/jobs/${job3}/controller`)
.reply(200, () => Promise.resolve([jobC3]))
.get(`/v1/jobs/${job3}/ex`)
.reply(200, () => Promise.resolve(jobEx3));

const alias = 'export_jobs1';

const config = buildCLIConfig(
action,
{
'job-id': ['all'],
jobId: ['all'],
clusterAlias: alias,
outdir: customDir,
yes: true,
y: true
}
);

const job = new Jobs(config);

await job.initialize();

await job.export();

// Since all 3 jobs have the same file name, and exportOne() is called concurrently on
// each job, 2 of the jobs may have to call createUniqueFilePath() a second or third time
// as the original file name given will be taken by file creation time. This means there is
// no order to the file names.
const jobFiles = [];
jobFiles.push(fs.readJsonSync(path.join(customDir, 'test-job.json')));
jobFiles.push(fs.readJsonSync(path.join(customDir, 'test-job-1.json')));
jobFiles.push(fs.readJsonSync(path.join(customDir, 'test-job-2.json')));

expect(jobFiles).toEqual(expect.arrayContaining([
expect.objectContaining({ __metadata: { cli: expect.objectContaining({ job_id: job1 })}}),
expect.objectContaining({ __metadata: { cli: expect.objectContaining({ job_id: job2 })}}),
expect.objectContaining({ __metadata: { cli: expect.objectContaining({ job_id: job3 })}})
]));
});

it('should not create a file with spaces', async () => {

});
});
});

0 comments on commit 439c093

Please sign in to comment.