From 7f4307cdd58ee73e9dda0e29b4064f1b857f724a Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Wed, 7 Jun 2023 10:25:40 -0600 Subject: [PATCH 01/10] initial changes --- qiita_db/handlers/tests/test_prep_template.py | 5 +++++ qiita_db/metadata_template/prep_template.py | 9 +++++++++ qiita_pet/handlers/study_handlers/prep_template.py | 6 +++--- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/qiita_db/handlers/tests/test_prep_template.py b/qiita_db/handlers/tests/test_prep_template.py index 3634d49c9..1ab0d312e 100644 --- a/qiita_db/handlers/tests/test_prep_template.py +++ b/qiita_db/handlers/tests/test_prep_template.py @@ -172,6 +172,11 @@ def test_post(self): self.assertEqual(pt.name, data['name']) self.assertEqual(pt.creation_job_id, data['job-id']) + # testing setter + jid = 'aaaaaaaa-aaaa-bbbb-aaaa-aaaaaaaaaaaa' + pt.creation_job_id = jid + self.assertEqual(pt.creation_job_id, jid) + if __name__ == '__main__': main() diff --git a/qiita_db/metadata_template/prep_template.py b/qiita_db/metadata_template/prep_template.py index 38b46f8d2..b23b6dc9f 100644 --- a/qiita_db/metadata_template/prep_template.py +++ b/qiita_db/metadata_template/prep_template.py @@ -970,3 +970,12 @@ def creation_job_id(self): WHERE prep_template_id = %s""" qdb.sql_connection.TRN.add(sql, [self.id]) return qdb.sql_connection.TRN.execute_fetchlast() + + @creation_job_id.setter + def creation_job_id(self, creation_job_id): + with qdb.sql_connection.TRN: + sql = """UPDATE qiita.prep_template + SET creation_job_id = %s + WHERE prep_template_id = %s""" + qdb.sql_connection.TRN.add(sql, [creation_job_id, self.id]) + qdb.sql_connection.TRN.execute() diff --git a/qiita_pet/handlers/study_handlers/prep_template.py b/qiita_pet/handlers/study_handlers/prep_template.py index eaae5fbb7..e02d2c477 100644 --- a/qiita_pet/handlers/study_handlers/prep_template.py +++ b/qiita_pet/handlers/study_handlers/prep_template.py @@ -76,9 +76,9 @@ def get(self): res['alert_message'] = url_escape(res['alert_message']) res['user_level'] = current_user.level if res['creation_job'] is not None: - vals = res['creation_job'].values - res['creation_job_filename'] = vals['filename'] - res['creation_job_filename_body'] = vals['body'] + fp = res['creation_job'].parameters.values['sample_sheet'] + res['creation_job_filename'] = fp['filename'] + res['creation_job_filename_body'] = fp['body'] self.render('study_ajax/prep_summary.html', **res) From cd347d6d550db45a65ffd64c30821d1f8d83ab9e Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Wed, 7 Jun 2023 11:20:16 -0600 Subject: [PATCH 02/10] fix /prep_summary.html [skip ci] --- qiita_pet/templates/study_ajax/prep_summary.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_pet/templates/study_ajax/prep_summary.html b/qiita_pet/templates/study_ajax/prep_summary.html index 954c27d25..51620085f 100644 --- a/qiita_pet/templates/study_ajax/prep_summary.html +++ b/qiita_pet/templates/study_ajax/prep_summary.html @@ -431,7 +431,7 @@

{{name}} - ID {{prep_id}} ({{data_type}}) {% if user_level in ('admin', 'wet-lab admin') and creation_job is not None %} - SampleSheet + SampleSheet {% end %} Edit name Prep info From 5b51aa1a33e0c905599b99169d076d92c07401e5 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Mon, 12 Jun 2023 07:53:17 -0600 Subject: [PATCH 03/10] restrict admin jobs to 14 days --- qiita_pet/handlers/admin_processing_job.py | 66 +++++++++++++--------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/qiita_pet/handlers/admin_processing_job.py b/qiita_pet/handlers/admin_processing_job.py index c740a4602..f712b781c 100644 --- a/qiita_pet/handlers/admin_processing_job.py +++ b/qiita_pet/handlers/admin_processing_job.py @@ -15,6 +15,8 @@ from qiita_db.software import Software from qiita_db.study import Study from qiita_db.exceptions import QiitaDBUnknownIDError +from qiita_db.sql_connection import TRN +from qiita_db.processing_job import ProcessingJob as PJ from json import dumps from collections import Counter @@ -57,35 +59,43 @@ def get(self): echo = self.get_argument('sEcho') command_id = int(self.get_argument('commandId')) - jobs = [] - for ps in self._get_private_software(): - for cmd in ps.commands: - if cmd.id != command_id: - continue + with TRN: + # different versions of the same plugin will have different + # command_id, this will make sure to get them all (commands) + sql = """SELECT processing_job_id FROM qiita.processing_job + WHERE command_id in ( + SELECT command_id FROM qiita.software_command + WHERE + name in ( + SELECT name FROM qiita.software_command + WHERE command_id = %s)) AND + (heartbeat > current_date - interval '14' day OR + heartbeat is NULL)""" + TRN.add(sql, [command_id]) + jids = TRN.execute_fetchflatten() - for job in cmd.processing_jobs: - if job.hidden: - continue - msg = '' - if job.status == 'error': - msg = job.log.msg - elif job.status == 'running': - msg = job.step - msg = msg.replace('\n', '
') - outputs = [] - if job.status == 'success': - outputs = [[k, v.id] for k, v in job.outputs.items()] - validator_jobs = [v.id for v in job.validator_jobs] - - if job.heartbeat is not None: - heartbeat = job.heartbeat.strftime('%Y-%m-%d %H:%M:%S') - else: - heartbeat = 'N/A' - - jobs.append([job.id, job.command.name, job.status, msg, - outputs, validator_jobs, heartbeat, - job.parameters.values, job.external_id, - job.user.email]) + jobs = [] + for jid in jids: + job = PJ(jid) + if job.status == 'error': + msg = job.log.msg + elif job.status == 'running': + msg = job.step + msg = msg.replace('\n', '
') + outputs = [] + if job.status == 'success': + outputs = [[k, v.id] for k, v in job.outputs.items()] + validator_jobs = [v.id for v in job.validator_jobs] + + if job.heartbeat is not None: + heartbeat = job.heartbeat.strftime('%Y-%m-%d %H:%M:%S') + else: + heartbeat = 'N/A' + + jobs.append([job.id, job.command.name, job.status, msg, + outputs, validator_jobs, heartbeat, + job.parameters.values, job.external_id, + job.user.email]) results = { "sEcho": echo, "recordsTotal": len(jobs), From 5a68e765b400a9a3138d795cea1d7764a28e0e5d Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Mon, 12 Jun 2023 08:32:49 -0600 Subject: [PATCH 04/10] commandId=1 -> commandId=2 --- qiita_pet/handlers/admin_processing_job.py | 2 ++ qiita_pet/test/test_admin_processing_job_handlers.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/qiita_pet/handlers/admin_processing_job.py b/qiita_pet/handlers/admin_processing_job.py index f712b781c..a4c641296 100644 --- a/qiita_pet/handlers/admin_processing_job.py +++ b/qiita_pet/handlers/admin_processing_job.py @@ -72,11 +72,13 @@ def get(self): (heartbeat > current_date - interval '14' day OR heartbeat is NULL)""" TRN.add(sql, [command_id]) + print (f'-----> {command_id}') jids = TRN.execute_fetchflatten() jobs = [] for jid in jids: job = PJ(jid) + msg = '' if job.status == 'error': msg = job.log.msg elif job.status == 'running': diff --git a/qiita_pet/test/test_admin_processing_job_handlers.py b/qiita_pet/test/test_admin_processing_job_handlers.py index 78ea3a022..4e45daa9e 100644 --- a/qiita_pet/test/test_admin_processing_job_handlers.py +++ b/qiita_pet/test/test_admin_processing_job_handlers.py @@ -32,7 +32,7 @@ def test_get(self): class TestAJAXAdminProcessingJobListing(BaseAdminTests): def test_get(self): - response = self.get('/admin/processing_jobs/list?sEcho=3&commandId=1') + response = self.get('/admin/processing_jobs/list?sEcho=3&commandId=2') self.assertEqual(response.code, 200) exp = {'sEcho': '3', 'recordsTotal': 0, 'recordsFiltered': 0, From 3e28abf5dc1bdf7e22e9e8ded7bd3fb4a5a0169e Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Mon, 12 Jun 2023 09:12:15 -0600 Subject: [PATCH 05/10] flake8 --- qiita_pet/handlers/admin_processing_job.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qiita_pet/handlers/admin_processing_job.py b/qiita_pet/handlers/admin_processing_job.py index a4c641296..7136b3477 100644 --- a/qiita_pet/handlers/admin_processing_job.py +++ b/qiita_pet/handlers/admin_processing_job.py @@ -72,7 +72,6 @@ def get(self): (heartbeat > current_date - interval '14' day OR heartbeat is NULL)""" TRN.add(sql, [command_id]) - print (f'-----> {command_id}') jids = TRN.execute_fetchflatten() jobs = [] From 47fb91ef81b849921c14a7e7e535bca83018a86f Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Tue, 13 Jun 2023 06:37:48 -0600 Subject: [PATCH 06/10] return dict in APIArtifactHandler --- qiita_db/handlers/artifact.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qiita_db/handlers/artifact.py b/qiita_db/handlers/artifact.py index 26ef5ca66..5080bae46 100644 --- a/qiita_db/handlers/artifact.py +++ b/qiita_db/handlers/artifact.py @@ -320,5 +320,4 @@ def post(self): r_client.set('prep_template_%d' % prep_id, dumps({'job_id': new_job.id, 'is_qiita_job': True})) - self.write(new_job.id) - self.finish() + self.finish({'job_id': new_job.id}) From 3639b0f7769e84c014b9e2626cee0036fe423107 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Tue, 13 Jun 2023 07:23:42 -0600 Subject: [PATCH 07/10] fix APIArtifactHandlerTests --- qiita_db/handlers/tests/test_artifact.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiita_db/handlers/tests/test_artifact.py b/qiita_db/handlers/tests/test_artifact.py index 4f66ae32c..009454b87 100644 --- a/qiita_db/handlers/tests/test_artifact.py +++ b/qiita_db/handlers/tests/test_artifact.py @@ -368,7 +368,7 @@ def test_post(self): # send the new data del data['prep_id'] obs = self.post('/qiita_db/artifact/', headers=self.header, data=data) - jid = obs.body.decode("utf-8") + jid = loads(obs.body)['job_id'] job = qdb.processing_job.ProcessingJob(jid) while job.status not in ('error', 'success'): @@ -404,7 +404,7 @@ def test_post(self): 'files': dumps({'biom': [fp]})} obs = self.post('/qiita_db/artifact/', headers=self.header, data=data) - jid = obs.body.decode("utf-8") + jid = loads(obs.body)['job_id'] job = qdb.processing_job.ProcessingJob(jid) while job.status not in ('error', 'success'): From b0ff2c7e0eaed4814479a1f494fd85a55b18677d Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Tue, 13 Jun 2023 09:13:22 -0600 Subject: [PATCH 08/10] fix validator_jobs error --- qiita_pet/handlers/admin_processing_job.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/qiita_pet/handlers/admin_processing_job.py b/qiita_pet/handlers/admin_processing_job.py index 7136b3477..c9d58ac7d 100644 --- a/qiita_pet/handlers/admin_processing_job.py +++ b/qiita_pet/handlers/admin_processing_job.py @@ -86,12 +86,12 @@ def get(self): outputs = [] if job.status == 'success': outputs = [[k, v.id] for k, v in job.outputs.items()] - validator_jobs = [v.id for v in job.validator_jobs] + validator_jobs = [v.id for v in job.validator_jobs] - if job.heartbeat is not None: - heartbeat = job.heartbeat.strftime('%Y-%m-%d %H:%M:%S') - else: - heartbeat = 'N/A' + if job.heartbeat is not None: + heartbeat = job.heartbeat.strftime('%Y-%m-%d %H:%M:%S') + else: + heartbeat = 'N/A' jobs.append([job.id, job.command.name, job.status, msg, outputs, validator_jobs, heartbeat, From 476ec3949d05516a5f2f03cff7c04f965a3ebb66 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Tue, 13 Jun 2023 09:24:26 -0600 Subject: [PATCH 09/10] hidden = false --- qiita_pet/handlers/admin_processing_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_pet/handlers/admin_processing_job.py b/qiita_pet/handlers/admin_processing_job.py index c9d58ac7d..5325558e5 100644 --- a/qiita_pet/handlers/admin_processing_job.py +++ b/qiita_pet/handlers/admin_processing_job.py @@ -63,7 +63,7 @@ def get(self): # different versions of the same plugin will have different # command_id, this will make sure to get them all (commands) sql = """SELECT processing_job_id FROM qiita.processing_job - WHERE command_id in ( + WHERE hidden = false and command_id in ( SELECT command_id FROM qiita.software_command WHERE name in ( From 7b807b1890f7b940c20aef97c0c4a0a1074bbcbf Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Wed, 14 Jun 2023 11:29:42 -0600 Subject: [PATCH 10/10] add external_id to admin jobs page --- qiita_pet/templates/admin_processing_job.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiita_pet/templates/admin_processing_job.html b/qiita_pet/templates/admin_processing_job.html index cd010d3ef..d341294d4 100644 --- a/qiita_pet/templates/admin_processing_job.html +++ b/qiita_pet/templates/admin_processing_job.html @@ -71,12 +71,12 @@ }; out.push('' + status + ' ' + - row[0] + ' [ ' + row[9] + ' ]
'); + row[0] + ' (' + row[8] + ') [ ' + row[9] + ' ]
'); if (status === 'running' || status === 'queued') { // row[0] is qiita job-id // row[3] is status 'Step n of 6' and other messages - out.push('[' + row[8] + ']
' + row[3] + '') + out.push('' + row[3] + '') } else { // We write a callback attribute on the link to be able to display a