From 42c63dceded7bbe681022a05687dfacd112b6979 Mon Sep 17 00:00:00 2001 From: Anandashankar Anil Date: Fri, 17 Jan 2025 14:03:33 +0100 Subject: [PATCH 1/7] Rename file to reflect that it handles more reports than just mqc --- status/multiqc_report.py | 20 ----- status/reports.py | 158 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+), 20 deletions(-) delete mode 100644 status/multiqc_report.py create mode 100644 status/reports.py diff --git a/status/multiqc_report.py b/status/multiqc_report.py deleted file mode 100644 index d0fe92008..000000000 --- a/status/multiqc_report.py +++ /dev/null @@ -1,20 +0,0 @@ -from status.util import SafeHandler - - -class MultiQCReportHandler(SafeHandler): - def get(self, project): - type = self.get_argument("type") - # get_multiqc() is defined in BaseHandler - multiqc_report = self.get_multiqc(project) - if multiqc_report: - self.write(multiqc_report[type]) - else: - t = self.application.loader.load("error_page.html") - self.write( - t.generate( - gs_globals=self.application.gs_globals, - status="404", - reason="MultiQC Report Not Found", - user=self.get_current_user(), - ) - ) diff --git a/status/reports.py b/status/reports.py new file mode 100644 index 000000000..0cb9df2f2 --- /dev/null +++ b/status/reports.py @@ -0,0 +1,158 @@ +import os + +from status.util import SafeHandler + + +class MultiQCReportHandler(SafeHandler): + def get(self, project): + report_type = self.get_argument("type") + multiqc_report = self.get_multiqc(self.application, project) + if multiqc_report: + self.write(multiqc_report[report_type]) + else: + t = self.application.loader.load("error_page.html") + self.write( + t.generate( + gs_globals=self.application.gs_globals, + status="404", + reason="MultiQC Report Not Found", + user=self.get_current_user(), + ) + ) + + @staticmethod + def get_multiqc(app, project_id, read_file=True): + """ + Getting multiqc reports for requested project from the filesystem + Returns a string containing html if report exists, otherwise None + If read_file is false, the value of the dictionary will be the path to the file + """ + + project_name = "" + multiqc_reports = {} + query_res = app.cloudant.post_view( + db="projects", ddoc="projects", view="id_to_name", key=project_id + ).get_result() + if query_res["rows"]: + project_name = query_res["rows"][0]["value"] + + if project_name: + multiqc_path = os.path.join(app.reports_path, "mqc_reports") or "" + for report_type in ["_", "_qc_", "_pipeline_"]: + multiqc_name = f"{project_name}{report_type}multiqc_report.html" + multiqc_file_path = os.path.join(multiqc_path, multiqc_name) + if os.path.exists(multiqc_file_path): + if read_file: + with open(multiqc_file_path, encoding="utf-8") as multiqc_file: + html = multiqc_file.read() + multiqc_reports[report_type] = html + else: + multiqc_reports[report_type] = multiqc_file_path + return multiqc_reports + + +class ProjectSummaryReportHandler(SafeHandler): + """Handler for project summary reports generated using yggdrasil""" + + def get(self, project_id): + report = self.get_summary_report(self.application, project_id) + if report: + self.write(report) + else: + t = self.application.loader.load("error_page.html") + self.write( + t.generate( + gs_globals=self.application.gs_globals, + status="404", + reason="Project Summary Report Not Found", + user=self.get_current_user(), + ) + ) + + @staticmethod + def get_summary_report(app, project_id, read_file=True): + """If read_file is false, the function will return True if the file exists, otherwise None + If read_file is True, it returns a string containing the report in html if it exists""" + project_name = "" + + query_res = app.cloudant.post_view( + db="projects", ddoc="projects", view="id_to_name", key=project_id + ).get_result() + + if query_res["rows"]: + project_name = query_res["rows"][0]["value"] + + if project_name: + report_path = os.path.join( + app.reports_path, + "yggdrasil", + project_id, + f"{project_name}_project_summary.html", + ) + if os.path.exists(report_path): + if read_file: + with open(report_path, encoding="utf-8") as report_file: + return report_file.read() + else: + return True + else: + return None + + +class SingleCellSampleSummaryReportHandler(SafeHandler): + """Handler for Single Cell sample summary reports generated using yggdrasil""" + + def get(self, project_id, sample_id): + report = self.get_sample_summary_report( + self.application, project_id, sample_id=sample_id + ) + if report: + self.set_header("Content-Type", "application/pdf") + self.set_header( + "Content-Disposition", + f"inline; filename={sample_id}_single_cell_sample_summary_report.pdf", + ) + self.write(report) + else: + t = self.application.loader.load("error_page.html") + self.write( + t.generate( + gs_globals=self.application.gs_globals, + status="404", + reason="Single Cell Sample Summary Report Not Found", + user=self.get_current_user(), + ) + ) + + @staticmethod + def get_sample_summary_report(app, project_id, sample_id=None): + """Returns a list of sample summary reports for the requested project if sample_id is None, + otherwise returns the report for the requested sample""" + + sample_summary_reports_path = os.path.join( + app.reports_path, "yggdrasil", project_id + ) + if sample_id: + report_path = os.path.join( + sample_summary_reports_path, sample_id, f"{sample_id}_report.pdf" + ) + if os.path.exists(report_path): + with open(report_path, "rb") as report_file: + return report_file.read() + else: + return None + + else: + reports = [] + for item in os.listdir(sample_summary_reports_path): + if os.path.isdir( + os.path.join(sample_summary_reports_path, item) + ) and item.startswith(f"{project_id}_"): + if os.path.exists( + os.path.join( + sample_summary_reports_path, item, f"{item}_report.pdf" + ) + ): + reports.append(item) + + return reports From 504d257fdee22e3df8c64edfb14bc4816751d2cd Mon Sep 17 00:00:00 2001 From: Anandashankar Anil Date: Fri, 17 Jan 2025 14:04:49 +0100 Subject: [PATCH 2/7] Add Yggdrasil reports to Project page --- run_dir/design/project_samples_old.html | 28 ++++++++++++++--------- status/projects.py | 30 ++++++++++++++++++++++--- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/run_dir/design/project_samples_old.html b/run_dir/design/project_samples_old.html index 60996fa6a..01af50be3 100644 --- a/run_dir/design/project_samples_old.html +++ b/run_dir/design/project_samples_old.html @@ -135,13 +135,11 @@

User project description
  • Running Notes
  • User communication
  • Links
    • - {% if not multiqc %} -
  • Open MultiQC reports
    • - {% else %} -
  • - -
    - {% for key in multiqc %} +
  • Reports +
    + {% if multiqc %} +
    MultiQC Reports
    + {% for key in multiqc %} {% if 'qc' in key %} {% set rep_name = ' QC' %} {% else %} @@ -149,9 +147,19 @@

    User project description {% end %} Open{{ rep_name }} MultiQC report {% end %} -

    -
    • - {% end %} + {% end %} + {% if 'project_summary' in reports %} +

    Project Summary Report
    + Open Project Summary report + {% end %} + {% if 'sample_summary_reports' in reports %} +
    Single Cell Sample Summary Reports
    + {% for sample in reports['sample_summary_reports'] %} + {{ sample}} + {% end %} + {% end %} + +
  • Open bioinfo tab
  • Agreements
    • diff --git a/status/projects.py b/status/projects.py index ac5bf13c7..a84284dfd 100644 --- a/status/projects.py +++ b/status/projects.py @@ -17,6 +17,11 @@ from ibm_cloud_sdk_core.api_exception import ApiException from zenpy import ZenpyException +from status.reports import ( + MultiQCReportHandler, + ProjectSummaryReportHandler, + SingleCellSampleSummaryReportHandler, +) from status.util import SafeHandler, dthandler lims = lims.Lims(BASEURI, USERNAME, PASSWORD) @@ -689,7 +694,9 @@ def project_info(self, project, view_with_sources=False): "_qc_": "QC MultiQC", "_pipeline_": "Pipeline MultiQC", } - for report_type in self.get_multiqc(project, read_file=False).keys(): + for report_type in MultiQCReportHandler.get_multiqc( + self.application, project, read_file=False + ).keys(): # Attempt to assign a name of the report type, otherwise default to the type itself report_name = type_to_name.get(report_type, report_type) reports[report_name] = f"/multiqc_report/{project}?type={report_type}" @@ -946,8 +953,24 @@ def get(self, project): worksets_view = self.application.worksets_db.view( "project/ws_name", descending=True ) - # to check if multiqc report exists (get_multiqc() is defined in util.BaseHandler) - multiqc = list(self.get_multiqc(project).keys()) + + reports = {} + multiqc = list( + MultiQCReportHandler.get_multiqc( + self.application, project, read_file=False + ).keys() + ) + if ProjectSummaryReportHandler.get_summary_report( + self.application, project, read_file=False + ): + reports["project_summary"] = True + sample_summary_reports = ( + SingleCellSampleSummaryReportHandler.get_sample_summary_report( + self.application, project + ) + ) + if sample_summary_reports: + reports["sample_summary_reports"] = sample_summary_reports self.write( t.generate( gs_globals=self.application.gs_globals, @@ -959,6 +982,7 @@ def get(self, project): prettify=prettify_css_names, worksets=worksets_view[project], multiqc=multiqc, + reports=reports, lims_uri=BASEURI, ) ) From f9a015badf568207b3efbc12b503d21bcc901a3c Mon Sep 17 00:00:00 2001 From: Anandashankar Anil Date: Fri, 17 Jan 2025 14:06:08 +0100 Subject: [PATCH 3/7] Cleanup code and add urls --- status/bioinfo_analysis.py | 5 +++-- status/util.py | 29 ----------------------------- status_app.py | 15 ++++++++++++--- 3 files changed, 15 insertions(+), 34 deletions(-) diff --git a/status/bioinfo_analysis.py b/status/bioinfo_analysis.py index bbf98849d..2b267a9de 100644 --- a/status/bioinfo_analysis.py +++ b/status/bioinfo_analysis.py @@ -4,6 +4,7 @@ import dateutil +from status.reports import MultiQCReportHandler from status.util import SafeHandler @@ -282,8 +283,8 @@ def get(self, project_id): if application in app_classes[key]: application = key break - # to check if multiqc report exists (get_multiqc() is defined in util.BaseHandler) - multiqc = self.get_multiqc(project_id) or "" + + multiqc = MultiQCReportHandler.get_multiqc(self.application, project_id) or "" self.write( t.generate( gs_globals=self.application.gs_globals, diff --git a/status/util.py b/status/util.py index 38fb741d4..584527b00 100644 --- a/status/util.py +++ b/status/util.py @@ -159,35 +159,6 @@ def write_error(self, status_code, **kwargs): ) ) - def get_multiqc(self, project_id, read_file=True): - """ - Getting multiqc reports for requested project from the filesystem - Returns a string containing html if report exists, otherwise None - If read_file is false, the value of the dictionary will be the path to the file - """ - view = self.application.projects_db.view("project/id_name_dates") - rows = view[project_id].rows - project_name = "" - multiqc_reports = {} - # get only the first one - for row in rows: - project_name = row.value.get("project_name", "") - break - - if project_name: - multiqc_path = self.application.multiqc_path or "" - for type in ["_", "_qc_", "_pipeline_"]: - multiqc_name = f"{project_name}{type}multiqc_report.html" - multiqc_file_path = os.path.join(multiqc_path, multiqc_name) - if os.path.exists(multiqc_file_path): - if read_file: - with open(multiqc_file_path, encoding="utf-8") as multiqc_file: - html = multiqc_file.read() - multiqc_reports[type] = html - else: - multiqc_reports[type] = multiqc_file_path - return multiqc_reports - @staticmethod def get_user_details(app, user_email): user_details = {} diff --git a/status_app.py b/status_app.py index fb51c651c..0c74a4bdd 100644 --- a/status_app.py +++ b/status_app.py @@ -65,7 +65,6 @@ SentInvoiceHandler, ) from status.lanes_ordered import LanesOrderedDataHandler, LanesOrderedHandler -from status.multiqc_report import MultiQCReportHandler from status.ngisweden_stats import NGISwedenHandler from status.ont_plot import ONTFlowcellPlotHandler, ONTFlowcellYieldHandler from status.pricing import ( @@ -124,6 +123,11 @@ qPCRPoolsHandler, ) from status.reads_plot import DataFlowcellYieldHandler, FlowcellPlotHandler +from status.reports import ( + MultiQCReportHandler, + ProjectSummaryReportHandler, + SingleCellSampleSummaryReportHandler, +) from status.running_notes import ( LatestStickyNoteHandler, LatestStickyNotesMultipleHandler, @@ -392,6 +396,7 @@ def __init__(self, settings): ("/projects", ProjectsHandler), ("/project_cards", ProjectCardsHandler), ("/proj_meta", ProjMetaCompareHandler), + ("/proj_summary_report/([^/]*)$", ProjectSummaryReportHandler), ("/reads_total/([^/]*)$", ReadsTotalHandler), ("/rec_ctrl_view/([^/]*)$", RecCtrlDataHandler), ("/sample_requirements", SampleRequirementsViewHandler), @@ -399,6 +404,10 @@ def __init__(self, settings): ("/sample_requirements_update", SampleRequirementsUpdateHandler), ("/sensorpush", SensorpushHandler), ("/sequencing_queues", SequencingQueuesHandler), + ( + "/singlecell_sample_summary_report/(P[^/]*)/([^/]*)$", + SingleCellSampleSummaryReportHandler, + ), ("/smartseq3_progress", SmartSeq3ProgressPageHandler), ("/suggestion_box", SuggestionBoxHandler), ("/user_management", UserManagementHandler), @@ -519,8 +528,8 @@ def __init__(self, settings): # to display instruments in the server status self.server_status = settings.get("server_status") - # project summary - multiqc tab - self.multiqc_path = settings.get("multiqc_path") + # project summary - reports tab, for multiqc reports and yggdrasil reports + self.reports_path = settings.get("reports_path") # MinKNOW reports self.minknow_reports_path = settings.get("minknow_reports_path") From 62b387b3d71870cb4f6136228b3f432b90c7b0d4 Mon Sep 17 00:00:00 2001 From: Anandashankar Anil Date: Fri, 17 Jan 2025 17:03:48 +0100 Subject: [PATCH 4/7] Add check to see it the report paths exist --- status/reports.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/status/reports.py b/status/reports.py index 0cb9df2f2..f12642ade 100644 --- a/status/reports.py +++ b/status/reports.py @@ -144,15 +144,16 @@ def get_sample_summary_report(app, project_id, sample_id=None): else: reports = [] - for item in os.listdir(sample_summary_reports_path): - if os.path.isdir( - os.path.join(sample_summary_reports_path, item) - ) and item.startswith(f"{project_id}_"): - if os.path.exists( - os.path.join( - sample_summary_reports_path, item, f"{item}_report.pdf" - ) - ): - reports.append(item) + if os.path.exists(sample_summary_reports_path): + for item in os.listdir(sample_summary_reports_path): + if os.path.isdir( + os.path.join(sample_summary_reports_path, item) + ) and item.startswith(f"{project_id}_"): + if os.path.exists( + os.path.join( + sample_summary_reports_path, item, f"{item}_report.pdf" + ) + ): + reports.append(item) return reports From c0c3affb2b8268faef99b214b7b6e4fd53990840 Mon Sep 17 00:00:00 2001 From: Anandashankar Anil Date: Fri, 17 Jan 2025 17:13:35 +0100 Subject: [PATCH 5/7] Add type hints --- status/reports.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/status/reports.py b/status/reports.py index f12642ade..480b0fcd8 100644 --- a/status/reports.py +++ b/status/reports.py @@ -1,10 +1,11 @@ import os +from typing import Any, Optional, Union from status.util import SafeHandler class MultiQCReportHandler(SafeHandler): - def get(self, project): + def get(self, project: str) -> None: report_type = self.get_argument("type") multiqc_report = self.get_multiqc(self.application, project) if multiqc_report: @@ -21,7 +22,9 @@ def get(self, project): ) @staticmethod - def get_multiqc(app, project_id, read_file=True): + def get_multiqc( + app: Any, project_id: str, read_file: bool = True + ) -> Union[str, dict, None]: """ Getting multiqc reports for requested project from the filesystem Returns a string containing html if report exists, otherwise None @@ -54,7 +57,7 @@ def get_multiqc(app, project_id, read_file=True): class ProjectSummaryReportHandler(SafeHandler): """Handler for project summary reports generated using yggdrasil""" - def get(self, project_id): + def get(self, project_id: str) -> None: report = self.get_summary_report(self.application, project_id) if report: self.write(report) @@ -70,7 +73,9 @@ def get(self, project_id): ) @staticmethod - def get_summary_report(app, project_id, read_file=True): + def get_summary_report( + app: Any, project_id: str, read_file: bool = True + ) -> Union[str, bool, None]: """If read_file is false, the function will return True if the file exists, otherwise None If read_file is True, it returns a string containing the report in html if it exists""" project_name = "" @@ -102,7 +107,7 @@ def get_summary_report(app, project_id, read_file=True): class SingleCellSampleSummaryReportHandler(SafeHandler): """Handler for Single Cell sample summary reports generated using yggdrasil""" - def get(self, project_id, sample_id): + def get(self, project_id: str, sample_id: str) -> None: report = self.get_sample_summary_report( self.application, project_id, sample_id=sample_id ) @@ -125,7 +130,9 @@ def get(self, project_id, sample_id): ) @staticmethod - def get_sample_summary_report(app, project_id, sample_id=None): + def get_sample_summary_report( + app: Any, project_id: str, sample_id: Optional[str] = None + ) -> Union[bytes, list[str], None]: """Returns a list of sample summary reports for the requested project if sample_id is None, otherwise returns the report for the requested sample""" From 687ebabc5fb65dd94f04926c6f184f491bd625a5 Mon Sep 17 00:00:00 2001 From: Anandashankar Anil Date: Fri, 17 Jan 2025 20:17:50 +0100 Subject: [PATCH 6/7] Subsume minknow and toulligqc report paths to common path --- status/flowcell.py | 6 ++++-- status_app.py | 15 ++++++++------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/status/flowcell.py b/status/flowcell.py index 413a38992..67e8dd092 100644 --- a/status/flowcell.py +++ b/status/flowcell.py @@ -470,7 +470,7 @@ def __init__(self, application, request, **kwargs): super(SafeHandler, self).__init__(application, request, **kwargs) def get(self, name): - reports_dir = self.application.minknow_reports_path + reports_dir = os.path.join(self.application.reports_path, "minknow_reports") report_path = os.path.join(reports_dir, f"report_{name}.html") self.write(open(report_path).read()) @@ -483,7 +483,9 @@ def __init__(self, application, request, **kwargs): super(SafeHandler, self).__init__(application, request, **kwargs) def get(self, name): - reports_dir = self.application.toulligqc_reports_path + reports_dir = os.path.join( + self.application.reports_path, "other_reports", "toulligqc_reports" + ) report_path = os.path.join(reports_dir, f"report_{name}.html") self.write(open(report_path).read()) diff --git a/status_app.py b/status_app.py index 0c74a4bdd..43b8f7506 100644 --- a/status_app.py +++ b/status_app.py @@ -528,15 +528,16 @@ def __init__(self, settings): # to display instruments in the server status self.server_status = settings.get("server_status") - # project summary - reports tab, for multiqc reports and yggdrasil reports + # project summary - reports tab + # Structure of the reports folder: + # / + # ├── other_reports/ + # │ └── toulligqc_reports/ + # ├── minknow_reports/ + # ├── mqc_reports/ + # └── yggdrasil// self.reports_path = settings.get("reports_path") - # MinKNOW reports - self.minknow_reports_path = settings.get("minknow_reports_path") - - # ToulligQC reports - self.toulligqc_reports_path = settings.get("toulligqc_reports_path") - # lims backend credentials limsbackend_cred_loc = Path( settings["lims_backend_credential_location"] From 07ba45966d50de100e01c8e3820b48e5692aeb1f Mon Sep 17 00:00:00 2001 From: Anandashankar Anil Date: Mon, 20 Jan 2025 08:21:01 +0100 Subject: [PATCH 7/7] Display msg if no reports; subsume multiqc into reports --- run_dir/design/project_samples_old.html | 7 +++++-- status/projects.py | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/run_dir/design/project_samples_old.html b/run_dir/design/project_samples_old.html index 01af50be3..12ad2e874 100644 --- a/run_dir/design/project_samples_old.html +++ b/run_dir/design/project_samples_old.html @@ -137,9 +137,12 @@

    User project description
  • Links
  • Reports
    - {% if multiqc %} + {% if not reports%} + No reports to show + {% end %} + {% if 'multiqc' in reports%}
    MultiQC Reports
    - {% for key in multiqc %} + {% for key in reports['multiqc'] %} {% if 'qc' in key %} {% set rep_name = ' QC' %} {% else %} diff --git a/status/projects.py b/status/projects.py index a84284dfd..718f9e3ea 100644 --- a/status/projects.py +++ b/status/projects.py @@ -960,6 +960,8 @@ def get(self, project): self.application, project, read_file=False ).keys() ) + if multiqc: + reports["multiqc"] = multiqc if ProjectSummaryReportHandler.get_summary_report( self.application, project, read_file=False ): @@ -981,7 +983,6 @@ def get(self, project): lims_dashboard_url=self.application.settings["lims_dashboard_url"], prettify=prettify_css_names, worksets=worksets_view[project], - multiqc=multiqc, reports=reports, lims_uri=BASEURI, )