diff --git a/.github/pr_labels.yml b/.github/pr_labels.yml deleted file mode 100644 index d04f24f38..000000000 --- a/.github/pr_labels.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: '1' -invalidStatus: "pending" -labelRule: - values: - - "validation" - - "no validation" diff --git a/.github/workflows/check_pr_label.yml b/.github/workflows/check_pr_label.yml new file mode 100644 index 000000000..cac9312cf --- /dev/null +++ b/.github/workflows/check_pr_label.yml @@ -0,0 +1,21 @@ +name: Check Label on PR +on: + pull_request: + types: [opened, synchronize, labeled, unlabeled] + +jobs: + check-pr_label: + runs-on: ubuntu-latest + steps: + - name: Checkout PR + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Fetch all history for all branches and tags + + - name: Check if the PR contains the label validation or no validation + id: prlabel_check + if: | + ! contains( github.event.pull_request.labels.*.name, 'validation') && ! contains( github.event.pull_request.labels.*.name, 'no validation') + run: | + echo "Neither 'validation' nor 'no validation' labels are present." + exit 1 # Exit with a failure \ No newline at end of file diff --git a/run_dir/design/deliveries.html b/run_dir/design/deliveries.html index 676e63379..8cc2fa8d7 100644 --- a/run_dir/design/deliveries.html +++ b/run_dir/design/deliveries.html @@ -145,7 +145,13 @@

- {{ flowcell_id }} + {% set url_addition = "" %} + {% if flowcell.get("instrument_type") == "element"%} + {% set url_addition = "_element" %} + {% elif flowcell.get("instrument_type") == "ont"%} + {% set url_addition = "_ont" %} + {% end %} + {{ flowcell_id }} {{ flowcell['flowcell_status'] }}
diff --git a/run_dir/design/project_samples_old.html b/run_dir/design/project_samples_old.html index 60996fa6a..ce5dcd88d 100644 --- a/run_dir/design/project_samples_old.html +++ b/run_dir/design/project_samples_old.html @@ -135,23 +135,34 @@

User project description
  • Running Notes
  • User communication
  • Links
    • - {% if not multiqc %} -
  • Open MultiQC reports
    • - {% else %} -
  • - -
    - {% for key in multiqc %} +
  • Reports +
    + {% if not reports%} + No reports to show + {% end %} + {% if 'multiqc' in reports%} + {% for key in reports['multiqc'] %} {% if 'qc' in key %} {% set rep_name = ' QC' %} {% else %} {% set rep_name = ' '+key.strip('_').capitalize() %} {% end %} - Open{{ rep_name }} MultiQC report + MultiQC {{ rep_name }} {% end %} -
    -
    • - {% end %} + {% end %} + {% if 'project_summary' in reports %} + Project Summary + {% end %} + {% if 'sample_summary_reports' in reports %} +

    Single Cell Sample Summary
    + {% for sample, values in reports['sample_summary_reports'].items() %} + {% for method, report in values.items() %} + {{ sample }} {{ method }} + {% end %} + {% end %} + {% end %} +
    +
  • Open bioinfo tab
  • Agreements
    • diff --git a/status/bioinfo_analysis.py b/status/bioinfo_analysis.py index bbf98849d..2b267a9de 100644 --- a/status/bioinfo_analysis.py +++ b/status/bioinfo_analysis.py @@ -4,6 +4,7 @@ import dateutil +from status.reports import MultiQCReportHandler from status.util import SafeHandler @@ -282,8 +283,8 @@ def get(self, project_id): if application in app_classes[key]: application = key break - # to check if multiqc report exists (get_multiqc() is defined in util.BaseHandler) - multiqc = self.get_multiqc(project_id) or "" + + multiqc = MultiQCReportHandler.get_multiqc(self.application, project_id) or "" self.write( t.generate( gs_globals=self.application.gs_globals, diff --git a/status/clone_project.py b/status/clone_project.py index 801980297..e58e23045 100644 --- a/status/clone_project.py +++ b/status/clone_project.py @@ -28,32 +28,21 @@ class LIMSProjectCloningHandler(SafeHandler): """ def get(self, project_identifier): - # Check if the project_identifier matches a project id. - # If not, assuming it's a project name, try to get the project id from the project name, - # since the LIMS API only accepts project ids - if not re.match("^(P[0-9]{3,7})", project_identifier): - try: - projectid = ( - self.application.projects_db.view("projects/name_to_id")[ - project_identifier - ] - .rows[0] - .value - ) - except IndexError: - self.set_status(404) - return self.write({"error": "Project not found"}) - else: - projectid = project_identifier + projectid = self.get_project_id(project_identifier) + if not projectid: + self.set_status(404) + return self.write({"error": "Project not found"}) + proj_values = self.get_project_data_from_lims(projectid, "get") if not proj_values: self.set_status(404) self.write({"error": "Project not found"}) return + self.set_header("Content-type", "application/json") self.write(proj_values) - def post(self, projectid): + def post(self, project_identifier): if not ( self.get_current_user().is_proj_coord or self.get_current_user().is_any_admin @@ -63,6 +52,11 @@ def post(self, projectid): "Error: You do not have the permissions for this operation!" ) + projectid = self.get_project_id(project_identifier) + if not projectid: + self.set_status(404) + return self.write({"error": "Project not found"}) + new_proj = self.get_project_data_from_lims(projectid, "post") if "error" in new_proj: self.set_status(400) @@ -72,7 +66,7 @@ def post(self, projectid): self.set_status(201) self.write(new_proj) - def get_project_data_from_lims(self, projectid, type): + def get_project_data_from_lims(self, projectid, req_type): copy_udfs = { "Customer project reference", "Project Comment", @@ -128,7 +122,7 @@ def get_project_data_from_lims(self, projectid, type): udfs[udf] = existing_project.udf[udf] proj_values["udfs"] = udfs - if type == "get": + if req_type == "get": return proj_values else: @@ -149,3 +143,24 @@ def get_project_data_from_lims(self, projectid, type): return {"error": e.message} return {"project_id": new_project.id, "project_name": new_project.name} + + def get_project_id(self, project_identifier): + """Return projectid for the provided identifier""" + # Check if the project_identifier matches a project id. + # If not, assuming it's a project name, try to get the project id from the project name, + # since the LIMS API only accepts project ids + projectid = None + if re.match("^(P[0-9]{3,7})", project_identifier): + projectid = project_identifier + else: + try: + projectid = ( + self.application.projects_db.view("projects/name_to_id")[ + project_identifier + ] + .rows[0] + .value + ) + except IndexError: + pass + return projectid diff --git a/status/deliveries.py b/status/deliveries.py index 7c586f86b..1f11f2825 100644 --- a/status/deliveries.py +++ b/status/deliveries.py @@ -159,6 +159,7 @@ def get(self): # define bioinfo checklist sample_data = flowcells[flowcell_id][lane_id][sample_id] + instrument_type = sample_data.get("instrument_type") checklist = self.__fill_checklist(sample_data) if checklist["total"] and len(checklist["total"]) == len( checklist["passed"] @@ -233,6 +234,7 @@ def get(self): flowcell_status = self.__aggregate_status(flowcell_statuses) runs_bioinfo[flowcell_id]["flowcell_status"] = flowcell_status runs_bioinfo[flowcell_id]["checklist"] = flowcell_checklists + runs_bioinfo[flowcell_id]["instrument_type"] = instrument_type # add flowcell_status to the status_list (needed for filtering) if flowcell_status not in status_list: diff --git a/status/flowcell.py b/status/flowcell.py index 413a38992..18613a6e9 100644 --- a/status/flowcell.py +++ b/status/flowcell.py @@ -470,8 +470,9 @@ def __init__(self, application, request, **kwargs): super(SafeHandler, self).__init__(application, request, **kwargs) def get(self, name): - reports_dir = self.application.minknow_reports_path - report_path = os.path.join(reports_dir, f"report_{name}.html") + report_path = os.path.join( + self.application.report_path["minknow"], f"report_{name}.html" + ) self.write(open(report_path).read()) @@ -483,8 +484,9 @@ def __init__(self, application, request, **kwargs): super(SafeHandler, self).__init__(application, request, **kwargs) def get(self, name): - reports_dir = self.application.toulligqc_reports_path - report_path = os.path.join(reports_dir, f"report_{name}.html") + report_path = os.path.join( + self.application.report_path["toulligqc"], f"report_{name}.html" + ) self.write(open(report_path).read()) @@ -758,10 +760,16 @@ def get(self, name): barcodes=self.fetch_barcodes(name), args=self.fetch_args(name), has_minknow_report=os.path.exists( - f"{self.application.minknow_reports_path}/report_{name}.html" + os.path.join( + self.application.report_path["minknow"], + f"report_{name}.html", + ) ), has_toulligqc_report=os.path.exists( - f"{self.application.toulligqc_reports_path}/report_{name}.html" + os.path.join( + self.application.report_path["tooulligqc"], + f"report_{name}.html", + ) ), user=self.get_current_user(), ) diff --git a/status/multiqc_report.py b/status/multiqc_report.py deleted file mode 100644 index d0fe92008..000000000 --- a/status/multiqc_report.py +++ /dev/null @@ -1,20 +0,0 @@ -from status.util import SafeHandler - - -class MultiQCReportHandler(SafeHandler): - def get(self, project): - type = self.get_argument("type") - # get_multiqc() is defined in BaseHandler - multiqc_report = self.get_multiqc(project) - if multiqc_report: - self.write(multiqc_report[type]) - else: - t = self.application.loader.load("error_page.html") - self.write( - t.generate( - gs_globals=self.application.gs_globals, - status="404", - reason="MultiQC Report Not Found", - user=self.get_current_user(), - ) - ) diff --git a/status/projects.py b/status/projects.py index b30264f37..59068ec97 100644 --- a/status/projects.py +++ b/status/projects.py @@ -5,6 +5,7 @@ import itertools import json import logging +import re from collections import OrderedDict import dateutil.parser @@ -13,9 +14,15 @@ from dateutil.relativedelta import relativedelta from genologics import lims from genologics.config import BASEURI, PASSWORD, USERNAME -from genologics.entities import Artifact, Project +from genologics.entities import Artifact +from ibm_cloud_sdk_core.api_exception import ApiException from zenpy import ZenpyException +from status.reports import ( + MultiQCReportHandler, + ProjectSummaryReportHandler, + SingleCellSampleSummaryReportHandler, +) from status.util import SafeHandler, dthandler lims = lims.Lims(BASEURI, USERNAME, PASSWORD) @@ -593,6 +600,13 @@ def search_project_names(self, search_string=""): } projects.append(project) + # Sort projects by project number so that the latest P-number is first + projects = sorted( + projects, + key=lambda x: int(x["name"].split(",")[0].replace("P", "")), + reverse=True, + ) + return projects @@ -693,7 +707,9 @@ def project_info(self, project, view_with_sources=False): "_qc_": "QC MultiQC", "_pipeline_": "Pipeline MultiQC", } - for report_type in self.get_multiqc(project, read_file=False).keys(): + for report_type in MultiQCReportHandler.get_multiqc( + self.application, project, read_file=False + ).keys(): # Attempt to assign a name of the report type, otherwise default to the type itself report_name = type_to_name.get(report_type, report_type) reports[report_name] = f"/multiqc_report/{project}?type={report_type}" @@ -962,8 +978,38 @@ def get(self, project): worksets_view = self.application.worksets_db.view( "project/ws_name", descending=True ) - # to check if multiqc report exists (get_multiqc() is defined in util.BaseHandler) - multiqc = list(self.get_multiqc(project).keys()) + + reports = {} + multiqc = list( + MultiQCReportHandler.get_multiqc( + self.application, project, read_file=False + ).keys() + ) + if multiqc: + reports["multiqc"] = multiqc + if ProjectSummaryReportHandler.get_summary_report( + self.application, project, read_file=False + ): + reports["project_summary"] = True + sample_summary_reports = ( + SingleCellSampleSummaryReportHandler.get_sample_summary_reports( + self.application, project + ) + ) + if sample_summary_reports: + group_summary_reports = {} + for report in sample_summary_reports: + # Match report names in the format _(_<(optional)>)_report.html/pdf + match = re.match( + rf"^({project}_\d+)_([^_]+(_[^_]+)?)_report\.(pdf|html)", report + ) + if match: + sample_id = match.group(1) + method = match.group(2) + if sample_id not in group_summary_reports: + group_summary_reports[sample_id] = {} + group_summary_reports[sample_id][method] = report + reports["sample_summary_reports"] = group_summary_reports self.write( t.generate( gs_globals=self.application.gs_globals, @@ -974,7 +1020,7 @@ def get(self, project): lims_dashboard_url=self.application.settings["lims_dashboard_url"], prettify=prettify_css_names, worksets=worksets_view[project], - multiqc=multiqc, + reports=reports, lims_uri=BASEURI, ) ) @@ -1025,11 +1071,15 @@ class LinksDataHandler(SafeHandler): """ def get(self, project): - self.set_header("Content-type", "application/json") - p = Project(lims, id=project) - p.get(force=True) - - links = json.loads(p.udf["Links"]) if "Links" in p.udf else {} + links_doc = {} + try: + links_doc = self.application.cloudant.get_document( + db="gs_links", doc_id=project + ).get_result() + except ApiException as e: + if e.message == "not_found": + pass + links = links_doc.get("links", {}) # Sort by descending date, then hopefully have deviations on top sorted_links = OrderedDict() @@ -1038,6 +1088,8 @@ def get(self, project): sorted_links = OrderedDict( sorted(sorted_links.items(), key=lambda k: k[1]["type"]) ) + + self.set_header("Content-type", "application/json") self.write(sorted_links) def post(self, project): @@ -1051,19 +1103,39 @@ def post(self, project): self.set_status(400) self.finish("Link title and type is required") else: - p = Project(lims, id=project) - p.get(force=True) - links = json.loads(p.udf["Links"]) if "Links" in p.udf else {} - links[str(datetime.datetime.now())] = { - "user": user.name, - "email": user.email, - "type": a_type, - "title": title, - "url": url, - "desc": desc, - } - p.udf["Links"] = json.dumps(links) - p.put() + links_doc = {} + links = {} + try: + links_doc = self.application.cloudant.get_document( + db="gs_links", doc_id=project + ).get_result() + except ApiException as e: + if e.message == "not_found": + links_doc["_id"] = project + links_doc["links"] = {} + links = links_doc.get("links", {}) + links.update( + { + str(datetime.datetime.now()): { + "user": user.name, + "email": user.email, + "type": a_type, + "title": title, + "url": url, + "desc": desc, + } + } + ) + links_doc["links"] = links + + response = self.application.cloudant.post_document( + db="gs_links", document=links_doc + ).get_result() + + if not response.get("ok"): + self.set_status(500) + return + self.set_status(200) # ajax cries if it does not get anything back self.set_header("Content-type", "application/json") diff --git a/status/reports.py b/status/reports.py new file mode 100644 index 000000000..3d61ee2a7 --- /dev/null +++ b/status/reports.py @@ -0,0 +1,168 @@ +import os +from typing import Any, Union + +from status.util import SafeHandler + + +class MultiQCReportHandler(SafeHandler): + def get(self, project: str) -> None: + report_type = self.get_argument("type") + multiqc_report = self.get_multiqc(self.application, project) + if multiqc_report: + self.write(multiqc_report[report_type]) + else: + t = self.application.loader.load("error_page.html") + self.write( + t.generate( + gs_globals=self.application.gs_globals, + status="404", + reason="MultiQC Report Not Found", + user=self.get_current_user(), + ) + ) + + @staticmethod + def get_multiqc( + app: Any, project_id: str, read_file: bool = True + ) -> Union[str, dict, None]: + """ + Getting multiqc reports for requested project from the filesystem + Returns a string containing html if report exists, otherwise None + If read_file is false, the value of the dictionary will be the path to the file + """ + + project_name = "" + multiqc_reports = {} + query_res = app.cloudant.post_view( + db="projects", ddoc="projects", view="id_to_name", key=project_id + ).get_result() + if query_res["rows"]: + project_name = query_res["rows"][0]["value"] + + if project_name: + multiqc_path = app.report_path["multiqc"] or "" + for report_type in ["_", "_qc_", "_pipeline_"]: + multiqc_name = f"{project_name}{report_type}multiqc_report.html" + multiqc_file_path = os.path.join(multiqc_path, multiqc_name) + if os.path.exists(multiqc_file_path): + if read_file: + with open(multiqc_file_path, encoding="utf-8") as multiqc_file: + html = multiqc_file.read() + multiqc_reports[report_type] = html + else: + multiqc_reports[report_type] = multiqc_file_path + return multiqc_reports + + +class ProjectSummaryReportHandler(SafeHandler): + """Handler for project summary reports generated using yggdrasil""" + + def get(self, project_id: str) -> None: + report = self.get_summary_report(self.application, project_id) + if report: + self.write(report) + else: + t = self.application.loader.load("error_page.html") + self.write( + t.generate( + gs_globals=self.application.gs_globals, + status="404", + reason="Project Summary Report Not Found", + user=self.get_current_user(), + ) + ) + + @staticmethod + def get_summary_report( + app: Any, project_id: str, read_file: bool = True + ) -> Union[str, bool, None]: + """If read_file is false, the function will return True if the file exists, otherwise None + If read_file is True, it returns a string containing the report in html if it exists""" + project_name = "" + + query_res = app.cloudant.post_view( + db="projects", ddoc="projects", view="id_to_name", key=project_id + ).get_result() + + if query_res["rows"]: + project_name = query_res["rows"][0]["value"] + + if project_name: + report_path = os.path.join( + app.report_path["yggdrasil"], + project_id, + f"{project_name}_project_summary.html", + ) + if os.path.exists(report_path): + if read_file: + with open(report_path, encoding="utf-8") as report_file: + return report_file.read() + else: + return True + else: + return None + + +class SingleCellSampleSummaryReportHandler(SafeHandler): + """Handler for Single Cell sample summary reports generated using yggdrasil""" + + def get(self, project_id: str, sample_id: str, rep_name: str) -> None: + proj_path = os.path.join(self.application.report_path["yggdrasil"], project_id) + file_type = rep_name.split(".")[-1] + mode = "rb" if file_type == "pdf" else "r" + encoding = None if file_type == "pdf" else "utf-8" + report_path = os.path.join(proj_path, sample_id, rep_name) + report = None + if os.path.exists(report_path): + with open(report_path, mode, encoding=encoding) as report_file: + report = report_file.read() + + if report: + if "pdf" in rep_name: + self.set_header("Content-Type", "application/pdf") + self.set_header( + "Content-Disposition", + f"inline; filename={sample_id}_single_cell_sample_summary_report.pdf", + ) + self.write(report) + else: + t = self.application.loader.load("error_page.html") + self.write( + t.generate( + gs_globals=self.application.gs_globals, + status="404", + reason="Single Cell Sample Summary Report Not Found", + user=self.get_current_user(), + ) + ) + + @staticmethod + def get_sample_summary_reports( + app: Any, + project_id: str, + ) -> Union[list[str], None]: + """Returns a list of sample summary reports for the requested project if sample_id is None, + otherwise returns the report for the requested sample""" + + proj_path = os.path.join(app.report_path["yggdrasil"], project_id) + reports = [] + + if os.path.exists(proj_path): + for item in os.listdir(proj_path): + if os.path.isdir(os.path.join(proj_path, item)) and item.startswith( + f"{project_id}_" + ): + sample_path = os.path.join(proj_path, item) + if os.path.exists(sample_path): + # Reports will be named as __<(optional)>_report.html/pdf + reports = [ + f + for f in os.listdir(sample_path) + if os.path.isfile(os.path.join(sample_path, f)) + and ( + f.startswith(f"{item}_") + and f.endswith(("_report.pdf", "_report.html")) + ) + ] + + return reports diff --git a/status/running_notes.py b/status/running_notes.py index 7516e3359..645ff1814 100644 --- a/status/running_notes.py +++ b/status/running_notes.py @@ -330,7 +330,7 @@ def notify_tagged_user( html = '\ \

    \ - {} in the project {}, {}! The note is as follows

    \ + {} in the project {}, {}
    The note is as follows

    \
    \
    \
    \ diff --git a/status/util.py b/status/util.py index 38fb741d4..584527b00 100644 --- a/status/util.py +++ b/status/util.py @@ -159,35 +159,6 @@ def write_error(self, status_code, **kwargs): ) ) - def get_multiqc(self, project_id, read_file=True): - """ - Getting multiqc reports for requested project from the filesystem - Returns a string containing html if report exists, otherwise None - If read_file is false, the value of the dictionary will be the path to the file - """ - view = self.application.projects_db.view("project/id_name_dates") - rows = view[project_id].rows - project_name = "" - multiqc_reports = {} - # get only the first one - for row in rows: - project_name = row.value.get("project_name", "") - break - - if project_name: - multiqc_path = self.application.multiqc_path or "" - for type in ["_", "_qc_", "_pipeline_"]: - multiqc_name = f"{project_name}{type}multiqc_report.html" - multiqc_file_path = os.path.join(multiqc_path, multiqc_name) - if os.path.exists(multiqc_file_path): - if read_file: - with open(multiqc_file_path, encoding="utf-8") as multiqc_file: - html = multiqc_file.read() - multiqc_reports[type] = html - else: - multiqc_reports[type] = multiqc_file_path - return multiqc_reports - @staticmethod def get_user_details(app, user_email): user_details = {} diff --git a/status_app.py b/status_app.py index e7759ffe8..f877d4af8 100644 --- a/status_app.py +++ b/status_app.py @@ -65,7 +65,6 @@ SentInvoiceHandler, ) from status.lanes_ordered import LanesOrderedDataHandler, LanesOrderedHandler -from status.multiqc_report import MultiQCReportHandler from status.ngisweden_stats import NGISwedenHandler from status.ont_plot import ONTFlowcellPlotHandler, ONTFlowcellYieldHandler from status.people_assignments import ( @@ -128,6 +127,11 @@ qPCRPoolsHandler, ) from status.reads_plot import DataFlowcellYieldHandler, FlowcellPlotHandler +from status.reports import ( + MultiQCReportHandler, + ProjectSummaryReportHandler, + SingleCellSampleSummaryReportHandler, +) from status.running_notes import ( LatestStickyNoteHandler, LatestStickyNotesMultipleHandler, @@ -402,6 +406,7 @@ def __init__(self, settings): ("/projects", ProjectsHandler), ("/project_cards", ProjectCardsHandler), ("/proj_meta", ProjMetaCompareHandler), + ("/proj_summary_report/([^/]*)$", ProjectSummaryReportHandler), ("/reads_total/([^/]*)$", ReadsTotalHandler), ("/rec_ctrl_view/([^/]*)$", RecCtrlDataHandler), ("/sample_requirements", SampleRequirementsViewHandler), @@ -409,6 +414,10 @@ def __init__(self, settings): ("/sample_requirements_update", SampleRequirementsUpdateHandler), ("/sensorpush", SensorpushHandler), ("/sequencing_queues", SequencingQueuesHandler), + ( + "/singlecell_sample_summary_report/(P[^/]*)/([^/]*)/([^/]*)$", + SingleCellSampleSummaryReportHandler, + ), ("/smartseq3_progress", SmartSeq3ProgressPageHandler), ("/suggestion_box", SuggestionBoxHandler), ("/user_management", UserManagementHandler), @@ -530,14 +539,22 @@ def __init__(self, settings): # to display instruments in the server status self.server_status = settings.get("server_status") - # project summary - multiqc tab - self.multiqc_path = settings.get("multiqc_path") - - # MinKNOW reports - self.minknow_reports_path = settings.get("minknow_reports_path") - - # ToulligQC reports - self.toulligqc_reports_path = settings.get("toulligqc_reports_path") + # project summary - reports tab + # Structure of the reports folder: + # / + # ├── other_reports/ + # │ └── toulligqc_reports/ + # ├── minknow_reports/ + # ├── mqc_reports/ + # └── yggdrasil// + self.reports_path = settings.get("reports_path") + self.report_path = {} + self.report_path["minknow"] = Path(self.reports_path, "minknow_reports") + self.report_path["multiqc"] = Path(self.reports_path, "mqc_reports") + self.report_path["toullingqc"] = Path( + self.reports_path, "other_reports", "toulligqc_reports" + ) + self.report_path["yggdrasil"] = Path(self.reports_path, "yggdrasil") # lims backend credentials limsbackend_cred_loc = Path(