From a51ed5199b2b1df4631eff1a1b30c68beac0f9cd Mon Sep 17 00:00:00 2001
From: William Allen <16820599+williamjallen@users.noreply.github.com>
Date: Wed, 22 May 2024 13:03:54 -0400
Subject: [PATCH] Cache submissions while migrations are running (#2195)

Some database migrations take several hours to complete for large
systems. To prevent unnecessary gaps in submission data during migration
periods, this PR expands our existing functionality for accepting
submissions while the database is offline to also cache submissions
while maintenance mode is enabled.

For users of Docker-based systems, maintenance mode is automatically
enabled when the container starts, and is disabled when the setup/update
process is complete. Administrators of bare-metal CDash systems should
enable maintenance mode with the command `php artisan down` when
upgrading, followed by `php artisan up` when maintenance is complete.
---
 .github/workflows/submit.sh                   |  8 +++++++
 app/Http/Controllers/SubmissionController.php |  5 +++-
 .../Middleware/CheckForMaintenanceMode.php    |  3 ++-
 app/Utils/UnparsedSubmissionProcessor.php     |  4 ++--
 docker/docker-entrypoint.sh                   | 24 ++++++++++++-------
 5 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/submit.sh b/.github/workflows/submit.sh
index e5a862d349..fdbe3ff6d3 100644
--- a/.github/workflows/submit.sh
+++ b/.github/workflows/submit.sh
@@ -19,6 +19,14 @@ echo "database=$database"
 echo "ctest_driver=$ctest_driver"
 echo "submit_type=$submit_type"
 
+# Wait for migrations to finish running by checking for maintenance mode to be lifted
+docker exec cdash bash -c "\
+  until [ ! -f /cdash/storage/framework/down ]; \
+  do \
+    sleep 1; \
+  done \
+"
+
 # Suppress any uncommitted changes left after the image build
 docker exec cdash bash -c "cd /cdash && /usr/bin/git checkout ."
 
diff --git a/app/Http/Controllers/SubmissionController.php b/app/Http/Controllers/SubmissionController.php
index 778aaed4ec..8471a2c0a6 100644
--- a/app/Http/Controllers/SubmissionController.php
+++ b/app/Http/Controllers/SubmissionController.php
@@ -100,7 +100,10 @@ private function submitProcess(): Response
         // Check if we can connect to the database before proceeding any further.
         try {
             DB::connection()->getPdo();
-        } catch (\Exception $e) {
+            if (app()->isDownForMaintenance()) {
+                throw new Exception();
+            }
+        } catch (\Exception) {
             // Write a marker file so we know to process these files when the DB comes back up.
             if (!Storage::exists("DB_WAS_DOWN")) {
                 Storage::put("DB_WAS_DOWN", "");
diff --git a/app/Http/Middleware/CheckForMaintenanceMode.php b/app/Http/Middleware/CheckForMaintenanceMode.php
index 35b9824bae..7ae23d7ab0 100755
--- a/app/Http/Middleware/CheckForMaintenanceMode.php
+++ b/app/Http/Middleware/CheckForMaintenanceMode.php
@@ -12,6 +12,7 @@ class CheckForMaintenanceMode extends Middleware
      * @var array
      */
     protected $except = [
-        //
+        '/submit.php',
+        '/ping',
     ];
 }
diff --git a/app/Utils/UnparsedSubmissionProcessor.php b/app/Utils/UnparsedSubmissionProcessor.php
index 83116e7815..155ffcb77c 100644
--- a/app/Utils/UnparsedSubmissionProcessor.php
+++ b/app/Utils/UnparsedSubmissionProcessor.php
@@ -83,7 +83,7 @@ public function postSubmit(): JsonResponse
         // Thus function will throw an exception if invalid data provided
         $this->parseBuildMetadata();
 
-        if ($this->checkDatabaseConnection()) {
+        if ($this->checkDatabaseConnection() && !app()->isDownForMaintenance()) {
             return $this->initializeBuild();
         }
 
@@ -215,7 +215,7 @@ public function putSubmitFile(): JsonResponse
         $this->parseDataFileParameters();
         $ext = pathinfo($this->backupfilename, PATHINFO_EXTENSION);
 
-        $db_up = $this->checkDatabaseConnection();
+        $db_up = $this->checkDatabaseConnection() && !app()->isDownForMaintenance();
         if ($db_up) {
             if (!is_numeric($this->buildid) || $this->buildid < 1) {
                 abort(Response::HTTP_NOT_FOUND, 'Build not found');
diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh
index f6b4773530..0007ede767 100755
--- a/docker/docker-entrypoint.sh
+++ b/docker/docker-entrypoint.sh
@@ -2,26 +2,34 @@
 
 set -e
 
+# Set job control so we can bg/fg processes
+set -m
+
 php artisan key:check || exit 1
 
 # If the "start-website" argument was provided, start the web server
 if [ "$1" = "start-website" ] ; then
-  if [ "$DEVELOPMENT_BUILD" = "1" ]; then
-    bash /cdash/install.sh --dev --initial-docker-install
-  else
-    bash /cdash/install.sh --initial-docker-install
-  fi
-
-  echo "Starting Apache..."
+   echo "Starting Apache..."
 
   # Start Apache under the current user, in case the current user isn't www-data.  Kubernetes-based systems
-  # typically run under a random user.
+  # typically run under a random user.  We start Apache before running the install scripts so the system can
+  # begin collecting submissions while database migrations run.  Apache starts in the background so the
+  # container gets killed if the migrations fail.
   if [ "$BASE_IMAGE" = "debian" ] ; then
     APACHE_RUN_USER=$(id -u -n) /usr/sbin/apache2ctl -D FOREGROUND
   elif [ "$BASE_IMAGE" = "ubi" ]; then
     /usr/libexec/s2i/run
+  fi & # & puts Apache in the background
+
+  if [ "$DEVELOPMENT_BUILD" = "1" ]; then
+    bash /cdash/install.sh --dev --initial-docker-install
+  else
+    bash /cdash/install.sh --initial-docker-install
   fi
 
+  # Bring Apache to the foreground so the container fails if Apache fails after this point.
+  fg
+
 # If the start-worker argument was provided, start a worker process instead
 elif [ "$1" = "start-worker" ] ; then
   php artisan queue:work