Cache submissions while migrations are running (#2195)

Some database migrations take several hours to complete for large
systems. To prevent unnecessary gaps in submission data during migration
periods, this PR expands our existing functionality for accepting
submissions while the database is offline to also cache submissions
while maintenance mode is enabled.

For users of Docker-based systems, maintenance mode is automatically
enabled when the container starts, and is disabled when the setup/update
process is complete. Administrators of bare-metal CDash systems should
enable maintenance mode with the command `php artisan down` when
upgrading, followed by `php artisan up` when maintenance is complete.

.github/workflows/submit.sh

@@ -19,6 +19,14 @@ echo "database=$database"
 echo "ctest_driver=$ctest_driver"
 echo "submit_type=$submit_type"
 
+# Wait for migrations to finish running by checking for maintenance mode to be lifted
+docker exec cdash bash -c "\
+  until [ ! -f /cdash/storage/framework/down ]; \
+  do \
+    sleep 1; \
+  done \
+"
+
 # Suppress any uncommitted changes left after the image build
 docker exec cdash bash -c "cd /cdash && /usr/bin/git checkout ."
 

app/Http/Controllers/SubmissionController.php

@@ -100,7 +100,10 @@ private function submitProcess(): Response
         // Check if we can connect to the database before proceeding any further.
         try {
             DB::connection()->getPdo();
-        } catch (\Exception $e) {
+            if (app()->isDownForMaintenance()) {
+                throw new Exception();
+            }
+        } catch (\Exception) {
             // Write a marker file so we know to process these files when the DB comes back up.
             if (!Storage::exists("DB_WAS_DOWN")) {
                 Storage::put("DB_WAS_DOWN", "");

app/Http/Middleware/CheckForMaintenanceMode.php

@@ -12,6 +12,7 @@ class CheckForMaintenanceMode extends Middleware
      * @var array
      */
     protected $except = [
-        //
+        '/submit.php',
+        '/ping',
     ];
 }

app/Utils/UnparsedSubmissionProcessor.php

@@ -83,7 +83,7 @@ public function postSubmit(): JsonResponse
         // Thus function will throw an exception if invalid data provided
         $this->parseBuildMetadata();
 
-        if ($this->checkDatabaseConnection()) {
+        if ($this->checkDatabaseConnection() && !app()->isDownForMaintenance()) {
             return $this->initializeBuild();
         }
 
@@ -215,7 +215,7 @@ public function putSubmitFile(): JsonResponse
         $this->parseDataFileParameters();
         $ext = pathinfo($this->backupfilename, PATHINFO_EXTENSION);
 
-        $db_up = $this->checkDatabaseConnection();
+        $db_up = $this->checkDatabaseConnection() && !app()->isDownForMaintenance();
         if ($db_up) {
             if (!is_numeric($this->buildid) || $this->buildid < 1) {
                 abort(Response::HTTP_NOT_FOUND, 'Build not found');

docker/docker-entrypoint.sh

@@ -2,26 +2,34 @@
 
 set -e
 
+# Set job control so we can bg/fg processes
+set -m
+
 php artisan key:check || exit 1
 
 # If the "start-website" argument was provided, start the web server
 if [ "$1" = "start-website" ] ; then
-  if [ "$DEVELOPMENT_BUILD" = "1" ]; then
-    bash /cdash/install.sh --dev --initial-docker-install
-  else
-    bash /cdash/install.sh --initial-docker-install
-  fi
-
-  echo "Starting Apache..."
+   echo "Starting Apache..."
 
   # Start Apache under the current user, in case the current user isn't www-data.  Kubernetes-based systems
-  # typically run under a random user.
+  # typically run under a random user.  We start Apache before running the install scripts so the system can
+  # begin collecting submissions while database migrations run.  Apache starts in the background so the
+  # container gets killed if the migrations fail.
   if [ "$BASE_IMAGE" = "debian" ] ; then
     APACHE_RUN_USER=$(id -u -n) /usr/sbin/apache2ctl -D FOREGROUND
   elif [ "$BASE_IMAGE" = "ubi" ]; then
     /usr/libexec/s2i/run
+  fi & # & puts Apache in the background
+
+  if [ "$DEVELOPMENT_BUILD" = "1" ]; then
+    bash /cdash/install.sh --dev --initial-docker-install
+  else
+    bash /cdash/install.sh --initial-docker-install
   fi
 
+  # Bring Apache to the foreground so the container fails if Apache fails after this point.
+  fg
+
 # If the start-worker argument was provided, start a worker process instead
 elif [ "$1" = "start-worker" ] ; then
   php artisan queue:work