Move "Cleanup Database" functionality to new artisan command

app/Console/Commands/CleanDatabase.php

@@ -0,0 +1,51 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Utils\DatabaseCleanupUtils;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Log;
+
+class CleanDatabase extends Command
+{
+    /**
+     * The name and signature of the console command.
+     */
+    protected $signature = 'db:clean';
+
+    /**
+     * The console command description.
+     */
+    protected $description = 'Prune unused records from the CDash database';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle(): void
+    {
+        // Reconfigure laravel to log to stderr for the rest of this command.
+        config(['logging.default' => 'stderr']);
+
+        Log::info("Deleting unused rows from `banner`");
+        $num_deleted = DB::delete("DELETE FROM banner WHERE projectid != 0 AND
+                NOT EXISTS (SELECT 1 FROM project WHERE project.id = banner.projectid)");
+        Log::info("{$num_deleted} rows deleted from `banner`");
+
+        DatabaseCleanupUtils::deleteUnusedRows('dailyupdate', 'projectid', 'project');
+
+        DatabaseCleanupUtils::deleteUnusedRows('buildfailuredetails', 'id', 'buildfailure', 'detailsid');
+        DatabaseCleanupUtils::deleteUnusedRows('configure', 'id', 'build2configure', 'configureid');
+        DatabaseCleanupUtils::deleteUnusedRows('coveragefile', 'id', 'coverage', 'fileid');
+        DatabaseCleanupUtils::deleteUnusedRows('dailyupdatefile', 'dailyupdateid', 'dailyupdate');
+        DatabaseCleanupUtils::deleteUnusedRows('note', 'id', 'build2note', 'noteid');
+        DatabaseCleanupUtils::deleteUnusedRows('testoutput', 'id', 'build2test', 'outputid');
+        DatabaseCleanupUtils::deleteUnusedRows('uploadfile', 'id', 'build2uploadfile', 'fileid');
+
+        Log::info("Deleting unused rows from `image`");
+        $num_deleted = DB::delete("DELETE FROM image WHERE
+                NOT EXISTS (SELECT 1 FROM project WHERE project.imageid = image.id) AND
+                NOT EXISTS (SELECT 1 FROM test2image WHERE test2image.imgid = image.id)");
+        Log::info("{$num_deleted} rows deleted from `image`");
+    }
+}

app/Http/Controllers/AdminController.php

@@ -331,7 +331,6 @@ public function upgrade()
         @$ComputeTestTiming = $_POST['ComputeTestTiming'];
         @$ComputeUpdateStatistics = $_POST['ComputeUpdateStatistics'];
 
-        @$Cleanup = $_POST['Cleanup'];
         @$Dependencies = $_POST['Dependencies'];
         @$Audit = $_POST['Audit'];
         @$ClearAudit = $_POST['Clear'];
@@ -377,55 +376,6 @@ public function upgrade()
             unlink($configFile);
         }
 
-
-        /* Cleanup the database */
-        if ($Cleanup) {
-            self::delete_unused_rows('banner', 'projectid', 'project');
-            self::delete_unused_rows('blockbuild', 'projectid', 'project');
-            self::delete_unused_rows('build', 'projectid', 'project');
-            self::delete_unused_rows('buildgroup', 'projectid', 'project');
-            self::delete_unused_rows('labelemail', 'projectid', 'project');
-            self::delete_unused_rows('project2repositories', 'projectid', 'project');
-            self::delete_unused_rows('dailyupdate', 'projectid', 'project');
-            self::delete_unused_rows('subproject', 'projectid', 'project');
-            self::delete_unused_rows('coveragefilepriority', 'projectid', 'project');
-            self::delete_unused_rows('user2project', 'projectid', 'project');
-            self::delete_unused_rows('userstatistics', 'projectid', 'project');
-
-            self::delete_unused_rows('build2configure', 'buildid', 'build');
-            self::delete_unused_rows('build2note', 'buildid', 'build');
-            self::delete_unused_rows('build2test', 'buildid', 'build');
-            self::delete_unused_rows('buildemail', 'buildid', 'build');
-            self::delete_unused_rows('builderror', 'buildid', 'build');
-            self::delete_unused_rows('builderrordiff', 'buildid', 'build');
-            self::delete_unused_rows('buildfailure', 'buildid', 'build');
-            self::delete_unused_rows('buildfailuredetails', 'id', 'buildfailure', 'detailsid');
-            self::delete_unused_rows('buildtesttime', 'buildid', 'build');
-            self::delete_unused_rows('configure', 'id', 'build2configure', 'configureid');
-            self::delete_unused_rows('configureerror', 'configureid', 'configure');
-            self::delete_unused_rows('configureerrordiff', 'buildid', 'build');
-            self::delete_unused_rows('coverage', 'buildid', 'build');
-            self::delete_unused_rows('coveragefilelog', 'buildid', 'build');
-            self::delete_unused_rows('coveragesummary', 'buildid', 'build');
-            self::delete_unused_rows('coveragesummarydiff', 'buildid', 'build');
-            self::delete_unused_rows('dynamicanalysis', 'buildid', 'build');
-            self::delete_unused_rows('label2build', 'buildid', 'build');
-            self::delete_unused_rows('subproject2build', 'buildid', 'build');
-            self::delete_unused_rows('summaryemail', 'buildid', 'build');
-            self::delete_unused_rows('testdiff', 'buildid', 'build');
-
-            self::delete_unused_rows('dynamicanalysisdefect', 'dynamicanalysisid', 'dynamicanalysis');
-            self::delete_unused_rows('subproject2subproject', 'subprojectid', 'subproject');
-
-            self::delete_unused_rows('dailyupdatefile', 'dailyupdateid', 'dailyupdate');
-            self::delete_unused_rows('coveragefile', 'id', 'coverage', 'fileid');
-
-            self::delete_unused_rows('dailyupdatefile', 'dailyupdateid', 'dailyupdate');
-            self::delete_unused_rows('test2image', 'outputid', 'testoutput');
-
-            $xml .= add_XML_value('alert', 'Database cleanup complete.');
-        }
-
         /* Check the builds with wrong date */
         if ($CheckBuildsWrongDate) {
             $currentdate = time() + 3600 * 24 * 3; // or 3 days away from now
@@ -506,11 +456,4 @@ public function userStatistics(): View
     {
         return $this->angular_view('userStatistics');
     }
-
-    /** Delete unused rows */
-    private static function delete_unused_rows($table, $field, $targettable, $selectfield = 'id'): void
-    {
-        DB::delete("DELETE FROM $table WHERE $field NOT IN (SELECT $selectfield AS $field FROM $targettable)");
-        echo pdo_error();
-    }
 }

app/Utils/DatabaseCleanupUtils.php

@@ -266,8 +266,6 @@ public static function removeBuild($buildid) : void
             // Use array_diff to get the list of tests that should be deleted.
             $testoutputs_to_delete = array_diff($all_outputids, $testoutputs_to_save);
             if (!empty($testoutputs_to_delete)) {
-                self::deleteRowsChunked('DELETE FROM testoutput WHERE id IN ', $testoutputs_to_delete);
-
                 $testoutputs_to_delete_prepare_array = $db->createPreparedArray(count($testoutputs_to_delete));
                 // Check if the images for the test are not shared
                 $test2image = DB::select("
@@ -291,6 +289,8 @@ public static function removeBuild($buildid) : void
                     $imgids_prepare_array = $db->createPreparedArray(count($imgids));
                     DB::delete("DELETE FROM image WHERE id IN $imgids_prepare_array", $imgids);
                 }
+
+                self::deleteRowsChunked('DELETE FROM testoutput WHERE id IN ', $testoutputs_to_delete);
             }
         }
 
@@ -366,4 +366,50 @@ private static function deleteRowsChunked(string $query, array $ids): void
             usleep(1);
         }
     }
+
+    /** Delete unused rows in batches */
+    public static function deleteUnusedRows(string $table, string $field, string $targettable, string $selectfield = 'id'): void
+    {
+        $start = DB::table($table)->min($field);
+        $max = DB::table($table)->max($field);
+        if (!is_numeric($start) || !is_numeric($max)) {
+            Log::info("Could not determine min and max for `{$field}` on `{$table}`");
+            return;
+        }
+
+        $start = intval($start);
+        $max = intval($max);
+        $total = $max - $start + 1;
+        if ($total < 1) {
+            Log::info("Invalid values found for min ({$start}) and/or max ({$max}) for `{$field}` on `{$table}`");
+            return;
+        }
+        $num_done = 0;
+        $num_deleted = 0;
+        $next_report = 10;
+        $done = false;
+        Log::info("Deleting unused rows from `{$table}`");
+        while (!$done) {
+            $end = $start + 49999;
+            $num_deleted += DB::delete("
+                DELETE FROM {$table}
+                WHERE {$field} BETWEEN {$start} AND {$end}
+                      AND NOT EXISTS
+                      (SELECT 1 FROM {$targettable} WHERE {$targettable}.{$selectfield} = {$table}.{$field})");
+            $num_done += 50000;
+            if ($end >= $max) {
+                $done = true;
+            } else {
+                usleep(1);
+                $start += 50000;
+                // Calculate percentage of work completed so far.
+                $percent = round(($num_done / $total) * 100, -1);
+                if ($percent > $next_report) {
+                    Log::info("Cleaning `{$table}`: {$next_report}%");
+                    $next_report = $next_report + 10;
+                }
+            }
+        }
+        Log::info("{$num_deleted} rows deleted from `{$table}`");
+    }
 }

app/cdash/public/upgrade.xsl

@@ -30,10 +30,6 @@
     <td><div align="right">Compute update statistics:</div></td>
     <td><div align="left">for the last <input type="text" name="UpdateStatisticsDays" size="2" value="4"/> days <input type="submit" name="ComputeUpdateStatistics" value="Compute update statistics"/></div></td>
   </tr>
-  <tr>
-    <td><div align="right">Cleanup CDash (can take a long time):</div></td>
-    <td><input type="submit" name="Cleanup" value="Cleanup database"/></td>
-  </tr>
   <tr>
     <td><div align="right">Manage CDash dependencies:</div></td>
     <td><input type="submit" name="Audit" value="Display audit report"/>