Kitware · zackgalbreath · Nov 15, 2024
diff --git a/app/Console/Commands/CleanupDatabase.php b/app/Console/Commands/CleanupDatabase.php
@@ -0,0 +1,95 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Log;
+
+class CleanupDatabase extends Command
+{
+    /**
+     * The name and signature of the console command.
+     */
+    protected $signature = 'db:cleanup';
+
+    /**
+     * The console command description.
+     */
+    protected $description = 'Prune unused records from the CDash database';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle(): void
+    {
+        $start = microtime(true);
+        DB::delete("DELETE FROM banner WHERE projectid != 0 AND projectid NOT IN (SELECT id FROM project)");
+        self::delete_unused_rows('dailyupdate', 'projectid', 'project');
+
+        self::delete_unused_rows('buildfailuredetails', 'id', 'buildfailure', 'detailsid');
+        self::delete_unused_rows('configure', 'id', 'build2configure', 'configureid');
+        self::delete_unused_rows('configureerror', 'configureid', 'configure');
+        self::delete_unused_rows('dailyupdatefile', 'dailyupdateid', 'dailyupdate');
+        self::delete_unused_rows('note', 'id', 'build2note', 'noteid');
+        self::delete_unused_rows('testoutput', 'id', 'build2test', 'outputid');
+        self::delete_unused_rows('updatefile', 'updateid', 'buildupdate');
+        self::delete_unused_rows('uploadfile', 'id', 'build2uploadfile', 'fileid');
+
+        self::delete_unused_rows('subproject2subproject', 'subprojectid', 'subproject');
+
+        self::delete_unused_rows('coveragefile', 'id', 'coverage', 'fileid');
+
+        self::delete_unused_rows('test2image', 'outputid', 'testoutput');
+
+        DB::delete("DELETE FROM image WHERE
+                      id NOT IN (SELECT imageid FROM project) AND
+                      id NOT IN (SELECT imgid FROM test2image)");
+        $end = microtime(true);
+        $duration = round($end - $start, 2);
+        Log::info("Database cleanup completed in {$duration} seconds");
+    }
+
+    /** Delete unused rows in batches */
+    private static function delete_unused_rows(string $table, string $field, string $targettable, string $selectfield = 'id'): void
+    {
+        $start = DB::table($table)->min($field);
+        $max = DB::table($table)->max($field);
+        if (!is_numeric($start) || !is_numeric($max)) {
+            echo "Could not determine min and max for $field on $table\n";
+            return;
+        }
+
+        $start = intval($start);
+        $max = intval($max);
+
+        $total = $max - $start;
+        if ($total < 1) {
+            return;
+        }
+        $num_done = 0;
+        $next_report = 10;
+        $done = false;
+        echo "Pruning unused rows from $table\n";
+        while (!$done) {
+            $end = $start + 49999;
+            DB::delete("
+                DELETE FROM $table
+                WHERE $field BETWEEN $start AND $end
+                      AND $field NOT IN (SELECT $selectfield FROM $targettable)");
+            $num_done += 50000;
+            if ($end >= $max) {
+                $done = true;
+            } else {
+                usleep(1);
+                $start += 50000;
+                // Calculate percentage of work completed so far.
+                $percent = round(($num_done / $total) * 100, -1);
+                if ($percent > $next_report) {
+                    echo "{$percent}%\n";
+                    $next_report = $next_report + 10;
+                }
+            }
+        }
+    }
+}
diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php
@@ -3,6 +3,8 @@
 namespace App\Console;
 
 use App\Jobs\PruneAuthTokens;
+use App\Jobs\PruneBuilds;
+use App\Jobs\PruneDatabase;
 use App\Jobs\PruneJobs;
 use Illuminate\Console\Scheduling\Schedule;
 use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
@@ -25,11 +27,19 @@ protected function schedule(Schedule $schedule)
             ->everySixHours()
             ->sendOutputTo($output_filename);
 
-        $schedule->job(new PruneJobs())
+        $schedule->job(new PruneAuthTokens(), 'low')
             ->hourly()
             ->withoutOverlapping();
 
-        $schedule->job(new PruneAuthTokens())
+        $schedule->job(new PruneBuilds(), 'low')
+            ->hourly()
+            ->withoutOverlapping();
+
+        $schedule->job(new PruneDatabase(), 'low')
+            ->dailyAt('03:00')
+            ->withoutOverlapping();
+
+        $schedule->job(new PruneJobs(), 'low')
             ->hourly()
             ->withoutOverlapping();
 

diff --git a/app/Http/Controllers/AdminController.php b/app/Http/Controllers/AdminController.php
@@ -2,6 +2,7 @@
 namespace App\Http\Controllers;
 
 use App\Models\User;
+use App\Utils\DatabaseCleanupUtils;
 use CDash\Model\Project;
 use Illuminate\Http\RedirectResponse;
 use Illuminate\Support\Facades\DB;
@@ -96,7 +97,7 @@ public function removeBuilds(): View|RedirectResponse
                 $builds[] = (int) $build_array->id;
             }
 
-            remove_build_chunked($builds);
+            DatabaseCleanupUtils::removeBuildChunked($builds);
             $alert = 'Removed ' . count($builds) . ' builds.';
         }
 
@@ -325,12 +326,9 @@ public function upgrade()
 
         @$AssignBuildToDefaultGroups = $_POST['AssignBuildToDefaultGroups'];
         @$FixBuildBasedOnRule = $_POST['FixBuildBasedOnRule'];
-        @$DeleteBuildsWrongDate = $_POST['DeleteBuildsWrongDate'];
-        @$CheckBuildsWrongDate = $_POST['CheckBuildsWrongDate'];
         @$ComputeTestTiming = $_POST['ComputeTestTiming'];
         @$ComputeUpdateStatistics = $_POST['ComputeUpdateStatistics'];
 
-        @$Cleanup = $_POST['Cleanup'];
         @$Dependencies = $_POST['Dependencies'];
         @$Audit = $_POST['Audit'];
         @$ClearAudit = $_POST['Clear'];
@@ -376,80 +374,6 @@ public function upgrade()
             unlink($configFile);
         }
 
-
-        /* Cleanup the database */
-        if ($Cleanup) {
-            self::delete_unused_rows('banner', 'projectid', 'project');
-            self::delete_unused_rows('blockbuild', 'projectid', 'project');
-            self::delete_unused_rows('build', 'projectid', 'project');
-            self::delete_unused_rows('buildgroup', 'projectid', 'project');
-            self::delete_unused_rows('labelemail', 'projectid', 'project');
-            self::delete_unused_rows('project2repositories', 'projectid', 'project');
-            self::delete_unused_rows('dailyupdate', 'projectid', 'project');
-            self::delete_unused_rows('subproject', 'projectid', 'project');
-            self::delete_unused_rows('coveragefilepriority', 'projectid', 'project');
-            self::delete_unused_rows('user2project', 'projectid', 'project');
-            self::delete_unused_rows('userstatistics', 'projectid', 'project');
-
-            self::delete_unused_rows('build2configure', 'buildid', 'build');
-            self::delete_unused_rows('build2note', 'buildid', 'build');
-            self::delete_unused_rows('build2test', 'buildid', 'build');
-            self::delete_unused_rows('buildemail', 'buildid', 'build');
-            self::delete_unused_rows('builderror', 'buildid', 'build');
-            self::delete_unused_rows('builderrordiff', 'buildid', 'build');
-            self::delete_unused_rows('buildfailure', 'buildid', 'build');
-            self::delete_unused_rows('buildfailuredetails', 'id', 'buildfailure', 'detailsid');
-            self::delete_unused_rows('buildtesttime', 'buildid', 'build');
-            self::delete_unused_rows('configure', 'id', 'build2configure', 'configureid');
-            self::delete_unused_rows('configureerror', 'configureid', 'configure');
-            self::delete_unused_rows('configureerrordiff', 'buildid', 'build');
-            self::delete_unused_rows('coverage', 'buildid', 'build');
-            self::delete_unused_rows('coveragefilelog', 'buildid', 'build');
-            self::delete_unused_rows('coveragesummary', 'buildid', 'build');
-            self::delete_unused_rows('coveragesummarydiff', 'buildid', 'build');
-            self::delete_unused_rows('dynamicanalysis', 'buildid', 'build');
-            self::delete_unused_rows('label2build', 'buildid', 'build');
-            self::delete_unused_rows('subproject2build', 'buildid', 'build');
-            self::delete_unused_rows('summaryemail', 'buildid', 'build');
-            self::delete_unused_rows('testdiff', 'buildid', 'build');
-
-            self::delete_unused_rows('dynamicanalysisdefect', 'dynamicanalysisid', 'dynamicanalysis');
-            self::delete_unused_rows('subproject2subproject', 'subprojectid', 'subproject');
-
-            self::delete_unused_rows('dailyupdatefile', 'dailyupdateid', 'dailyupdate');
-            self::delete_unused_rows('coveragefile', 'id', 'coverage', 'fileid');
-
-            self::delete_unused_rows('dailyupdatefile', 'dailyupdateid', 'dailyupdate');
-            self::delete_unused_rows('test2image', 'outputid', 'testoutput');
-
-            $xml .= add_XML_value('alert', 'Database cleanup complete.');
-        }
-
-        /* Check the builds with wrong date */
-        if ($CheckBuildsWrongDate) {
-            $currentdate = time() + 3600 * 24 * 3; // or 3 days away from now
-            $forwarddate = date(FMT_DATETIME, $currentdate);
-
-            $builds = pdo_query("SELECT id,name,starttime FROM build WHERE starttime<'1975-12-31 23:59:59' OR starttime>'$forwarddate'");
-            while ($builds_array = pdo_fetch_array($builds)) {
-                echo $builds_array['name'] . '-' . $builds_array['starttime'] . '<br>';
-            }
-        }
-
-        /* Delete the builds with wrong date */
-        if ($DeleteBuildsWrongDate) {
-            $currentdate = time() + 3600 * 24 * 3; // or 3 days away from now
-            $forwarddate = date(FMT_DATETIME, $currentdate);
-
-            $builds = pdo_query(
-                "SELECT id FROM build WHERE parentid IN (0, -1) AND
-          starttime<'1975-12-31 23:59:59' OR starttime>'$forwarddate'");
-            while ($builds_array = pdo_fetch_array($builds)) {
-                $buildid = $builds_array['id'];
-                remove_build($buildid);
-            }
-        }
-
         if ($FixBuildBasedOnRule) {
             // loop through the list of build2group
             $buildgroups = pdo_query('SELECT * from build2group');
@@ -505,11 +429,4 @@ public function userStatistics(): View
     {
         return $this->angular_view('userStatistics');
     }
-
-    /** Delete unused rows */
-    private static function delete_unused_rows($table, $field, $targettable, $selectfield = 'id'): void
-    {
-        DB::delete("DELETE FROM $table WHERE $field NOT IN (SELECT $selectfield AS $field FROM $targettable)");
-        echo pdo_error();
-    }
 }
diff --git a/app/Http/Controllers/BuildController.php b/app/Http/Controllers/BuildController.php
@@ -4,6 +4,7 @@
 use App\Models\Comment;
 use App\Models\User;
 use App\Models\Build as EloquentBuild;
+use App\Utils\DatabaseCleanupUtils;
 use App\Utils\PageTimer;
 use App\Utils\RepositoryUtils;
 use App\Utils\TestingDay;
@@ -1470,7 +1471,7 @@ private function restApiPost(): JsonResponse
     private function restApiDelete(): JsonResponse
     {
         Log::info("Build #{$this->build->Id} removed manually.");
-        remove_build($this->build->Id);
+        DatabaseCleanupUtils::removeBuild($this->build->Id);
         return response()->json();
     }
 }
diff --git a/app/Jobs/PruneBuilds.php b/app/Jobs/PruneBuilds.php
@@ -0,0 +1,30 @@
+<?php
+
+namespace App\Jobs;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Artisan;
+
+/**
+ * Removes builds that have expired according to per-project and
+ * per-buildgroup settings.
+ */
+class PruneBuilds implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        if (!(bool) config('cdash.autoremove_builds')) {
+            return;
+        }
+        Artisan::call('build:remove all');
+    }
+}
diff --git a/app/Jobs/PruneDatabase.php b/app/Jobs/PruneDatabase.php
@@ -0,0 +1,26 @@
+<?php
+
+namespace App\Jobs;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Artisan;
+
+/**
+ * Remove unreferenced database rows.
+ */
+class PruneDatabase implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        Artisan::call('db:cleanup');
+    }
+}