Skip to content

Commit

Permalink
Move "Cleanup Database" functionality to new artisan command (#2580)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zackgalbreath
zackgalbreath authored Nov 25, 2024
1 parent 74ee534 commit 62feae3
Show file tree
Hide file tree
Showing 7 changed files with 324 additions and 324 deletions.
51 changes: 51 additions & 0 deletions app/Console/Commands/CleanDatabase.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
<?php

namespace App\Console\Commands;

use App\Utils\DatabaseCleanupUtils;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;

class CleanDatabase extends Command
{
/**
* The name and signature of the console command.
*/
protected $signature = 'db:clean';

/**
* The console command description.
*/
protected $description = 'Prune unused records from the CDash database';

/**
* Execute the console command.
*/
public function handle(): void
{
// Reconfigure laravel to log to stderr for the rest of this command.
config(['logging.default' => 'stderr']);

Log::info("Deleting unused rows from `banner`");
$num_deleted = DB::delete("DELETE FROM banner WHERE projectid != 0 AND
NOT EXISTS (SELECT 1 FROM project WHERE project.id = banner.projectid)");
Log::info("{$num_deleted} rows deleted from `banner`");

DatabaseCleanupUtils::deleteUnusedRows('dailyupdate', 'projectid', 'project');

DatabaseCleanupUtils::deleteUnusedRows('buildfailuredetails', 'id', 'buildfailure', 'detailsid');
DatabaseCleanupUtils::deleteUnusedRows('configure', 'id', 'build2configure', 'configureid');
DatabaseCleanupUtils::deleteUnusedRows('coveragefile', 'id', 'coverage', 'fileid');
DatabaseCleanupUtils::deleteUnusedRows('dailyupdatefile', 'dailyupdateid', 'dailyupdate');
DatabaseCleanupUtils::deleteUnusedRows('note', 'id', 'build2note', 'noteid');
DatabaseCleanupUtils::deleteUnusedRows('testoutput', 'id', 'build2test', 'outputid');
DatabaseCleanupUtils::deleteUnusedRows('uploadfile', 'id', 'build2uploadfile', 'fileid');

Log::info("Deleting unused rows from `image`");
$num_deleted = DB::delete("DELETE FROM image WHERE
NOT EXISTS (SELECT 1 FROM project WHERE project.imageid = image.id) AND
NOT EXISTS (SELECT 1 FROM test2image WHERE test2image.imgid = image.id)");
Log::info("{$num_deleted} rows deleted from `image`");
}
}
57 changes: 0 additions & 57 deletions app/Http/Controllers/AdminController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,6 @@ public function upgrade()
@$ComputeTestTiming = $_POST['ComputeTestTiming'];
@$ComputeUpdateStatistics = $_POST['ComputeUpdateStatistics'];

@$Cleanup = $_POST['Cleanup'];
@$Dependencies = $_POST['Dependencies'];
@$Audit = $_POST['Audit'];
@$ClearAudit = $_POST['Clear'];
Expand Down Expand Up @@ -377,55 +376,6 @@ public function upgrade()
unlink($configFile);
}


/* Cleanup the database */
if ($Cleanup) {
self::delete_unused_rows('banner', 'projectid', 'project');
self::delete_unused_rows('blockbuild', 'projectid', 'project');
self::delete_unused_rows('build', 'projectid', 'project');
self::delete_unused_rows('buildgroup', 'projectid', 'project');
self::delete_unused_rows('labelemail', 'projectid', 'project');
self::delete_unused_rows('project2repositories', 'projectid', 'project');
self::delete_unused_rows('dailyupdate', 'projectid', 'project');
self::delete_unused_rows('subproject', 'projectid', 'project');
self::delete_unused_rows('coveragefilepriority', 'projectid', 'project');
self::delete_unused_rows('user2project', 'projectid', 'project');
self::delete_unused_rows('userstatistics', 'projectid', 'project');

self::delete_unused_rows('build2configure', 'buildid', 'build');
self::delete_unused_rows('build2note', 'buildid', 'build');
self::delete_unused_rows('build2test', 'buildid', 'build');
self::delete_unused_rows('buildemail', 'buildid', 'build');
self::delete_unused_rows('builderror', 'buildid', 'build');
self::delete_unused_rows('builderrordiff', 'buildid', 'build');
self::delete_unused_rows('buildfailure', 'buildid', 'build');
self::delete_unused_rows('buildfailuredetails', 'id', 'buildfailure', 'detailsid');
self::delete_unused_rows('buildtesttime', 'buildid', 'build');
self::delete_unused_rows('configure', 'id', 'build2configure', 'configureid');
self::delete_unused_rows('configureerror', 'configureid', 'configure');
self::delete_unused_rows('configureerrordiff', 'buildid', 'build');
self::delete_unused_rows('coverage', 'buildid', 'build');
self::delete_unused_rows('coveragefilelog', 'buildid', 'build');
self::delete_unused_rows('coveragesummary', 'buildid', 'build');
self::delete_unused_rows('coveragesummarydiff', 'buildid', 'build');
self::delete_unused_rows('dynamicanalysis', 'buildid', 'build');
self::delete_unused_rows('label2build', 'buildid', 'build');
self::delete_unused_rows('subproject2build', 'buildid', 'build');
self::delete_unused_rows('summaryemail', 'buildid', 'build');
self::delete_unused_rows('testdiff', 'buildid', 'build');

self::delete_unused_rows('dynamicanalysisdefect', 'dynamicanalysisid', 'dynamicanalysis');
self::delete_unused_rows('subproject2subproject', 'subprojectid', 'subproject');

self::delete_unused_rows('dailyupdatefile', 'dailyupdateid', 'dailyupdate');
self::delete_unused_rows('coveragefile', 'id', 'coverage', 'fileid');

self::delete_unused_rows('dailyupdatefile', 'dailyupdateid', 'dailyupdate');
self::delete_unused_rows('test2image', 'outputid', 'testoutput');

$xml .= add_XML_value('alert', 'Database cleanup complete.');
}

/* Check the builds with wrong date */
if ($CheckBuildsWrongDate) {
$currentdate = time() + 3600 * 24 * 3; // or 3 days away from now
Expand Down Expand Up @@ -506,11 +456,4 @@ public function userStatistics(): View
{
return $this->angular_view('userStatistics');
}

/** Delete unused rows */
private static function delete_unused_rows($table, $field, $targettable, $selectfield = 'id'): void
{
DB::delete("DELETE FROM $table WHERE $field NOT IN (SELECT $selectfield AS $field FROM $targettable)");
echo pdo_error();
}
}
50 changes: 48 additions & 2 deletions app/Utils/DatabaseCleanupUtils.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,8 +266,6 @@ public static function removeBuild($buildid) : void
// Use array_diff to get the list of tests that should be deleted.
$testoutputs_to_delete = array_diff($all_outputids, $testoutputs_to_save);
if (!empty($testoutputs_to_delete)) {
self::deleteRowsChunked('DELETE FROM testoutput WHERE id IN ', $testoutputs_to_delete);

$testoutputs_to_delete_prepare_array = $db->createPreparedArray(count($testoutputs_to_delete));
// Check if the images for the test are not shared
$test2image = DB::select("
Expand All @@ -291,6 +289,8 @@ public static function removeBuild($buildid) : void
$imgids_prepare_array = $db->createPreparedArray(count($imgids));
DB::delete("DELETE FROM image WHERE id IN $imgids_prepare_array", $imgids);
}

self::deleteRowsChunked('DELETE FROM testoutput WHERE id IN ', $testoutputs_to_delete);
}
}

Expand Down Expand Up @@ -366,4 +366,50 @@ private static function deleteRowsChunked(string $query, array $ids): void
usleep(1);
}
}

/** Delete unused rows in batches */
public static function deleteUnusedRows(string $table, string $field, string $targettable, string $selectfield = 'id'): void
{
$start = DB::table($table)->min($field);
$max = DB::table($table)->max($field);
if (!is_numeric($start) || !is_numeric($max)) {
Log::info("Could not determine min and max for `{$field}` on `{$table}`");
return;
}

$start = intval($start);
$max = intval($max);
$total = $max - $start + 1;
if ($total < 1) {
Log::info("Invalid values found for min ({$start}) and/or max ({$max}) for `{$field}` on `{$table}`");
return;
}
$num_done = 0;
$num_deleted = 0;
$next_report = 10;
$done = false;
Log::info("Deleting unused rows from `{$table}`");
while (!$done) {
$end = $start + 49999;
$num_deleted += DB::delete("
DELETE FROM {$table}
WHERE {$field} BETWEEN {$start} AND {$end}
AND NOT EXISTS
(SELECT 1 FROM {$targettable} WHERE {$targettable}.{$selectfield} = {$table}.{$field})");
$num_done += 50000;
if ($end >= $max) {
$done = true;
} else {
usleep(1);
$start += 50000;
// Calculate percentage of work completed so far.
$percent = round(($num_done / $total) * 100, -1);
if ($percent > $next_report) {
Log::info("Cleaning `{$table}`: {$next_report}%");
$next_report = $next_report + 10;
}
}
}
Log::info("{$num_deleted} rows deleted from `{$table}`");
}
}
4 changes: 0 additions & 4 deletions app/cdash/public/upgrade.xsl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,6 @@
<td><div align="right">Compute update statistics:</div></td>
<td><div align="left">for the last <input type="text" name="UpdateStatisticsDays" size="2" value="4"/> days <input type="submit" name="ComputeUpdateStatistics" value="Compute update statistics"/></div></td>
</tr>
<tr>
<td><div align="right">Cleanup CDash (can take a long time):</div></td>
<td><input type="submit" name="Cleanup" value="Cleanup database"/></td>
</tr>
<tr>
<td><div align="right">Manage CDash dependencies:</div></td>
<td><input type="submit" name="Audit" value="Display audit report"/>
Expand Down
Loading

0 comments on commit 62feae3

Please sign in to comment.