Skip to content

Commit

Permalink
Merge pull request #15 from adrianreber/age-fix
Browse files Browse the repository at this point in the history 
Only age files with the same name
  • Loading branch information
@adrianreber
adrianreber authored Jun 2, 2021
2 parents 4b17e10 + f6f58ee commit a3ffb03
Show file tree
Hide file tree
Showing 2 changed files with 92 additions and 7 deletions.
21 changes: 19 additions & 2 deletions src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ struct UpdateDirectory {
fn age_file_details(
c: &PgConnection,
fds: &mut Vec<db::models::FileDetail>,
dirs: &[db::models::Directory],
max_stale_days: i64,
max_propagation_days: i64,
) -> Result<(), diesel::result::Error> {
Expand All @@ -122,6 +123,7 @@ fn age_file_details(

let mut old_id: i32 = -1;
let mut old_ts: i64 = -1;
let mut old_name = String::from("");
let now = chrono::offset::Local::now().timestamp();
let stale = now - (60 * 60 * 24 * max_stale_days);
let propagation = now - (60 * 60 * 24 * max_propagation_days);
Expand All @@ -130,11 +132,23 @@ fn age_file_details(
let mut delete_list: Vec<i32> = Vec::new();

for fd in fds {
let mut in_dirs = false;
for d in dirs {
if fd.directory_id == d.id {
in_dirs = true;
break;
}
}

if !in_dirs {
continue;
}

let ts = match fd.timestamp {
Some(t) => t,
_ => continue,
};
if fd.directory_id == old_id {
if fd.directory_id == old_id && fd.filename == old_name {
same_entries += 1;
if ts < stale && (same_entries > 2 || old_ts < propagation) {
delete_list.push(fd.id);
Expand All @@ -144,6 +158,7 @@ fn age_file_details(
old_ts = ts;
}
old_id = fd.directory_id;
old_name = fd.filename.clone();
}

if !delete_list.is_empty() {
Expand Down Expand Up @@ -423,7 +438,8 @@ fn cleanup_database(
db::schema::host_category_dir::dsl::host_category_dir
.filter(db::schema::host_category_dir::dsl::directory_id.eq(d.id)),
);
let debug_host_category_dir = diesel::debug_query::<diesel::pg::Pg, _>(&delete_host_category_dir);
let debug_host_category_dir =
diesel::debug_query::<diesel::pg::Pg, _>(&delete_host_category_dir);
debug::print_step(debug_host_category_dir.to_string());
delete_host_category_dir.execute(c)?;

Expand Down Expand Up @@ -1556,6 +1572,7 @@ fn main() {
if let Err(e) = age_file_details(
&connection,
&mut fds,
&d,
match settings.max_stale_days {
Some(m) => m,
_ => 3,
Expand Down
78 changes: 73 additions & 5 deletions src/scan_primary_mirror_test.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,30 @@ fn age_file_details_test() {
}
};

let dirs = vec![
db::models::Directory {
id: 7,
name: "dirname7".to_string(),
files: Vec::new(),
readable: true,
ctime: 0,
},
db::models::Directory {
id: 8,
name: "dirname8".to_string(),
files: Vec::new(),
readable: true,
ctime: 0,
},
db::models::Directory {
id: 9,
name: "dirname9".to_string(),
files: Vec::new(),
readable: true,
ctime: 0,
},
];

assert!(!diesel::delete(db::schema::file_detail::dsl::file_detail)
.execute(&c)
.is_err());
Expand All @@ -224,6 +248,16 @@ fn age_file_details_test() {
sha256: Some(String::from("sha256")),
sha512: Some(String::from("sha512")),
};
let fd_7_5_other = db::models::InsertFileDetail {
directory_id: 7,
filename: String::from("other_name"),
timestamp: Some(five),
size: Some(40),
sha1: Some(String::from("sha1")),
md5: Some(String::from("md5")),
sha256: Some(String::from("sha256")),
sha512: Some(String::from("sha512")),
};
let fd_8_3 = db::models::InsertFileDetail {
directory_id: 8,
filename: String::from("repomd.xml"),
Expand Down Expand Up @@ -254,11 +288,45 @@ fn age_file_details_test() {
sha256: Some(String::from("sha256")),
sha512: Some(String::from("sha512")),
};
let fd_10_5 = db::models::InsertFileDetail {
directory_id: 10,
filename: String::from("repomd.xml"),
timestamp: Some(five),
size: Some(40),
sha1: Some(String::from("sha1")),
md5: Some(String::from("md5")),
sha256: Some(String::from("sha256")),
sha512: Some(String::from("sha512")),
};
let fd_10_4 = db::models::InsertFileDetail {
directory_id: 10,
filename: String::from("repomd.xml"),
timestamp: Some(four),
size: Some(40),
sha1: Some(String::from("sha1")),
md5: Some(String::from("md5")),
sha256: Some(String::from("sha256")),
sha512: Some(String::from("sha512")),
};
let fd_10_3 = db::models::InsertFileDetail {
directory_id: 10,
filename: String::from("repomd.xml"),
timestamp: Some(three),
size: Some(40),
sha1: Some(String::from("sha1")),
md5: Some(String::from("md5")),
sha256: Some(String::from("sha256")),
sha512: Some(String::from("sha512")),
};

ifds.push(fd_7_5);
ifds.push(fd_7_5_other);
ifds.push(fd_8_3);
ifds.push(fd_7_4);
ifds.push(fd_7_3);
ifds.push(fd_10_5);
ifds.push(fd_10_4);
ifds.push(fd_10_3);

if let Err(e) = diesel::insert_into(db::schema::file_detail::dsl::file_detail)
.values(&ifds)
Expand All @@ -270,7 +338,7 @@ fn age_file_details_test() {

let mut fds = db::functions::get_file_details(&c);
let fds_org = db::functions::get_file_details(&c);
if let Err(e) = age_file_details(&c, &mut fds, 6, 5) {
if let Err(e) = age_file_details(&c, &mut fds, &dirs, 6, 5) {
println!("Running age_file_details() failed {}", e);
panic!();
}
Expand All @@ -279,18 +347,18 @@ fn age_file_details_test() {
.eq(db::functions::get_file_details(&c).iter()));

fds = db::functions::get_file_details(&c);
if let Err(e) = age_file_details(&c, &mut fds, 4, 3) {
if let Err(e) = age_file_details(&c, &mut fds, &dirs, 4, 3) {
println!("Running age_file_details() failed {}", e);
panic!();
}
assert_eq!(3, db::functions::get_file_details(&c).len());
assert_eq!(7, db::functions::get_file_details(&c).len());

fds = db::functions::get_file_details(&c);
if let Err(e) = age_file_details(&c, &mut fds, 1, 0) {
if let Err(e) = age_file_details(&c, &mut fds, &dirs, 1, 0) {
println!("Running age_file_details() failed {}", e);
panic!();
}
assert_eq!(2, db::functions::get_file_details(&c).len());
assert_eq!(6, db::functions::get_file_details(&c).len());
}

#[test]
Expand Down

0 comments on commit a3ffb03

Please sign in to comment.