diff --git a/tools/harbor-garbage-collector/src/main.rs b/tools/harbor-garbage-collector/src/main.rs index 9203cf3..f723d2a 100644 --- a/tools/harbor-garbage-collector/src/main.rs +++ b/tools/harbor-garbage-collector/src/main.rs @@ -1,6 +1,7 @@ use regex::Regex; use serde::Deserialize; use std::env; +use std::time::Duration; use urlencoding::encode; #[derive(Deserialize, Debug)] @@ -25,6 +26,8 @@ struct Artifact { references: Option>, } +const ROBOT_USER: &str = "robot$stackable-cleanup"; + #[tokio::main] async fn main() -> Result<(), Box> { let registry_hostname = "oci.stackable.tech"; @@ -33,6 +36,17 @@ async fn main() -> Result<(), Box> { let mut page = 1; let attestation_tag_regex = Regex::new(r"^sha256-[0-9a-f]{64}.att$").unwrap(); + // Fail fast if the cleanup credential is missing. Previously this used + // `.ok()`, so a missing/renamed variable silently produced unauthenticated + // DELETEs that Harbor rejects while the tool still exits 0. + let robot_password = env::var("HARBOR_ROBOT_PASSWORD") + .map_err(|_| "HARBOR_ROBOT_PASSWORD environment variable must be set")?; + + // One shared client with a timeout, so a hung registry can't block forever. + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(30)) + .build()?; + loop { // Loop over all repositories, paged let url = format!( @@ -40,7 +54,7 @@ async fn main() -> Result<(), Box> { base_url, page_size, page ); - let repositories: Vec = reqwest::get(&url).await?.json().await?; + let repositories: Vec = client.get(&url).send().await?.json().await?; for repository in &repositories { let (project_name, repository_name) = repository.name.split_once('/').unwrap(); @@ -58,17 +72,19 @@ async fn main() -> Result<(), Box> { loop { // Loop over pages to get all artifacts - let artifacts_page: Vec = reqwest::get(format!( - "{}/projects/{}/repositories/{}/artifacts?page_size={}&page={}", - base_url, - encode(project_name), - encode(repository_name), - page_size, - page - )) - .await? - .json() - .await?; + let artifacts_page: Vec = client + .get(format!( + "{}/projects/{}/repositories/{}/artifacts?page_size={}&page={}", + base_url, + encode(project_name), + encode(repository_name), + page_size, + page + )) + .send() + .await? + .json() + .await?; let number_of_returned_artifacts = artifacts_page.len(); artifacts.extend(artifacts_page); @@ -104,7 +120,7 @@ async fn main() -> Result<(), Box> { if !potentially_attested_artifacts.contains(&attestation_referenced_sha) { let attestation_tag = format!("sha256-{}.att", attestation_referenced_sha); println!("Removing dangling attestation {}", attestation_tag); - reqwest::Client::new() + client .delete(format!( "{}/projects/{}/repositories/{}/artifacts/{}", base_url, @@ -112,12 +128,10 @@ async fn main() -> Result<(), Box> { encode(repository_name), attestation_tag )) - .basic_auth( - "robot$stackable-cleanup", - env::var("HARBOR_ROBOT_PASSWORD").ok(), - ) + .basic_auth(ROBOT_USER, Some(&robot_password)) .send() - .await?; + .await? + .error_for_status()?; } } } @@ -133,53 +147,75 @@ async fn main() -> Result<(), Box> { // Only keep the "latest" manifest and its referenced artifacts for ubi_version in &["ubi8", "ubi9"] { - let latest_rust_builder_manifest_list: Artifact = reqwest::Client::new() - .get(format!( - "{}/projects/sdp/repositories/{}-rust-builder/artifacts/latest", - base_url, - ubi_version - )) - .send() - .await? - .json() - .await?; + let latest_url = format!( + "{}/projects/sdp/repositories/{}-rust-builder/artifacts/latest", + base_url, ubi_version + ); + let latest_rust_builder_manifest_list: Artifact = + client.get(&latest_url).send().await?.json().await?; + + let latest_digest = latest_rust_builder_manifest_list.digest.clone(); + // Tolerate a "latest" that is a plain manifest without references + // (e.g. temporarily single-arch) instead of panicking. let referenced_digests = latest_rust_builder_manifest_list .references - .unwrap() + .unwrap_or_default() .into_iter() .map(|reference| reference.child_digest) .collect::>(); - let rust_builder_artifacts: Vec = reqwest::get(format!( - "{}/projects/sdp/repositories/{}-rust-builder/artifacts?page_size=100", - base_url, - ubi_version - )) - .await? - .json() - .await?; + // Fetch the full artifact list, paginated (a single page_size=100 + // request silently ignored anything beyond the first 100 artifacts). + let mut rust_builder_artifacts: Vec = Vec::new(); + let mut page = 1; + let page_size = 100; + loop { + let artifacts_page: Vec = client + .get(format!( + "{}/projects/sdp/repositories/{}-rust-builder/artifacts?page_size={}&page={}", + base_url, ubi_version, page_size, page + )) + .send() + .await? + .json() + .await?; + let number_of_returned_artifacts = artifacts_page.len(); + rust_builder_artifacts.extend(artifacts_page); + if number_of_returned_artifacts < page_size { + break; + } + page += 1; + } + + // Guard against a race: if a new image was tagged "latest" between the + // snapshot above and now, our referenced-digests set is stale and we + // could delete the *current* latest. Re-check and skip if it moved. + let latest_after: Artifact = client.get(&latest_url).send().await?.json().await?; + if latest_after.digest != latest_digest { + println!( + "'latest' for {}-rust-builder changed during the run; skipping cleanup to avoid deleting the current latest.", + ubi_version + ); + continue; + } for artifact in rust_builder_artifacts { // Keep "latest" and its referenced artifacts - if artifact.digest != latest_rust_builder_manifest_list.digest - && !referenced_digests.contains(&artifact.digest) - { + if artifact.digest != latest_digest && !referenced_digests.contains(&artifact.digest) { println!( "Removing dangling rust builder artifact {}", artifact.digest ); - reqwest::Client::new() + client .delete(format!( "{}/projects/sdp/repositories/{}-rust-builder/artifacts/{}", base_url, ubi_version, artifact.digest, )) - .basic_auth( - "robot$stackable-cleanup", - env::var("HARBOR_ROBOT_PASSWORD").ok(), - ) + .basic_auth(ROBOT_USER, Some(&robot_password)) .send() - .await?; + .await? + .error_for_status()?; } } }