Skip to content

fix(harbor-gc): make the garbage collector fail safe #149

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lfrancke wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(harbor-gc): make the garbage collector fail safe #149

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 82 additions & 46 deletions tools/harbor-garbage-collector/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use regex::Regex;
use serde::Deserialize;
use std::env;
use std::time::Duration;
use urlencoding::encode;

#[derive(Deserialize, Debug)]
Expand All @@ -25,6 +26,8 @@ struct Artifact {
references: Option<Vec<ArtifactReference>>,
}

const ROBOT_USER: &str = "robot$stackable-cleanup";

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let registry_hostname = "oci.stackable.tech";
Expand All @@ -33,14 +36,25 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let mut page = 1;
let attestation_tag_regex = Regex::new(r"^sha256-[0-9a-f]{64}.att$").unwrap();

// Fail fast if the cleanup credential is missing. Previously this used
// `.ok()`, so a missing/renamed variable silently produced unauthenticated
// DELETEs that Harbor rejects while the tool still exits 0.
let robot_password = env::var("HARBOR_ROBOT_PASSWORD")
.map_err(|_| "HARBOR_ROBOT_PASSWORD environment variable must be set")?;

// One shared client with a timeout, so a hung registry can't block forever.
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(30))
.build()?;

loop {
// Loop over all repositories, paged
let url = format!(
"{}/repositories?page_size={}&page={}",
base_url, page_size, page
);

let repositories: Vec<Repository> = reqwest::get(&url).await?.json().await?;
let repositories: Vec<Repository> = client.get(&url).send().await?.json().await?;

for repository in &repositories {
let (project_name, repository_name) = repository.name.split_once('/').unwrap();
Expand All @@ -58,17 +72,19 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {

loop {
// Loop over pages to get all artifacts
let artifacts_page: Vec<Artifact> = reqwest::get(format!(
"{}/projects/{}/repositories/{}/artifacts?page_size={}&page={}",
base_url,
encode(project_name),
encode(repository_name),
page_size,
page
))
.await?
.json()
.await?;
let artifacts_page: Vec<Artifact> = client
.get(format!(
"{}/projects/{}/repositories/{}/artifacts?page_size={}&page={}",
base_url,
encode(project_name),
encode(repository_name),
page_size,
page
))
.send()
.await?
.json()
.await?;

let number_of_returned_artifacts = artifacts_page.len();
artifacts.extend(artifacts_page);
Expand Down Expand Up @@ -104,20 +120,18 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
if !potentially_attested_artifacts.contains(&attestation_referenced_sha) {
let attestation_tag = format!("sha256-{}.att", attestation_referenced_sha);
println!("Removing dangling attestation {}", attestation_tag);
reqwest::Client::new()
client
.delete(format!(
"{}/projects/{}/repositories/{}/artifacts/{}",
base_url,
encode(project_name),
encode(repository_name),
attestation_tag
))
.basic_auth(
"robot$stackable-cleanup",
env::var("HARBOR_ROBOT_PASSWORD").ok(),
)
.basic_auth(ROBOT_USER, Some(&robot_password))
.send()
.await?;
.await?
.error_for_status()?;
}
}
}
Expand All @@ -133,53 +147,75 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Only keep the "latest" manifest and its referenced artifacts

for ubi_version in &["ubi8", "ubi9"] {
let latest_rust_builder_manifest_list: Artifact = reqwest::Client::new()
.get(format!(
"{}/projects/sdp/repositories/{}-rust-builder/artifacts/latest",
base_url,
ubi_version
))
.send()
.await?
.json()
.await?;
let latest_url = format!(
"{}/projects/sdp/repositories/{}-rust-builder/artifacts/latest",
base_url, ubi_version
);

let latest_rust_builder_manifest_list: Artifact =
client.get(&latest_url).send().await?.json().await?;

let latest_digest = latest_rust_builder_manifest_list.digest.clone();
// Tolerate a "latest" that is a plain manifest without references
// (e.g. temporarily single-arch) instead of panicking.
let referenced_digests = latest_rust_builder_manifest_list
.references
.unwrap()
.unwrap_or_default()
.into_iter()
.map(|reference| reference.child_digest)
.collect::<Vec<String>>();

let rust_builder_artifacts: Vec<Artifact> = reqwest::get(format!(
"{}/projects/sdp/repositories/{}-rust-builder/artifacts?page_size=100",
base_url,
ubi_version
))
.await?
.json()
.await?;
// Fetch the full artifact list, paginated (a single page_size=100
// request silently ignored anything beyond the first 100 artifacts).
let mut rust_builder_artifacts: Vec<Artifact> = Vec::new();
let mut page = 1;
let page_size = 100;
loop {
let artifacts_page: Vec<Artifact> = client
.get(format!(
"{}/projects/sdp/repositories/{}-rust-builder/artifacts?page_size={}&page={}",
base_url, ubi_version, page_size, page
))
.send()
.await?
.json()
.await?;
let number_of_returned_artifacts = artifacts_page.len();
rust_builder_artifacts.extend(artifacts_page);
if number_of_returned_artifacts < page_size {
break;
}
page += 1;
}

// Guard against a race: if a new image was tagged "latest" between the
// snapshot above and now, our referenced-digests set is stale and we
// could delete the *current* latest. Re-check and skip if it moved.
let latest_after: Artifact = client.get(&latest_url).send().await?.json().await?;
if latest_after.digest != latest_digest {
println!(
"'latest' for {}-rust-builder changed during the run; skipping cleanup to avoid deleting the current latest.",
ubi_version
);
continue;
}

for artifact in rust_builder_artifacts {
// Keep "latest" and its referenced artifacts
if artifact.digest != latest_rust_builder_manifest_list.digest
&& !referenced_digests.contains(&artifact.digest)
{
if artifact.digest != latest_digest && !referenced_digests.contains(&artifact.digest) {
println!(
"Removing dangling rust builder artifact {}",
artifact.digest
);
reqwest::Client::new()
client
.delete(format!(
"{}/projects/sdp/repositories/{}-rust-builder/artifacts/{}",
base_url, ubi_version, artifact.digest,
))
.basic_auth(
"robot$stackable-cleanup",
env::var("HARBOR_ROBOT_PASSWORD").ok(),
)
.basic_auth(ROBOT_USER, Some(&robot_password))
.send()
.await?;
.await?
.error_for_status()?;
}
}
}
Expand Down