ci(mirror): 每资源计时 + 2m 上传上限(超时 warn 并跳过)

.github/tools/mirror_res.sh

@@ -42,8 +42,58 @@ read -r -a ASSETS <<< "${ASSETS:-$DEFAULT_ASSETS}"
 
 info() { echo "[mirror] $*"; }
 
+# Per-asset upload cap. Exceeding it WARNs and abandons that asset (no retry,
+# no delete) so one slow cross-border PUT can't eat the whole job budget — the
+# v0.0.94 release lost `publish-ecosystem` to a 30min job timeout with zero
+# per-asset visibility, and the two biggest linux tarballs had to be pushed by
+# hand afterwards.
+#
+# CAUTION (v0.0.90 postmortem, still binding): a cap that kills a
+# slow-but-PROGRESSING upload and then RETRIES it is strictly worse than no cap
+# — every restart resumes from byte zero and the mirror never converges. This
+# cap is safe only because a capped asset is SKIPPED, never re-uploaded. The
+# completeness gate at the bottom is still the pass/fail, so a skipped asset
+# fails the release loudly instead of silently shipping a half mirror.
+#
+# Sizing: mcpp's largest asset is ~30MB (no package exceeds 100MB). 180s is a
+# generous ceiling for that — an upload still running at 3min is not "slow", it
+# is stuck, and the right move is to stop paying CI for it and push that one
+# asset by hand (the gate below prints the exact command).
+: "${MIRROR_UPLOAD_TIMEOUT:=180}"
+
 DL="$(mktemp -d)"; trap 'rm -rf "$DL"' EXIT
 
+human_size() { # path → e.g. 31.9MB
+  local b; b=$(stat -c %s "$1" 2>/dev/null || stat -f %z "$1" 2>/dev/null || echo 0)
+  awk -v b="$b" 'BEGIN{ if (b>=1048576) printf "%.1fMB", b/1048576; else if (b>=1024) printf "%.1fKB", b/1024; else printf "%dB", b }'
+}
+
+host_label() { [[ "$1" == gh ]] && echo github || echo gitcode; }
+
+# Upload one asset under the cap, timing it. 0 = the command returned within
+# the cap (NOT proof it landed — gtc's exit code lies both ways, so the probe /
+# gate remains the only source of truth); 1 = the cap fired, asset abandoned.
+upload_asset() { # kind(gh|gtc) asset → 0 returned / 1 capped
+  local kind="$1" a="$2" start elapsed rc=0 sz host
+  sz=$(human_size "$DL/$a")
+  host=$(host_label "$kind")
+  start=$SECONDS
+  if [[ "$kind" == gh ]]; then
+    GH_TOKEN="${XLINGS_RES_TOKEN:-}" timeout "$MIRROR_UPLOAD_TIMEOUT" \
+      gh release upload "$VER" "$DL/$a" -R "$GH_DST" --clobber >/dev/null 2>&1 || rc=$?
+  else
+    timeout "$MIRROR_UPLOAD_TIMEOUT" gtc release upload "$GTC_DST" "$DL/$a" --tag "$VER" \
+      >/dev/null 2>&1 || rc=$?
+  fi
+  elapsed=$((SECONDS - start))
+  if [[ $rc == 124 || $rc == 137 ]]; then
+    info "WARN: $host $a ($sz) exceeded the ${MIRROR_UPLOAD_TIMEOUT}s cap after ${elapsed}s — skipping (not retried; the verify gate below decides the release)"
+    return 1
+  fi
+  info "$host $a ($sz) uploaded in ${elapsed}s"
+  return 0
+}
+
 info "downloading $SRC_REPO v$VER assets ($PROJ)"
 for a in "${ASSETS[@]}"; do
   gh release download "v$VER" -R "$SRC_REPO" -D "$DL" -p "$a" 2>/dev/null || { echo "[mirror] FAIL: missing $a in $SRC_REPO v$VER" >&2; exit 1; }
@@ -93,36 +143,43 @@ verify_batch() { # base_url asset... → prints assets still not serving
 # Hard-won rules (0.0.86 / 0.0.89 / 0.0.90 postmortems):
 #  - NEVER delete on a verify timeout — the eager 404→delete loop repeatedly
 #    deleted GOOD uploads whose propagation was merely slow.
-#  - NEVER kill a slow-but-progressing upload: the outer `timeout` MUST
-#    exceed gtc's inner PUT timeout (600s). v0.0.90 wrapped uploads in
-#    `timeout 300`, so every cross-border PUT >5min was SIGKILLed at 60%%
-#    and restarted from byte zero — the 20min job ceiling fell to this.
+#  - NEVER kill a slow-but-progressing upload AND RETRY IT. v0.0.90 wrapped
+#    uploads in `timeout 300`, so every cross-border PUT >5min was SIGKILLed
+#    at 60%% and restarted from byte zero — the 20min job ceiling fell to
+#    this. MIRROR_UPLOAD_TIMEOUT keeps a cap but ABANDONS the asset instead of
+#    retrying it, which is what makes the cap safe; the gate then fails the
+#    release loudly rather than thrashing until the job is killed.
 #  - gtc's exit code lies both ways (obs_callback flakiness); the download
 #    probe is the only source of truth.
 mirror_host() { # kind(gh|gtc) base_url
   local kind="$1" base="$2" try a
   local pending failed
+  local -A capped=()
+  local host_start=$SECONDS
+  local host; host=$(host_label "$kind")
   for try in 1 2 3; do
     pending=()
     for a in "${ASSETS[@]}"; do
+      # A capped asset is never re-attempted (see MIRROR_UPLOAD_TIMEOUT).
+      [[ -n "${capped[$a]:-}" ]] && continue
+      # Step 1: already serving? then it's mirrored — never re-upload it.
       if probe "${base}/${a}"; then
-        [[ $try == 1 ]] && info "$kind $a already mirrored, skipping"
+        [[ $try == 1 ]] && info "$host $a already mirrored, skipping"
         continue
       fi
-      pending+=("$a")
-      if [[ "$kind" == gh ]]; then
-        GH_TOKEN="${XLINGS_RES_TOKEN:-}" timeout 900 \
-          gh release upload "$VER" "$DL/$a" -R "$GH_DST" --clobber || true
+      if upload_asset "$kind" "$a"; then
+        pending+=("$a")
       else
-        timeout 900 gtc release upload "$GTC_DST" "$DL/$a" --tag "$VER" \
-          >/dev/null 2>&1 || true
+        capped[$a]=1
       fi
     done
-    [[ ${#pending[@]} == 0 ]] && return 0
+    [[ ${#pending[@]} == 0 ]] && break
     failed=$(verify_batch "$base" "${pending[@]}")
-    [[ -z "$failed" ]] && return 0
-    echo "[mirror] $kind not serving after patience (try $try): $failed — re-uploading (no delete)"
+    [[ -z "$failed" ]] && break
+    info "$host not serving after patience (try $try): $failed — re-uploading (no delete)"
   done
+  ((${#capped[@]})) && info "WARN: $host abandoned ${#capped[@]} asset(s) at the ${MIRROR_UPLOAD_TIMEOUT}s cap: ${!capped[*]}"
+  info "$host mirror leg finished in $((SECONDS - host_start))s"
   return 0  # the completeness gate below is the real pass/fail
 }
 
@@ -174,5 +231,9 @@ for host in "${hosts[@]}"; do
     [[ "$code" == 200 ]] || { rc=1; echo "[mirror] FAIL: missing/unverified: https://${host}/releases/download/${VER}/${a}" >&2; }
   done
 done
-[[ $rc == 0 ]] && info "all assets mirrored + verified on ${#hosts[@]} host(s)"
+if [[ $rc != 0 ]]; then
+  echo "[mirror] hint: if the asset above was WARNed as capped at ${MIRROR_UPLOAD_TIMEOUT}s, either raise MIRROR_UPLOAD_TIMEOUT for this run or push it by hand:" >&2
+  echo "[mirror]       gh release download v$VER -R $SRC_REPO -p '<asset>' && gtc release upload $GTC_DST '<asset>' --tag $VER" >&2
+fi
+[[ $rc == 0 ]] && info "all assets mirrored + verified on ${#hosts[@]} host(s) in ${SECONDS}s"
 exit $rc

.github/workflows/release.yml

@@ -725,6 +725,13 @@ jobs:
 
       - name: Mirror binaries to xlings-res/mcpp (gh + gtc)
         if: ${{ env.XLINGS_RES_TOKEN != '' }}
+        # Hard ceiling for the whole mirror segment. The script caps each asset
+        # at MIRROR_UPLOAD_TIMEOUT (180s) and abandons anything slower, so a
+        # healthy run lands well inside this; 10min is the backstop that keeps a
+        # pathological host from burning the job's budget the way v0.0.94 did
+        # (30min, killed, zero per-asset visibility). Whatever gets skipped is
+        # reported by name + size and pushed by hand.
+        timeout-minutes: 10
         env:
           GH_TOKEN: ${{ secrets.XLINGS_RES_TOKEN }}
         run: |