| 1 | //! Admin-only dashboard pages (site-level), gated by `User.is_admin`. |
| 2 | |
| 3 | use std::time::Duration; |
| 4 | |
| 5 | use anvil_core::{ |
| 6 | App, |
| 7 | ci, |
| 8 | jobs::{ |
| 9 | RUNNER_TTL, |
| 10 | RunnerStatus, |
| 11 | }, |
| 12 | repos, |
| 13 | usage, |
| 14 | users, |
| 15 | }; |
| 16 | use axum::{ |
| 17 | Router, |
| 18 | extract::State, |
| 19 | response::{ |
| 20 | IntoResponse, |
| 21 | Response, |
| 22 | }, |
| 23 | routing::get, |
| 24 | }; |
| 25 | use maud::{ |
| 26 | Markup, |
| 27 | PreEscaped, |
| 28 | html, |
| 29 | }; |
| 30 | |
| 31 | use crate::{ |
| 32 | auth::CurrentUser, |
| 33 | ui::{ |
| 34 | fmt_size, |
| 35 | layout, |
| 36 | not_found, |
| 37 | server_error, |
| 38 | }, |
| 39 | }; |
| 40 | |
| 41 | pub fn routes(router: Router<App>) -> Router<App> { |
| 42 | router |
| 43 | .route("/-/admin/usage", get(usage_page)) |
| 44 | .route("/-/admin/runners", get(runners_page)) |
| 45 | } |
| 46 | |
| 47 | /// `GET /-/admin/usage` — site-wide disk usage by user and content type. |
| 48 | /// Non-admins (including anonymous) get a 404, so the page's existence isn't |
| 49 | /// leaked. Tries to use cached value first; computes on-demand if not found. |
| 50 | async fn usage_page(State(app): State<App>, CurrentUser(user): CurrentUser) -> Response { |
| 51 | if !user.as_ref().is_some_and(|u| u.is_admin) { |
| 52 | return not_found("not found"); |
| 53 | } |
| 54 | |
| 55 | // Try to get cached disk usage first |
| 56 | let data = if let Ok(Some(cached)) = anvil_core::admin_cache::get(&app.db, "disk_usage").await { |
| 57 | match serde_json::from_str(&cached.value) { |
| 58 | Ok(d) => d, |
| 59 | Err(_) => { |
| 60 | // Cache corrupted, recompute |
| 61 | match usage::compute(&app).await { |
| 62 | Ok(d) => d, |
| 63 | Err(e) => return server_error(e), |
| 64 | } |
| 65 | } |
| 66 | } |
| 67 | } else { |
| 68 | // No cached value, compute on demand |
| 69 | match usage::compute(&app).await { |
| 70 | Ok(d) => d, |
| 71 | Err(e) => return server_error(e), |
| 72 | } |
| 73 | }; |
| 74 | |
| 75 | layout("Disk usage", user.as_ref(), render(&data)).into_response() |
| 76 | } |
| 77 | |
| 78 | fn render(u: &usage::Usage) -> Markup { |
| 79 | html! { |
| 80 | h1 { "Disk usage" } |
| 81 | p.muted { |
| 82 | "Actual on-disk bytes per user, by content type — " |
| 83 | (fmt_size(u.total() as i64)) " total across the instance." |
| 84 | } |
| 85 | table.usage { |
| 86 | thead { tr { |
| 87 | th { "User" } |
| 88 | th.num { "Repositories" } |
| 89 | th.num { "CI artifacts" } |
| 90 | th.num { "Attachments" } |
| 91 | th.num { "Total" } |
| 92 | } } |
| 93 | tbody { |
| 94 | @for row in &u.per_user { |
| 95 | tr { |
| 96 | td { (row.username) } |
| 97 | td.num { (fmt_size(row.git as i64)) } |
| 98 | td.num { (fmt_size(row.artifacts as i64)) } |
| 99 | td.num { (fmt_size(row.attachments as i64)) } |
| 100 | td.num { (fmt_size(row.total() as i64)) } |
| 101 | } |
| 102 | } |
| 103 | } |
| 104 | tfoot { tr { |
| 105 | td { "All users" } |
| 106 | td.num { (fmt_size(u.git as i64)) } |
| 107 | td.num { (fmt_size(u.artifacts as i64)) } |
| 108 | td.num { (fmt_size(u.attachments as i64)) } |
| 109 | td.num { (fmt_size(u.total() as i64)) } |
| 110 | } } |
| 111 | } |
| 112 | } |
| 113 | } |
| 114 | |
| 115 | /// When a present runner starts reading as late. |
| 116 | /// |
| 117 | /// Liveness is not a dedicated ping: it is the ordinary traffic a working |
| 118 | /// runner already generates. An idle one refreshes itself once per parked |
| 119 | /// claim ([`anvil_job::CLAIM_POLL`]), a busy one every |
| 120 | /// [`HEARTBEAT_INTERVAL`](anvil_job::HEARTBEAT_INTERVAL). So a runner has to |
| 121 | /// have missed a whole poll window *and* a heartbeat before silence means |
| 122 | /// anything, and this is that sum. Between here and `RUNNER_TTL` a runner is |
| 123 | /// still routed to — the page says "late", not "gone", because that is |
| 124 | /// precisely what the dispatcher still believes. |
| 125 | const STALE_AFTER: Duration = |
| 126 | Duration::from_secs(anvil_job::CLAIM_POLL.as_secs() + anvil_job::HEARTBEAT_INTERVAL.as_secs()); |
| 127 | |
| 128 | /// How often the page reloads itself. Comfortably under `STALE_AFTER`, so a |
| 129 | /// runner that goes quiet is visible as late without anyone pressing reload. |
| 130 | const REFRESH_SECS: u64 = 15; |
| 131 | |
| 132 | const REFRESH_JS: &str = "setTimeout(function(){ location.reload(); }, 15000);"; |
| 133 | |
| 134 | /// `GET /-/admin/runners` — the CI runners currently dialled in. |
| 135 | /// |
| 136 | /// Non-admins (including anonymous) get a 404, same as the usage page: the |
| 137 | /// list names the machines that build this instance's code and how idle they |
| 138 | /// are, which is not something to leak by letting the route 403. |
| 139 | async fn runners_page(State(app): State<App>, CurrentUser(user): CurrentUser) -> Response { |
| 140 | if !user.as_ref().is_some_and(|u| u.is_admin) { |
| 141 | return not_found("not found"); |
| 142 | } |
| 143 | |
| 144 | let runners = app.jobs.runners(); |
| 145 | // Queued runs are the other half of the diagnosis: runners but no queue is |
| 146 | // a healthy idle instance, a queue but no runners is a stuck one. |
| 147 | let queued = ci::queued_ids(&app.db).await.unwrap_or_default(); |
| 148 | |
| 149 | // In-flight runs are keyed by id alone (the lease knows nothing of repos), |
| 150 | // so resolve each to its page. A handful at most — one per busy runner. |
| 151 | let mut links: Vec<(i64, String)> = Vec::new(); |
| 152 | for id in runners.iter().flat_map(|r| r.running.iter().copied()) { |
| 153 | if let Some(href) = run_href(&app, id).await { |
| 154 | links.push((id, href)); |
| 155 | } |
| 156 | } |
| 157 | |
| 158 | layout( |
| 159 | "CI runners", |
| 160 | user.as_ref(), |
| 161 | render_runners( |
| 162 | &runners, |
| 163 | queued.len(), |
| 164 | !app.config.ci.runner_token.is_empty(), |
| 165 | &links, |
| 166 | ), |
| 167 | ) |
| 168 | .into_response() |
| 169 | } |
| 170 | |
| 171 | /// The canonical page for a run id, or `None` if any part of the chain is |
| 172 | /// missing (a run deleted mid-flight, most plausibly). |
| 173 | async fn run_href(app: &App, run_id: i64) -> Option<String> { |
| 174 | let run = ci::get(&app.db, run_id).await.ok()??; |
| 175 | let repo = repos::find_by_id(&app.db, run.repo_id).await.ok()??; |
| 176 | let owner = users::find_by_id(&app.db, repo.owner_id).await.ok()??; |
| 177 | Some(format!("/{}/{}/ci/{run_id}", owner.username, repo.name)) |
| 178 | } |
| 179 | |
| 180 | fn render_runners( |
| 181 | runners: &[RunnerStatus], |
| 182 | queued: usize, |
| 183 | token_set: bool, |
| 184 | links: &[(i64, String)], |
| 185 | ) -> Markup { |
| 186 | html! { |
| 187 | h1 { "CI runners" } |
| 188 | p.muted { |
| 189 | "anvil dispatches CI but does not execute it: runners dial in, claim jobs " |
| 190 | "and run them on their own Docker daemon. This is who has been in touch " |
| 191 | "within " (fmt_span(RUNNER_TTL)) " — the window the dispatcher itself " |
| 192 | "treats as connected. Reloads every " (REFRESH_SECS) "s." |
| 193 | } |
| 194 | |
| 195 | @if !token_set { |
| 196 | p.error-msg { |
| 197 | "[ci] runner_token is empty, so every claim is refused with a 503 and " |
| 198 | "no runner can connect at all. Set it in the config and restart." |
| 199 | } |
| 200 | } |
| 201 | |
| 202 | @if runners.is_empty() { |
| 203 | p.muted { |
| 204 | "No runner connected. " |
| 205 | @if queued > 0 { |
| 206 | b { (queued) " run" @if queued != 1 { "s" } " queued" } |
| 207 | " and nothing to run " @if queued == 1 { "it" } @else { "them" } "." |
| 208 | } @else { |
| 209 | "Nothing is queued, so nothing is stuck yet — but the next push has nowhere to go." |
| 210 | } |
| 211 | } |
| 212 | } @else { |
| 213 | table.usage { |
| 214 | thead { tr { |
| 215 | th { "Runner" } |
| 216 | th { "Platform" } |
| 217 | th { "Version" } |
| 218 | th { "Last seen" } |
| 219 | th { "Connected" } |
| 220 | th { "Running" } |
| 221 | } } |
| 222 | tbody { |
| 223 | @for r in runners { |
| 224 | tr { |
| 225 | td { |
| 226 | (r.name) |
| 227 | " " |
| 228 | @if r.last_seen >= STALE_AFTER { |
| 229 | span class="st failure" { "late" } |
| 230 | } @else if r.running.is_empty() { |
| 231 | span class="st success" { "idle" } |
| 232 | } @else { |
| 233 | span class="st running" { "busy" } |
| 234 | } |
| 235 | } |
| 236 | td { @if r.platform.is_empty() { span.muted { "unstated" } } @else { code { (r.platform) } } } |
| 237 | td { @if r.version.is_empty() { span.muted { "—" } } @else { (r.version) } } |
| 238 | td.num { (fmt_span(r.last_seen)) " ago" } |
| 239 | td.num { (fmt_span(r.connected_for)) } |
| 240 | td { |
| 241 | @if r.running.is_empty() { |
| 242 | span.muted { "—" } |
| 243 | } @else { |
| 244 | @for id in &r.running { |
| 245 | @match links.iter().find(|(i, _)| i == id) { |
| 246 | Some((_, href)) => { a href=(href) { "#" (id) } " " } |
| 247 | None => { span { "#" (id) } " " } |
| 248 | } |
| 249 | } |
| 250 | } |
| 251 | } |
| 252 | } |
| 253 | } |
| 254 | } |
| 255 | } |
| 256 | p.muted { |
| 257 | (queued) " run" @if queued != 1 { "s" } " queued." |
| 258 | @if queued > 0 && runners.iter().all(|r| !r.running.is_empty()) { |
| 259 | " Every runner is busy, so the queue is waiting on capacity rather than on a missing runner." |
| 260 | } |
| 261 | } |
| 262 | } |
| 263 | |
| 264 | p.muted { |
| 265 | "\"Connected\" counts from this process's first contact, so an anvild " |
| 266 | "restart resets it — it is not the runner's own uptime." |
| 267 | } |
| 268 | script { (PreEscaped(REFRESH_JS)) } |
| 269 | } |
| 270 | } |
| 271 | |
| 272 | /// A duration as a short span (`4s`, `3m`, `2h 5m`). Deliberately finer than |
| 273 | /// `fmt_relative`, whose floor is "just now": the whole point of the last-seen |
| 274 | /// column is telling 4 seconds from 90. |
| 275 | fn fmt_span(d: Duration) -> String { |
| 276 | // A zero remainder is noise, not precision: the TTL should read "5m", not |
| 277 | // "5m 0s". |
| 278 | let pair = |big: u64, bu: &str, small: u64, su: &str| match small { |
| 279 | 0 => format!("{big}{bu}"), |
| 280 | _ => format!("{big}{bu} {small}{su}"), |
| 281 | }; |
| 282 | match d.as_secs() { |
| 283 | s if s < 60 => format!("{s}s"), |
| 284 | s if s < 3600 => pair(s / 60, "m", s % 60, "s"), |
| 285 | s if s < 86_400 => pair(s / 3600, "h", (s % 3600) / 60, "m"), |
| 286 | s => pair(s / 86_400, "d", (s % 86_400) / 3600, "h"), |
| 287 | } |
| 288 | } |
| 289 | |
| 290 | #[cfg(test)] |
| 291 | mod tests { |
| 292 | use super::*; |
| 293 | |
| 294 | #[test] |
| 295 | fn spans_read_as_ages() { |
| 296 | assert_eq!(fmt_span(Duration::from_secs(0)), "0s"); |
| 297 | assert_eq!(fmt_span(Duration::from_secs(31)), "31s"); |
| 298 | assert_eq!(fmt_span(Duration::from_secs(86)), "1m 26s"); |
| 299 | assert_eq!(fmt_span(RUNNER_TTL), "5m"); |
| 300 | assert_eq!(fmt_span(Duration::from_secs(7_500)), "2h 5m"); |
| 301 | assert_eq!(fmt_span(Duration::from_secs(90_000)), "1d 1h"); |
| 302 | } |
| 303 | |
| 304 | /// The threshold has to sit above a full idle cycle, or every runner that |
| 305 | /// is simply parked in a claim reads as late. |
| 306 | #[test] |
| 307 | fn stale_after_allows_a_whole_claim_poll() { |
| 308 | assert!(STALE_AFTER > anvil_job::CLAIM_POLL); |
| 309 | assert!(STALE_AFTER < RUNNER_TTL); |
| 310 | } |
| 311 | } |