anvilsign in

collin/anvil

1//! Docker specifics for an agent session: creating the container, seeding it,
2//! attaching a terminal to the tmux session inside, and tearing it down.
3//!
4//! The containment story is deliberately the CI one (see
5//! `docs/untrusted-mode.md` §1): all capabilities dropped, `no-new-privileges`,
6//! pids/memory/cpu caps, **no Docker socket, no bind mounts, no volumes**.
7//! Everything the container needs — the checkout and the agent's credentials —
8//! is uploaded as a tar through the Docker API, so nothing on the anvil host is
9//! ever exposed to it.
10
11use anvil_core::config::AgentConfig;
12use bollard::{
13 Docker,
14 body_full,
15 exec::{
16 CreateExecOptions,
17 ResizeExecOptions,
18 StartExecOptions,
19 StartExecResults,
20 },
21 models::{
22 ContainerCreateBody,
23 HostConfig,
24 },
25 query_parameters::{
26 CreateContainerOptions,
27 DownloadFromContainerOptions,
28 RemoveContainerOptions,
29 StartContainerOptions,
30 UploadToContainerOptions,
31 },
32};
33use futures_util::StreamExt;
34
35/// Label carrying the session id, so containers can be found again after a
36/// restart — the registry is in-memory and does not survive one.
37pub const LABEL_SESSION: &str = "anvil.session";
38
39/// Working directory inside the container. Under [`RUN_AS`]'s home rather
40/// than a bare `/workspace` (which is what the CI runner still uses — CI runs
41/// as root, so it has no home directory to prefer) so tools that assume a
42/// project lives under `$HOME` behave.
43pub const WORKDIR: &str = "/home/ubuntu/workspace";
44
45/// The unprivileged user sessions run as rather than root; CI keeps the
46/// image's default (root) because plenty of pipelines expect to `apt-get`.
47/// This is `ubuntu`, not a purpose-made account: `docker/runner/Dockerfile`'s
48/// base image already ships a uid-1000 user by that name, so reusing it saves
49/// a `useradd`. If the base image ever moves off Ubuntu, this needs an actual
50/// account created again.
51pub const RUN_AS: &str = "ubuntu";
52
53/// That user's uid/gid, needed when building tars so the uploaded files are
54/// owned by the account that has to write them.
55const RUN_AS_UID: u64 = 1000;
56
57/// Home directory of [`RUN_AS`]; the agent CLI's config lives under it.
58pub const HOME: &str = "/home/ubuntu";
59
60/// tmux session name, matching `session-entrypoint.sh`.
61pub const TMUX_SESSION: &str = "agent";
62
63/// Create (but do not start) a session container.
64pub async fn create(
65 docker: &Docker,
66 cfg: &AgentConfig,
67 session_id: i64,
68 env: &[(String, String)],
69 command: &[String],
70) -> Result<String, String> {
71 // The same sandbox CI uses. Limits of 0 mean "unlimited" and omit the cap.
72 //
73 // Note there is no `network_mode` opt-out: unlike a CI job, a session is
74 // useless without network — it has to reach the model API, and from M2 it
75 // clones and pushes back to anvil.
76 let host_config = HostConfig {
77 cap_drop: Some(vec!["ALL".to_string()]),
78 security_opt: Some(vec!["no-new-privileges:true".to_string()]),
79 pids_limit: (cfg.pids_limit > 0).then_some(cfg.pids_limit),
80 memory: (cfg.memory_mb > 0).then(|| cfg.memory_mb * 1024 * 1024),
81 memory_swap: (cfg.memory_mb > 0).then(|| cfg.memory_mb * 1024 * 1024),
82 nano_cpus: (cfg.cpus > 0.0).then_some((cfg.cpus * 1e9) as i64),
83 ..Default::default()
84 };
85
86 let mut cmd = vec!["anvil-session".to_string()];
87 cmd.extend(command.iter().cloned());
88
89 let config = ContainerCreateBody {
90 image: Some(cfg.image.clone()),
91 cmd: Some(cmd),
92 env: Some(
93 env.iter()
94 .map(|(k, v)| format!("{k}={v}"))
95 .chain([
96 format!("HOME={HOME}"),
97 format!("ANVIL_WORKDIR={WORKDIR}"),
98 "TERM=xterm-256color".to_string(),
99 ])
100 .collect(),
101 ),
102 working_dir: Some(WORKDIR.to_string()),
103 user: Some(RUN_AS.to_string()),
104 labels: Some(
105 [(LABEL_SESSION.to_string(), session_id.to_string())]
106 .into_iter()
107 .collect(),
108 ),
109 // A terminal program needs a tty even before anyone attaches, or tmux
110 // starts with a 80x24 dumb terminal and never recovers.
111 tty: Some(true),
112 open_stdin: Some(true),
113 host_config: Some(host_config),
114 ..Default::default()
115 };
116
117 let created = docker
118 .create_container(None::<CreateContainerOptions>, config)
119 .await
120 .map_err(|e| format!("create session container: {e}"))?;
121 Ok(created.id)
122}
123
124/// Upload a tar into the container, rooted at `/`.
125pub async fn upload(docker: &Docker, id: &str, tar: Vec<u8>) -> Result<(), String> {
126 docker
127 .upload_to_container(
128 id,
129 Some(UploadToContainerOptions {
130 path: "/".to_string(),
131 ..Default::default()
132 }),
133 body_full(tar.into()),
134 )
135 .await
136 .map_err(|e| format!("upload to session container: {e}"))
137}
138
139/// Read one file's current bytes out of the container (which must already
140/// exist, though it need not be started — the filesystem is there either
141/// way). `None` covers every reason it might not be readable yet: the file
142/// does not exist, an intermediate directory does not exist, anything —
143/// a "json" secret starts from `{}` in all of those cases alike, the same as
144/// a brand new file would.
145pub async fn download_file(docker: &Docker, id: &str, absolute_path: &str) -> Option<Vec<u8>> {
146 let mut stream = docker.download_from_container(
147 id,
148 Some(DownloadFromContainerOptions {
149 path: absolute_path.to_string(),
150 }),
151 );
152 let mut tar = Vec::new();
153 while let Some(chunk) = stream.next().await {
154 tar.extend_from_slice(&chunk.ok()?);
155 }
156 let mut archive = tar::Archive::new(tar.as_slice());
157 let mut entries = archive.entries().ok()?;
158 let mut entry = entries.next()?.ok()?;
159 let mut contents = Vec::new();
160 std::io::Read::read_to_end(&mut entry, &mut contents).ok()?;
161 Some(contents)
162}
163
164/// Build a tar of `(path, contents, executable)` entries, rooted at `prefix`
165/// (no leading slash) and owned by the session user.
166///
167/// Emits an explicit entry for every intermediate **directory**, which matters
168/// more than it looks: a tar of files alone makes Docker create the parent
169/// directories itself, owned by root. The session runs as [`RUN_AS`], so
170/// `src/` would come out root-owned and the agent could edit existing files
171/// but never add one — which is most of what a coding agent does.
172pub fn build_tar(prefix: &str, files: &[(String, Vec<u8>, bool)]) -> Vec<u8> {
173 let mut builder = tar::Builder::new(Vec::new());
174 let mtime = anvil_core::agent::now_secs().max(0) as u64;
175
176 let header_for = |size: u64, mode: u32, entry_type: tar::EntryType| {
177 let mut header = tar::Header::new_gnu();
178 header.set_size(size);
179 header.set_mode(mode);
180 header.set_entry_type(entry_type);
181 // Ownership matters: the container runs as [`RUN_AS`], and a checkout
182 // it cannot write is not a workspace.
183 header.set_uid(RUN_AS_UID);
184 header.set_gid(RUN_AS_UID);
185 // Without this every file lands in 1970, which upsets anything that
186 // compares timestamps (make, and incremental builds generally).
187 header.set_mtime(mtime);
188 header
189 };
190
191 // Every ancestor directory, deduplicated and shortest-first so parents are
192 // created before their children.
193 let mut dirs: Vec<String> = Vec::new();
194 for (path, _, _) in files {
195 let mut parts: Vec<&str> = path.split('/').collect();
196 parts.pop(); // the file itself
197 let mut acc = String::new();
198 for part in parts {
199 if !acc.is_empty() {
200 acc.push('/');
201 }
202 acc.push_str(part);
203 if !dirs.contains(&acc) {
204 dirs.push(acc.clone());
205 }
206 }
207 }
208 dirs.sort_by_key(|d| d.matches('/').count());
209
210 // The root itself, so `prefix` is agent-owned even when empty.
211 let mut header = header_for(0, 0o755, tar::EntryType::Directory);
212 let _ = builder.append_data(&mut header, format!("{prefix}/"), std::io::empty());
213 for dir in &dirs {
214 let mut header = header_for(0, 0o755, tar::EntryType::Directory);
215 let _ = builder.append_data(&mut header, format!("{prefix}/{dir}/"), std::io::empty());
216 }
217
218 for (path, content, executable) in files {
219 let mode = if *executable { 0o755 } else { 0o644 };
220 let mut header = header_for(content.len() as u64, mode, tar::EntryType::Regular);
221 let full = format!("{prefix}/{path}");
222 if builder
223 .append_data(&mut header, &full, content.as_slice())
224 .is_err()
225 {
226 continue;
227 }
228 }
229 builder.into_inner().unwrap_or_default()
230}
231
232/// Read a host directory into `(relative path, bytes, executable)` entries.
233///
234/// Used for the agent CLI's credentials directory, which is uploaded rather
235/// than bind-mounted so the no-mounts invariant survives.
236pub fn read_dir_recursive(root: &std::path::Path) -> Result<Vec<(String, Vec<u8>, bool)>, String> {
237 fn walk(
238 base: &std::path::Path,
239 dir: &std::path::Path,
240 out: &mut Vec<(String, Vec<u8>, bool)>,
241 ) -> std::io::Result<()> {
242 for entry in std::fs::read_dir(dir)? {
243 let entry = entry?;
244 let path = entry.path();
245 let meta = entry.metadata()?;
246 if meta.is_dir() {
247 walk(base, &path, out)?;
248 } else if meta.is_file() {
249 let Ok(rel) = path.strip_prefix(base) else {
250 continue;
251 };
252 let executable = {
253 #[cfg(unix)]
254 {
255 use std::os::unix::fs::PermissionsExt;
256 meta.permissions().mode() & 0o111 != 0
257 }
258 #[cfg(not(unix))]
259 {
260 false
261 }
262 };
263 out.push((
264 rel.to_string_lossy().replace('\\', "/"),
265 std::fs::read(&path)?,
266 executable,
267 ));
268 }
269 }
270 Ok(())
271 }
272
273 let mut out = Vec::new();
274 walk(root, root, &mut out).map_err(|e| format!("reading {}: {e}", root.display()))?;
275 Ok(out)
276}
277
278/// Start a created container.
279pub async fn start(docker: &Docker, id: &str) -> Result<(), String> {
280 docker
281 .start_container(id, None::<StartContainerOptions>)
282 .await
283 .map_err(|e| format!("start session container: {e}"))
284}
285
286/// A live terminal attached to the container's tmux session.
287pub struct Terminal {
288 /// Docker exec id, needed to resize the pty.
289 pub exec_id: String,
290 /// Bytes the terminal produces.
291 pub output: std::pin::Pin<
292 Box<
293 dyn futures_util::Stream<
294 Item = Result<bollard::container::LogOutput, bollard::errors::Error>,
295 > + Send,
296 >,
297 >,
298 /// Keystrokes go here.
299 pub input: std::pin::Pin<Box<dyn tokio::io::AsyncWrite + Send>>,
300}
301
302/// Attach a new tmux client to the container's session.
303///
304/// Each caller gets its own `docker exec`, which is the point: a dropped
305/// websocket kills that client only, never the agent, because the agent is a
306/// process inside tmux rather than a child of the exec.
307pub async fn attach(docker: &Docker, id: &str, cols: u16, rows: u16) -> Result<Terminal, String> {
308 let exec = docker
309 .create_exec(
310 id,
311 CreateExecOptions {
312 attach_stdin: Some(true),
313 attach_stdout: Some(true),
314 attach_stderr: Some(true),
315 tty: Some(true),
316 user: Some(RUN_AS.to_string()),
317 env: Some(vec![
318 "TERM=xterm-256color".to_string(),
319 format!("HOME={HOME}"),
320 ]),
321 cmd: Some(vec![
322 "tmux".to_string(),
323 "-f".to_string(),
324 "/etc/anvil/tmux.conf".to_string(),
325 "attach-session".to_string(),
326 "-t".to_string(),
327 TMUX_SESSION.to_string(),
328 ]),
329 ..Default::default()
330 },
331 )
332 .await
333 .map_err(|e| format!("create attach exec: {e}"))?;
334
335 let started = docker
336 .start_exec(
337 &exec.id,
338 Some(StartExecOptions {
339 detach: false,
340 tty: true,
341 ..Default::default()
342 }),
343 )
344 .await
345 .map_err(|e| format!("start attach exec: {e}"))?;
346
347 let StartExecResults::Attached { output, input } = started else {
348 return Err("attach exec detached unexpectedly".to_string());
349 };
350
351 // Size the pty before the first byte, so the TUI lays out correctly rather
352 // than redrawing from an 80x24 assumption.
353 let terminal = Terminal {
354 exec_id: exec.id,
355 output,
356 input,
357 };
358 resize(docker, &terminal.exec_id, cols, rows).await;
359 Ok(terminal)
360}
361
362/// Resize an attached terminal. Best-effort: a resize racing the exec's exit is
363/// routine and not worth failing an attach over.
364pub async fn resize(docker: &Docker, exec_id: &str, cols: u16, rows: u16) {
365 if cols == 0 || rows == 0 {
366 return;
367 }
368 if let Err(e) = docker
369 .resize_exec(
370 exec_id,
371 ResizeExecOptions {
372 height: rows,
373 width: cols,
374 },
375 )
376 .await
377 {
378 tracing::debug!("resize exec {exec_id}: {e}");
379 }
380}
381
382/// Run a one-shot command in the container and collect its stdout.
383///
384/// This is the `capture-pane` / `send-keys` path: short, non-interactive tmux
385/// control commands rather than a terminal.
386pub async fn exec_capture(docker: &Docker, id: &str, cmd: &[&str]) -> Result<Vec<u8>, String> {
387 use futures_util::StreamExt;
388
389 let exec = docker
390 .create_exec(
391 id,
392 CreateExecOptions {
393 attach_stdout: Some(true),
394 attach_stderr: Some(true),
395 tty: Some(false),
396 user: Some(RUN_AS.to_string()),
397 env: Some(vec![format!("HOME={HOME}")]),
398 cmd: Some(cmd.iter().map(|s| s.to_string()).collect()),
399 ..Default::default()
400 },
401 )
402 .await
403 .map_err(|e| format!("create exec {cmd:?}: {e}"))?;
404
405 let started = docker
406 .start_exec(&exec.id, None)
407 .await
408 .map_err(|e| format!("start exec {cmd:?}: {e}"))?;
409
410 let StartExecResults::Attached { mut output, .. } = started else {
411 return Ok(Vec::new());
412 };
413
414 let mut buf = Vec::new();
415 while let Some(chunk) = output.next().await {
416 match chunk {
417 Ok(log) => buf.extend_from_slice(log.into_bytes().as_ref()),
418 Err(e) => return Err(format!("exec {cmd:?}: {e}")),
419 }
420 }
421 Ok(buf)
422}
423
424/// The tmux pane's current screen plus scrollback, with escape sequences kept.
425///
426/// This is what a reconnecting browser is sent before the live stream, so it
427/// resumes with the screen it left rather than a blank one.
428pub async fn capture_pane(docker: &Docker, id: &str) -> Result<Vec<u8>, String> {
429 exec_capture(
430 docker,
431 id,
432 &[
433 "tmux",
434 "capture-pane",
435 "-p",
436 "-e",
437 "-S",
438 "-",
439 "-t",
440 TMUX_SESSION,
441 ],
442 )
443 .await
444}
445
446/// Type a line into the tmux session, as if the user had.
447///
448/// This is how an autonomous session is driven: rather than a headless
449/// `-p` invocation, the same interactive agent gets its prompt typed at it, so
450/// a human can take over mid-run just by attaching.
451pub async fn send_keys(docker: &Docker, id: &str, text: &str) -> Result<(), String> {
452 exec_capture(
453 docker,
454 id,
455 &["tmux", "send-keys", "-t", TMUX_SESSION, text, "Enter"],
456 )
457 .await
458 .map(|_| ())
459}
460
461/// Force-remove the container. Best-effort; a container already gone is a
462/// success as far as callers are concerned.
463pub async fn remove(docker: &Docker, id: &str) {
464 if let Err(e) = docker
465 .remove_container(
466 id,
467 Some(RemoveContainerOptions {
468 force: true,
469 ..Default::default()
470 }),
471 )
472 .await
473 {
474 tracing::debug!("removing session container {id}: {e}");
475 }
476}
477
478/// Session containers Docker still knows about, as `(container id, session id)`.
479///
480/// Startup uses this to reconcile rows against reality: the registry is
481/// in-memory, so after a restart every one of these is an orphan.
482pub async fn list_sessions(docker: &Docker) -> Result<Vec<(String, i64)>, String> {
483 use bollard::query_parameters::ListContainersOptions;
484
485 let containers = docker
486 .list_containers(Some(ListContainersOptions {
487 all: true,
488 filters: Some(
489 [("label".to_string(), vec![LABEL_SESSION.to_string()])]
490 .into_iter()
491 .collect(),
492 ),
493 ..Default::default()
494 }))
495 .await
496 .map_err(|e| format!("listing session containers: {e}"))?;
497
498 Ok(containers
499 .into_iter()
500 .filter_map(|c| {
501 let id = c.id?;
502 let session_id = c.labels?.get(LABEL_SESSION)?.parse().ok()?;
503 Some((id, session_id))
504 })
505 .collect())
506}
507
508#[cfg(test)]
509mod tests {
510 use super::*;
511
512 fn entries(tar: &[u8]) -> Vec<(String, tar::EntryType, u64, u32)> {
513 let mut archive = tar::Archive::new(tar);
514 archive
515 .entries()
516 .unwrap()
517 .map(|e| {
518 let e = e.unwrap();
519 let header = e.header();
520 (
521 e.path().unwrap().to_string_lossy().into_owned(),
522 header.entry_type(),
523 header.uid().unwrap(),
524 header.mode().unwrap(),
525 )
526 })
527 .collect()
528 }
529
530 /// The bug this guards: a tar of files alone leaves Docker to create the
531 /// parent directories, owned by root. The session runs unprivileged, so
532 /// `src/` came out root-owned and the agent could edit `src/main.rs` but
533 /// never add `src/lib.rs` — most of what a coding agent does.
534 #[test]
535 fn intermediate_directories_are_present_and_agent_owned() {
536 let files = vec![
537 ("README.md".to_string(), b"hi".to_vec(), false),
538 ("src/main.rs".to_string(), b"fn main() {}".to_vec(), false),
539 ("a/b/c/deep.txt".to_string(), b"deep".to_vec(), false),
540 ];
541 let entries = entries(&build_tar("workspace", &files));
542
543 let dirs: Vec<_> = entries
544 .iter()
545 .filter(|(_, t, _, _)| *t == tar::EntryType::Directory)
546 .map(|(p, _, _, _)| p.trim_end_matches('/').to_string())
547 .collect();
548 for expected in [
549 "workspace",
550 "workspace/src",
551 "workspace/a",
552 "workspace/a/b",
553 "workspace/a/b/c",
554 ] {
555 assert!(
556 dirs.contains(&expected.to_string()),
557 "missing dir {expected}: {dirs:?}"
558 );
559 }
560
561 // Everything, files and directories alike, must belong to the session
562 // user or the workspace is read-only in practice.
563 for (path, _, uid, _) in &entries {
564 assert_eq!(*uid, RUN_AS_UID, "{path} is not owned by the session user");
565 }
566 }
567
568 /// Parents must precede their children, or the ownership set on a
569 /// directory entry is applied to one Docker already made as root.
570 #[test]
571 fn parents_are_written_before_their_children() {
572 let files = vec![("a/b/c/deep.txt".to_string(), b"x".to_vec(), false)];
573 let entries = entries(&build_tar("workspace", &files));
574 let order: Vec<_> = entries.iter().map(|(p, _, _, _)| p.as_str()).collect();
575
576 let index = |needle: &str| order.iter().position(|p| p.trim_end_matches('/') == needle);
577 let root = index("workspace").expect("root dir");
578 let a = index("workspace/a").expect("a");
579 let b = index("workspace/a/b").expect("b");
580 let c = index("workspace/a/b/c").expect("c");
581 let file = index("workspace/a/b/c/deep.txt").expect("file");
582 assert!(
583 root < a && a < b && b < c && c < file,
584 "wrong order: {order:?}"
585 );
586 }
587
588 #[test]
589 fn executables_keep_their_bit() {
590 let files = vec![
591 ("run.sh".to_string(), b"#!/bin/sh\n".to_vec(), true),
592 ("plain.txt".to_string(), b"x".to_vec(), false),
593 ];
594 let entries = entries(&build_tar("workspace", &files));
595 let mode = |name: &str| {
596 entries
597 .iter()
598 .find(|(p, _, _, _)| p == name)
599 .map(|(_, _, _, m)| *m)
600 .unwrap()
601 };
602 assert_eq!(mode("workspace/run.sh") & 0o111, 0o111);
603 assert_eq!(mode("workspace/plain.txt") & 0o111, 0);
604 }
605}