anvilsign in

collin/anvil

1//! Docker specifics for an agent session: creating the container, seeding it,
2//! attaching a terminal to the tmux session inside, and tearing it down.
3//!
4//! The containment story is deliberately the CI one (see
5//! `docs/untrusted-mode.md` §1): all capabilities dropped, `no-new-privileges`,
6//! pids/memory/cpu caps, **no Docker socket, no bind mounts, no volumes**.
7//! Everything the container needs — the checkout and the agent's credentials —
8//! is uploaded as a tar through the Docker API, so nothing on the anvil host is
9//! ever exposed to it.
10
11use anvil_core::config::AgentConfig;
12use bollard::{
13 Docker,
14 container::{
15 Config,
16 CreateContainerOptions,
17 DownloadFromContainerOptions,
18 RemoveContainerOptions,
19 StartContainerOptions,
20 UploadToContainerOptions,
21 },
22 exec::{
23 CreateExecOptions,
24 ResizeExecOptions,
25 StartExecOptions,
26 StartExecResults,
27 },
28 models::HostConfig,
29};
30use futures_util::StreamExt;
31
32/// Label carrying the session id, so containers can be found again after a
33/// restart — the registry is in-memory and does not survive one.
34pub const LABEL_SESSION: &str = "anvil.session";
35
36/// Working directory inside the container. Under [`RUN_AS`]'s home rather
37/// than a bare `/workspace` (which is what the CI runner still uses — CI runs
38/// as root, so it has no home directory to prefer) so tools that assume a
39/// project lives under `$HOME` behave.
40pub const WORKDIR: &str = "/home/ubuntu/workspace";
41
42/// The unprivileged user sessions run as rather than root; CI keeps the
43/// image's default (root) because plenty of pipelines expect to `apt-get`.
44/// This is `ubuntu`, not a purpose-made account: `deploy/runner/Dockerfile`'s
45/// base image already ships a uid-1000 user by that name, so reusing it saves
46/// a `useradd`. If the base image ever moves off Ubuntu, this needs an actual
47/// account created again.
48pub const RUN_AS: &str = "ubuntu";
49
50/// That user's uid/gid, needed when building tars so the uploaded files are
51/// owned by the account that has to write them.
52const RUN_AS_UID: u64 = 1000;
53
54/// Home directory of [`RUN_AS`]; the agent CLI's config lives under it.
55pub const HOME: &str = "/home/ubuntu";
56
57/// tmux session name, matching `session-entrypoint.sh`.
58pub const TMUX_SESSION: &str = "agent";
59
60/// Create (but do not start) a session container.
61pub async fn create(
62 docker: &Docker,
63 cfg: &AgentConfig,
64 session_id: i64,
65 env: &[(String, String)],
66 command: &[String],
67) -> Result<String, String> {
68 // The same sandbox CI uses. Limits of 0 mean "unlimited" and omit the cap.
69 //
70 // Note there is no `network_mode` opt-out: unlike a CI job, a session is
71 // useless without network — it has to reach the model API, and from M2 it
72 // clones and pushes back to anvil.
73 let host_config = HostConfig {
74 cap_drop: Some(vec!["ALL".to_string()]),
75 security_opt: Some(vec!["no-new-privileges:true".to_string()]),
76 pids_limit: (cfg.pids_limit > 0).then_some(cfg.pids_limit),
77 memory: (cfg.memory_mb > 0).then(|| cfg.memory_mb * 1024 * 1024),
78 memory_swap: (cfg.memory_mb > 0).then(|| cfg.memory_mb * 1024 * 1024),
79 nano_cpus: (cfg.cpus > 0.0).then_some((cfg.cpus * 1e9) as i64),
80 ..Default::default()
81 };
82
83 let mut cmd = vec!["anvil-session".to_string()];
84 cmd.extend(command.iter().cloned());
85
86 let config = Config {
87 image: Some(cfg.image.clone()),
88 cmd: Some(cmd),
89 env: Some(
90 env.iter()
91 .map(|(k, v)| format!("{k}={v}"))
92 .chain([
93 format!("HOME={HOME}"),
94 format!("ANVIL_WORKDIR={WORKDIR}"),
95 "TERM=xterm-256color".to_string(),
96 ])
97 .collect(),
98 ),
99 working_dir: Some(WORKDIR.to_string()),
100 user: Some(RUN_AS.to_string()),
101 labels: Some(
102 [(LABEL_SESSION.to_string(), session_id.to_string())]
103 .into_iter()
104 .collect(),
105 ),
106 // A terminal program needs a tty even before anyone attaches, or tmux
107 // starts with a 80x24 dumb terminal and never recovers.
108 tty: Some(true),
109 open_stdin: Some(true),
110 host_config: Some(host_config),
111 ..Default::default()
112 };
113
114 let created = docker
115 .create_container(None::<CreateContainerOptions<String>>, config)
116 .await
117 .map_err(|e| format!("create session container: {e}"))?;
118 Ok(created.id)
119}
120
121/// Upload a tar into the container, rooted at `/`.
122pub async fn upload(docker: &Docker, id: &str, tar: Vec<u8>) -> Result<(), String> {
123 docker
124 .upload_to_container(
125 id,
126 Some(UploadToContainerOptions {
127 path: "/".to_string(),
128 ..Default::default()
129 }),
130 tar.into(),
131 )
132 .await
133 .map_err(|e| format!("upload to session container: {e}"))
134}
135
136/// Read one file's current bytes out of the container (which must already
137/// exist, though it need not be started — the filesystem is there either
138/// way). `None` covers every reason it might not be readable yet: the file
139/// does not exist, an intermediate directory does not exist, anything —
140/// a "json" secret starts from `{}` in all of those cases alike, the same as
141/// a brand new file would.
142pub async fn download_file(docker: &Docker, id: &str, absolute_path: &str) -> Option<Vec<u8>> {
143 let mut stream = docker.download_from_container(
144 id,
145 Some(DownloadFromContainerOptions {
146 path: absolute_path.to_string(),
147 }),
148 );
149 let mut tar = Vec::new();
150 while let Some(chunk) = stream.next().await {
151 tar.extend_from_slice(&chunk.ok()?);
152 }
153 let mut archive = tar::Archive::new(tar.as_slice());
154 let mut entries = archive.entries().ok()?;
155 let mut entry = entries.next()?.ok()?;
156 let mut contents = Vec::new();
157 std::io::Read::read_to_end(&mut entry, &mut contents).ok()?;
158 Some(contents)
159}
160
161/// Build a tar of `(path, contents, executable)` entries, rooted at `prefix`
162/// (no leading slash) and owned by the session user.
163///
164/// Emits an explicit entry for every intermediate **directory**, which matters
165/// more than it looks: a tar of files alone makes Docker create the parent
166/// directories itself, owned by root. The session runs as [`RUN_AS`], so
167/// `src/` would come out root-owned and the agent could edit existing files
168/// but never add one — which is most of what a coding agent does.
169pub fn build_tar(prefix: &str, files: &[(String, Vec<u8>, bool)]) -> Vec<u8> {
170 let mut builder = tar::Builder::new(Vec::new());
171 let mtime = anvil_core::agent::now_secs().max(0) as u64;
172
173 let header_for = |size: u64, mode: u32, entry_type: tar::EntryType| {
174 let mut header = tar::Header::new_gnu();
175 header.set_size(size);
176 header.set_mode(mode);
177 header.set_entry_type(entry_type);
178 // Ownership matters: the container runs as [`RUN_AS`], and a checkout
179 // it cannot write is not a workspace.
180 header.set_uid(RUN_AS_UID);
181 header.set_gid(RUN_AS_UID);
182 // Without this every file lands in 1970, which upsets anything that
183 // compares timestamps (make, and incremental builds generally).
184 header.set_mtime(mtime);
185 header
186 };
187
188 // Every ancestor directory, deduplicated and shortest-first so parents are
189 // created before their children.
190 let mut dirs: Vec<String> = Vec::new();
191 for (path, _, _) in files {
192 let mut parts: Vec<&str> = path.split('/').collect();
193 parts.pop(); // the file itself
194 let mut acc = String::new();
195 for part in parts {
196 if !acc.is_empty() {
197 acc.push('/');
198 }
199 acc.push_str(part);
200 if !dirs.contains(&acc) {
201 dirs.push(acc.clone());
202 }
203 }
204 }
205 dirs.sort_by_key(|d| d.matches('/').count());
206
207 // The root itself, so `prefix` is agent-owned even when empty.
208 let mut header = header_for(0, 0o755, tar::EntryType::Directory);
209 let _ = builder.append_data(&mut header, format!("{prefix}/"), std::io::empty());
210 for dir in &dirs {
211 let mut header = header_for(0, 0o755, tar::EntryType::Directory);
212 let _ = builder.append_data(&mut header, format!("{prefix}/{dir}/"), std::io::empty());
213 }
214
215 for (path, content, executable) in files {
216 let mode = if *executable { 0o755 } else { 0o644 };
217 let mut header = header_for(content.len() as u64, mode, tar::EntryType::Regular);
218 let full = format!("{prefix}/{path}");
219 if builder
220 .append_data(&mut header, &full, content.as_slice())
221 .is_err()
222 {
223 continue;
224 }
225 }
226 builder.into_inner().unwrap_or_default()
227}
228
229/// Read a host directory into `(relative path, bytes, executable)` entries.
230///
231/// Used for the agent CLI's credentials directory, which is uploaded rather
232/// than bind-mounted so the no-mounts invariant survives.
233pub fn read_dir_recursive(root: &std::path::Path) -> Result<Vec<(String, Vec<u8>, bool)>, String> {
234 fn walk(
235 base: &std::path::Path,
236 dir: &std::path::Path,
237 out: &mut Vec<(String, Vec<u8>, bool)>,
238 ) -> std::io::Result<()> {
239 for entry in std::fs::read_dir(dir)? {
240 let entry = entry?;
241 let path = entry.path();
242 let meta = entry.metadata()?;
243 if meta.is_dir() {
244 walk(base, &path, out)?;
245 } else if meta.is_file() {
246 let Ok(rel) = path.strip_prefix(base) else {
247 continue;
248 };
249 let executable = {
250 #[cfg(unix)]
251 {
252 use std::os::unix::fs::PermissionsExt;
253 meta.permissions().mode() & 0o111 != 0
254 }
255 #[cfg(not(unix))]
256 {
257 false
258 }
259 };
260 out.push((
261 rel.to_string_lossy().replace('\\', "/"),
262 std::fs::read(&path)?,
263 executable,
264 ));
265 }
266 }
267 Ok(())
268 }
269
270 let mut out = Vec::new();
271 walk(root, root, &mut out).map_err(|e| format!("reading {}: {e}", root.display()))?;
272 Ok(out)
273}
274
275/// Start a created container.
276pub async fn start(docker: &Docker, id: &str) -> Result<(), String> {
277 docker
278 .start_container(id, None::<StartContainerOptions<String>>)
279 .await
280 .map_err(|e| format!("start session container: {e}"))
281}
282
283/// A live terminal attached to the container's tmux session.
284pub struct Terminal {
285 /// Docker exec id, needed to resize the pty.
286 pub exec_id: String,
287 /// Bytes the terminal produces.
288 pub output: std::pin::Pin<
289 Box<
290 dyn futures_util::Stream<
291 Item = Result<bollard::container::LogOutput, bollard::errors::Error>,
292 > + Send,
293 >,
294 >,
295 /// Keystrokes go here.
296 pub input: std::pin::Pin<Box<dyn tokio::io::AsyncWrite + Send>>,
297}
298
299/// Attach a new tmux client to the container's session.
300///
301/// Each caller gets its own `docker exec`, which is the point: a dropped
302/// websocket kills that client only, never the agent, because the agent is a
303/// process inside tmux rather than a child of the exec.
304pub async fn attach(docker: &Docker, id: &str, cols: u16, rows: u16) -> Result<Terminal, String> {
305 let exec = docker
306 .create_exec(
307 id,
308 CreateExecOptions {
309 attach_stdin: Some(true),
310 attach_stdout: Some(true),
311 attach_stderr: Some(true),
312 tty: Some(true),
313 user: Some(RUN_AS.to_string()),
314 env: Some(vec![
315 "TERM=xterm-256color".to_string(),
316 format!("HOME={HOME}"),
317 ]),
318 cmd: Some(vec![
319 "tmux".to_string(),
320 "-f".to_string(),
321 "/etc/anvil/tmux.conf".to_string(),
322 "attach-session".to_string(),
323 "-t".to_string(),
324 TMUX_SESSION.to_string(),
325 ]),
326 ..Default::default()
327 },
328 )
329 .await
330 .map_err(|e| format!("create attach exec: {e}"))?;
331
332 let started = docker
333 .start_exec(
334 &exec.id,
335 Some(StartExecOptions {
336 detach: false,
337 tty: true,
338 ..Default::default()
339 }),
340 )
341 .await
342 .map_err(|e| format!("start attach exec: {e}"))?;
343
344 let StartExecResults::Attached { output, input } = started else {
345 return Err("attach exec detached unexpectedly".to_string());
346 };
347
348 // Size the pty before the first byte, so the TUI lays out correctly rather
349 // than redrawing from an 80x24 assumption.
350 let terminal = Terminal {
351 exec_id: exec.id,
352 output,
353 input,
354 };
355 resize(docker, &terminal.exec_id, cols, rows).await;
356 Ok(terminal)
357}
358
359/// Resize an attached terminal. Best-effort: a resize racing the exec's exit is
360/// routine and not worth failing an attach over.
361pub async fn resize(docker: &Docker, exec_id: &str, cols: u16, rows: u16) {
362 if cols == 0 || rows == 0 {
363 return;
364 }
365 if let Err(e) = docker
366 .resize_exec(
367 exec_id,
368 ResizeExecOptions {
369 height: rows,
370 width: cols,
371 },
372 )
373 .await
374 {
375 tracing::debug!("resize exec {exec_id}: {e}");
376 }
377}
378
379/// Run a one-shot command in the container and collect its stdout.
380///
381/// This is the `capture-pane` / `send-keys` path: short, non-interactive tmux
382/// control commands rather than a terminal.
383pub async fn exec_capture(docker: &Docker, id: &str, cmd: &[&str]) -> Result<Vec<u8>, String> {
384 use futures_util::StreamExt;
385
386 let exec = docker
387 .create_exec(
388 id,
389 CreateExecOptions {
390 attach_stdout: Some(true),
391 attach_stderr: Some(true),
392 tty: Some(false),
393 user: Some(RUN_AS.to_string()),
394 env: Some(vec![format!("HOME={HOME}")]),
395 cmd: Some(cmd.iter().map(|s| s.to_string()).collect()),
396 ..Default::default()
397 },
398 )
399 .await
400 .map_err(|e| format!("create exec {cmd:?}: {e}"))?;
401
402 let started = docker
403 .start_exec(&exec.id, None)
404 .await
405 .map_err(|e| format!("start exec {cmd:?}: {e}"))?;
406
407 let StartExecResults::Attached { mut output, .. } = started else {
408 return Ok(Vec::new());
409 };
410
411 let mut buf = Vec::new();
412 while let Some(chunk) = output.next().await {
413 match chunk {
414 Ok(log) => buf.extend_from_slice(log.into_bytes().as_ref()),
415 Err(e) => return Err(format!("exec {cmd:?}: {e}")),
416 }
417 }
418 Ok(buf)
419}
420
421/// The tmux pane's current screen plus scrollback, with escape sequences kept.
422///
423/// This is what a reconnecting browser is sent before the live stream, so it
424/// resumes with the screen it left rather than a blank one.
425pub async fn capture_pane(docker: &Docker, id: &str) -> Result<Vec<u8>, String> {
426 exec_capture(
427 docker,
428 id,
429 &[
430 "tmux",
431 "capture-pane",
432 "-p",
433 "-e",
434 "-S",
435 "-",
436 "-t",
437 TMUX_SESSION,
438 ],
439 )
440 .await
441}
442
443/// Type a line into the tmux session, as if the user had.
444///
445/// This is how an autonomous session is driven: rather than a headless
446/// `-p` invocation, the same interactive agent gets its prompt typed at it, so
447/// a human can take over mid-run just by attaching.
448pub async fn send_keys(docker: &Docker, id: &str, text: &str) -> Result<(), String> {
449 exec_capture(
450 docker,
451 id,
452 &["tmux", "send-keys", "-t", TMUX_SESSION, text, "Enter"],
453 )
454 .await
455 .map(|_| ())
456}
457
458/// Force-remove the container. Best-effort; a container already gone is a
459/// success as far as callers are concerned.
460pub async fn remove(docker: &Docker, id: &str) {
461 if let Err(e) = docker
462 .remove_container(
463 id,
464 Some(RemoveContainerOptions {
465 force: true,
466 ..Default::default()
467 }),
468 )
469 .await
470 {
471 tracing::debug!("removing session container {id}: {e}");
472 }
473}
474
475/// Session containers Docker still knows about, as `(container id, session id)`.
476///
477/// Startup uses this to reconcile rows against reality: the registry is
478/// in-memory, so after a restart every one of these is an orphan.
479pub async fn list_sessions(docker: &Docker) -> Result<Vec<(String, i64)>, String> {
480 use bollard::container::ListContainersOptions;
481
482 let containers = docker
483 .list_containers(Some(ListContainersOptions::<String> {
484 all: true,
485 filters: [("label".to_string(), vec![LABEL_SESSION.to_string()])]
486 .into_iter()
487 .collect(),
488 ..Default::default()
489 }))
490 .await
491 .map_err(|e| format!("listing session containers: {e}"))?;
492
493 Ok(containers
494 .into_iter()
495 .filter_map(|c| {
496 let id = c.id?;
497 let session_id = c.labels?.get(LABEL_SESSION)?.parse().ok()?;
498 Some((id, session_id))
499 })
500 .collect())
501}
502
503#[cfg(test)]
504mod tests {
505 use super::*;
506
507 fn entries(tar: &[u8]) -> Vec<(String, tar::EntryType, u64, u32)> {
508 let mut archive = tar::Archive::new(tar);
509 archive
510 .entries()
511 .unwrap()
512 .map(|e| {
513 let e = e.unwrap();
514 let header = e.header();
515 (
516 e.path().unwrap().to_string_lossy().into_owned(),
517 header.entry_type(),
518 header.uid().unwrap(),
519 header.mode().unwrap(),
520 )
521 })
522 .collect()
523 }
524
525 /// The bug this guards: a tar of files alone leaves Docker to create the
526 /// parent directories, owned by root. The session runs unprivileged, so
527 /// `src/` came out root-owned and the agent could edit `src/main.rs` but
528 /// never add `src/lib.rs` — most of what a coding agent does.
529 #[test]
530 fn intermediate_directories_are_present_and_agent_owned() {
531 let files = vec![
532 ("README.md".to_string(), b"hi".to_vec(), false),
533 ("src/main.rs".to_string(), b"fn main() {}".to_vec(), false),
534 ("a/b/c/deep.txt".to_string(), b"deep".to_vec(), false),
535 ];
536 let entries = entries(&build_tar("workspace", &files));
537
538 let dirs: Vec<_> = entries
539 .iter()
540 .filter(|(_, t, _, _)| *t == tar::EntryType::Directory)
541 .map(|(p, _, _, _)| p.trim_end_matches('/').to_string())
542 .collect();
543 for expected in [
544 "workspace",
545 "workspace/src",
546 "workspace/a",
547 "workspace/a/b",
548 "workspace/a/b/c",
549 ] {
550 assert!(
551 dirs.contains(&expected.to_string()),
552 "missing dir {expected}: {dirs:?}"
553 );
554 }
555
556 // Everything, files and directories alike, must belong to the session
557 // user or the workspace is read-only in practice.
558 for (path, _, uid, _) in &entries {
559 assert_eq!(*uid, RUN_AS_UID, "{path} is not owned by the session user");
560 }
561 }
562
563 /// Parents must precede their children, or the ownership set on a
564 /// directory entry is applied to one Docker already made as root.
565 #[test]
566 fn parents_are_written_before_their_children() {
567 let files = vec![("a/b/c/deep.txt".to_string(), b"x".to_vec(), false)];
568 let entries = entries(&build_tar("workspace", &files));
569 let order: Vec<_> = entries.iter().map(|(p, _, _, _)| p.as_str()).collect();
570
571 let index = |needle: &str| order.iter().position(|p| p.trim_end_matches('/') == needle);
572 let root = index("workspace").expect("root dir");
573 let a = index("workspace/a").expect("a");
574 let b = index("workspace/a/b").expect("b");
575 let c = index("workspace/a/b/c").expect("c");
576 let file = index("workspace/a/b/c/deep.txt").expect("file");
577 assert!(
578 root < a && a < b && b < c && c < file,
579 "wrong order: {order:?}"
580 );
581 }
582
583 #[test]
584 fn executables_keep_their_bit() {
585 let files = vec![
586 ("run.sh".to_string(), b"#!/bin/sh\n".to_vec(), true),
587 ("plain.txt".to_string(), b"x".to_vec(), false),
588 ];
589 let entries = entries(&build_tar("workspace", &files));
590 let mode = |name: &str| {
591 entries
592 .iter()
593 .find(|(p, _, _, _)| p == name)
594 .map(|(_, _, _, m)| *m)
595 .unwrap()
596 };
597 assert_eq!(mode("workspace/run.sh") & 0o111, 0o111);
598 assert_eq!(mode("workspace/plain.txt") & 0o111, 0);
599 }
600}