eb_pipeline/
run.rs

1//! Execution of a plan as subprocesses.
2
3use crate::exit::{self, Fail};
4use crate::plan::{
5    build_plan, read_units, shell_line, valid_job_id, Cmd, Deselected, Planner, StemSeq, Unit,
6    PHASES,
7};
8use crate::resolve::Role;
9use crate::tools::{help_lists, map_rotate, map_staged, RotateOutcome, StagedOutcome, Tool};
10use crate::util::ListLine;
11use serde::Serialize;
12use std::collections::BTreeMap;
13use std::fs::File;
14use std::path::{Path, PathBuf};
15use std::process::{Command, Stdio};
16use std::sync::atomic::{AtomicI32, Ordering};
17use std::time::Duration;
18
19static SIGNAL: AtomicI32 = AtomicI32::new(0);
20
21extern "C" fn on_signal(sig: libc::c_int) {
22    SIGNAL.store(sig, Ordering::SeqCst);
23}
24
25/// Install SIGTERM and SIGINT handlers that record the signal.
26pub fn install_handlers() {
27    for s in [libc::SIGTERM, libc::SIGINT] {
28        // SAFETY: the handler only stores into an atomic.
29        unsafe {
30            let mut sa: libc::sigaction = std::mem::zeroed();
31            sa.sa_sigaction = on_signal as extern "C" fn(libc::c_int) as usize;
32            libc::sigemptyset(&mut sa.sa_mask);
33            sa.sa_flags = 0;
34            libc::sigaction(s, &sa, std::ptr::null_mut());
35        }
36    }
37}
38
39fn interrupted() -> Option<i32> {
40    match SIGNAL.load(Ordering::SeqCst) {
41        0 => None,
42        s => Some(s),
43    }
44}
45
46/// The wait before the next admission attempt. It starts at `wait_seconds`
47/// and is never longer than `max_wait_seconds`, including when that maximum
48/// is the smaller of the two.
49fn capped_wait(wait_seconds: u64, max_wait_seconds: u64) -> u64 {
50    wait_seconds.min(max_wait_seconds)
51}
52
53/// The wait after a refused attempt: double the previous one, still capped
54/// at `max_wait_seconds`.
55fn doubled_wait(wait: u64, max_wait_seconds: u64) -> u64 {
56    wait.saturating_mul(2).min(max_wait_seconds)
57}
58
59/// Sleep in short slices; returns true when a signal arrived.
60fn sleep_interruptible(secs: u64) -> bool {
61    let mut left = Duration::from_secs(secs);
62    let slice = Duration::from_millis(50);
63    while !left.is_zero() {
64        if interrupted().is_some() {
65            return true;
66        }
67        let d = left.min(slice);
68        std::thread::sleep(d);
69        left -= d;
70    }
71    interrupted().is_some()
72}
73
74/// Print a log line on stderr: `<level> [<phase>] <target>: <msg>`.
75pub fn log(level: &str, phase: &str, target: &str, msg: &str) {
76    eprintln!("{level} [{phase}] {target}: {msg}");
77}
78
79/// What one phase came to.
80#[derive(Debug, Serialize, Clone)]
81pub struct PhaseResult {
82    /// The phase, one of [`PHASES`].
83    pub name: String,
84    /// `ok`, `failed`, `refused`, `skipped`, `unsharded`, `interrupted` or
85    /// `not configured`.
86    pub status: String,
87    /// The exit code behind the status, when there is one.
88    #[serde(skip_serializing_if = "Option::is_none")]
89    pub exit_code: Option<i32>,
90    /// Why the phase came to its status, when that needs saying.
91    #[serde(skip_serializing_if = "Option::is_none")]
92    pub note: Option<String>,
93}
94
95/// What one build list line came to.
96#[derive(Debug, Serialize, Clone)]
97pub struct LineResult {
98    /// The line's easyconfig.
99    pub easyconfig: String,
100    /// The unit it belongs to: `all`, `cpu-prerequisites`, `prelude`, or a
101    /// shard's name.
102    pub unit: String,
103    /// `build`, or `sources` when its source fetch failed.
104    pub phase: String,
105    /// `success`, `failed`, `refused`, `pending`, `skipped`.
106    pub status: String,
107    /// The exit code the line counts as.
108    #[serde(skip_serializing_if = "Option::is_none")]
109    pub exit_code: Option<i32>,
110    /// The command's own exit code, when `exit_code` translates it, as 78
111    /// does eb-staged-rebuild's 2.
112    #[serde(skip_serializing_if = "Option::is_none")]
113    pub translated_from: Option<i32>,
114    /// The log file of the line's command.
115    pub log: String,
116    /// Why the line came to its status, when that needs saying.
117    #[serde(skip_serializing_if = "Option::is_none")]
118    pub note: Option<String>,
119}
120
121/// What one unit's admission came to.
122#[derive(Debug, Serialize, Clone)]
123pub struct AdmissionResult {
124    /// The unit, as in [`LineResult::unit`].
125    pub unit: String,
126    /// How many times `stack admit` ran.
127    pub attempts: u32,
128    /// `admitted`, `refused` or `failed`.
129    pub outcome: String,
130    /// The last JSON line `stack admit` printed.
131    #[serde(skip_serializing_if = "Option::is_none")]
132    pub last_json: Option<String>,
133}
134
135/// The result file `run` writes: what every phase and line came to.
136#[derive(Debug, Serialize, Clone)]
137pub struct RunResult {
138    /// Version of this format, 1.
139    pub schema_version: u32,
140    /// `result`.
141    pub kind: String,
142    /// `site.name`.
143    pub site: String,
144    /// The target's name.
145    pub target: String,
146    /// The build type, `default` without one.
147    pub build_type: String,
148    /// The run id.
149    pub run_id: String,
150    /// The build list, as given.
151    pub list: String,
152    /// Where the run writes its logs.
153    pub log_dir: String,
154    /// When the run started, in RFC 3339, in UTC.
155    pub started_at: String,
156    /// When the run ended, in RFC 3339, in UTC.
157    pub ended_at: String,
158    /// The process exit code.
159    pub exit_code: i32,
160    /// Whether the lines were built in the units of a shard plan.
161    pub sharded: bool,
162    /// How many lines came to each status.
163    pub counts: BTreeMap<String, usize>,
164    /// The phases the site file leaves out.
165    pub not_configured: Vec<String>,
166    /// The admission of each unit that went through it, in order.
167    pub admission: Vec<AdmissionResult>,
168    /// Every phase that was reached, in [`PHASES`] order.
169    pub phases: Vec<PhaseResult>,
170    /// Every line, in the order the run reached them.
171    pub lines: Vec<LineResult>,
172    /// Lines the target's selector left out; not counted as results.
173    #[serde(skip_serializing_if = "Vec::is_empty")]
174    pub deselected: Vec<Deselected>,
175    /// The error that stopped the run, when one did.
176    #[serde(skip_serializing_if = "Option::is_none")]
177    pub message: Option<String>,
178    /// `SIGINT` or `SIGTERM`, when a signal stopped the run.
179    #[serde(skip_serializing_if = "Option::is_none")]
180    pub interrupted_by: Option<String>,
181}
182
183struct Exec {
184    code: i32,
185    log: PathBuf,
186}
187
188/// Run one command to completion with output in a log file. A signal that
189/// arrives is forwarded to the child once.
190fn exec(cmd: &Cmd, base_env: &BTreeMap<String, String>, log: &Path) -> Result<Exec, Fail> {
191    if let Some(dir) = log.parent() {
192        std::fs::create_dir_all(dir).map_err(|e| {
193            Fail::new(
194                exit::FAILED,
195                format!("cannot create {}: {e}", dir.display()),
196            )
197        })?;
198    }
199    let out = File::create(log).map_err(|e| {
200        Fail::new(
201            exit::FAILED,
202            format!("cannot create {}: {e}", log.display()),
203        )
204    })?;
205    let err = out
206        .try_clone()
207        .map_err(|e| Fail::new(exit::FAILED, format!("cannot duplicate log handle: {e}")))?;
208    let mut c = Command::new(&cmd.argv[0]);
209    c.args(&cmd.argv[1..])
210        .envs(base_env)
211        .envs(&cmd.env)
212        .stdin(Stdio::null())
213        .stdout(Stdio::from(out))
214        .stderr(Stdio::from(err));
215    let mut child = match c.spawn() {
216        Ok(ch) => ch,
217        Err(e) => {
218            let _ = std::fs::write(log, format!("cannot start {}: {e}\n", cmd.argv[0]));
219            return Ok(Exec {
220                code: 127,
221                log: log.to_path_buf(),
222            });
223        }
224    };
225    let mut forwarded = 0;
226    let status = loop {
227        match child.try_wait() {
228            Ok(Some(s)) => break s,
229            Ok(None) => {}
230            Err(e) => {
231                return Err(Fail::new(exit::FAILED, format!("wait failed: {e}")));
232            }
233        }
234        if let Some(sig) = interrupted() {
235            if forwarded != sig {
236                // SAFETY: the pid belongs to a child this process started.
237                unsafe {
238                    libc::kill(child.id() as libc::pid_t, sig);
239                }
240                forwarded = sig;
241            }
242        }
243        std::thread::sleep(Duration::from_millis(20));
244    };
245    use std::os::unix::process::ExitStatusExt;
246    let code = status
247        .code()
248        .unwrap_or_else(|| 128 + status.signal().unwrap_or(0));
249    Ok(Exec {
250        code,
251        log: log.to_path_buf(),
252    })
253}
254
255fn last_json_line(path: &Path) -> Option<String> {
256    let text = std::fs::read_to_string(path).ok()?;
257    text.lines()
258        .rev()
259        .map(str::trim)
260        .find(|l| l.starts_with('{') && l.ends_with('}'))
261        .map(str::to_string)
262}
263
264/// The lock eb-staged-rebuild names when it gives up waiting for another
265/// writer.
266fn held_lock(path: &Path) -> Option<String> {
267    const HELD: &str = "another eb-staged-rebuild holds the lock at ";
268    let bytes = std::fs::read(path).ok()?;
269    String::from_utf8_lossy(&bytes)
270        .lines()
271        .rev()
272        .find_map(|l| l.split_once(HELD).map(|(_, lock)| lock.trim().to_string()))
273}
274
275/// Why a staged line was refused, when eb-staged-rebuild found a lock held:
276/// the lock, and whether `rebuild.lock_timeout` ran out waiting for it.
277/// Without `rebuild.state_root`, a module the line loads can set the state
278/// root in EB_STAGED_REBUILD_STATE_ROOT, out of eb-pipeline's sight, so a
279/// lock away from the expected state root may still be the state root's.
280fn lock_refusal(
281    p: &Planner,
282    cmd: &Cmd,
283    env: &BTreeMap<String, String>,
284    log: &Path,
285) -> Option<String> {
286    const ROOT: &str = "EB_STAGED_REBUILD_STATE_ROOT";
287    let lock = held_lock(log)?;
288    let env_root = cmd
289        .env
290        .get(ROOT)
291        .or(env.get(ROOT))
292        .cloned()
293        .or_else(|| std::env::var(ROOT).ok());
294    let ours = p.state_root_lock(env_root.as_deref());
295    let n = p.scope.site.rebuild_on().map_or(0, |r| r.lock_timeout);
296    if Path::new(&lock).components().eq(ours.components()) {
297        return Some(format!(
298            "another eb-staged-rebuild holds the lock at {lock}, and rebuild.lock_timeout ({n} s) ran out"
299        ));
300    }
301    if p.scope.state_root().is_some() {
302        return Some(format!(
303            "another eb-staged-rebuild holds the lock at {lock}"
304        ));
305    }
306    Some(format!(
307        "another eb-staged-rebuild holds the lock at {lock}, and rebuild.lock_timeout ({n} s) ran out if that is the state root's lock: rebuild.state_root is unset, and eb-pipeline cannot see a state root that a module sets in {ROOT}"
308    ))
309}
310
311/// The reservation `stack admit` wrote for this job, `<reservations>/<job>.json`.
312struct Reservation {
313    path: PathBuf,
314    job: String,
315    lock: Option<PathBuf>,
316    posix: bool,
317}
318
319/// Take the admission lock the way `stack admit` takes it: `flock`, or a
320/// POSIX record lock over the whole file. Dropping the file releases it.
321fn admission_lock(path: &Path, posix: bool) -> std::io::Result<File> {
322    use std::os::fd::AsRawFd;
323    let f = std::fs::OpenOptions::new()
324        .create(true)
325        .truncate(false)
326        .read(true)
327        .write(true)
328        .open(path)?;
329    let fd = f.as_raw_fd();
330    // SAFETY: `fd` is open for the whole call, and a zeroed `libc::flock` is
331    // a valid value of that plain C struct.
332    let rc = unsafe {
333        if posix {
334            let mut l: libc::flock = std::mem::zeroed();
335            l.l_type = libc::F_WRLCK as _;
336            l.l_whence = libc::SEEK_SET as _;
337            libc::fcntl(fd, libc::F_SETLKW, &raw const l)
338        } else {
339            libc::flock(fd, libc::LOCK_EX)
340        }
341    };
342    if rc == 0 {
343        Ok(f)
344    } else {
345        Err(std::io::Error::last_os_error())
346    }
347}
348
349/// Hold the rotation lock shared for the whole run, so that another run's
350/// rotation does not compress this run's logs while they are written: a run
351/// that began on an earlier day in UTC is old enough. Waits while another
352/// run rotates; `Ok(None)` when the lock cannot be had, and the rotation then
353/// goes ahead without it, as it did before there was one.
354fn hold_rotate_lock(st: &mut State, path: &Path) -> Result<Option<File>, i32> {
355    let target = st.res.target.clone();
356    let without = |e: std::io::Error| {
357        log(
358            "WARN",
359            "logs",
360            &target,
361            &format!(
362                "cannot lock {}: {e}; the rotation does not wait for other runs",
363                path.display()
364            ),
365        );
366        Ok(None)
367    };
368    let opened = path
369        .parent()
370        .map_or(Ok(()), std::fs::create_dir_all)
371        .and_then(|()| {
372            std::fs::OpenOptions::new()
373                .create(true)
374                .truncate(false)
375                .read(true)
376                .write(true)
377                .open(path)
378        });
379    let f = match opened {
380        Ok(f) => f,
381        Err(e) => return without(e),
382    };
383    let mut waiting = false;
384    loop {
385        match f.try_lock_shared() {
386            Ok(()) => return Ok(Some(f)),
387            Err(std::fs::TryLockError::WouldBlock) => {
388                if !waiting {
389                    let msg = format!("waiting: another run rotates and holds {}", path.display());
390                    log("INFO", "logs", &target, &msg);
391                    waiting = true;
392                }
393                if sleep_interruptible(1) {
394                    if let Some(c) = stop_if_signalled(st, "logs") {
395                        return Err(c);
396                    }
397                }
398            }
399            Err(std::fs::TryLockError::Error(e)) => return without(e),
400        }
401    }
402}
403
404/// The rotation lock taken alone in place of this run's shared hold;
405/// `Ok(None)` while another run holds it, live or rotating.
406fn rotate_alone(f: File) -> std::io::Result<Option<File>> {
407    f.unlock()?;
408    match f.try_lock() {
409        Ok(()) => Ok(Some(f)),
410        Err(std::fs::TryLockError::WouldBlock) => Ok(None),
411        Err(std::fs::TryLockError::Error(e)) => Err(e),
412    }
413}
414
415/// Remove this job's reservation under the admission lock. A record that is
416/// no longer the one `stack admit` wrote for this job is kept: a forecast or
417/// waiting record belongs to another writer.
418fn release(r: Reservation, target: &str) {
419    let kept = |why: String| {
420        log(
421            "WARN",
422            "admission",
423            target,
424            &format!("reservation {} kept: {why}", r.path.display()),
425        )
426    };
427    let _lock = match r.lock.as_deref().map(|l| admission_lock(l, r.posix)) {
428        Some(Err(e)) => return kept(format!("cannot take the lock: {e}")),
429        held => held,
430    };
431    let text = match std::fs::read_to_string(&r.path) {
432        Ok(t) => t,
433        Err(e) if e.kind() == std::io::ErrorKind::NotFound => return,
434        Err(e) => return kept(e.to_string()),
435    };
436    let ours = serde_json::from_str::<serde_json::Value>(&text).is_ok_and(|v| {
437        v["job"] == r.job.as_str() && v.get("phase").is_none_or(|p| p == "admitted")
438    });
439    if !ours {
440        return kept("it is no longer this job's admitted reservation".into());
441    }
442    match std::fs::remove_file(&r.path) {
443        Ok(()) => log(
444            "INFO",
445            "admission",
446            target,
447            &format!("released {}", r.path.display()),
448        ),
449        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
450        Err(e) => kept(e.to_string()),
451    }
452}
453
454/// How [`run`] goes through the plan.
455pub struct Options {
456    /// Print each planned command, run none, and return exit 0.
457    pub dry_run: bool,
458}
459
460struct State {
461    res: RunResult,
462    refused: bool,
463    pending: bool,
464    failed: bool,
465    forced_code: Option<i32>,
466    /// Released when the admitted unit ends, or by `finish`.
467    reservation: Option<Reservation>,
468}
469
470impl State {
471    fn phase(&mut self, name: &str, status: &str, code: Option<i32>, note: Option<String>) {
472        self.res.phases.push(PhaseResult {
473            name: name.into(),
474            status: status.into(),
475            exit_code: code,
476            note,
477        });
478    }
479}
480
481/// Execute the plan. Returns the result; its `exit_code` is the process
482/// exit code.
483pub fn run(planner: &Planner, lines: &[ListLine], opts: &Options) -> RunResult {
484    let mut owned = planner.clone();
485    let p = &mut owned;
486    let site = p.scope.site;
487    let target = p.scope.target.name.clone();
488    let env = p.scope.env();
489    let log_dir = PathBuf::from(p.scope.log_dir());
490    let mut st = State {
491        res: RunResult {
492            schema_version: 1,
493            kind: "result".into(),
494            site: site.site.name.clone(),
495            target: target.clone(),
496            build_type: p.scope.build_name().to_string(),
497            run_id: p.scope.run_id.clone(),
498            list: p.list_path.clone(),
499            log_dir: p.scope.log_dir(),
500            started_at: chrono::Utc::now().to_rfc3339(),
501            ended_at: String::new(),
502            exit_code: 0,
503            sharded: false,
504            counts: BTreeMap::new(),
505            not_configured: vec![],
506            admission: vec![],
507            phases: vec![],
508            lines: vec![],
509            deselected: vec![],
510            message: None,
511            interrupted_by: None,
512        },
513        refused: false,
514        pending: false,
515        failed: false,
516        forced_code: None,
517        reservation: None,
518    };
519
520    let plan = match build_plan(p, lines, None) {
521        Ok(pl) => pl,
522        Err(f) => {
523            st.res.message = Some(f.msg.clone());
524            return finish(st, Some(f.code));
525        }
526    };
527
528    if opts.dry_run {
529        for ph in &plan.phases {
530            if ph.status != "planned" {
531                println!("# [{}] {}", ph.name, ph.status);
532                continue;
533            }
534            for c in &ph.commands {
535                println!("[{}] {}", ph.name, shell_line(c));
536            }
537        }
538        log("INFO", "run", &target, "dry run: nothing executed");
539        return finish(st, None);
540    }
541
542    // Phases that are not configured are recorded, not failed.
543    for ph in &plan.phases {
544        match ph.status.as_str() {
545            "not configured" => {
546                log("INFO", &ph.name, &target, "not configured");
547                st.res.not_configured.push(ph.name.clone());
548                st.phase(&ph.name, "not configured", None, None);
549            }
550            "refused" => {
551                let msg = ph.reason.clone().unwrap_or_default();
552                log("ERROR", &ph.name, &target, &msg);
553                st.phase(&ph.name, "refused", Some(exit::REFUSED), Some(msg.clone()));
554                st.res.message = Some(msg);
555                return finish(st, Some(exit::REFUSED));
556            }
557            _ => {}
558        }
559    }
560    if let Some(a) = site
561        .admission_on()
562        .filter(|a| a.quota_cmd.is_some() && a.reservations.is_some())
563    {
564        match std::env::var(&a.job_id_env) {
565            Ok(v) if valid_job_id(&v) => p.job_id = Some(v),
566            _ => {
567                let msg = format!(
568                    "admission: {} is unset or not a positive integer without a leading zero",
569                    a.job_id_env
570                );
571                log("ERROR", "preflight", &target, &msg);
572                st.phase(
573                    "preflight",
574                    "refused",
575                    Some(exit::REFUSED),
576                    Some(msg.clone()),
577                );
578                st.res.message = Some(msg);
579                return finish(st, Some(exit::REFUSED));
580            }
581        }
582    }
583    if let Some(sh) = site.shards_on() {
584        let mut vars = BTreeMap::new();
585        for n in &sh.installed_modules_dirs_from_env {
586            match std::env::var(n) {
587                Ok(v) => {
588                    vars.insert(n.clone(), v);
589                }
590                Err(_) => {
591                    let msg = format!("shards.installed_modules_dirs_from_env: {n} is unset");
592                    log("ERROR", "preflight", &target, &msg);
593                    st.phase(
594                        "preflight",
595                        "refused",
596                        Some(exit::REFUSED),
597                        Some(msg.clone()),
598                    );
599                    st.res.message = Some(msg);
600                    return finish(st, Some(exit::REFUSED));
601                }
602            }
603        }
604        p.runtime_env = Some(vars);
605    }
606    let p: &Planner = p;
607    st.phase("preflight", "ok", None, None);
608    if site.licences_on().is_some() {
609        st.phase("licences", "ok", None, None);
610    }
611    let rotate_lock = match p.rotate_lock().map(|l| hold_rotate_lock(&mut st, &l)) {
612        Some(Err(c)) => return finish(st, Some(c)),
613        Some(Ok(f)) => f,
614        None => None,
615    };
616
617    // selection: lines the target's selector leaves out are recorded, not built;
618    // the kept lines go to a file that the shard planner reads
619    let (kept, deselected) = p.select_lines(lines);
620    let has_select = p.selected_list_path().is_some();
621    if let Some(path) = p.selected_list_path() {
622        let mut text = kept
623            .iter()
624            .map(|l| l.raw.as_str())
625            .collect::<Vec<_>>()
626            .join("\n");
627        if !kept.is_empty() {
628            text.push('\n');
629        }
630        let written = path
631            .parent()
632            .map_or(Ok(()), std::fs::create_dir_all)
633            .and_then(|()| std::fs::write(&path, text));
634        if let Err(e) = written {
635            let msg = format!("cannot write {}: {e}", path.display());
636            log("ERROR", "preflight", &target, &msg);
637            st.res.message = Some(msg);
638            return finish(st, Some(exit::FAILED));
639        }
640    }
641    if !deselected.is_empty() {
642        log(
643            "INFO",
644            "build",
645            &target,
646            &format!("{} line(s) deselected for this target", deselected.len()),
647        );
648    }
649    st.res.deselected = deselected;
650
651    // shard
652    let mut units: Vec<Unit> = if has_select && kept.is_empty() {
653        vec![]
654    } else {
655        vec![Unit {
656            name: "all".into(),
657            role: Role::Build,
658            lines: kept,
659            footprint: None,
660        }]
661    };
662    if let Some(sh) = site.shards_on().filter(|_| !units.is_empty()) {
663        match p.shard_cmd() {
664            Err(f) => {
665                st.res.message = Some(f.msg);
666                return finish(st, Some(f.code));
667            }
668            Ok(cmd) => {
669                for d in p.missing_installed_modules_dirs() {
670                    log(
671                        "WARN",
672                        "shard",
673                        &target,
674                        &format!("{d} does not exist; no module counts as installed there"),
675                    );
676                }
677                log(
678                    "INFO",
679                    "shard",
680                    &target,
681                    &format!("planning {} shards", sh.count),
682                );
683                let ex = match exec(&cmd, &env, &log_dir.join("shard.log")) {
684                    Ok(e) => e,
685                    Err(f) => {
686                        st.res.message = Some(f.msg);
687                        return finish(st, Some(f.code));
688                    }
689                };
690                if let Some(c) = stop_if_signalled(&mut st, "shard") {
691                    return finish(st, Some(c));
692                }
693                if ex.code == 0 {
694                    match read_units(Path::new(&p.shard_dir())) {
695                        Ok(u) if !u.is_empty() => {
696                            units = u;
697                            st.res.sharded = true;
698                            st.phase("shard", "ok", Some(0), None);
699                        }
700                        Ok(_) => {
701                            st.phase(
702                                "shard",
703                                "ok",
704                                Some(0),
705                                Some("plan holds no lines; building the list as one unit".into()),
706                            );
707                        }
708                        Err(f) => {
709                            log("ERROR", "shard", &target, &f.msg);
710                            st.phase("shard", "failed", Some(0), Some(f.msg.clone()));
711                            st.res.message = Some(f.msg);
712                            return finish(st, Some(exit::FAILED));
713                        }
714                    }
715                } else if ex.code == sh.planner_exit_unsharded {
716                    log(
717                        "INFO",
718                        "shard",
719                        &target,
720                        &format!(
721                            "planner declined to shard (exit {}); building unsharded",
722                            ex.code
723                        ),
724                    );
725                    st.phase(
726                        "shard",
727                        "unsharded",
728                        Some(ex.code),
729                        Some(format!(
730                            "exit {} is shards.planner_exit_unsharded: planner declined to shard; plan built unsharded",
731                            ex.code
732                        )),
733                    );
734                } else {
735                    let code = if ex.code == 2 {
736                        exit::USAGE
737                    } else {
738                        exit::FAILED
739                    };
740                    let msg = format!("stack shard exited {} (see {})", ex.code, ex.log.display());
741                    log("ERROR", "shard", &target, &msg);
742                    st.phase("shard", "failed", Some(ex.code), Some(msg.clone()));
743                    st.res.message = Some(msg);
744                    return finish(st, Some(code));
745                }
746            }
747        }
748    }
749
750    // the route's receipt digest is read once a CPU-role unit needs it
751    let mut routed = p.clone();
752    if units.iter().any(|u| u.role == Role::Cpu) {
753        if let Some(n) = p
754            .scope
755            .target
756            .route
757            .as_ref()
758            .filter(|r| r.kind == "command")
759            .and_then(|r| r.receipt_digest_env.as_ref())
760        {
761            match std::env::var(n) {
762                Ok(v) if !v.is_empty() => routed.route_digest = Some(v),
763                _ => {
764                    let msg = format!("route.receipt_digest_env: {n} is unset or empty");
765                    log("ERROR", "build", &target, &msg);
766                    st.phase("build", "refused", Some(exit::REFUSED), Some(msg.clone()));
767                    st.res.message = Some(msg);
768                    return finish(st, Some(exit::REFUSED));
769                }
770            }
771        }
772    }
773    let p = &routed;
774
775    // sources: prefetch every line once; a failed fetch fails the line.
776    let mut stems = StemSeq::default();
777    let mut unit_stems: Vec<Vec<String>> = Vec::new();
778    for u in &units {
779        unit_stems.push(u.lines.iter().map(|l| stems.next(&l.easyconfig)).collect());
780    }
781    let mut fetch_failed: Vec<String> = Vec::new();
782    if site.sources_on().is_some() {
783        let mut any_failed = false;
784        for (u, ss) in units.iter().zip(&unit_stems) {
785            for (l, s) in u.lines.iter().zip(ss) {
786                make_route_dirs(p, u.role, "sources");
787                let cmd = match p.prefetch_cmd(l, s, u.role) {
788                    Ok(c) => c,
789                    Err(f) => {
790                        st.res.message = Some(f.msg);
791                        return finish(st, Some(f.code));
792                    }
793                };
794                let ex = match exec(&cmd, &env, &log_dir.join(format!("{s}.fetch.log"))) {
795                    Ok(e) => e,
796                    Err(f) => {
797                        st.res.message = Some(f.msg);
798                        return finish(st, Some(f.code));
799                    }
800                };
801                if let Some(c) = stop_if_signalled(&mut st, "sources") {
802                    return finish(st, Some(c));
803                }
804                if ex.code != 0 {
805                    any_failed = true;
806                    fetch_failed.push(s.clone());
807                    log(
808                        "ERROR",
809                        "sources",
810                        &target,
811                        &format!("{s}: fetch exited {}", ex.code),
812                    );
813                    st.res.lines.push(LineResult {
814                        easyconfig: l.easyconfig.clone(),
815                        unit: u.name.clone(),
816                        phase: "sources".into(),
817                        status: "failed".into(),
818                        exit_code: Some(ex.code),
819                        translated_from: None,
820                        log: ex.log.display().to_string(),
821                        note: Some("source fetch failed; line not built".into()),
822                    });
823                    st.failed = true;
824                }
825            }
826        }
827        st.phase(
828            "sources",
829            if any_failed { "failed" } else { "ok" },
830            None,
831            None,
832        );
833    }
834
835    // admission and builds, unit by unit
836    let adm = site
837        .admission_on()
838        .filter(|a| a.quota_cmd.is_some() && a.reservations.is_some());
839    let keep_failed = site.logs_on().is_some_and(|l| l.keep_failed);
840    let mut stopped = false;
841    for (ui, (u, ss)) in units.iter().zip(&unit_stems).enumerate() {
842        if stopped {
843            for (l, s) in u.lines.iter().zip(ss) {
844                st.res.lines.push(LineResult {
845                    easyconfig: l.easyconfig.clone(),
846                    unit: u.name.clone(),
847                    phase: "build".into(),
848                    status: "skipped".into(),
849                    exit_code: None,
850                    translated_from: None,
851                    log: p.log_stdout(s),
852                    note: Some("an earlier unit was refused".into()),
853                });
854            }
855            continue;
856        }
857        if let Some(a) = adm {
858            let fp = u.footprint.or(a.default_footprint).unwrap_or(0);
859            let cmd = match p.admission_cmd(&u.name, fp) {
860                Ok(c) => c,
861                Err(f) => {
862                    st.res.message = Some(f.msg);
863                    return finish(st, Some(f.code));
864                }
865            };
866            let mut attempt = 0u32;
867            let mut wait = capped_wait(a.wait_seconds, a.max_wait_seconds);
868            let outcome = loop {
869                attempt += 1;
870                let ex = match exec(
871                    &cmd,
872                    &env,
873                    &log_dir.join(format!("admission-{}.{attempt}.log", u.name)),
874                ) {
875                    Ok(e) => e,
876                    Err(f) => {
877                        st.res.message = Some(f.msg);
878                        return finish(st, Some(f.code));
879                    }
880                };
881                if let Some(c) = stop_if_signalled(&mut st, "admission") {
882                    return finish(st, Some(c));
883                }
884                let last = last_json_line(&ex.log);
885                match ex.code {
886                    0 => {
887                        log(
888                            "INFO",
889                            "admission",
890                            &target,
891                            &format!("{} admitted on attempt {attempt}", u.name),
892                        );
893                        break ("admitted", None, last, attempt, 0);
894                    }
895                    78 => {
896                        if attempt >= a.attempts {
897                            break ("refused", None, last, attempt, 78);
898                        }
899                        log(
900                            "WARN",
901                            "admission",
902                            &target,
903                            &format!(
904                                "{} refused on attempt {attempt} of {}; waiting {wait}s",
905                                u.name, a.attempts
906                            ),
907                        );
908                        if sleep_interruptible(wait) {
909                            if let Some(c) = stop_if_signalled(&mut st, "admission") {
910                                return finish(st, Some(c));
911                            }
912                        }
913                        wait = doubled_wait(wait, a.max_wait_seconds);
914                    }
915                    c => {
916                        let code = if c == 2 { exit::USAGE } else { exit::FAILED };
917                        break (
918                            "failed",
919                            Some(format!("stack admit exited {c}")),
920                            last,
921                            attempt,
922                            code,
923                        );
924                    }
925                }
926            };
927            let (kind, note, last, attempts, code) = outcome;
928            st.res.admission.push(AdmissionResult {
929                unit: u.name.clone(),
930                attempts,
931                outcome: kind.into(),
932                last_json: last.clone(),
933            });
934            if kind == "admitted" {
935                st.reservation = p.job_id.as_ref().map(|job| Reservation {
936                    path: Path::new(a.reservations.as_deref().unwrap_or_default())
937                        .join(format!("{job}.json")),
938                    job: job.clone(),
939                    lock: a.lock.as_ref().map(PathBuf::from),
940                    posix: a.lock_kind == "posix",
941                });
942            }
943            if kind != "admitted" {
944                let msg = if kind == "refused" {
945                    format!(
946                        "admission refused after {attempts} attempts: {}",
947                        last.clone().unwrap_or_else(|| "(no JSON line)".into())
948                    )
949                } else {
950                    format!("admission failed: {}", note.unwrap_or_default())
951                };
952                log("ERROR", "admission", &target, &msg);
953                st.phase("admission", kind, Some(code), Some(msg.clone()));
954                st.res.message = Some(msg);
955                if kind == "refused" {
956                    st.refused = true;
957                    stopped = true;
958                    for (l, s) in u.lines.iter().zip(ss) {
959                        st.res.lines.push(LineResult {
960                            easyconfig: l.easyconfig.clone(),
961                            unit: u.name.clone(),
962                            phase: "build".into(),
963                            status: "skipped".into(),
964                            exit_code: None,
965                            translated_from: None,
966                            log: p.log_stdout(s),
967                            note: Some("admission refused".into()),
968                        });
969                    }
970                    continue;
971                }
972                return finish(st, Some(code));
973            }
974        }
975
976        for (li, (l, s)) in u.lines.iter().zip(ss).enumerate() {
977            if fetch_failed.contains(s) {
978                continue;
979            }
980            if interrupted().is_some() {
981                note_skipped(&mut st, &units, &unit_stems, &fetch_failed, ui, li, p);
982                break;
983            }
984            make_route_dirs(p, u.role, "build");
985            // Before the build, which may install the module file it looks for.
986            let staged = p.is_staged(l);
987            let cmd = match p.build_cmd(u.role, l, s) {
988                Ok(c) => c,
989                Err(f) => {
990                    st.res.message = Some(f.msg);
991                    return finish(st, Some(f.code));
992                }
993            };
994            if let Some(w) = p.in_place_reason(l) {
995                log("WARN", "build", &target, &format!("{s}: {w}"));
996            }
997            log("INFO", "build", &target, &format!("{s}: start"));
998            let ex = match exec(&cmd, &env, Path::new(&p.log_stdout(s))) {
999                Ok(e) => e,
1000                Err(f) => {
1001                    st.res.message = Some(f.msg);
1002                    return finish(st, Some(f.code));
1003                }
1004            };
1005            let mut lr = LineResult {
1006                easyconfig: l.easyconfig.clone(),
1007                unit: u.name.clone(),
1008                phase: "build".into(),
1009                status: "success".into(),
1010                exit_code: Some(ex.code),
1011                translated_from: None,
1012                log: ex.log.display().to_string(),
1013                note: staged.then(|| "ran through eb-staged-rebuild".into()),
1014            };
1015            if ex.code != 0 {
1016                let outcome = if staged {
1017                    map_staged(ex.code)
1018                } else {
1019                    StagedOutcome::Failed
1020                };
1021                let mut detail = None;
1022                match outcome {
1023                    StagedOutcome::Refused => {
1024                        lr.status = "refused".into();
1025                        lr.translated_from = Some(ex.code);
1026                        lr.exit_code = Some(exit::REFUSED);
1027                        detail = lock_refusal(p, &cmd, &env, &ex.log);
1028                        lr.note = Some(match &detail {
1029                            Some(d) => {
1030                                format!("eb-staged-rebuild exit 2: {d}; translated to refused (78)")
1031                            }
1032                            None => {
1033                                "eb-staged-rebuild exit 2 is a refusal; translated to refused (78)"
1034                                    .into()
1035                            }
1036                        });
1037                        st.refused = true;
1038                    }
1039                    StagedOutcome::Pending => {
1040                        lr.status = "pending".into();
1041                        lr.note = Some(
1042                            "eb-staged-rebuild exit 75: a required ledger event is pending".into(),
1043                        );
1044                        st.pending = true;
1045                    }
1046                    _ => {
1047                        lr.status = "failed".into();
1048                        st.failed = true;
1049                    }
1050                }
1051                let detail = detail.map(|d| format!(": {d}")).unwrap_or_default();
1052                log(
1053                    "ERROR",
1054                    "build",
1055                    &target,
1056                    &format!("{s}: {} (exit {}){detail}", lr.status, ex.code),
1057                );
1058                if interrupted().is_none() && keep_failed && lr.status == "failed" {
1059                    if let Ok(kc) = p.keep_failed_cmd(s, &ex.code.to_string()) {
1060                        let kx = exec(&kc, &env, &log_dir.join(format!("{s}.keep-logs.log")));
1061                        match kx {
1062                            Ok(k) if k.code == 0 => {}
1063                            Ok(k) => log(
1064                                "WARN",
1065                                "logs",
1066                                &target,
1067                                &format!("{s}: eb-keep-logs failed exited {}", k.code),
1068                            ),
1069                            Err(f) => log("WARN", "logs", &target, &f.msg),
1070                        }
1071                    }
1072                }
1073            } else {
1074                log("INFO", "build", &target, &format!("{s}: ok"));
1075            }
1076            st.res.lines.push(lr);
1077            if interrupted().is_some() {
1078                note_skipped(&mut st, &units, &unit_stems, &fetch_failed, ui, li + 1, p);
1079                break;
1080            }
1081        }
1082        if let Some(r) = st.reservation.take() {
1083            release(r, &target);
1084        }
1085        if interrupted().is_some() {
1086            break;
1087        }
1088    }
1089    if let Some(c) = stop_if_signalled(&mut st, "build") {
1090        return finish(st, Some(c));
1091    }
1092    if site.admission_on().is_some() && !st.res.admission.is_empty() && !st.refused {
1093        st.phase("admission", "ok", Some(0), None);
1094    }
1095    let build: Vec<&str> = st
1096        .res
1097        .lines
1098        .iter()
1099        .filter(|l| l.phase == "build")
1100        .map(|l| l.status.as_str())
1101        .collect();
1102    let (status, note) = if build.contains(&"failed") {
1103        ("failed", None)
1104    } else if st.refused && build.iter().all(|s| *s == "skipped") {
1105        (
1106            "skipped",
1107            Some("admission refused before any build line started".to_string()),
1108        )
1109    } else if st.refused && !build.contains(&"success") {
1110        ("refused", None)
1111    } else {
1112        ("ok", None)
1113    };
1114    st.phase("build", status, None, note);
1115
1116    // caches only after a clean build
1117    if !st.failed && !st.refused && !st.pending {
1118        match p.cache_cmd() {
1119            Ok(Some(cmd)) => {
1120                log("INFO", "caches", &target, "start");
1121                match exec(&cmd, &env, &log_dir.join("caches.log")) {
1122                    Ok(ex) => {
1123                        if ex.code == 0 {
1124                            st.phase("caches", "ok", Some(0), None);
1125                        } else {
1126                            let msg = format!("cache command exited {}", ex.code);
1127                            log("ERROR", "caches", &target, &msg);
1128                            st.phase("caches", "failed", Some(ex.code), Some(msg));
1129                            st.failed = true;
1130                        }
1131                    }
1132                    Err(f) => {
1133                        st.res.message = Some(f.msg);
1134                        return finish(st, Some(f.code));
1135                    }
1136                }
1137                if let Some(c) = stop_if_signalled(&mut st, "caches") {
1138                    return finish(st, Some(c));
1139                }
1140            }
1141            Ok(None) => {}
1142            Err(f) => {
1143                st.res.message = Some(f.msg);
1144                return finish(st, Some(f.code));
1145            }
1146        }
1147    } else if p.cache_cmd().ok().flatten().is_some() {
1148        st.phase(
1149            "caches",
1150            "skipped",
1151            None,
1152            Some("the build did not finish cleanly".into()),
1153        );
1154    }
1155
1156    // rotation, by a run that holds the rotation lock alone
1157    if let Ok(Some(cmd)) = p.rotate_cmd(true) {
1158        let lock = p.rotate_lock().unwrap_or_default();
1159        match rotate_lock.map(rotate_alone) {
1160            Some(Ok(None)) => {
1161                let msg = format!(
1162                    "another run under {} holds {}, live or rotating; the rotation is left to a run that ends alone",
1163                    site.filesystems.log_root,
1164                    lock.display()
1165                );
1166                log("INFO", "logs", &target, &format!("rotation skipped: {msg}"));
1167                st.phase("logs", "skipped", None, Some(msg));
1168            }
1169            alone => {
1170                if let Some(Err(e)) = &alone {
1171                    let msg = format!(
1172                        "cannot take {} alone: {e}; rotating without it",
1173                        lock.display()
1174                    );
1175                    log("WARN", "logs", &target, &msg);
1176                }
1177                let bin = Tool::EbKeepLogs.bin(site);
1178                let mut probe_env = env.clone();
1179                probe_env.extend(cmd.env.clone());
1180                let note = match help_lists(&bin, &probe_env, "--lock-timeout") {
1181                    Ok(true) => None,
1182                    Ok(false) => Some(format!(
1183                        "`{bin} --help` does not list it, as in eb-keep-logs 0.2.1, which takes no lock of its own"
1184                    )),
1185                    Err(e) => Some(e),
1186                }
1187                .map(|why| format!("no --lock-timeout for eb-keep-logs rotate: {why}"));
1188                let cmd = match &note {
1189                    None => cmd,
1190                    Some(n) => {
1191                        log("INFO", "logs", &target, n);
1192                        p.rotate_cmd(false).ok().flatten().unwrap_or(cmd)
1193                    }
1194                };
1195                match exec(&cmd, &env, &log_dir.join("rotate.log")) {
1196                    Ok(ex) => match map_rotate(ex.code) {
1197                        RotateOutcome::Ok => st.phase("logs", "ok", Some(0), note),
1198                        RotateOutcome::Locked => {
1199                            let msg = format!(
1200                                "eb-keep-logs rotate exited 75: another rotate holds {}, and this one changed nothing; the rotation is left to it",
1201                                p.keep_logs_lock().display()
1202                            );
1203                            log("INFO", "logs", &target, &format!("rotation skipped: {msg}"));
1204                            st.phase("logs", "skipped", Some(ex.code), Some(msg));
1205                        }
1206                        RotateOutcome::Failed => {
1207                            let msg = format!("eb-keep-logs rotate exited {}", ex.code);
1208                            log("ERROR", "logs", &target, &msg);
1209                            st.phase("logs", "failed", Some(ex.code), Some(msg));
1210                            st.failed = true;
1211                        }
1212                    },
1213                    Err(f) => {
1214                        st.res.message = Some(f.msg);
1215                        return finish(st, Some(f.code));
1216                    }
1217                }
1218                if let Some(c) = stop_if_signalled(&mut st, "logs") {
1219                    return finish(st, Some(c));
1220                }
1221            }
1222        }
1223    }
1224
1225    finish(st, None)
1226}
1227
1228/// Lines from `(from_unit, from_line)` onward that have not started, recorded
1229/// as skipped. A fetch that already failed stays the fetch result.
1230fn note_skipped(
1231    st: &mut State,
1232    units: &[Unit],
1233    stems: &[Vec<String>],
1234    fetch_failed: &[String],
1235    from_unit: usize,
1236    from_line: usize,
1237    p: &Planner,
1238) {
1239    for (ui, (u, ss)) in units.iter().zip(stems).enumerate().skip(from_unit) {
1240        let start = if ui == from_unit { from_line } else { 0 };
1241        for (l, s) in u.lines.iter().zip(ss).skip(start) {
1242            if fetch_failed.iter().any(|f| f == s) {
1243                continue;
1244            }
1245            st.res.lines.push(LineResult {
1246                easyconfig: l.easyconfig.clone(),
1247                unit: u.name.clone(),
1248                phase: "build".into(),
1249                status: "skipped".into(),
1250                exit_code: None,
1251                translated_from: None,
1252                log: p.log_stdout(s),
1253                note: Some("interrupted before this line started".into()),
1254            });
1255        }
1256    }
1257}
1258
1259/// Create the directories a routed command expects; a failure surfaces when
1260/// the command itself fails.
1261fn make_route_dirs(p: &Planner, role: Role, phase: &str) {
1262    if role == Role::Cpu {
1263        for d in p.scope.route_dirs(phase) {
1264            let _ = std::fs::create_dir_all(d);
1265        }
1266    }
1267}
1268
1269fn stop_if_signalled(st: &mut State, phase: &str) -> Option<i32> {
1270    let sig = interrupted()?;
1271    let (name, code) = if sig == libc::SIGINT {
1272        ("SIGINT", exit::SIGINT)
1273    } else {
1274        ("SIGTERM", exit::SIGTERM)
1275    };
1276    log(
1277        "WARN",
1278        phase,
1279        &st.res.target.clone(),
1280        &format!("{name} received; child ended, stopping"),
1281    );
1282    st.res.interrupted_by = Some(name.into());
1283    st.phase(phase, "interrupted", None, None);
1284    st.forced_code = Some(code);
1285    Some(code)
1286}
1287
1288fn finish(mut st: State, forced: Option<i32>) -> RunResult {
1289    if let Some(r) = st.reservation.take() {
1290        release(r, &st.res.target);
1291    }
1292    st.res.ended_at = chrono::Utc::now().to_rfc3339();
1293    let mut counts: BTreeMap<String, usize> = BTreeMap::new();
1294    for l in &st.res.lines {
1295        *counts.entry(l.status.clone()).or_default() += 1;
1296    }
1297    st.res.counts = counts;
1298    st.res.phases.sort_by_key(|ph| {
1299        PHASES
1300            .iter()
1301            .position(|n| *n == ph.name)
1302            .unwrap_or(PHASES.len())
1303    });
1304    st.res.exit_code = match forced.or(st.forced_code) {
1305        Some(c) => c,
1306        None if st.failed => exit::FAILED,
1307        None if st.refused => exit::REFUSED,
1308        None if st.pending => exit::TEMPFAIL,
1309        None => exit::OK,
1310    };
1311    st.res
1312}
1313
1314#[cfg(test)]
1315mod tests {
1316    use super::{capped_wait, doubled_wait};
1317
1318    #[test]
1319    fn admission_wait_never_exceeds_the_configured_maximum() {
1320        assert_eq!(capped_wait(60, 600), 60);
1321        assert_eq!(doubled_wait(60, 600), 120);
1322        assert_eq!(doubled_wait(480, 600), 600);
1323        assert_eq!(doubled_wait(600, 600), 600);
1324        // A maximum below the initial wait is still the maximum.
1325        assert_eq!(capped_wait(120, 50), 50);
1326        assert_eq!(doubled_wait(50, 50), 50);
1327        assert_eq!(doubled_wait(120, 50), 50);
1328        assert_eq!(capped_wait(5, 0), 0);
1329    }
1330}