1use crate::exit::{self, Fail};
4use crate::plan::{
5 build_plan, read_units, shell_line, valid_job_id, Cmd, Deselected, Planner, StemSeq, Unit,
6 PHASES,
7};
8use crate::resolve::Role;
9use crate::tools::{help_lists, map_rotate, map_staged, RotateOutcome, StagedOutcome, Tool};
10use crate::util::ListLine;
11use serde::Serialize;
12use std::collections::BTreeMap;
13use std::fs::File;
14use std::path::{Path, PathBuf};
15use std::process::{Command, Stdio};
16use std::sync::atomic::{AtomicI32, Ordering};
17use std::time::Duration;
18
19static SIGNAL: AtomicI32 = AtomicI32::new(0);
20
21extern "C" fn on_signal(sig: libc::c_int) {
22 SIGNAL.store(sig, Ordering::SeqCst);
23}
24
25pub fn install_handlers() {
27 for s in [libc::SIGTERM, libc::SIGINT] {
28 unsafe {
30 let mut sa: libc::sigaction = std::mem::zeroed();
31 sa.sa_sigaction = on_signal as extern "C" fn(libc::c_int) as usize;
32 libc::sigemptyset(&mut sa.sa_mask);
33 sa.sa_flags = 0;
34 libc::sigaction(s, &sa, std::ptr::null_mut());
35 }
36 }
37}
38
39fn interrupted() -> Option<i32> {
40 match SIGNAL.load(Ordering::SeqCst) {
41 0 => None,
42 s => Some(s),
43 }
44}
45
46fn capped_wait(wait_seconds: u64, max_wait_seconds: u64) -> u64 {
50 wait_seconds.min(max_wait_seconds)
51}
52
53fn doubled_wait(wait: u64, max_wait_seconds: u64) -> u64 {
56 wait.saturating_mul(2).min(max_wait_seconds)
57}
58
59fn sleep_interruptible(secs: u64) -> bool {
61 let mut left = Duration::from_secs(secs);
62 let slice = Duration::from_millis(50);
63 while !left.is_zero() {
64 if interrupted().is_some() {
65 return true;
66 }
67 let d = left.min(slice);
68 std::thread::sleep(d);
69 left -= d;
70 }
71 interrupted().is_some()
72}
73
74pub fn log(level: &str, phase: &str, target: &str, msg: &str) {
76 eprintln!("{level} [{phase}] {target}: {msg}");
77}
78
79#[derive(Debug, Serialize, Clone)]
81pub struct PhaseResult {
82 pub name: String,
84 pub status: String,
87 #[serde(skip_serializing_if = "Option::is_none")]
89 pub exit_code: Option<i32>,
90 #[serde(skip_serializing_if = "Option::is_none")]
92 pub note: Option<String>,
93}
94
95#[derive(Debug, Serialize, Clone)]
97pub struct LineResult {
98 pub easyconfig: String,
100 pub unit: String,
103 pub phase: String,
105 pub status: String,
107 #[serde(skip_serializing_if = "Option::is_none")]
109 pub exit_code: Option<i32>,
110 #[serde(skip_serializing_if = "Option::is_none")]
113 pub translated_from: Option<i32>,
114 pub log: String,
116 #[serde(skip_serializing_if = "Option::is_none")]
118 pub note: Option<String>,
119}
120
121#[derive(Debug, Serialize, Clone)]
123pub struct AdmissionResult {
124 pub unit: String,
126 pub attempts: u32,
128 pub outcome: String,
130 #[serde(skip_serializing_if = "Option::is_none")]
132 pub last_json: Option<String>,
133}
134
135#[derive(Debug, Serialize, Clone)]
137pub struct RunResult {
138 pub schema_version: u32,
140 pub kind: String,
142 pub site: String,
144 pub target: String,
146 pub build_type: String,
148 pub run_id: String,
150 pub list: String,
152 pub log_dir: String,
154 pub started_at: String,
156 pub ended_at: String,
158 pub exit_code: i32,
160 pub sharded: bool,
162 pub counts: BTreeMap<String, usize>,
164 pub not_configured: Vec<String>,
166 pub admission: Vec<AdmissionResult>,
168 pub phases: Vec<PhaseResult>,
170 pub lines: Vec<LineResult>,
172 #[serde(skip_serializing_if = "Vec::is_empty")]
174 pub deselected: Vec<Deselected>,
175 #[serde(skip_serializing_if = "Option::is_none")]
177 pub message: Option<String>,
178 #[serde(skip_serializing_if = "Option::is_none")]
180 pub interrupted_by: Option<String>,
181}
182
183struct Exec {
184 code: i32,
185 log: PathBuf,
186}
187
188fn exec(cmd: &Cmd, base_env: &BTreeMap<String, String>, log: &Path) -> Result<Exec, Fail> {
191 if let Some(dir) = log.parent() {
192 std::fs::create_dir_all(dir).map_err(|e| {
193 Fail::new(
194 exit::FAILED,
195 format!("cannot create {}: {e}", dir.display()),
196 )
197 })?;
198 }
199 let out = File::create(log).map_err(|e| {
200 Fail::new(
201 exit::FAILED,
202 format!("cannot create {}: {e}", log.display()),
203 )
204 })?;
205 let err = out
206 .try_clone()
207 .map_err(|e| Fail::new(exit::FAILED, format!("cannot duplicate log handle: {e}")))?;
208 let mut c = Command::new(&cmd.argv[0]);
209 c.args(&cmd.argv[1..])
210 .envs(base_env)
211 .envs(&cmd.env)
212 .stdin(Stdio::null())
213 .stdout(Stdio::from(out))
214 .stderr(Stdio::from(err));
215 let mut child = match c.spawn() {
216 Ok(ch) => ch,
217 Err(e) => {
218 let _ = std::fs::write(log, format!("cannot start {}: {e}\n", cmd.argv[0]));
219 return Ok(Exec {
220 code: 127,
221 log: log.to_path_buf(),
222 });
223 }
224 };
225 let mut forwarded = 0;
226 let status = loop {
227 match child.try_wait() {
228 Ok(Some(s)) => break s,
229 Ok(None) => {}
230 Err(e) => {
231 return Err(Fail::new(exit::FAILED, format!("wait failed: {e}")));
232 }
233 }
234 if let Some(sig) = interrupted() {
235 if forwarded != sig {
236 unsafe {
238 libc::kill(child.id() as libc::pid_t, sig);
239 }
240 forwarded = sig;
241 }
242 }
243 std::thread::sleep(Duration::from_millis(20));
244 };
245 use std::os::unix::process::ExitStatusExt;
246 let code = status
247 .code()
248 .unwrap_or_else(|| 128 + status.signal().unwrap_or(0));
249 Ok(Exec {
250 code,
251 log: log.to_path_buf(),
252 })
253}
254
255fn last_json_line(path: &Path) -> Option<String> {
256 let text = std::fs::read_to_string(path).ok()?;
257 text.lines()
258 .rev()
259 .map(str::trim)
260 .find(|l| l.starts_with('{') && l.ends_with('}'))
261 .map(str::to_string)
262}
263
264fn held_lock(path: &Path) -> Option<String> {
267 const HELD: &str = "another eb-staged-rebuild holds the lock at ";
268 let bytes = std::fs::read(path).ok()?;
269 String::from_utf8_lossy(&bytes)
270 .lines()
271 .rev()
272 .find_map(|l| l.split_once(HELD).map(|(_, lock)| lock.trim().to_string()))
273}
274
275fn lock_refusal(
281 p: &Planner,
282 cmd: &Cmd,
283 env: &BTreeMap<String, String>,
284 log: &Path,
285) -> Option<String> {
286 const ROOT: &str = "EB_STAGED_REBUILD_STATE_ROOT";
287 let lock = held_lock(log)?;
288 let env_root = cmd
289 .env
290 .get(ROOT)
291 .or(env.get(ROOT))
292 .cloned()
293 .or_else(|| std::env::var(ROOT).ok());
294 let ours = p.state_root_lock(env_root.as_deref());
295 let n = p.scope.site.rebuild_on().map_or(0, |r| r.lock_timeout);
296 if Path::new(&lock).components().eq(ours.components()) {
297 return Some(format!(
298 "another eb-staged-rebuild holds the lock at {lock}, and rebuild.lock_timeout ({n} s) ran out"
299 ));
300 }
301 if p.scope.state_root().is_some() {
302 return Some(format!(
303 "another eb-staged-rebuild holds the lock at {lock}"
304 ));
305 }
306 Some(format!(
307 "another eb-staged-rebuild holds the lock at {lock}, and rebuild.lock_timeout ({n} s) ran out if that is the state root's lock: rebuild.state_root is unset, and eb-pipeline cannot see a state root that a module sets in {ROOT}"
308 ))
309}
310
311struct Reservation {
313 path: PathBuf,
314 job: String,
315 lock: Option<PathBuf>,
316 posix: bool,
317}
318
319fn admission_lock(path: &Path, posix: bool) -> std::io::Result<File> {
322 use std::os::fd::AsRawFd;
323 let f = std::fs::OpenOptions::new()
324 .create(true)
325 .truncate(false)
326 .read(true)
327 .write(true)
328 .open(path)?;
329 let fd = f.as_raw_fd();
330 let rc = unsafe {
333 if posix {
334 let mut l: libc::flock = std::mem::zeroed();
335 l.l_type = libc::F_WRLCK as _;
336 l.l_whence = libc::SEEK_SET as _;
337 libc::fcntl(fd, libc::F_SETLKW, &raw const l)
338 } else {
339 libc::flock(fd, libc::LOCK_EX)
340 }
341 };
342 if rc == 0 {
343 Ok(f)
344 } else {
345 Err(std::io::Error::last_os_error())
346 }
347}
348
349fn hold_rotate_lock(st: &mut State, path: &Path) -> Result<Option<File>, i32> {
355 let target = st.res.target.clone();
356 let without = |e: std::io::Error| {
357 log(
358 "WARN",
359 "logs",
360 &target,
361 &format!(
362 "cannot lock {}: {e}; the rotation does not wait for other runs",
363 path.display()
364 ),
365 );
366 Ok(None)
367 };
368 let opened = path
369 .parent()
370 .map_or(Ok(()), std::fs::create_dir_all)
371 .and_then(|()| {
372 std::fs::OpenOptions::new()
373 .create(true)
374 .truncate(false)
375 .read(true)
376 .write(true)
377 .open(path)
378 });
379 let f = match opened {
380 Ok(f) => f,
381 Err(e) => return without(e),
382 };
383 let mut waiting = false;
384 loop {
385 match f.try_lock_shared() {
386 Ok(()) => return Ok(Some(f)),
387 Err(std::fs::TryLockError::WouldBlock) => {
388 if !waiting {
389 let msg = format!("waiting: another run rotates and holds {}", path.display());
390 log("INFO", "logs", &target, &msg);
391 waiting = true;
392 }
393 if sleep_interruptible(1) {
394 if let Some(c) = stop_if_signalled(st, "logs") {
395 return Err(c);
396 }
397 }
398 }
399 Err(std::fs::TryLockError::Error(e)) => return without(e),
400 }
401 }
402}
403
404fn rotate_alone(f: File) -> std::io::Result<Option<File>> {
407 f.unlock()?;
408 match f.try_lock() {
409 Ok(()) => Ok(Some(f)),
410 Err(std::fs::TryLockError::WouldBlock) => Ok(None),
411 Err(std::fs::TryLockError::Error(e)) => Err(e),
412 }
413}
414
415fn release(r: Reservation, target: &str) {
419 let kept = |why: String| {
420 log(
421 "WARN",
422 "admission",
423 target,
424 &format!("reservation {} kept: {why}", r.path.display()),
425 )
426 };
427 let _lock = match r.lock.as_deref().map(|l| admission_lock(l, r.posix)) {
428 Some(Err(e)) => return kept(format!("cannot take the lock: {e}")),
429 held => held,
430 };
431 let text = match std::fs::read_to_string(&r.path) {
432 Ok(t) => t,
433 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return,
434 Err(e) => return kept(e.to_string()),
435 };
436 let ours = serde_json::from_str::<serde_json::Value>(&text).is_ok_and(|v| {
437 v["job"] == r.job.as_str() && v.get("phase").is_none_or(|p| p == "admitted")
438 });
439 if !ours {
440 return kept("it is no longer this job's admitted reservation".into());
441 }
442 match std::fs::remove_file(&r.path) {
443 Ok(()) => log(
444 "INFO",
445 "admission",
446 target,
447 &format!("released {}", r.path.display()),
448 ),
449 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
450 Err(e) => kept(e.to_string()),
451 }
452}
453
454pub struct Options {
456 pub dry_run: bool,
458}
459
460struct State {
461 res: RunResult,
462 refused: bool,
463 pending: bool,
464 failed: bool,
465 forced_code: Option<i32>,
466 reservation: Option<Reservation>,
468}
469
470impl State {
471 fn phase(&mut self, name: &str, status: &str, code: Option<i32>, note: Option<String>) {
472 self.res.phases.push(PhaseResult {
473 name: name.into(),
474 status: status.into(),
475 exit_code: code,
476 note,
477 });
478 }
479}
480
481pub fn run(planner: &Planner, lines: &[ListLine], opts: &Options) -> RunResult {
484 let mut owned = planner.clone();
485 let p = &mut owned;
486 let site = p.scope.site;
487 let target = p.scope.target.name.clone();
488 let env = p.scope.env();
489 let log_dir = PathBuf::from(p.scope.log_dir());
490 let mut st = State {
491 res: RunResult {
492 schema_version: 1,
493 kind: "result".into(),
494 site: site.site.name.clone(),
495 target: target.clone(),
496 build_type: p.scope.build_name().to_string(),
497 run_id: p.scope.run_id.clone(),
498 list: p.list_path.clone(),
499 log_dir: p.scope.log_dir(),
500 started_at: chrono::Utc::now().to_rfc3339(),
501 ended_at: String::new(),
502 exit_code: 0,
503 sharded: false,
504 counts: BTreeMap::new(),
505 not_configured: vec![],
506 admission: vec![],
507 phases: vec![],
508 lines: vec![],
509 deselected: vec![],
510 message: None,
511 interrupted_by: None,
512 },
513 refused: false,
514 pending: false,
515 failed: false,
516 forced_code: None,
517 reservation: None,
518 };
519
520 let plan = match build_plan(p, lines, None) {
521 Ok(pl) => pl,
522 Err(f) => {
523 st.res.message = Some(f.msg.clone());
524 return finish(st, Some(f.code));
525 }
526 };
527
528 if opts.dry_run {
529 for ph in &plan.phases {
530 if ph.status != "planned" {
531 println!("# [{}] {}", ph.name, ph.status);
532 continue;
533 }
534 for c in &ph.commands {
535 println!("[{}] {}", ph.name, shell_line(c));
536 }
537 }
538 log("INFO", "run", &target, "dry run: nothing executed");
539 return finish(st, None);
540 }
541
542 for ph in &plan.phases {
544 match ph.status.as_str() {
545 "not configured" => {
546 log("INFO", &ph.name, &target, "not configured");
547 st.res.not_configured.push(ph.name.clone());
548 st.phase(&ph.name, "not configured", None, None);
549 }
550 "refused" => {
551 let msg = ph.reason.clone().unwrap_or_default();
552 log("ERROR", &ph.name, &target, &msg);
553 st.phase(&ph.name, "refused", Some(exit::REFUSED), Some(msg.clone()));
554 st.res.message = Some(msg);
555 return finish(st, Some(exit::REFUSED));
556 }
557 _ => {}
558 }
559 }
560 if let Some(a) = site
561 .admission_on()
562 .filter(|a| a.quota_cmd.is_some() && a.reservations.is_some())
563 {
564 match std::env::var(&a.job_id_env) {
565 Ok(v) if valid_job_id(&v) => p.job_id = Some(v),
566 _ => {
567 let msg = format!(
568 "admission: {} is unset or not a positive integer without a leading zero",
569 a.job_id_env
570 );
571 log("ERROR", "preflight", &target, &msg);
572 st.phase(
573 "preflight",
574 "refused",
575 Some(exit::REFUSED),
576 Some(msg.clone()),
577 );
578 st.res.message = Some(msg);
579 return finish(st, Some(exit::REFUSED));
580 }
581 }
582 }
583 if let Some(sh) = site.shards_on() {
584 let mut vars = BTreeMap::new();
585 for n in &sh.installed_modules_dirs_from_env {
586 match std::env::var(n) {
587 Ok(v) => {
588 vars.insert(n.clone(), v);
589 }
590 Err(_) => {
591 let msg = format!("shards.installed_modules_dirs_from_env: {n} is unset");
592 log("ERROR", "preflight", &target, &msg);
593 st.phase(
594 "preflight",
595 "refused",
596 Some(exit::REFUSED),
597 Some(msg.clone()),
598 );
599 st.res.message = Some(msg);
600 return finish(st, Some(exit::REFUSED));
601 }
602 }
603 }
604 p.runtime_env = Some(vars);
605 }
606 let p: &Planner = p;
607 st.phase("preflight", "ok", None, None);
608 if site.licences_on().is_some() {
609 st.phase("licences", "ok", None, None);
610 }
611 let rotate_lock = match p.rotate_lock().map(|l| hold_rotate_lock(&mut st, &l)) {
612 Some(Err(c)) => return finish(st, Some(c)),
613 Some(Ok(f)) => f,
614 None => None,
615 };
616
617 let (kept, deselected) = p.select_lines(lines);
620 let has_select = p.selected_list_path().is_some();
621 if let Some(path) = p.selected_list_path() {
622 let mut text = kept
623 .iter()
624 .map(|l| l.raw.as_str())
625 .collect::<Vec<_>>()
626 .join("\n");
627 if !kept.is_empty() {
628 text.push('\n');
629 }
630 let written = path
631 .parent()
632 .map_or(Ok(()), std::fs::create_dir_all)
633 .and_then(|()| std::fs::write(&path, text));
634 if let Err(e) = written {
635 let msg = format!("cannot write {}: {e}", path.display());
636 log("ERROR", "preflight", &target, &msg);
637 st.res.message = Some(msg);
638 return finish(st, Some(exit::FAILED));
639 }
640 }
641 if !deselected.is_empty() {
642 log(
643 "INFO",
644 "build",
645 &target,
646 &format!("{} line(s) deselected for this target", deselected.len()),
647 );
648 }
649 st.res.deselected = deselected;
650
651 let mut units: Vec<Unit> = if has_select && kept.is_empty() {
653 vec![]
654 } else {
655 vec![Unit {
656 name: "all".into(),
657 role: Role::Build,
658 lines: kept,
659 footprint: None,
660 }]
661 };
662 if let Some(sh) = site.shards_on().filter(|_| !units.is_empty()) {
663 match p.shard_cmd() {
664 Err(f) => {
665 st.res.message = Some(f.msg);
666 return finish(st, Some(f.code));
667 }
668 Ok(cmd) => {
669 for d in p.missing_installed_modules_dirs() {
670 log(
671 "WARN",
672 "shard",
673 &target,
674 &format!("{d} does not exist; no module counts as installed there"),
675 );
676 }
677 log(
678 "INFO",
679 "shard",
680 &target,
681 &format!("planning {} shards", sh.count),
682 );
683 let ex = match exec(&cmd, &env, &log_dir.join("shard.log")) {
684 Ok(e) => e,
685 Err(f) => {
686 st.res.message = Some(f.msg);
687 return finish(st, Some(f.code));
688 }
689 };
690 if let Some(c) = stop_if_signalled(&mut st, "shard") {
691 return finish(st, Some(c));
692 }
693 if ex.code == 0 {
694 match read_units(Path::new(&p.shard_dir())) {
695 Ok(u) if !u.is_empty() => {
696 units = u;
697 st.res.sharded = true;
698 st.phase("shard", "ok", Some(0), None);
699 }
700 Ok(_) => {
701 st.phase(
702 "shard",
703 "ok",
704 Some(0),
705 Some("plan holds no lines; building the list as one unit".into()),
706 );
707 }
708 Err(f) => {
709 log("ERROR", "shard", &target, &f.msg);
710 st.phase("shard", "failed", Some(0), Some(f.msg.clone()));
711 st.res.message = Some(f.msg);
712 return finish(st, Some(exit::FAILED));
713 }
714 }
715 } else if ex.code == sh.planner_exit_unsharded {
716 log(
717 "INFO",
718 "shard",
719 &target,
720 &format!(
721 "planner declined to shard (exit {}); building unsharded",
722 ex.code
723 ),
724 );
725 st.phase(
726 "shard",
727 "unsharded",
728 Some(ex.code),
729 Some(format!(
730 "exit {} is shards.planner_exit_unsharded: planner declined to shard; plan built unsharded",
731 ex.code
732 )),
733 );
734 } else {
735 let code = if ex.code == 2 {
736 exit::USAGE
737 } else {
738 exit::FAILED
739 };
740 let msg = format!("stack shard exited {} (see {})", ex.code, ex.log.display());
741 log("ERROR", "shard", &target, &msg);
742 st.phase("shard", "failed", Some(ex.code), Some(msg.clone()));
743 st.res.message = Some(msg);
744 return finish(st, Some(code));
745 }
746 }
747 }
748 }
749
750 let mut routed = p.clone();
752 if units.iter().any(|u| u.role == Role::Cpu) {
753 if let Some(n) = p
754 .scope
755 .target
756 .route
757 .as_ref()
758 .filter(|r| r.kind == "command")
759 .and_then(|r| r.receipt_digest_env.as_ref())
760 {
761 match std::env::var(n) {
762 Ok(v) if !v.is_empty() => routed.route_digest = Some(v),
763 _ => {
764 let msg = format!("route.receipt_digest_env: {n} is unset or empty");
765 log("ERROR", "build", &target, &msg);
766 st.phase("build", "refused", Some(exit::REFUSED), Some(msg.clone()));
767 st.res.message = Some(msg);
768 return finish(st, Some(exit::REFUSED));
769 }
770 }
771 }
772 }
773 let p = &routed;
774
775 let mut stems = StemSeq::default();
777 let mut unit_stems: Vec<Vec<String>> = Vec::new();
778 for u in &units {
779 unit_stems.push(u.lines.iter().map(|l| stems.next(&l.easyconfig)).collect());
780 }
781 let mut fetch_failed: Vec<String> = Vec::new();
782 if site.sources_on().is_some() {
783 let mut any_failed = false;
784 for (u, ss) in units.iter().zip(&unit_stems) {
785 for (l, s) in u.lines.iter().zip(ss) {
786 make_route_dirs(p, u.role, "sources");
787 let cmd = match p.prefetch_cmd(l, s, u.role) {
788 Ok(c) => c,
789 Err(f) => {
790 st.res.message = Some(f.msg);
791 return finish(st, Some(f.code));
792 }
793 };
794 let ex = match exec(&cmd, &env, &log_dir.join(format!("{s}.fetch.log"))) {
795 Ok(e) => e,
796 Err(f) => {
797 st.res.message = Some(f.msg);
798 return finish(st, Some(f.code));
799 }
800 };
801 if let Some(c) = stop_if_signalled(&mut st, "sources") {
802 return finish(st, Some(c));
803 }
804 if ex.code != 0 {
805 any_failed = true;
806 fetch_failed.push(s.clone());
807 log(
808 "ERROR",
809 "sources",
810 &target,
811 &format!("{s}: fetch exited {}", ex.code),
812 );
813 st.res.lines.push(LineResult {
814 easyconfig: l.easyconfig.clone(),
815 unit: u.name.clone(),
816 phase: "sources".into(),
817 status: "failed".into(),
818 exit_code: Some(ex.code),
819 translated_from: None,
820 log: ex.log.display().to_string(),
821 note: Some("source fetch failed; line not built".into()),
822 });
823 st.failed = true;
824 }
825 }
826 }
827 st.phase(
828 "sources",
829 if any_failed { "failed" } else { "ok" },
830 None,
831 None,
832 );
833 }
834
835 let adm = site
837 .admission_on()
838 .filter(|a| a.quota_cmd.is_some() && a.reservations.is_some());
839 let keep_failed = site.logs_on().is_some_and(|l| l.keep_failed);
840 let mut stopped = false;
841 for (ui, (u, ss)) in units.iter().zip(&unit_stems).enumerate() {
842 if stopped {
843 for (l, s) in u.lines.iter().zip(ss) {
844 st.res.lines.push(LineResult {
845 easyconfig: l.easyconfig.clone(),
846 unit: u.name.clone(),
847 phase: "build".into(),
848 status: "skipped".into(),
849 exit_code: None,
850 translated_from: None,
851 log: p.log_stdout(s),
852 note: Some("an earlier unit was refused".into()),
853 });
854 }
855 continue;
856 }
857 if let Some(a) = adm {
858 let fp = u.footprint.or(a.default_footprint).unwrap_or(0);
859 let cmd = match p.admission_cmd(&u.name, fp) {
860 Ok(c) => c,
861 Err(f) => {
862 st.res.message = Some(f.msg);
863 return finish(st, Some(f.code));
864 }
865 };
866 let mut attempt = 0u32;
867 let mut wait = capped_wait(a.wait_seconds, a.max_wait_seconds);
868 let outcome = loop {
869 attempt += 1;
870 let ex = match exec(
871 &cmd,
872 &env,
873 &log_dir.join(format!("admission-{}.{attempt}.log", u.name)),
874 ) {
875 Ok(e) => e,
876 Err(f) => {
877 st.res.message = Some(f.msg);
878 return finish(st, Some(f.code));
879 }
880 };
881 if let Some(c) = stop_if_signalled(&mut st, "admission") {
882 return finish(st, Some(c));
883 }
884 let last = last_json_line(&ex.log);
885 match ex.code {
886 0 => {
887 log(
888 "INFO",
889 "admission",
890 &target,
891 &format!("{} admitted on attempt {attempt}", u.name),
892 );
893 break ("admitted", None, last, attempt, 0);
894 }
895 78 => {
896 if attempt >= a.attempts {
897 break ("refused", None, last, attempt, 78);
898 }
899 log(
900 "WARN",
901 "admission",
902 &target,
903 &format!(
904 "{} refused on attempt {attempt} of {}; waiting {wait}s",
905 u.name, a.attempts
906 ),
907 );
908 if sleep_interruptible(wait) {
909 if let Some(c) = stop_if_signalled(&mut st, "admission") {
910 return finish(st, Some(c));
911 }
912 }
913 wait = doubled_wait(wait, a.max_wait_seconds);
914 }
915 c => {
916 let code = if c == 2 { exit::USAGE } else { exit::FAILED };
917 break (
918 "failed",
919 Some(format!("stack admit exited {c}")),
920 last,
921 attempt,
922 code,
923 );
924 }
925 }
926 };
927 let (kind, note, last, attempts, code) = outcome;
928 st.res.admission.push(AdmissionResult {
929 unit: u.name.clone(),
930 attempts,
931 outcome: kind.into(),
932 last_json: last.clone(),
933 });
934 if kind == "admitted" {
935 st.reservation = p.job_id.as_ref().map(|job| Reservation {
936 path: Path::new(a.reservations.as_deref().unwrap_or_default())
937 .join(format!("{job}.json")),
938 job: job.clone(),
939 lock: a.lock.as_ref().map(PathBuf::from),
940 posix: a.lock_kind == "posix",
941 });
942 }
943 if kind != "admitted" {
944 let msg = if kind == "refused" {
945 format!(
946 "admission refused after {attempts} attempts: {}",
947 last.clone().unwrap_or_else(|| "(no JSON line)".into())
948 )
949 } else {
950 format!("admission failed: {}", note.unwrap_or_default())
951 };
952 log("ERROR", "admission", &target, &msg);
953 st.phase("admission", kind, Some(code), Some(msg.clone()));
954 st.res.message = Some(msg);
955 if kind == "refused" {
956 st.refused = true;
957 stopped = true;
958 for (l, s) in u.lines.iter().zip(ss) {
959 st.res.lines.push(LineResult {
960 easyconfig: l.easyconfig.clone(),
961 unit: u.name.clone(),
962 phase: "build".into(),
963 status: "skipped".into(),
964 exit_code: None,
965 translated_from: None,
966 log: p.log_stdout(s),
967 note: Some("admission refused".into()),
968 });
969 }
970 continue;
971 }
972 return finish(st, Some(code));
973 }
974 }
975
976 for (li, (l, s)) in u.lines.iter().zip(ss).enumerate() {
977 if fetch_failed.contains(s) {
978 continue;
979 }
980 if interrupted().is_some() {
981 note_skipped(&mut st, &units, &unit_stems, &fetch_failed, ui, li, p);
982 break;
983 }
984 make_route_dirs(p, u.role, "build");
985 let staged = p.is_staged(l);
987 let cmd = match p.build_cmd(u.role, l, s) {
988 Ok(c) => c,
989 Err(f) => {
990 st.res.message = Some(f.msg);
991 return finish(st, Some(f.code));
992 }
993 };
994 if let Some(w) = p.in_place_reason(l) {
995 log("WARN", "build", &target, &format!("{s}: {w}"));
996 }
997 log("INFO", "build", &target, &format!("{s}: start"));
998 let ex = match exec(&cmd, &env, Path::new(&p.log_stdout(s))) {
999 Ok(e) => e,
1000 Err(f) => {
1001 st.res.message = Some(f.msg);
1002 return finish(st, Some(f.code));
1003 }
1004 };
1005 let mut lr = LineResult {
1006 easyconfig: l.easyconfig.clone(),
1007 unit: u.name.clone(),
1008 phase: "build".into(),
1009 status: "success".into(),
1010 exit_code: Some(ex.code),
1011 translated_from: None,
1012 log: ex.log.display().to_string(),
1013 note: staged.then(|| "ran through eb-staged-rebuild".into()),
1014 };
1015 if ex.code != 0 {
1016 let outcome = if staged {
1017 map_staged(ex.code)
1018 } else {
1019 StagedOutcome::Failed
1020 };
1021 let mut detail = None;
1022 match outcome {
1023 StagedOutcome::Refused => {
1024 lr.status = "refused".into();
1025 lr.translated_from = Some(ex.code);
1026 lr.exit_code = Some(exit::REFUSED);
1027 detail = lock_refusal(p, &cmd, &env, &ex.log);
1028 lr.note = Some(match &detail {
1029 Some(d) => {
1030 format!("eb-staged-rebuild exit 2: {d}; translated to refused (78)")
1031 }
1032 None => {
1033 "eb-staged-rebuild exit 2 is a refusal; translated to refused (78)"
1034 .into()
1035 }
1036 });
1037 st.refused = true;
1038 }
1039 StagedOutcome::Pending => {
1040 lr.status = "pending".into();
1041 lr.note = Some(
1042 "eb-staged-rebuild exit 75: a required ledger event is pending".into(),
1043 );
1044 st.pending = true;
1045 }
1046 _ => {
1047 lr.status = "failed".into();
1048 st.failed = true;
1049 }
1050 }
1051 let detail = detail.map(|d| format!(": {d}")).unwrap_or_default();
1052 log(
1053 "ERROR",
1054 "build",
1055 &target,
1056 &format!("{s}: {} (exit {}){detail}", lr.status, ex.code),
1057 );
1058 if interrupted().is_none() && keep_failed && lr.status == "failed" {
1059 if let Ok(kc) = p.keep_failed_cmd(s, &ex.code.to_string()) {
1060 let kx = exec(&kc, &env, &log_dir.join(format!("{s}.keep-logs.log")));
1061 match kx {
1062 Ok(k) if k.code == 0 => {}
1063 Ok(k) => log(
1064 "WARN",
1065 "logs",
1066 &target,
1067 &format!("{s}: eb-keep-logs failed exited {}", k.code),
1068 ),
1069 Err(f) => log("WARN", "logs", &target, &f.msg),
1070 }
1071 }
1072 }
1073 } else {
1074 log("INFO", "build", &target, &format!("{s}: ok"));
1075 }
1076 st.res.lines.push(lr);
1077 if interrupted().is_some() {
1078 note_skipped(&mut st, &units, &unit_stems, &fetch_failed, ui, li + 1, p);
1079 break;
1080 }
1081 }
1082 if let Some(r) = st.reservation.take() {
1083 release(r, &target);
1084 }
1085 if interrupted().is_some() {
1086 break;
1087 }
1088 }
1089 if let Some(c) = stop_if_signalled(&mut st, "build") {
1090 return finish(st, Some(c));
1091 }
1092 if site.admission_on().is_some() && !st.res.admission.is_empty() && !st.refused {
1093 st.phase("admission", "ok", Some(0), None);
1094 }
1095 let build: Vec<&str> = st
1096 .res
1097 .lines
1098 .iter()
1099 .filter(|l| l.phase == "build")
1100 .map(|l| l.status.as_str())
1101 .collect();
1102 let (status, note) = if build.contains(&"failed") {
1103 ("failed", None)
1104 } else if st.refused && build.iter().all(|s| *s == "skipped") {
1105 (
1106 "skipped",
1107 Some("admission refused before any build line started".to_string()),
1108 )
1109 } else if st.refused && !build.contains(&"success") {
1110 ("refused", None)
1111 } else {
1112 ("ok", None)
1113 };
1114 st.phase("build", status, None, note);
1115
1116 if !st.failed && !st.refused && !st.pending {
1118 match p.cache_cmd() {
1119 Ok(Some(cmd)) => {
1120 log("INFO", "caches", &target, "start");
1121 match exec(&cmd, &env, &log_dir.join("caches.log")) {
1122 Ok(ex) => {
1123 if ex.code == 0 {
1124 st.phase("caches", "ok", Some(0), None);
1125 } else {
1126 let msg = format!("cache command exited {}", ex.code);
1127 log("ERROR", "caches", &target, &msg);
1128 st.phase("caches", "failed", Some(ex.code), Some(msg));
1129 st.failed = true;
1130 }
1131 }
1132 Err(f) => {
1133 st.res.message = Some(f.msg);
1134 return finish(st, Some(f.code));
1135 }
1136 }
1137 if let Some(c) = stop_if_signalled(&mut st, "caches") {
1138 return finish(st, Some(c));
1139 }
1140 }
1141 Ok(None) => {}
1142 Err(f) => {
1143 st.res.message = Some(f.msg);
1144 return finish(st, Some(f.code));
1145 }
1146 }
1147 } else if p.cache_cmd().ok().flatten().is_some() {
1148 st.phase(
1149 "caches",
1150 "skipped",
1151 None,
1152 Some("the build did not finish cleanly".into()),
1153 );
1154 }
1155
1156 if let Ok(Some(cmd)) = p.rotate_cmd(true) {
1158 let lock = p.rotate_lock().unwrap_or_default();
1159 match rotate_lock.map(rotate_alone) {
1160 Some(Ok(None)) => {
1161 let msg = format!(
1162 "another run under {} holds {}, live or rotating; the rotation is left to a run that ends alone",
1163 site.filesystems.log_root,
1164 lock.display()
1165 );
1166 log("INFO", "logs", &target, &format!("rotation skipped: {msg}"));
1167 st.phase("logs", "skipped", None, Some(msg));
1168 }
1169 alone => {
1170 if let Some(Err(e)) = &alone {
1171 let msg = format!(
1172 "cannot take {} alone: {e}; rotating without it",
1173 lock.display()
1174 );
1175 log("WARN", "logs", &target, &msg);
1176 }
1177 let bin = Tool::EbKeepLogs.bin(site);
1178 let mut probe_env = env.clone();
1179 probe_env.extend(cmd.env.clone());
1180 let note = match help_lists(&bin, &probe_env, "--lock-timeout") {
1181 Ok(true) => None,
1182 Ok(false) => Some(format!(
1183 "`{bin} --help` does not list it, as in eb-keep-logs 0.2.1, which takes no lock of its own"
1184 )),
1185 Err(e) => Some(e),
1186 }
1187 .map(|why| format!("no --lock-timeout for eb-keep-logs rotate: {why}"));
1188 let cmd = match ¬e {
1189 None => cmd,
1190 Some(n) => {
1191 log("INFO", "logs", &target, n);
1192 p.rotate_cmd(false).ok().flatten().unwrap_or(cmd)
1193 }
1194 };
1195 match exec(&cmd, &env, &log_dir.join("rotate.log")) {
1196 Ok(ex) => match map_rotate(ex.code) {
1197 RotateOutcome::Ok => st.phase("logs", "ok", Some(0), note),
1198 RotateOutcome::Locked => {
1199 let msg = format!(
1200 "eb-keep-logs rotate exited 75: another rotate holds {}, and this one changed nothing; the rotation is left to it",
1201 p.keep_logs_lock().display()
1202 );
1203 log("INFO", "logs", &target, &format!("rotation skipped: {msg}"));
1204 st.phase("logs", "skipped", Some(ex.code), Some(msg));
1205 }
1206 RotateOutcome::Failed => {
1207 let msg = format!("eb-keep-logs rotate exited {}", ex.code);
1208 log("ERROR", "logs", &target, &msg);
1209 st.phase("logs", "failed", Some(ex.code), Some(msg));
1210 st.failed = true;
1211 }
1212 },
1213 Err(f) => {
1214 st.res.message = Some(f.msg);
1215 return finish(st, Some(f.code));
1216 }
1217 }
1218 if let Some(c) = stop_if_signalled(&mut st, "logs") {
1219 return finish(st, Some(c));
1220 }
1221 }
1222 }
1223 }
1224
1225 finish(st, None)
1226}
1227
1228fn note_skipped(
1231 st: &mut State,
1232 units: &[Unit],
1233 stems: &[Vec<String>],
1234 fetch_failed: &[String],
1235 from_unit: usize,
1236 from_line: usize,
1237 p: &Planner,
1238) {
1239 for (ui, (u, ss)) in units.iter().zip(stems).enumerate().skip(from_unit) {
1240 let start = if ui == from_unit { from_line } else { 0 };
1241 for (l, s) in u.lines.iter().zip(ss).skip(start) {
1242 if fetch_failed.iter().any(|f| f == s) {
1243 continue;
1244 }
1245 st.res.lines.push(LineResult {
1246 easyconfig: l.easyconfig.clone(),
1247 unit: u.name.clone(),
1248 phase: "build".into(),
1249 status: "skipped".into(),
1250 exit_code: None,
1251 translated_from: None,
1252 log: p.log_stdout(s),
1253 note: Some("interrupted before this line started".into()),
1254 });
1255 }
1256 }
1257}
1258
1259fn make_route_dirs(p: &Planner, role: Role, phase: &str) {
1262 if role == Role::Cpu {
1263 for d in p.scope.route_dirs(phase) {
1264 let _ = std::fs::create_dir_all(d);
1265 }
1266 }
1267}
1268
1269fn stop_if_signalled(st: &mut State, phase: &str) -> Option<i32> {
1270 let sig = interrupted()?;
1271 let (name, code) = if sig == libc::SIGINT {
1272 ("SIGINT", exit::SIGINT)
1273 } else {
1274 ("SIGTERM", exit::SIGTERM)
1275 };
1276 log(
1277 "WARN",
1278 phase,
1279 &st.res.target.clone(),
1280 &format!("{name} received; child ended, stopping"),
1281 );
1282 st.res.interrupted_by = Some(name.into());
1283 st.phase(phase, "interrupted", None, None);
1284 st.forced_code = Some(code);
1285 Some(code)
1286}
1287
1288fn finish(mut st: State, forced: Option<i32>) -> RunResult {
1289 if let Some(r) = st.reservation.take() {
1290 release(r, &st.res.target);
1291 }
1292 st.res.ended_at = chrono::Utc::now().to_rfc3339();
1293 let mut counts: BTreeMap<String, usize> = BTreeMap::new();
1294 for l in &st.res.lines {
1295 *counts.entry(l.status.clone()).or_default() += 1;
1296 }
1297 st.res.counts = counts;
1298 st.res.phases.sort_by_key(|ph| {
1299 PHASES
1300 .iter()
1301 .position(|n| *n == ph.name)
1302 .unwrap_or(PHASES.len())
1303 });
1304 st.res.exit_code = match forced.or(st.forced_code) {
1305 Some(c) => c,
1306 None if st.failed => exit::FAILED,
1307 None if st.refused => exit::REFUSED,
1308 None if st.pending => exit::TEMPFAIL,
1309 None => exit::OK,
1310 };
1311 st.res
1312}
1313
1314#[cfg(test)]
1315mod tests {
1316 use super::{capped_wait, doubled_wait};
1317
1318 #[test]
1319 fn admission_wait_never_exceeds_the_configured_maximum() {
1320 assert_eq!(capped_wait(60, 600), 60);
1321 assert_eq!(doubled_wait(60, 600), 120);
1322 assert_eq!(doubled_wait(480, 600), 600);
1323 assert_eq!(doubled_wait(600, 600), 600);
1324 assert_eq!(capped_wait(120, 50), 50);
1326 assert_eq!(doubled_wait(50, 50), 50);
1327 assert_eq!(doubled_wait(120, 50), 50);
1328 assert_eq!(capped_wait(5, 0), 0);
1329 }
1330}