1use crate::exit::{self, Fail};
6use crate::resolve::{Role, Scope};
7use crate::tools::Tool;
8use crate::util::{parse_list, ListLine};
9use serde::Serialize;
10use std::collections::BTreeMap;
11use std::path::{Path, PathBuf};
12
13#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
15pub struct Cmd {
16 pub name: String,
19 pub argv: Vec<String>,
21 #[serde(skip_serializing_if = "BTreeMap::is_empty")]
23 pub env: BTreeMap<String, String>,
24 #[serde(skip_serializing_if = "Option::is_none")]
26 pub note: Option<String>,
27}
28
29#[derive(Debug, Clone, Serialize)]
31pub struct PlanPhase {
32 pub name: String,
34 pub status: String,
37 #[serde(skip_serializing_if = "Option::is_none")]
39 pub reason: Option<String>,
40 #[serde(skip_serializing_if = "Option::is_none")]
42 pub role: Option<String>,
43 pub env: BTreeMap<String, String>,
45 pub commands: Vec<Cmd>,
47}
48
49#[derive(Debug, Clone, Serialize)]
51pub struct Plan {
52 pub schema_version: u32,
54 pub kind: String,
56 pub site: String,
58 pub target: String,
60 pub build_type: String,
62 pub run_id: String,
64 pub list: String,
66 pub log_dir: String,
68 pub lines: usize,
70 #[serde(skip_serializing_if = "Option::is_none")]
72 pub selected: Option<usize>,
73 #[serde(skip_serializing_if = "Option::is_none")]
75 pub selected_list: Option<String>,
76 pub phases: Vec<PlanPhase>,
78 #[serde(skip_serializing_if = "Vec::is_empty")]
80 pub deselected: Vec<Deselected>,
81}
82
83#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
85pub struct Deselected {
86 pub easyconfig: String,
88 pub unit: String,
90}
91
92pub const PHASES: [&str; 8] = [
94 "preflight",
95 "licences",
96 "shard",
97 "sources",
98 "admission",
99 "build",
100 "caches",
101 "logs",
102];
103
104#[derive(Debug, Clone)]
106pub struct Unit {
107 pub name: String,
109 pub role: Role,
111 pub lines: Vec<ListLine>,
113 pub footprint: Option<u64>,
115}
116
117#[derive(Clone)]
119pub struct Planner<'a> {
120 pub scope: Scope<'a>,
122 pub list_path: String,
124 pub job_id: Option<String>,
126 pub route_digest: Option<String>,
128 pub runtime_env: Option<BTreeMap<String, String>>,
132}
133
134pub fn module_rel(easyconfig: &str) -> Option<String> {
136 let (name, _) = crate::util::package_of(easyconfig);
137 let base = easyconfig.rsplit('/').next().unwrap_or(easyconfig);
138 let stem = base.strip_suffix(".eb").unwrap_or(base);
139 let version = stem.strip_prefix(&format!("{name}-"))?;
140 Some(format!("{name}/{version}"))
141}
142
143pub fn wants_rebuild(line: &ListLine) -> bool {
145 line.args
146 .iter()
147 .any(|a| a == "--rebuild" || a == "-f" || a == "--force")
148}
149
150pub fn without_robot<'a>(args: impl Iterator<Item = &'a String>) -> Vec<String> {
156 let mut robot: Option<&str> = None;
157 let mut out: Vec<String> = Vec::new();
158 for a in args {
159 if a == "--robot" {
160 robot = Some("");
161 } else if let Some(p) = a.strip_prefix("--robot=") {
162 robot = Some(p);
163 } else {
164 out.push(a.clone());
165 }
166 }
167 let Some(lead) = robot.filter(|p| !p.is_empty()) else {
168 return out;
169 };
170 match out.iter().rposition(|a| a.starts_with("--robot-paths=")) {
171 Some(i) => out[i] = format!("--robot-paths={lead}:{}", &out[i]["--robot-paths=".len()..]),
172 None => out.push(format!("--robot-paths={lead}:")),
173 }
174 out
175}
176
177pub fn valid_job_id(s: &str) -> bool {
179 !s.is_empty() && !s.starts_with('0') && s.chars().all(|c| c.is_ascii_digit())
180}
181
182fn stem(easyconfig: &str) -> String {
183 let base = easyconfig.rsplit('/').next().unwrap_or(easyconfig);
184 base.strip_suffix(".eb").unwrap_or(base).to_string()
185}
186
187#[derive(Default)]
189pub struct StemSeq(BTreeMap<String, usize>);
190
191impl StemSeq {
192 pub fn next(&mut self, easyconfig: &str) -> String {
195 let s = stem(easyconfig);
196 let n = self.0.entry(s.clone()).or_insert(0);
197 *n += 1;
198 if *n == 1 {
199 s
200 } else {
201 format!("{s}.{n}")
202 }
203 }
204}
205
206pub fn unique_stems(lines: &[ListLine]) -> Vec<String> {
208 let mut seq = StemSeq::default();
209 lines.iter().map(|l| seq.next(&l.easyconfig)).collect()
210}
211
212impl<'a> Planner<'a> {
213 pub fn new(scope: Scope<'a>, list_path: String) -> Self {
215 Planner {
216 scope,
217 list_path,
218 job_id: None,
219 route_digest: None,
220 runtime_env: None,
221 }
222 }
223
224 pub fn select_lines(&self, lines: &[ListLine]) -> (Vec<ListLine>, Vec<Deselected>) {
228 let Some(sel) = &self.scope.target.select else {
229 return (lines.to_vec(), Vec::new());
230 };
231 let mut kept = Vec::new();
232 let mut out = Vec::new();
233 for l in lines {
234 let r = crate::select::select_line(sel, l);
235 if r.planned {
236 let raw = if sel.strip {
237 std::iter::once(&l.easyconfig)
238 .chain(&r.args)
239 .map(|a| quote(a))
240 .collect::<Vec<_>>()
241 .join(" ")
242 } else {
243 l.raw.clone()
244 };
245 kept.push(ListLine {
246 raw,
247 args: r.args,
248 ..l.clone()
249 });
250 } else {
251 out.push(Deselected {
252 easyconfig: l.easyconfig.clone(),
253 unit: "all".into(),
254 });
255 }
256 }
257 (kept, out)
258 }
259
260 pub fn selected_list_path(&self) -> Option<PathBuf> {
262 self.scope.target.select.as_ref()?;
263 Some(
264 PathBuf::from(self.scope.log_dir())
265 .join(format!("selected-{}.list", self.scope.target.name)),
266 )
267 }
268
269 fn shard_list(&self) -> String {
271 match self.selected_list_path() {
272 Some(p) => p.display().to_string(),
273 None => self.list_path.clone(),
274 }
275 }
276
277 fn site(&self) -> &crate::schema::Site {
278 self.scope.site
279 }
280
281 fn wrapped(
282 &self,
283 role: Role,
284 phase: &str,
285 inner: Vec<String>,
286 modules: bool,
287 ) -> Result<Vec<String>, Fail> {
288 self.wrapped_routed(
289 role,
290 phase,
291 inner,
292 modules,
293 (role == Role::Cpu).then_some(phase),
294 )
295 }
296
297 fn wrapped_routed(
298 &self,
299 role: Role,
300 phase: &str,
301 inner: Vec<String>,
302 modules: bool,
303 route_phase: Option<&str>,
304 ) -> Result<Vec<String>, Fail> {
305 let mut v = self.scope.launch_routed(role, phase, None, route_phase)?;
306 if modules {
307 v.extend(self.scope.module_wrap());
308 }
309 v.extend(inner);
310 Ok(v)
311 }
312
313 pub fn shard_dir(&self) -> String {
316 let base = self
317 .site()
318 .filesystems
319 .shard_dir
320 .clone()
321 .unwrap_or_else(|| format!("{}/shards", self.site().filesystems.scratch));
322 format!("{}/{}", base.trim_end_matches('/'), self.scope.run_id)
323 }
324
325 fn installed_modules_dirs(&self) -> Vec<String> {
328 let s = self.site().shards_on().expect("shards active");
329 let mut v = s.installed_modules_dirs.clone();
330 for n in &s.installed_modules_dirs_from_env {
331 match &self.runtime_env {
332 None => v.push(format!("${{{n}}}")),
333 Some(vars) => v.extend(
334 vars.get(n)
335 .map(String::as_str)
336 .unwrap_or("")
337 .split(':')
338 .filter(|d| !d.is_empty())
339 .map(str::to_string),
340 ),
341 }
342 }
343 v
344 }
345
346 pub fn missing_installed_modules_dirs(&self) -> Vec<String> {
350 if self.runtime_env.is_none() {
351 return Vec::new();
352 }
353 self.installed_modules_dirs()
354 .into_iter()
355 .filter(|d| {
356 std::fs::metadata(d).is_err_and(|e| e.kind() == std::io::ErrorKind::NotFound)
357 })
358 .collect()
359 }
360
361 pub fn shard_cmd(&self) -> Result<Cmd, Fail> {
364 let s = self.site().shards_on().expect("shards active");
365 let mut a = vec![
366 Tool::EbStack.bin(self.site()),
367 "stack".into(),
368 "shard".into(),
369 "--build-list".into(),
370 self.shard_list(),
371 "--shards".into(),
372 s.count.to_string(),
373 "--out-dir".into(),
374 self.shard_dir(),
375 ];
376 for e in &s.easyconfigs {
377 a.push("--easyconfigs".into());
378 a.push(e.clone());
379 }
380 let missing = self.missing_installed_modules_dirs();
381 for d in self.installed_modules_dirs() {
382 if !missing.contains(&d) {
383 a.push("--installed-modules-dir".into());
384 a.push(d);
385 }
386 }
387 if let Some(w) = &s.weights {
388 a.push("--weights".into());
389 a.push(w.clone());
390 }
391 if s.lift_cpu_prerequisites {
392 a.push("--lift-cpu-prerequisites".into());
393 }
394 if let Some(adm) = self.site().admission_on() {
395 if let Some(q) = adm.quota_total {
396 a.push("--inode-quota".into());
397 a.push(q.to_string());
398 if let Some(m) = adm.margin {
399 a.push("--inode-margin".into());
400 a.push(m.to_string());
401 }
402 }
403 if let Some(f) = &adm.footprints {
404 a.push("--inode-footprints".into());
405 a.push(f.clone());
406 }
407 if let Some(o) = adm.line_overhead {
408 a.push("--inode-line-overhead".into());
409 a.push(o.to_string());
410 }
411 }
412 Ok(Cmd {
413 name: "shard".into(),
414 argv: self.wrapped(Role::Plan, "shard", a, true)?,
415 env: BTreeMap::new(),
416 note: Some(format!(
417 "exit {} means the plan was built unsharded and the list builds as one unit",
418 s.planner_exit_unsharded
419 )),
420 })
421 }
422
423 fn route_phase(role: Role, phase: &'static str) -> Option<&'static str> {
425 (role == Role::Cpu).then_some(phase)
426 }
427
428 fn route_env(&self, role: Role, phase: &'static str) -> BTreeMap<String, String> {
430 match Self::route_phase(role, phase) {
431 Some(p) => self.scope.route_env(p, self.route_digest.as_deref()),
432 None => BTreeMap::new(),
433 }
434 }
435
436 pub fn prefetch_cmd(&self, line: &ListLine, name: &str, unit_role: Role) -> Result<Cmd, Fail> {
439 let extra = self
440 .site()
441 .sources_on()
442 .map(|s| s.extra_args.clone())
443 .unwrap_or_default();
444 let args = line.args.iter().filter(|x| !x.starts_with("--rebuild"));
445 let mut a = vec!["eb".to_string(), line.easyconfig.clone(), "--fetch".into()];
446 if extra.iter().any(|x| x == "--disable-robot") {
447 a.extend(without_robot(args));
448 } else {
449 a.push("--robot".into());
450 a.extend(args.cloned());
451 }
452 a.extend(extra);
453 let mut env = self.scope.line_env(&line.easyconfig);
454 env.extend(self.scope.fetch_env());
455 env.extend(self.scope.hardware_env(unit_role));
456 env.extend(self.route_env(unit_role, "sources"));
457 Ok(Cmd {
458 name: format!("fetch:{name}"),
459 argv: self.wrapped_routed(
460 Role::Short,
461 "sources",
462 a,
463 true,
464 Self::route_phase(unit_role, "sources"),
465 )?,
466 env,
467 note: None,
468 })
469 }
470
471 pub fn admission_cmd(&self, unit: &str, footprint: u64) -> Result<Cmd, Fail> {
474 let adm = self.site().admission_on().expect("admission active");
475 let job = self
476 .job_id
477 .clone()
478 .unwrap_or_else(|| format!("${{{}}}", adm.job_id_env));
479 let mut a = vec![
480 Tool::EbStack.bin(self.site()),
481 "stack".into(),
482 "admit".into(),
483 "--quota-cmd".into(),
484 adm.quota_cmd.clone().unwrap_or_default(),
485 "--reservations".into(),
486 adm.reservations.clone().unwrap_or_default(),
487 "--footprint".into(),
488 footprint.to_string(),
489 "--job".into(),
490 job,
491 "--lock-kind".into(),
492 adm.lock_kind.clone(),
493 ];
494 if let Some(l) = &adm.lock {
495 a.push("--lock".into());
496 a.push(l.clone());
497 }
498 if let Some(m) = adm.margin {
499 a.push("--margin".into());
500 a.push(m.to_string());
501 }
502 if let Some(u) = adm.unknown_reserve {
503 a.push("--unknown-reserve".into());
504 a.push(u.to_string());
505 }
506 if let Some(c) = &adm.live_jobs_cmd {
507 a.push("--live-jobs-cmd".into());
508 a.push(c.clone());
509 }
510 Ok(Cmd {
511 name: format!("admit:{unit}"),
512 argv: self.wrapped(Role::Short, "admission", a, false)?,
513 env: BTreeMap::new(),
514 note: Some(format!(
515 "exit 78 retries up to {} attempts, waiting {}s doubling to at most {}s",
516 adm.attempts, adm.wait_seconds, adm.max_wait_seconds
517 )),
518 })
519 }
520
521 pub fn staged_enabled(&self) -> bool {
523 match self.site().rebuild_on() {
524 Some(r) => self.scope.target.staged.unwrap_or(r.staged),
525 None => false,
526 }
527 }
528
529 pub fn installed(&self, line: &ListLine) -> bool {
531 let Some(rel) = module_rel(&line.easyconfig) else {
532 return false;
533 };
534 let base = format!("{}/all/{}", self.scope.modules_root(), rel);
535 std::path::Path::new(&format!("{base}.lua")).is_file()
536 || std::path::Path::new(&base).is_file()
537 }
538
539 pub fn is_staged(&self, line: &ListLine) -> bool {
542 self.staged_enabled() && wants_rebuild(line) && self.installed(line)
543 }
544
545 pub fn staged_run_id(&self, stem: &str) -> String {
548 format!("{}-{stem}", self.scope.run_id)
549 }
550
551 pub fn state_root_lock(&self, env_root: Option<&str>) -> PathBuf {
557 let root = self
558 .scope
559 .state_root()
560 .or_else(|| env_root.filter(|r| !r.is_empty()).map(str::to_string))
561 .map(PathBuf::from)
562 .unwrap_or_else(|| {
563 let software = PathBuf::from(self.scope.software_root());
564 let parent = software.parent().unwrap_or(Path::new("/"));
565 parent.join(".eb-staged-rebuild")
566 });
567 root.join(".lock")
568 }
569
570 pub fn in_place_reason(&self, line: &ListLine) -> Option<String> {
573 if !self.staged_enabled() || !wants_rebuild(line) || self.installed(line) {
574 return None;
575 }
576 let rel = module_rel(&line.easyconfig).unwrap_or_else(|| "<Name>/<version>".into());
577 Some(format!(
578 "no module file {}/all/{rel}[.lua]; the rebuild runs as plain eb",
579 self.scope.modules_root()
580 ))
581 }
582
583 pub fn log_stdout(&self, stem: &str) -> String {
585 format!("{}/{}.stdout.log", self.scope.log_dir(), stem)
586 }
587
588 pub fn tmp_logdir(&self, stem: &str) -> String {
591 format!("{}/easybuild-logs/{}", self.scope.log_dir(), stem)
592 }
593
594 pub fn build_cmd(&self, role: Role, line: &ListLine, stem: &str) -> Result<Cmd, Fail> {
598 let mut inner: Vec<String>;
599 let note;
600 if self.is_staged(line) {
601 let r = self.site().rebuild_on().expect("rebuild active");
602 inner = vec![
603 Tool::EbStagedRebuild.bin(self.site()),
604 "run".into(),
605 "--software".into(),
606 self.scope.software_root(),
607 "--modules".into(),
608 self.scope.modules_root(),
609 "--backend".into(),
610 r.backend.clone(),
611 "--run-id".into(),
612 self.staged_run_id(stem),
613 ];
614 if let Some(s) = self.scope.state_root() {
615 inner.push("--state-root".into());
616 inner.push(s);
617 }
618 inner.push("--lock-timeout".into());
619 inner.push(r.lock_timeout.to_string());
620 if let Some(g) = r.grace_hours {
621 inner.push("--grace-hours".into());
622 inner.push(g.to_string());
623 }
624 if let Some(p) = &r.prepare {
625 inner.push("--prepare".into());
626 inner.push(p.clone());
627 }
628 if let Some(c) = self.scope.ledger_config() {
629 inner.push("--ledger-config".into());
630 inner.push(c);
631 }
632 inner.push("--".into());
633 note = Some("staged rebuild: exit 2 is a refusal, 75 is pending".to_string());
634 } else {
635 inner = vec!["eb".into()];
636 note = self.in_place_reason(line);
637 }
638 inner.push(line.easyconfig.clone());
639 inner.extend(line.args.iter().cloned());
640 if self.site().logs_on().is_some_and(|l| l.keep_failed) {
641 inner.push(format!("--tmp-logdir={}", self.tmp_logdir(stem)));
642 }
643 let mut env = self.scope.line_env(&line.easyconfig);
644 env.extend(self.scope.hardware_env(role));
645 env.extend(self.route_env(role, "build"));
646 Ok(Cmd {
647 name: format!("build:{stem}"),
648 argv: self.wrapped(role, "build", inner, true)?,
649 env,
650 note,
651 })
652 }
653
654 pub fn keep_failed_cmd(&self, stem: &str, status: &str) -> Result<Cmd, Fail> {
657 let a = vec![
658 Tool::EbKeepLogs.bin(self.site()),
659 "failed".into(),
660 status.to_string(),
661 stem.to_string(),
662 self.scope.log_dir(),
663 self.tmp_logdir(stem),
664 ];
665 let env = BTreeMap::from([("EASYBUILD_OUTPUT_STYLE".to_string(), "basic".to_string())]);
668 Ok(Cmd {
669 name: format!("keep-failed:{stem}"),
670 argv: self.wrapped(Role::Short, "logs", a, true)?,
671 env,
672 note: Some("runs after a failed line".into()),
673 })
674 }
675
676 pub fn rotate_cmd(&self, lock_timeout: bool) -> Result<Option<Cmd>, Fail> {
680 let Some(l) = self.site().logs_on() else {
681 return Ok(None);
682 };
683 let Some(r) = &l.rotate else {
684 return Ok(None);
685 };
686 let mut a = vec![
687 Tool::EbKeepLogs.bin(self.site()),
688 "rotate".into(),
689 self.site().filesystems.log_root.clone(),
690 "--compress-after".into(),
691 r.compress_after_days.to_string(),
692 ];
693 if let Some(d) = r.delete_after_days {
694 a.push("--delete-after".into());
695 a.push(d.to_string());
696 }
697 if lock_timeout {
698 a.push("--lock-timeout".into());
703 a.push("0".into());
704 }
705 Ok(Some(Cmd {
706 name: "rotate".into(),
707 argv: self.wrapped(Role::Short, "logs", a, false)?,
708 env: BTreeMap::from([("TZ".to_string(), "UTC".to_string())]),
711 note: lock_timeout.then(|| {
712 "without --lock-timeout 0 when `eb-keep-logs --help` does not list it, as in eb-keep-logs 0.2.1".into()
713 }),
714 }))
715 }
716
717 pub fn rotate_lock(&self) -> Option<PathBuf> {
724 self.site().logs_on()?.rotate.as_ref()?;
725 Some(PathBuf::from(&self.site().filesystems.log_root).join(".eb-pipeline-rotate.lock"))
726 }
727
728 pub fn keep_logs_lock(&self) -> PathBuf {
731 PathBuf::from(&self.site().filesystems.log_root).join(".eb-keep-logs-rotate.lock")
732 }
733
734 pub fn cache_cmd(&self) -> Result<Option<Cmd>, Fail> {
737 let Some(c) = self.site().caches_on() else {
738 return Ok(None);
739 };
740 let Some(command) = &c.command else {
741 return Ok(None);
742 };
743 let Some(entry) = c
744 .targets
745 .iter()
746 .find(|t| t.target == self.scope.target.name)
747 else {
748 return Ok(None);
749 };
750 let mut env = BTreeMap::new();
751 env.insert("EB_PIPELINE_CACHE_CONFIG".to_string(), entry.config.clone());
752 if let Some(b) = &c.bindings {
753 env.insert("EB_PIPELINE_CACHE_BINDINGS".into(), b.clone());
754 }
755 if let Some(m) = entry.init_module.as_ref().or(c.init_module.as_ref()) {
756 env.insert("EB_PIPELINE_CACHE_INIT_MODULE".into(), m.clone());
757 }
758 let mut v = self
759 .scope
760 .launch_with(Role::Cache, "caches", entry.alloc.as_deref())?;
761 v.extend(self.scope.module_wrap());
762 v.extend(crate::util::split_args(command)?);
763 Ok(Some(Cmd {
764 name: "cache".into(),
765 argv: v,
766 env,
767 note: None,
768 }))
769 }
770}
771
772pub fn build_plan(
774 p: &Planner,
775 lines: &[ListLine],
776 shard_units: Option<&[Unit]>,
777) -> Result<Plan, Fail> {
778 let site = p.scope.site;
779 let env = p.scope.env();
780 let mut phases = Vec::new();
781 let skip = |name: &str, reason: &str| PlanPhase {
782 name: name.into(),
783 status: "not configured".into(),
784 reason: Some(reason.into()),
785 role: None,
786 env: BTreeMap::new(),
787 commands: vec![],
788 };
789 let planned = |name: &str, role: Option<Role>, commands: Vec<Cmd>| PlanPhase {
790 name: name.into(),
791 status: "planned".into(),
792 reason: None,
793 role: role.map(|r| r.name().to_string()),
794 env: env.clone(),
795 commands,
796 };
797
798 let probes: Vec<Cmd> = crate::tools::needed(site)
800 .into_iter()
801 .filter_map(|t| {
802 let min = t.min_version(site)?;
803 Some(Cmd {
804 name: format!("probe:{}", t.key()),
805 argv: vec![t.bin(site), "--version".into()],
806 env: BTreeMap::new(),
807 note: Some(format!("min_version {min}")),
808 })
809 })
810 .collect();
811 phases.push(PlanPhase {
812 name: "preflight".into(),
813 status: "planned".into(),
814 reason: None,
815 role: None,
816 env: BTreeMap::new(),
817 commands: probes,
818 });
819
820 phases.push(if site.licences_on().is_some() {
821 PlanPhase {
822 name: "licences".into(),
823 status: "planned".into(),
824 reason: None,
825 role: None,
826 env: BTreeMap::new(),
827 commands: vec![],
828 }
829 } else {
830 skip("licences", "not configured")
831 });
832
833 phases.push(if site.shards_on().is_some() {
834 planned("shard", Some(Role::Plan), vec![p.shard_cmd()?])
835 } else {
836 skip("shard", "not configured")
837 });
838
839 let (kept, deselected) = p.select_lines(lines);
841 let units: Vec<Unit> = match shard_units {
842 Some(u) if !u.is_empty() => u.to_vec(),
843 _ if p.scope.target.select.is_some() && kept.is_empty() => vec![],
844 _ => vec![Unit {
845 name: "all".into(),
846 role: Role::Build,
847 lines: kept.clone(),
848 footprint: None,
849 }],
850 };
851 let mut seq = StemSeq::default();
852 let stems: Vec<Vec<String>> = units
853 .iter()
854 .map(|u| u.lines.iter().map(|l| seq.next(&l.easyconfig)).collect())
855 .collect();
856 phases.push(if site.sources_on().is_some() {
857 let mut cmds = Vec::new();
858 for (u, ss) in units.iter().zip(&stems) {
859 for (l, s) in u.lines.iter().zip(ss) {
860 cmds.push(p.prefetch_cmd(l, s, u.role)?);
861 }
862 }
863 planned("sources", Some(Role::Short), cmds)
864 } else {
865 skip("sources", "not configured")
866 });
867
868 phases.push(match site.admission_on() {
869 None => skip("admission", "not configured"),
870 Some(a) if a.quota_cmd.is_none() || a.reservations.is_none() => {
871 if a.required {
872 PlanPhase {
873 name: "admission".into(),
874 status: "refused".into(),
875 reason: Some(
876 "admission.required is set but quota_cmd or reservations is missing".into(),
877 ),
878 role: None,
879 env: BTreeMap::new(),
880 commands: vec![],
881 }
882 } else {
883 skip("admission", "not configured")
884 }
885 }
886 Some(a) => {
887 let mut cmds = Vec::new();
888 for u in &units {
889 let fp = u.footprint.or(a.default_footprint).unwrap_or(0);
890 cmds.push(p.admission_cmd(&u.name, fp)?);
891 }
892 planned("admission", Some(Role::Short), cmds)
893 }
894 });
895
896 let mut cmds = Vec::new();
897 for (u, ss) in units.iter().zip(&stems) {
898 for (l, s) in u.lines.iter().zip(ss) {
899 cmds.push(p.build_cmd(u.role, l, s)?);
900 if site.logs_on().is_some_and(|l| l.keep_failed) {
901 cmds.push(p.keep_failed_cmd(s, "<exit-status>")?);
902 }
903 }
904 }
905 phases.push(planned("build", Some(Role::Build), cmds));
906
907 phases.push(match p.cache_cmd()? {
908 Some(c) => planned("caches", Some(Role::Cache), vec![c]),
909 None => skip("caches", "not configured"),
910 });
911
912 phases.push(match p.rotate_cmd(true)? {
913 Some(c) => planned("logs", Some(Role::Short), vec![c]),
914 None => skip("logs", "not configured"),
915 });
916
917 Ok(Plan {
918 schema_version: 1,
919 kind: "plan".into(),
920 site: site.site.name.clone(),
921 target: p.scope.target.name.clone(),
922 build_type: p.scope.build_name().to_string(),
923 run_id: p.scope.run_id.clone(),
924 list: p.list_path.clone(),
925 log_dir: p.scope.log_dir(),
926 lines: lines.len(),
927 selected: p.scope.target.select.as_ref().map(|_| kept.len()),
928 selected_list: p.selected_list_path().map(|x| x.display().to_string()),
929 phases,
930 deselected,
931 })
932}
933
934pub fn shell_line(c: &Cmd) -> String {
936 let mut s = String::new();
937 for (k, v) in &c.env {
938 s.push_str(&format!("{k}={} ", quote(v)));
939 }
940 s.push_str(
941 &c.argv
942 .iter()
943 .map(|a| quote(a))
944 .collect::<Vec<_>>()
945 .join(" "),
946 );
947 s
948}
949
950fn quote(a: &str) -> String {
951 if !a.is_empty()
952 && a.chars()
953 .all(|c| c.is_ascii_alphanumeric() || "-_./=:,@%+".contains(c))
954 {
955 a.to_string()
956 } else {
957 format!("'{}'", a.replace('\'', "'\\''"))
958 }
959}
960
961fn shard_list_path(dir: &Path, file: &str) -> Result<PathBuf, Fail> {
965 let path = Path::new(file);
966 let one_name = !file.is_empty()
967 && !file.contains('\0')
968 && !file.contains('\\')
969 && path.components().count() == 1
970 && matches!(
971 path.components().next(),
972 Some(std::path::Component::Normal(_))
973 );
974 if !one_name {
975 return Err(Fail::new(
976 exit::FAILED,
977 format!(
978 "shard plan file `{file}` is not a file name in {}",
979 dir.display()
980 ),
981 ));
982 }
983 Ok(dir.join(path))
984}
985
986pub fn read_units(dir: &Path) -> Result<Vec<Unit>, Fail> {
988 let plan_path = dir.join("plan.json");
989 let text = std::fs::read_to_string(&plan_path).map_err(|e| {
990 Fail::new(
991 exit::FAILED,
992 format!("cannot read {}: {e}", plan_path.display()),
993 )
994 })?;
995 let v: serde_json::Value = serde_json::from_str(&text)
996 .map_err(|e| Fail::new(exit::FAILED, format!("{}: {e}", plan_path.display())))?;
997 let mut units = Vec::new();
998 let mut add = |name: String, role: Role, entry: &serde_json::Value| -> Result<(), Fail> {
999 let Some(file) = entry["file"].as_str() else {
1000 return Ok(());
1001 };
1002 let path = shard_list_path(dir, file)?;
1003 let list = std::fs::read_to_string(&path)
1004 .map_err(|e| Fail::new(exit::FAILED, format!("cannot read {}: {e}", path.display())))?;
1005 let lines = parse_list(&list)?;
1006 if lines.is_empty() {
1007 return Ok(());
1008 }
1009 units.push(Unit {
1010 name,
1011 role,
1012 lines,
1013 footprint: entry["inode_footprint"].as_u64(),
1014 });
1015 Ok(())
1016 };
1017 add(
1018 "cpu-prerequisites".into(),
1019 Role::Cpu,
1020 &v["cpu_prerequisites"],
1021 )?;
1022 add("prelude".into(), Role::Build, &v["prelude"])?;
1023 if let Some(shards) = v["shards"].as_array() {
1024 for (i, s) in shards.iter().enumerate() {
1025 let name = s["name"]
1026 .as_str()
1027 .map(str::to_string)
1028 .unwrap_or_else(|| format!("shard-{:02}", i + 1));
1029 add(name, Role::Build, s)?;
1030 }
1031 }
1032 Ok(units)
1033}
1034
1035#[cfg(test)]
1036mod read_tests {
1037 use super::read_units;
1038
1039 #[test]
1040 fn a_shard_plan_cannot_name_a_file_outside_its_directory() {
1041 let root = tempfile::tempdir().unwrap();
1042 let shard = root.path().join("out");
1043 std::fs::create_dir(&shard).unwrap();
1044 std::fs::write(root.path().join("secret.list"), "Secret-1.0.eb\n").unwrap();
1045 std::fs::write(shard.join("local.list"), "Foo-1.0.eb\n").unwrap();
1046
1047 std::fs::write(
1048 shard.join("plan.json"),
1049 r#"{"shards":[{"name":"s","file":"../secret.list"}]}"#,
1050 )
1051 .unwrap();
1052 let err = read_units(&shard).unwrap_err();
1053 assert!(err.msg.contains("not a file name"), "{}", err.msg);
1054
1055 let abs = root.path().join("secret.list");
1056 std::fs::write(
1057 shard.join("plan.json"),
1058 format!(
1059 r#"{{"shards":[{{"name":"s","file":"{}"}}]}}"#,
1060 abs.display()
1061 ),
1062 )
1063 .unwrap();
1064 assert!(read_units(&shard).is_err());
1065
1066 std::fs::write(
1067 shard.join("plan.json"),
1068 r#"{"shards":[{"name":"s","file":"local.list","inode_footprint":3}]}"#,
1069 )
1070 .unwrap();
1071 let units = read_units(&shard).unwrap();
1072 assert_eq!(units.len(), 1);
1073 assert_eq!(units[0].lines[0].easyconfig, "Foo-1.0.eb");
1074 assert_eq!(units[0].footprint, Some(3));
1075 }
1076}