eb_pipeline/
schema.rs

1//! Site file schema.
2//!
3//! Every struct is declared through `doc_struct!`, which emits the serde
4//! struct and its field documentation from one declaration. The parser, the
5//! `schema` reference and the documentation test all read the same table.
6
7use serde::{Deserialize, Serialize};
8use std::collections::BTreeMap;
9
10/// One documented key of the site file.
11#[derive(Debug, Clone, Copy)]
12pub struct FieldDoc {
13    /// The key, as the site file spells it.
14    pub key: &'static str,
15    /// The type of its value: `string`, `integer`, `table`, `array of string`
16    /// and the like.
17    pub ty: &'static str,
18    /// The default, as the reference prints it; empty when it prints none.
19    pub default: &'static str,
20    /// Whether the site file must set the key.
21    pub required: bool,
22    /// What the key means; also the field's rustdoc.
23    pub doc: &'static str,
24}
25
26macro_rules! doc_struct {
27    (
28        $(#[$m:meta])*
29        $name:ident {
30            $( $(#[$fm:meta])* $f:ident : $t:ty => ($ty:expr, $def:expr, $req:expr, $doc:expr) ),* $(,)?
31        }
32    ) => {
33        $(#[$m])*
34        #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
35        #[serde(deny_unknown_fields)]
36        pub struct $name {
37            $( $(#[$fm])* #[doc = $doc] pub $f: $t, )*
38        }
39        impl $name {
40            /// The documentation of every key of this table, in declaration
41            /// order.
42            pub const FIELDS: &'static [FieldDoc] = &[
43                $( FieldDoc { key: stringify!($f), ty: $ty, default: $def, required: $req, doc: $doc } ),*
44            ];
45        }
46    };
47}
48
49fn yes() -> bool {
50    true
51}
52fn d_launch() -> String {
53    "srun".into()
54}
55fn d_template() -> String {
56    "{project}-{build}-{phase}".into()
57}
58fn d_lock_kind() -> String {
59    "flock".into()
60}
61fn d_attempts() -> u32 {
62    5
63}
64fn d_wait() -> u64 {
65    60
66}
67fn d_max_wait() -> u64 {
68    600
69}
70fn d_one() -> u32 {
71    1
72}
73fn d_unsharded() -> i32 {
74    3
75}
76fn d_shell() -> String {
77    "bash".into()
78}
79fn d_backend() -> String {
80    "auto".into()
81}
82fn d_lock_timeout() -> u64 {
83    600
84}
85fn d_route_kind() -> String {
86    "none".into()
87}
88fn d_sep() -> String {
89    ",".into()
90}
91fn d_all() -> String {
92    "all".into()
93}
94fn d_route_phases() -> Vec<String> {
95    vec!["build".into()]
96}
97fn d_single() -> String {
98    "single".into()
99}
100fn d_ro_value() -> String {
101    "1".into()
102}
103fn d_job_env() -> String {
104    "SLURM_JOB_ID".into()
105}
106fn d_cpu_var() -> String {
107    "EESSI_SOFTWARE_SUBDIR_OVERRIDE".into()
108}
109fn d_accel_var() -> String {
110    "EESSI_ACCELERATOR_TARGET_OVERRIDE".into()
111}
112
113doc_struct! {
114    /// A sibling tool: binary and minimum version.
115    ToolSpec {
116        bin: Option<String> => ("string", "the tool's own name, looked up on PATH", false,
117            "Executable path or name."),
118        min_version: Option<String> => ("string", "none", false,
119            "Minimum version as dotted numbers. The tool's `--version` output is probed and a lower version stops the run with exit 2 before any work."),
120    }
121}
122
123doc_struct! {
124    /// Sibling tool table.
125    Tools {
126        eb_stack: Option<ToolSpec> => ("table", "bin eb-stack", false,
127            "eb-stack, called for `stack shard` and `stack admit`."),
128        eb_keep_logs: Option<ToolSpec> => ("table", "bin eb-keep-logs", false,
129            "eb-keep-logs, called for `failed` and `rotate`."),
130        eb_staged_rebuild: Option<ToolSpec> => ("table", "bin eb-staged-rebuild", false,
131            "eb-staged-rebuild, called for lines that rebuild a live install."),
132    }
133}
134
135doc_struct! {
136    /// Site identity.
137    SiteInfo {
138        name: String => ("string", "", true, "Site label used in receipts and job names."),
139        project: Option<String> => ("string", "site.name", false,
140            "Value of the `{project}` placeholder in `scheduler.job_name_template`."),
141        build_account: Option<String> => ("string", "none", false,
142            "Account that owns installs; informational, exported as EB_PIPELINE_BUILD_ACCOUNT."),
143        default_group: Option<String> => ("string", "none", false,
144            "Default unix group of installs; exported as EB_PIPELINE_DEFAULT_GROUP."),
145        stack_generation: Option<String> => ("string", "none", false,
146            "Label of the software generation; not interpreted."),
147        tools: Option<Tools> => ("table", "all tools looked up on PATH, no minimum version", false,
148            "Tool binaries and minimum versions, `[site.tools.<tool>]`."),
149    }
150}
151
152doc_struct! {
153    /// How a phase becomes a job.
154    Scheduler {
155        kind: String => ("string", "", true,
156            "`slurm` (phases launch through `launch`) or `local` (phases run directly, phase arguments ignored)."),
157        #[serde(default = "d_launch")]
158        launch: String => ("string", "srun", false, "Launch command for `slurm`."),
159        #[serde(default)]
160        common_args: Vec<String> => ("array of string", "[]", false,
161            "Arguments added to every launch line, before the per-phase arguments."),
162        #[serde(default)]
163        env: BTreeMap<String, String> => ("table of string", "{}", false,
164            "Environment exported to every phase."),
165        #[serde(default = "d_template")]
166        job_name_template: String => ("string", "{project}-{build}-{phase}", false,
167            "Job name. Placeholders: {project}, {build}, {phase}, {target}."),
168    }
169}
170
171doc_struct! {
172    /// Launch arguments per phase role.
173    PhaseArgs {
174        plan: Option<String> => ("string", "none", false,
175            "Launch arguments of the planning phase (`stack shard`), split on whitespace with quotes honoured."),
176        build: Option<String> => ("string", "none", false,
177            "Launch arguments of build phases."),
178        short: Option<String> => ("string", "none", false,
179            "Launch arguments of short phases: admission, source prefetch, log handling."),
180        cpu: Option<String> => ("string", "none", false,
181            "Launch arguments of CPU-only prerequisite builds; the target's `route` wraps these."),
182        cache: Option<String> => ("string", "none", false,
183            "Launch arguments of the module cache phase when a cache entry sets none."),
184    }
185}
186
187doc_struct! {
188    /// Wrapper for CPU phases on a host that is not native to the target.
189    Route {
190        #[serde(default = "d_route_kind")]
191        kind: String => ("string", "none", false, "`none` or `command`."),
192        command: Option<String> => ("string", "none", false,
193            "Wrapper command placed before the launch line of CPU-role phases; required for kind `command`."),
194        #[serde(default)]
195        args: Vec<String> => ("array of string", "[]", false,
196            "Argument templates placed after `command`. When present, a `--` follows them and the phase command comes after it. Placeholders: {phase_dir}, {log_dir}, {work_dir}, {target}, {phase}, {runtime_root}, {shares_ro}, {shares_rw}. {phase_dir} is `<log_dir>/<phase>` and is created before the command starts; {work_dir} is the target's `filesystems.build_path`; the share placeholders join the rendered lists with `:`."),
197        #[serde(default)]
198        env: BTreeMap<String, String> => ("table of string", "{}", false,
199            "Environment exported to the routed command; values accept the same placeholders as `args`."),
200        runtime_root: Option<String> => ("string", "none", false,
201            "Root of the route's runtime; exported as EB_PIPELINE_ROUTE_RUNTIME_ROOT and available as {runtime_root}."),
202        #[serde(default)]
203        shares_ro: Vec<String> => ("array of string", "[]", false,
204            "Paths shared read-only into the route; placeholders {phase_dir}, {log_dir}, {work_dir}, {target}, {phase} allowed, as in `args`. Exported joined with `:` as EB_PIPELINE_ROUTE_SHARES_RO."),
205        #[serde(default)]
206        shares_rw: Vec<String> => ("array of string", "[]", false,
207            "Paths shared read-write into the route; same placeholders as `shares_ro`. Exported as EB_PIPELINE_ROUTE_SHARES_RW."),
208        receipt_digest_env: Option<String> => ("string", "none", false,
209            "Name of an environment variable that holds a receipt digest. `run` refuses with exit 78 when it is unset or empty and passes the value to the routed command as EB_PIPELINE_ROUTE_RECEIPT_DIGEST; `plan` shows `${NAME}`."),
210        #[serde(default = "d_route_phases")]
211        phases: Vec<String> => ("array of string", "[\"build\"]", false,
212            "Phases whose CPU-role commands go through the route: `build`, `sources`."),
213    }
214}
215
216doc_struct! {
217    /// Per-target line selection by a build list option.
218    Select {
219        option: String => ("string", "", true,
220            "Long option that carries the selection on a build list line, leading dashes included: `option = \"--name\"` reads `--name=a,b` or `--name a,b`."),
221        value: String => ("string", "", true, "This target's token in the option's value list."),
222        #[serde(default = "d_sep")]
223        separator: String => ("string", ",", false, "Separator between tokens in the option's value."),
224        #[serde(default = "d_all")]
225        when_absent: String => ("string", "all", false,
226            "`all` plans a line that lacks the option for this target; `none` skips it."),
227        #[serde(default)]
228        strip: bool => ("boolean", "false", false,
229            "Remove the option from the line before `eb` is called."),
230    }
231}
232
233doc_struct! {
234    /// Hardware identity of a target.
235    Hardware {
236        vendor_id: Option<String> => ("string", "none", false, "CPU vendor id."),
237        cpu_family: Option<u32> => ("integer", "none", false, "CPU family."),
238        model: Option<u32> => ("integer", "none", false, "CPU model."),
239    }
240}
241
242doc_struct! {
243    /// Per build type override of a target.
244    BuildTypeOverride {
245        eb_optarch: Option<String> => ("string", "target value", false,
246            "EASYBUILD_OPTARCH for this build type."),
247        max_parallel: Option<u32> => ("integer", "target value", false,
248            "EASYBUILD_MAX_PARALLEL for this build type."),
249    }
250}
251
252doc_struct! {
253    /// One node type.
254    Target {
255        name: String => ("string", "", true, "Target name; unique across the file."),
256        #[serde(default)]
257        build_lists: Vec<String> => ("array of string", "[]", false,
258            "Names of build list files that belong to this target; checked against `--lists-dir`."),
259        eb_optarch: Option<String> => ("string", "none", false, "EASYBUILD_OPTARCH."),
260        max_parallel: Option<u32> => ("integer", "none", false, "EASYBUILD_MAX_PARALLEL."),
261        cuda_compute_capabilities: Option<String> => ("string", "none", false,
262            "EASYBUILD_CUDA_COMPUTE_CAPABILITIES."),
263        cpu_target: Option<String> => ("string", "none", false,
264            "CPU software subdirectory in EESSI form (`x86_64/amd/zen4`, `aarch64/neoverse_v1`), exported through `modules.eessi.cpu_override_var`."),
265        accel_target: Option<String> => ("string", "none", false,
266            "Accelerator subdirectory in EESSI form (`accel/nvidia/cc90`, `accel/amd/gfx90a`), exported through `modules.eessi.accel_override_var`; empty means none."),
267        site_prefix: Option<String> => ("string", "none", false,
268            "Install prefix of the target; EASYBUILD_INSTALLPATH and EESSI_SITE_SOFTWARE_PREFIX. Falls back to `filesystems.install_root`. With `modules.eessi.site_install`, EESSI-extend replaces EASYBUILD_INSTALLPATH when it loads; `explain` shows its path as `install_note`."),
269        software_root: Option<String> => ("string", "<install prefix>/software", false,
270            "Live software root of the target; passed to eb-staged-rebuild `--software`. Required for staged rebuilds with `modules.eessi.site_install`: the `software` directory under the EESSI-extend install path."),
271        modules_root: Option<String> => ("string", "<install prefix>/modules", false,
272            "Live modules root of the target; passed to eb-staged-rebuild `--modules` and searched for installed modules at `<modules_root>/all/<Name>/<version>[.lua]`. Required for staged rebuilds with `modules.eessi.site_install`: the `modules` directory under the EESSI-extend install path."),
273        staged: Option<bool> => ("boolean", "rebuild.staged", false,
274            "Overrides `rebuild.staged` for this target."),
275        ledger_config: Option<String> => ("string", "rebuild.ledger_config", false,
276            "Overrides `rebuild.ledger_config` for this target. A ledger configuration names one `target`, so two targets that take staged rebuilds cannot share one."),
277        #[serde(default)]
278        default: bool => ("boolean", "false", false, "Marks the default target of the site; informational."),
279        phases: Option<PhaseArgs> => ("table", "phase_defaults", false,
280            "Overrides of `[phase_defaults]` for this target."),
281        route: Option<Route> => ("table", "none", false, "CPU phase wrapper."),
282        select: Option<Select> => ("table", "none", false,
283            "Per-line target selection; without it every line is planned for every target."),
284        hardware: Option<Hardware> => ("table", "none", false,
285            "Hardware identity of the target's build nodes; exported only when `cpu_hardware` is set."),
286        cpu_hardware: Option<Hardware> => ("table", "none", false,
287            "Hardware identity of the nodes that build CPU-role units. When set, the source prefetch and build commands of each line export a class as EB_PIPELINE_HW_VENDOR_ID, EB_PIPELINE_HW_CPU_FAMILY and EB_PIPELINE_HW_MODEL: `cpu_hardware` for a line of a CPU-role unit, `hardware` for the rest. No other command exports them."),
288        fetch_profile: Option<toml::Table> => ("table", "none", false,
289            "Free-form table passed to source prefetch commands as EB_PIPELINE_FETCH_PROFILE, serialised as JSON with sorted keys."),
290        #[serde(default)]
291        package_env: BTreeMap<String, BTreeMap<String, String>> => ("table of tables", "{}", false,
292            "Environment per package name for this target, `[targets.package_env.<Name>]`; per key it wins over `modules.package_env`."),
293        #[serde(default)]
294        build_types: BTreeMap<String, BuildTypeOverride> => ("table of tables", "{}", false,
295            "Per build type overrides, `[targets.build_types.<name>]`."),
296    }
297}
298
299doc_struct! {
300    /// Paths.
301    Filesystems {
302        scratch: String => ("string", "", true, "Scratch root."),
303        tmp: String => ("string", "", true, "EASYBUILD_TMPDIR."),
304        build_path: String => ("string", "", true,
305            "EASYBUILD_BUILDPATH. `{target}` expands to the target name, which gives each target a build directory of its own; no other placeholder is allowed."),
306        #[serde(default)]
307        source_paths: Vec<String> => ("array of string", "[]", false,
308            "EASYBUILD_SOURCEPATH entries, joined with `:`. EasyBuild looks for a source in every entry and downloads a missing one into the first; `{target}` expands to the target name, so `{target}` in the first entry gives each target a download directory of its own. No other placeholder is allowed."),
309        install_root: String => ("string", "", true, "EASYBUILD_INSTALLPATH unless a target sets `site_prefix`."),
310        #[serde(default = "d_single")]
311        export_install_vars: String => ("string", "single", false,
312            "`single` exports EASYBUILD_INSTALLPATH; `split` exports EASYBUILD_INSTALLPATH_SOFTWARE and EASYBUILD_INSTALLPATH_MODULES from the target's software and modules roots instead."),
313        log_root: String => ("string", "", true,
314            "Root of per-run log directories, `<log_root>/<log-stamp>/<target>/`."),
315        shard_dir: Option<String> => ("string", "<scratch>/shards", false,
316            "Output directory of `stack shard`; one subdirectory per run id."),
317        #[serde(default)]
318        read_only_installdir: bool => ("boolean", "false", false, "EASYBUILD_READ_ONLY_INSTALLDIR."),
319        #[serde(default = "d_ro_value")]
320        read_only_installdir_value: String => ("string", "1", false,
321            "Value exported as EASYBUILD_READ_ONLY_INSTALLDIR when `read_only_installdir` is set."),
322        configs_repo: Option<String> => ("string", "none", false, "EASYBUILD_CONFIGS_REPOPATH."),
323    }
324}
325
326doc_struct! {
327    /// Inode admission before a build unit starts.
328    Admission {
329        #[serde(default = "yes")]
330        enabled: bool => ("boolean", "true", false, "Set false to skip the phase."),
331        #[serde(default)]
332        required: bool => ("boolean", "false", false,
333            "When true, a missing `quota_cmd` or `reservations` refuses the run with exit 78. When false, the phase is skipped as not configured."),
334        quota_cmd: Option<String> => ("string", "none", false,
335            "Shell command printing the quota JSON, passed to `stack admit --quota-cmd`."),
336        live_jobs_cmd: Option<String> => ("string", "none", false,
337            "Shell command printing live job ids, passed to `stack admit --live-jobs-cmd`. Without it every other job's reservation counts, including those of jobs that ended without removing theirs; `check` warns."),
338        lock: Option<String> => ("string", "none", false, "Lock file shared by all admission controllers; `run` also takes it to remove the job's reservation."),
339        #[serde(default = "d_lock_kind")]
340        lock_kind: String => ("string", "flock", false, "`flock` or `posix`; must match the other lock holders."),
341        reservations: Option<String> => ("string", "none", false, "Directory of reservations."),
342        margin: Option<u64> => ("integer", "tool default", false, "Inodes kept free."),
343        unknown_reserve: Option<u64> => ("integer", "0", false,
344            "Inodes counted per live job that has no reservation record (`stack admit --unknown-reserve`)."),
345        default_footprint: Option<u64> => ("integer", "none", false,
346            "Inode footprint of a unit when the shard plan gives none. Required when admission is enabled; `check` exits 2 without it."),
347        #[serde(default = "d_job_env")]
348        job_id_env: String => ("string", "SLURM_JOB_ID", false,
349            "Environment variable that holds the scheduler job id at run time. `run` refuses with exit 78 before any build when it is unset or not a positive integer without a leading zero; `plan` shows `${NAME}`."),
350        quota_total: Option<u64> => ("integer", "none", false,
351            "Planning cap, passed to `stack shard --inode-quota`."),
352        footprints: Option<String> => ("string", "none", false,
353            "footprints.json for `stack shard --inode-footprints`."),
354        line_overhead: Option<u64> => ("integer", "tool default", false,
355            "Inodes per build list line beyond its build tree."),
356        #[serde(default = "d_attempts")]
357        attempts: u32 => ("integer", "5", false, "Admission attempts before the run is refused."),
358        #[serde(default = "d_wait")]
359        wait_seconds: u64 => ("integer", "60", false,
360            "Seconds before the second attempt; doubles on each further attempt."),
361        #[serde(default = "d_max_wait")]
362        max_wait_seconds: u64 => ("integer", "600", false, "Upper bound of the wait between attempts."),
363    }
364}
365
366doc_struct! {
367    /// Shard planning.
368    Shards {
369        #[serde(default = "yes")]
370        enabled: bool => ("boolean", "true", false, "Set false to skip the phase."),
371        #[serde(default = "d_one")]
372        count: u32 => ("integer", "1", false,
373            "Most shards to plan. A count below 2 skips the phase and builds the list as one unit."),
374        #[serde(default = "d_unsharded")]
375        planner_exit_unsharded: i32 => ("integer", "3", false,
376            "Exit code of `stack shard` that means the plan was built unsharded; the run continues unsharded."),
377        #[serde(default)]
378        lift_cpu_prerequisites: bool => ("boolean", "false", false,
379            "Passes `--lift-cpu-prerequisites` to `stack shard`."),
380        #[serde(default)]
381        easyconfigs: Vec<String> => ("array of string", "[]", false,
382            "Robot path trees for `stack shard --easyconfigs`; at least one is required when the phase is active."),
383        weights: Option<String> => ("string", "none", false, "Build time table for `stack shard --weights`."),
384        #[serde(default)]
385        installed_modules_dirs: Vec<String> => ("array of string", "[]", false,
386            "Module trees counted as installed, `--installed-modules-dir`. `run` leaves out a tree that does not exist yet, with a WARN."),
387        #[serde(default)]
388        installed_modules_dirs_from_env: Vec<String> => ("array of string", "[]", false,
389            "Environment variables whose colon-separated values are appended to `installed_modules_dirs` at run time; `run` refuses with exit 78 when one is unset, an empty value adds nothing, and `plan` shows `${NAME}`."),
390    }
391}
392
393doc_struct! {
394    /// Source prefetch.
395    Sources {
396        #[serde(default = "yes")]
397        enabled: bool => ("boolean", "true", false, "Set false to skip the phase."),
398        #[serde(default = "yes")]
399        prefetch: bool => ("boolean", "true", false,
400            "Run `eb --fetch --robot` for every list line before building."),
401        #[serde(default)]
402        extra_args: Vec<String> => ("array of string", "[]", false,
403            "Extra arguments for the prefetch command, after the line's own. `--disable-robot` drops `--robot`, so a line fetches its own sources only; a line's `--robot=PATH` moves to the front of `--robot-paths`."),
404    }
405}
406
407doc_struct! {
408    /// EESSI layer.
409    Eessi {
410        #[serde(default = "yes")]
411        enabled: bool => ("boolean", "true", false, "Set false to ignore the table."),
412        #[serde(default)]
413        site_install: bool => ("boolean", "false", false, "Exports EESSI_SITE_INSTALL=1."),
414        bootstrap_modulepath: Option<String> => ("string", "none", false,
415            "Exported as EESSI_SITE_BOOTSTRAP_MODULEPATH."),
416        #[serde(default = "d_cpu_var")]
417        cpu_override_var: String => ("string", "EESSI_SOFTWARE_SUBDIR_OVERRIDE", false,
418            "Variable that carries the target's `cpu_target`."),
419        #[serde(default = "d_accel_var")]
420        accel_override_var: String => ("string", "EESSI_ACCELERATOR_TARGET_OVERRIDE", false,
421            "Variable that carries the target's `accel_target`."),
422    }
423}
424
425doc_struct! {
426    /// Environment modules.
427    Modules {
428        init: Vec<String> => ("array of string", "", true,
429            "Modules loaded, in order, after `module purge` in every EasyBuild phase and around `stack shard`, one name per entry, without white space. May be empty."),
430        #[serde(default)]
431        unuse_path: bool => ("boolean", "false", false, "Clears MODULEPATH after `shell_init` and before loading `init`, so the first `init` module needs an absolute path."),
432        #[serde(default = "d_shell")]
433        shell: String => ("string", "bash", false,
434            "POSIX shell that runs the module wrapper, by name or by path: `sh`, `bash`, `dash`, `ash`, `ksh`, `ksh93`, `mksh` or `yash`."),
435        shell_init: Option<String> => ("string", "none", false,
436            "File sourced before `module` is called, for shells that do not define it. It is sourced without `set -e` or `set -u`; a `module` call that fails still stops the step."),
437        eessi: Option<Eessi> => ("table", "none", false, "EESSI site layer."),
438        #[serde(default)]
439        package_env: BTreeMap<String, BTreeMap<String, String>> => ("table of tables", "{}", false,
440            "Environment per package name, `[modules.package_env.<Name>]`, exported for lines of that package."),
441    }
442}
443
444doc_struct! {
445    /// One module cache entry.
446    CacheTarget {
447        target: String => ("string", "", true, "Name of a `[[targets]]` entry."),
448        config: String => ("string", "", true, "Cache configuration file; EB_PIPELINE_CACHE_CONFIG."),
449        alloc: Option<String> => ("string", "phase_defaults.cache", false, "Launch arguments of the cache phase."),
450        init_module: Option<String> => ("string", "none", false, "EB_PIPELINE_CACHE_INIT_MODULE."),
451    }
452}
453
454doc_struct! {
455    /// Module caches, one finaliser per target.
456    Caches {
457        #[serde(default = "yes")]
458        enabled: bool => ("boolean", "true", false, "Set false to skip the phase."),
459        command: Option<String> => ("string", "none", false,
460            "Cache writer, run once per run for the selected target with EB_PIPELINE_CACHE_* in the environment. Without it the phase is not configured."),
461        bindings: Option<String> => ("string", "none", false, "EB_PIPELINE_CACHE_BINDINGS."),
462        init_module: Option<String> => ("string", "none", false,
463            "Default EB_PIPELINE_CACHE_INIT_MODULE for entries that set none."),
464        targets: Vec<CacheTarget> => ("array of table", "[]", false, "`[[caches.targets]]` entries."),
465    }
466}
467
468doc_struct! {
469    /// Rotation limits.
470    Rotate {
471        compress_after_days: u32 => ("integer", "", true, "`eb-keep-logs rotate --compress-after`."),
472        delete_after_days: Option<u32> => ("integer", "none", false,
473            "`eb-keep-logs rotate --delete-after`; greater than compress_after_days."),
474    }
475}
476
477doc_struct! {
478    /// Log handling.
479    Logs {
480        #[serde(default = "yes")]
481        enabled: bool => ("boolean", "true", false, "Set false to skip the phase."),
482        #[serde(default = "yes")]
483        keep_failed: bool => ("boolean", "true", false,
484            "Run `eb-keep-logs failed` after a failed line, with EASYBUILD_OUTPUT_STYLE=basic so that `eb --last-log` prints the log path on one line."),
485        rotate: Option<Rotate> => ("table", "none", false,
486            "Run `eb-keep-logs rotate <log_root>` at the end of the run, with TZ=UTC: run directories are named in UTC. Only a run that ends with no other run under log_root live rotates: each run holds `<log_root>/.eb-pipeline-rotate.lock` shared, and rotates when it can take it alone. Where `eb-keep-logs --help` lists `--lock-timeout`, the rotate passes `--lock-timeout 0`, and while another rotate holds `<log_root>/.eb-keep-logs-rotate.lock` it changes nothing and exits 75, recorded as skipped."),
487    }
488}
489
490doc_struct! {
491    /// Staged rebuild.
492    Rebuild {
493        #[serde(default = "yes")]
494        enabled: bool => ("boolean", "true", false, "Set false to skip the phase."),
495        #[serde(default = "yes")]
496        staged: bool => ("boolean", "true", false,
497            "Route a line through `eb-staged-rebuild run` when it carries `--rebuild`, `-f` or `--force` as a whole option and its module is already installed. A target can override this with `staged`."),
498        state_root: Option<String> => ("string", "tool default", false,
499            "`--state-root`. `{target}` expands to the target name, which gives each target a state root and a lock of its own; `check` refuses it when two staged targets share a software root, and any other placeholder. Unset, eb-staged-rebuild takes EB_STAGED_REBUILD_STATE_ROOT, which a module can set where eb-pipeline cannot see it, else `.eb-staged-rebuild` next to the software tree."),
500        #[serde(default = "d_lock_timeout")]
501        lock_timeout: u64 => ("integer", "600", false,
502            "`--lock-timeout`: seconds a staged line waits for the state root's lock, which another eb-staged-rebuild holds for the whole of its build and publish; 0 refuses at once. A line whose wait runs out is `refused`."),
503        #[serde(default = "d_backend")]
504        backend: String => ("string", "auto", false, "`auto`, `bwrap`, `userns` or `installpath`."),
505        grace_hours: Option<u32> => ("integer", "tool default", false, "`--grace-hours`."),
506        prepare: Option<String> => ("string", "none", false, "Executable passed as `--prepare`."),
507        ledger_config: Option<String> => ("string", "none", false,
508            "JSON file passed as `--ledger-config`. A target can override this with `ledger_config`."),
509    }
510}
511
512doc_struct! {
513    /// Required licensed software.
514    LicenceRequired {
515        #[serde(default)]
516        software: Vec<String> => ("array of string", "[]", false,
517            "Package names that must each have a group, a threshold or an exemption."),
518    }
519}
520
521doc_struct! {
522    /// Licence enforcement mechanism; the data stays in the site file.
523    Licences {
524        #[serde(default = "yes")]
525        enabled: bool => ("boolean", "true", false, "Set false to skip the gate."),
526        #[serde(default = "yes")]
527        enforce: bool => ("boolean", "true", false,
528            "Run the gate: every required name needs a group, a threshold or an exemption, and an empty configuration is refused."),
529        #[serde(default)]
530        groups: BTreeMap<String, String> => ("table of string", "{}", false,
531            "Package name to unix group, exported as EB_PIPELINE_LICENCE_GROUP for lines of that package."),
532        #[serde(default)]
533        below_version: BTreeMap<String, String> => ("table of string", "{}", false,
534            "Package name to version; the group applies below that version only."),
535        required: Option<LicenceRequired> => ("table", "none", false,
536            "Independent second list of licensed packages."),
537        #[serde(default)]
538        exempt: Vec<String> => ("array of string", "[]", false,
539            "Required names that deliberately have no group."),
540        #[serde(default)]
541        env: BTreeMap<String, BTreeMap<String, String>> => ("table of tables", "{}", false,
542            "Environment per package name, `[licences.env.<Name>]`."),
543    }
544}
545
546doc_struct! {
547    /// A site file.
548    Site {
549        schema_version: u32 => ("integer", "", true, "Schema version; 1."),
550        kind: String => ("string", "", true, "File kind; `site`."),
551        site: SiteInfo => ("table", "", true, "Site identity and tools."),
552        scheduler: Scheduler => ("table", "", true, "Scheduler launch settings."),
553        phase_defaults: Option<PhaseArgs> => ("table", "none", false, "Launch arguments per phase role."),
554        targets: Vec<Target> => ("array of table", "", true, "Node types; at least one."),
555        filesystems: Filesystems => ("table", "", true, "Paths."),
556        admission: Option<Admission> => ("table", "none", false, "Inode admission."),
557        shards: Option<Shards> => ("table", "none", false, "Shard planning."),
558        sources: Option<Sources> => ("table", "none", false, "Source prefetch."),
559        modules: Modules => ("table", "", true, "Environment modules."),
560        caches: Option<Caches> => ("table", "none", false, "Module caches."),
561        logs: Option<Logs> => ("table", "none", false, "Log handling."),
562        rebuild: Option<Rebuild> => ("table", "none", false, "Staged rebuild."),
563        licences: Option<Licences> => ("table", "none", false, "Licence enforcement."),
564    }
565}
566
567/// Every documented table, by its path in the file.
568pub const SECTIONS: &[(&str, &[FieldDoc])] = &[
569    ("", Site::FIELDS),
570    ("site", SiteInfo::FIELDS),
571    ("site.tools", Tools::FIELDS),
572    ("site.tools.<tool>", ToolSpec::FIELDS),
573    ("scheduler", Scheduler::FIELDS),
574    ("phase_defaults", PhaseArgs::FIELDS),
575    ("targets[]", Target::FIELDS),
576    ("targets[].phases", PhaseArgs::FIELDS),
577    ("targets[].route", Route::FIELDS),
578    ("targets[].select", Select::FIELDS),
579    ("targets[].hardware", Hardware::FIELDS),
580    ("targets[].cpu_hardware", Hardware::FIELDS),
581    ("targets[].build_types.<name>", BuildTypeOverride::FIELDS),
582    ("filesystems", Filesystems::FIELDS),
583    ("admission", Admission::FIELDS),
584    ("shards", Shards::FIELDS),
585    ("sources", Sources::FIELDS),
586    ("modules", Modules::FIELDS),
587    ("modules.eessi", Eessi::FIELDS),
588    ("caches", Caches::FIELDS),
589    ("caches.targets[]", CacheTarget::FIELDS),
590    ("logs", Logs::FIELDS),
591    ("logs.rotate", Rotate::FIELDS),
592    ("rebuild", Rebuild::FIELDS),
593    ("licences", Licences::FIELDS),
594    ("licences.required", LicenceRequired::FIELDS),
595];
596
597/// Render the key reference as plain text.
598pub fn reference() -> String {
599    let mut out = String::new();
600    out.push_str("Site file reference (kind = \"site\", schema_version = 1)\n");
601    out.push_str("Unknown keys are errors. Absent optional sections are skipped.\n");
602    for (path, fields) in SECTIONS {
603        out.push('\n');
604        out.push_str(&format!(
605            "[{}]\n",
606            if path.is_empty() { "top level" } else { path }
607        ));
608        for f in *fields {
609            out.push_str(&format!(
610                "  {} ({}, {}{})\n      {}\n",
611                f.key,
612                f.ty,
613                if f.required { "required" } else { "optional" },
614                if f.default.is_empty() {
615                    String::new()
616                } else {
617                    format!(", default: {}", f.default)
618                },
619                f.doc
620            ));
621        }
622    }
623    out
624}
625
626#[cfg(test)]
627mod tests {
628    use super::*;
629
630    /// Keys a struct accepts, read from serde's unknown-field error.
631    fn accepted<T: serde::de::DeserializeOwned>() -> Vec<String> {
632        let err = match toml::from_str::<T>("zz_probe = 1") {
633            Ok(_) => panic!("probe key accepted"),
634            Err(e) => e.to_string(),
635        };
636        let line = err
637            .lines()
638            .find(|l| l.contains("expected one of") || l.contains("expected `"))
639            .unwrap_or_else(|| panic!("no key list in: {err}"));
640        let rest = line.split("expected").nth(1).unwrap();
641        let mut keys: Vec<String> = rest
642            .split('`')
643            .skip(1)
644            .step_by(2)
645            .map(str::to_string)
646            .collect();
647        keys.sort();
648        keys
649    }
650
651    fn documented(f: &[FieldDoc]) -> Vec<String> {
652        let mut k: Vec<String> = f.iter().map(|d| d.key.to_string()).collect();
653        k.sort();
654        k
655    }
656
657    macro_rules! check_all {
658        ($($t:ident),* $(,)?) => {{
659            let mut seen: Vec<Vec<String>> = Vec::new();
660            $(
661                assert_eq!(
662                    accepted::<$t>(),
663                    documented($t::FIELDS),
664                    "{} fields and documentation differ",
665                    stringify!($t)
666                );
667                seen.push(documented($t::FIELDS));
668            )*
669            seen
670        }};
671    }
672
673    #[test]
674    fn every_field_is_documented() {
675        let seen = check_all!(
676            Site,
677            SiteInfo,
678            Tools,
679            ToolSpec,
680            Scheduler,
681            PhaseArgs,
682            Target,
683            Route,
684            Select,
685            Hardware,
686            BuildTypeOverride,
687            Filesystems,
688            Admission,
689            Shards,
690            Sources,
691            Modules,
692            Eessi,
693            Caches,
694            CacheTarget,
695            Logs,
696            Rotate,
697            Rebuild,
698            Licences,
699            LicenceRequired,
700        );
701        // Every documented table appears in the printed reference.
702        for (_, fields) in SECTIONS {
703            assert!(seen.contains(&documented(fields)));
704        }
705        let text = reference();
706        for (_, fields) in SECTIONS {
707            for f in *fields {
708                assert!(!f.doc.is_empty() && !f.ty.is_empty(), "{}", f.key);
709                assert!(text.contains(f.doc), "{} missing from reference", f.key);
710            }
711        }
712    }
713
714    #[test]
715    fn required_keys_match_the_contract() {
716        let req: Vec<&str> = Site::FIELDS
717            .iter()
718            .filter(|f| f.required)
719            .map(|f| f.key)
720            .collect();
721        for k in ["site", "scheduler", "targets", "filesystems", "modules"] {
722            assert!(req.contains(&k), "{k} must be required");
723        }
724        for k in [
725            "admission",
726            "shards",
727            "sources",
728            "caches",
729            "logs",
730            "rebuild",
731            "licences",
732            "phase_defaults",
733        ] {
734            assert!(!req.contains(&k), "{k} must be optional");
735        }
736    }
737}