diff --git a/_assembler/tests/determinism.rs b/_assembler/tests/determinism.rs new file mode 100644 index 0000000..b0c7e0f --- /dev/null +++ b/_assembler/tests/determinism.rs @@ -0,0 +1,96 @@ +//! Determinism + ordering tests for the assembler. +//! +//! The assembler module docstring promises: +//! > Output is deterministic: same manifest + blocks → byte-identical .md +//! +//! These tests actually verify that promise. Catches any accidental +//! `HashMap`-iteration leak, embedded timestamp, or non-stable sort. + +mod common; + +use common::{assemble_one, seed_tempdir}; +use std::fs; + +/// Same input, two runs, byte-identical output. +#[test] +fn determinism_same_input_byte_identical() { + let (_tmp1, root1) = seed_tempdir(); + let first = assemble_one(&root1, "code-implementer"); + + let (_tmp2, root2) = seed_tempdir(); + let second = assemble_one(&root2, "code-implementer"); + + assert_eq!( + first.as_bytes(), + second.as_bytes(), + "two independent runs produced different bytes" + ); +} + +/// Same input, ten runs, all byte-identical. Higher chance to catch +/// hash-map iteration nondeterminism that escapes a 2-run check. +#[test] +fn determinism_ten_runs_all_identical() { + let mut seen: Option = None; + for i in 0..10 { + let (_tmp, root) = seed_tempdir(); + let out = assemble_one(&root, "researcher"); + match &seen { + None => seen = Some(out), + Some(prev) => assert_eq!( + prev.as_bytes(), + out.as_bytes(), + "run {i} diverged from run 0" + ), + } + } +} + +/// Block ordering: the order in `manifest.blocks` defines the order +/// in the output. Reorder the blocks list → output changes, and the +/// change is localized to the block region (not to frontmatter or +/// trailing sections). +#[test] +fn block_order_controls_output_order() { + let (_tmp, root) = seed_tempdir(); + + // Baseline: default researcher (baseline, evidence-grading, memory-protocol). + let default_out = assemble_one(&root, "researcher"); + + // Swap two blocks — write a modified manifest into the same tempdir. + let manifest_src = fs::read_to_string(root.join("_manifests/researcher.toml")).unwrap(); + let swapped = manifest_src.replace( + "blocks = [\n \"baseline\", # OBLIGATORY\n \"evidence-grading\", # OBLIGATORY\n \"memory-protocol\", # OBLIGATORY\n]", + "blocks = [\n \"baseline\",\n \"memory-protocol\",\n \"evidence-grading\",\n]", + ); + assert_ne!( + manifest_src, swapped, + "blocks-list replacement did not match — test fixture drifted" + ); + fs::write(root.join("_manifests/researcher.toml"), &swapped).unwrap(); + + let swapped_out = assemble_one(&root, "researcher"); + + // 1. Output is different. + assert_ne!( + default_out, swapped_out, + "swapping block order did not change output" + ); + + // 2. Frontmatter unchanged (first `---` through the trailing `---\n\n` + // ends identically — compare the first 500 bytes, which cover + // frontmatter for all our fixtures). + let prefix_len = default_out + .find("# BASELINE") + .expect("BASELINE marker missing in default output"); + assert_eq!( + &default_out[..prefix_len], + &swapped_out[..prefix_len], + "frontmatter + role drifted when only blocks were reordered" + ); + + // 3. The "# DOMAIN SCOPE" marker appears in both (tail section unchanged + // by block reordering). + assert!(default_out.contains("# DOMAIN SCOPE")); + assert!(swapped_out.contains("# DOMAIN SCOPE")); +} diff --git a/_assembler/tests/roundtrip.rs b/_assembler/tests/roundtrip.rs new file mode 100644 index 0000000..972fdd5 --- /dev/null +++ b/_assembler/tests/roundtrip.rs @@ -0,0 +1,90 @@ +//! Roundtrip / data-preservation tests. +//! +//! The assembler projects the Manifest struct into a Markdown file. +//! We cannot re-parse a Markdown file back into a Manifest (the +//! projection is lossy: comments / blank lines / heading formatting), +//! but we CAN assert that every user-visible string from the manifest +//! appears verbatim in the generated output — i.e. no field is +//! silently dropped by a refactor. + +mod common; + +use common::{assemble_one, seed_tempdir}; +use std::fs; + +/// Every `domain_in` bullet, every `forbidden_domain` bullet, every +/// handoff target + trigger, and the agent name must appear in the +/// generated output. Covers the code-implementer manifest which has +/// the richest field population. +#[test] +fn every_manifest_string_appears_in_output() { + let (_tmp, root) = seed_tempdir(); + let out = assemble_one(&root, "code-implementer"); + + // Parse the same manifest independently with toml crate so we + // can iterate its fields without reaching into the private + // Manifest struct from main.rs. + let toml_text = + fs::read_to_string(root.join("_manifests/code-implementer.toml")).unwrap(); + let parsed: toml::Value = toml::from_str(&toml_text).unwrap(); + + let name = parsed["name"].as_str().unwrap(); + assert!( + out.contains(&format!("name: {name}")), + "frontmatter missing name" + ); + + let model = parsed["model"].as_str().unwrap(); + assert!( + out.contains(&format!("model: {model}")), + "frontmatter missing model" + ); + + // Tools are joined with ", ". + let tools: Vec<&str> = parsed["tools"] + .as_array() + .unwrap() + .iter() + .map(|v| v.as_str().unwrap()) + .collect(); + let tools_line = format!("tools: {}", tools.join(", ")); + assert!( + out.contains(&tools_line), + "frontmatter tools line missing or wrong order" + ); + + // domain_in bullets. + for item in parsed["domain_in"].as_array().unwrap() { + let s = item.as_str().unwrap(); + assert!(out.contains(s), "domain_in entry missing: {s}"); + } + + // forbidden_domain bullets. + for item in parsed["forbidden_domain"].as_array().unwrap() { + let s = item.as_str().unwrap(); + assert!(out.contains(s), "forbidden_domain entry missing: {s}"); + } + + // Handoffs: each target AND each trigger appears. + for h in parsed["handoff"].as_array().unwrap() { + let target = h["target"].as_str().unwrap(); + let trigger = h["trigger"].as_str().unwrap(); + assert!(out.contains(target), "handoff target missing: {target}"); + assert!(out.contains(trigger), "handoff trigger missing: {trigger}"); + } +} + +/// Double-assembly determinism at the text level: parse + assemble +/// twice from the very same tempdir (not two separate tempdirs) — +/// catches any caching or mutable-global drift inside the binary. +#[test] +fn double_assembly_same_tempdir_identical() { + let (_tmp, root) = seed_tempdir(); + let first = assemble_one(&root, "patent-compliance"); + let second = assemble_one(&root, "patent-compliance"); + assert_eq!( + first.as_bytes(), + second.as_bytes(), + "consecutive runs in same tempdir diverged" + ); +} diff --git a/_assembler/tests/validator_negative.rs b/_assembler/tests/validator_negative.rs new file mode 100644 index 0000000..4c0445f --- /dev/null +++ b/_assembler/tests/validator_negative.rs @@ -0,0 +1,158 @@ +//! Validator negative-path tests. +//! +//! Locks the error contract of validator.rs: each flavour of bad +//! manifest produces a non-zero exit status AND a stderr message +//! that names the offending invariant. +//! +//! Note: the unsubstituted-`{{placeholder}}` check is being added +//! in a parallel PR (fix/remaining-findings). That specific test +//! is deliberately NOT included here; when the check lands, add a +//! case here and re-run. + +mod common; + +use common::{run_assemble, seed_tempdir}; +use std::fs; +use std::path::Path; + +/// Write a minimal valid manifest then mutate one field to break it. +/// Returns the tempdir guard (keeps it alive) and the manifest path. +fn write_broken( + root: &Path, + filename: &str, + mutate: impl FnOnce(&mut String), +) -> std::path::PathBuf { + let src = fs::read_to_string(root.join("_manifests/researcher.toml")).unwrap(); + let mut buf = src; + mutate(&mut buf); + let target = root.join("_manifests").join(filename); + fs::write(&target, buf).unwrap(); + target +} + +fn assert_fails_with(root: &Path, manifest: &Path, needle: &str) { + let out = run_assemble(root, &[manifest.to_str().unwrap()]); + assert!( + !out.status.success(), + "expected non-zero exit for broken manifest {}; stdout={:?} stderr={:?}", + manifest.display(), + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr), + ); + let combined = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + assert!( + combined.contains(needle), + "stderr did not mention {needle:?}; full output:\n{combined}" + ); +} + +#[test] +fn validator_rejects_unknown_block_ref() { + let (_tmp, root) = seed_tempdir(); + // Add an extra block name that doesn't exist on disk. + let manifest = write_broken(&root, "broken-unknown-block.toml", |s| { + *s = s.replace( + "\"memory-protocol\", # OBLIGATORY\n]", + "\"memory-protocol\",\n \"this-block-does-not-exist\",\n]", + ); + }); + assert_fails_with(&root, &manifest, "this-block-does-not-exist"); +} + +#[test] +fn validator_rejects_missing_obligatory_block() { + let (_tmp, root) = seed_tempdir(); + // Drop "memory-protocol" from the blocks list. + let manifest = write_broken(&root, "broken-missing-obligatory.toml", |s| { + *s = s.replace("\"memory-protocol\", # OBLIGATORY\n", ""); + }); + assert_fails_with(&root, &manifest, "memory-protocol"); +} + +#[test] +fn validator_rejects_empty_handoff() { + let (_tmp, root) = seed_tempdir(); + // Strip every `[[handoff]]` table from the manifest. + let manifest = write_broken(&root, "broken-no-handoff.toml", |s| { + let mut out = String::new(); + let mut skip = false; + for line in s.lines() { + if line.trim_start().starts_with("[[handoff]]") { + skip = true; + continue; + } + if skip && (line.trim_start().starts_with("[") || line.trim().is_empty()) { + // End of the handoff block (next [table] or blank-line gap). + if line.trim_start().starts_with("[") && !line.trim_start().starts_with("[[handoff]]") { + skip = false; + } else if line.trim().is_empty() { + // Tolerate blank line inside handoff table separator. + continue; + } + } + if !skip { + out.push_str(line); + out.push('\n'); + } + } + *s = out; + }); + assert_fails_with(&root, &manifest, "handoff"); +} + +#[test] +fn validator_rejects_empty_role() { + let (_tmp, root) = seed_tempdir(); + // Replace the role with whitespace only. + let manifest = write_broken(&root, "broken-empty-role.toml", |s| { + // The researcher manifest uses triple-quoted `role = """..."""`. + let start = s.find("role = \"\"\"").expect("role block marker missing"); + let end_rel = s[start..] + .find("\"\"\"\n") + .and_then(|_| s[start + 10..].find("\"\"\"")) + .expect("role closing marker missing"); + let end = start + 10 + end_rel + 3; + let before = &s[..start]; + let after = &s[end..]; + *s = format!("{before}role = \" \"\n{after}"); + }); + assert_fails_with(&root, &manifest, "role"); +} + +#[test] +fn validator_rejects_empty_domain_in() { + let (_tmp, root) = seed_tempdir(); + // Replace domain_in array with an empty one. + let manifest = write_broken(&root, "broken-empty-domain-in.toml", |s| { + let start = s.find("domain_in = [").expect("domain_in marker missing"); + let end_rel = s[start..].find("]\n").expect("domain_in close marker missing"); + let end = start + end_rel + 2; + let before = &s[..start]; + let after = &s[end..]; + *s = format!("{before}domain_in = []\n{after}"); + }); + assert_fails_with(&root, &manifest, "domain_in"); +} + +#[test] +fn validate_only_flag_skips_write() { + // --validate must NOT write anything under _generated/. + let (_tmp, root) = seed_tempdir(); + let manifest = root.join("_manifests/researcher.toml"); + let out = run_assemble(&root, &["--validate", manifest.to_str().unwrap()]); + assert!( + out.status.success(), + "--validate on a valid manifest failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + let generated = root.join("_generated/researcher.md"); + assert!( + !generated.exists(), + "--validate wrote an output file at {}", + generated.display() + ); +}