0be354a920
Single-commit clean baseline after security scrub of niche-tells, project codenames, internal jargon, and contributor-email leaks. Contents: - 100 Rust crates (_primitives/_rust/) - 37 agent manifests (_manifests/) + generated specs (_generated/) - 67 user-invocable skills (skills/) - 33 hooks (hooks/) - Composition blocks (_blocks/) - Documentation (docs/, README.md) - TS adapter packages (_ts_packages/) - Assembler (_assembler/) - Roles (_roles/) - Templates (_templates/) - Forgejo CI (.forgejo/) Author: Denis Parfionovich <info@greendragon.info> License: see LICENSE.
29 lines
980 B
Rust
29 lines
980 B
Rust
//! Stream — NDJSON chunk parser.
|
|
//!
|
|
//! Three token chunks + a terminal `done` marker should yield 4 chunks
|
|
//! and `concat_chunks` should reconstruct the full string.
|
|
|
|
use kei_llm_mlx::stream::{concat_chunks, parse_stream};
|
|
|
|
const STREAM_SAMPLE: &str = r#"{"delta": "Hello", "tokens_so_far": 1}
|
|
{"delta": ", ", "tokens_so_far": 2}
|
|
{"delta": "world!", "tokens_so_far": 3}
|
|
{"delta": "", "done": true, "tokens_so_far": 3}
|
|
"#;
|
|
|
|
#[test]
|
|
fn ndjson_yields_n_plus_one_chunks() {
|
|
let chunks = parse_stream(STREAM_SAMPLE).expect("parse");
|
|
assert_eq!(chunks.len(), 4);
|
|
assert!(chunks.last().unwrap().done);
|
|
assert_eq!(chunks[0].tokens_so_far, Some(1));
|
|
assert_eq!(concat_chunks(&chunks), "Hello, world!");
|
|
}
|
|
|
|
#[test]
|
|
fn non_json_lines_are_skipped() {
|
|
let mixed = "loading model...\n{\"delta\": \"X\", \"tokens_so_far\": 1}\nready.\n";
|
|
let chunks = parse_stream(mixed).expect("parse");
|
|
assert_eq!(chunks.len(), 1);
|
|
assert_eq!(chunks[0].delta, "X");
|
|
}
|