Merge commit '44718e8a' into feat/8jk6.3-date-hierarchy
# Conflicts: # crates/improvise-io/src/import/wizard.rs # tests/headless_cli.rs
This commit is contained in:
Claude (agent-hier)
@@ -1236,7 +1236,12 @@ impl App {
|
||||
}
|
||||
|
||||
pub fn start_import_wizard(&mut self, json: serde_json::Value) {
|
||||
self.wizard = Some(ImportWizard::new(json));
|
||||
self.start_import(crate::import::wizard::ImportPipeline::new(json));
|
||||
}
|
||||
|
||||
/// Open the wizard over a pipeline the host built, sources and all.
|
||||
pub fn start_import(&mut self, pipeline: crate::import::wizard::ImportPipeline) {
|
||||
self.wizard = Some(ImportWizard::from_pipeline(pipeline));
|
||||
self.view_state.mode = AppMode::ImportWizard;
|
||||
}
|
||||
|
||||
|
||||
@@ -1323,6 +1323,7 @@ impl Effect for ImportJsonHeadless {
|
||||
formulas: vec![],
|
||||
// Headless: nobody to ask, so a collision refuses (improvise-cgqt).
|
||||
on_duplicate: DuplicatePolicy::Refuse,
|
||||
sources: vec![],
|
||||
};
|
||||
|
||||
match pipeline.build_model_with_warnings() {
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use csv::ReaderBuilder;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::storage::{PhysicalFs, Storage};
|
||||
|
||||
pub fn csv_path_p(path: &Path) -> bool {
|
||||
path.extension()
|
||||
.is_some_and(|ext| ext.eq_ignore_ascii_case("csv"))
|
||||
@@ -31,6 +33,12 @@ pub fn parse_csv_with_warnings(path: &Path) -> Result<(Vec<Value>, Vec<String>)>
|
||||
/// As [`parse_csv_with_warnings`] but over bytes, so import can come from any
|
||||
/// [`Storage`](crate::storage::Storage) backend, not only a file (improvise-fuj).
|
||||
pub fn parse_csv_bytes_with_warnings(bytes: &[u8]) -> Result<(Vec<Value>, Vec<String>)> {
|
||||
let (_, records, warnings) = parse_csv_bytes(bytes)?;
|
||||
Ok((records, warnings))
|
||||
}
|
||||
|
||||
/// The header (after renaming), the records and the warnings.
|
||||
fn parse_csv_bytes(bytes: &[u8]) -> Result<(Vec<String>, Vec<Value>, Vec<String>)> {
|
||||
let mut reader = ReaderBuilder::new()
|
||||
.has_headers(true)
|
||||
.flexible(true)
|
||||
@@ -110,7 +118,7 @@ pub fn parse_csv_bytes_with_warnings(bytes: &[u8]) -> Result<(Vec<Value>, Vec<St
|
||||
}),
|
||||
);
|
||||
|
||||
Ok((records, warnings))
|
||||
Ok((headers, records, warnings))
|
||||
}
|
||||
|
||||
/// A header name keys the record object, so two equal names were one column and
|
||||
@@ -145,7 +153,41 @@ fn disambiguate_headers(raw: Vec<String>) -> (Vec<String>, Vec<String>) {
|
||||
/// Parse multiple CSV files and merge into a single JSON array.
|
||||
/// Each record gets a "File" field set to the filename stem (e.g., "sales" from "sales.csv").
|
||||
pub fn merge_csvs(paths: &[impl AsRef<Path>]) -> Result<Vec<Value>> {
|
||||
let mut all_records = Vec::new();
|
||||
let merged = merge_csvs_from(&PhysicalFs, paths)?;
|
||||
for warning in &merged.warnings {
|
||||
eprintln!("warning: {warning}");
|
||||
}
|
||||
Ok(merged.records)
|
||||
}
|
||||
|
||||
/// The category every merged record names its source file by.
|
||||
pub const FILE_CATEGORY: &str = "File";
|
||||
|
||||
/// One merged file: its `File` item and the columns its header named.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SourceFile {
|
||||
pub item: String,
|
||||
pub columns: Vec<String>,
|
||||
}
|
||||
|
||||
/// Several CSV files as one record set, with each file's own columns kept.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Merged {
|
||||
pub records: Vec<Value>,
|
||||
/// One per distinct `File` item, in the order the files were read.
|
||||
pub sources: Vec<SourceFile>,
|
||||
/// Each file's parse warnings, prefixed with its path.
|
||||
pub warnings: Vec<String>,
|
||||
}
|
||||
|
||||
/// As [`merge_csvs`] over any [`Storage`]. Files may have different headers,
|
||||
/// and two files with one stem share its item and its columns (improvise-lpzu).
|
||||
pub fn merge_csvs_from(fs: &dyn Storage, paths: &[impl AsRef<Path>]) -> Result<Merged> {
|
||||
let mut merged = Merged {
|
||||
records: Vec::new(),
|
||||
sources: Vec::new(),
|
||||
warnings: Vec::new(),
|
||||
};
|
||||
for path in paths {
|
||||
let path = path.as_ref();
|
||||
let stem = path
|
||||
@@ -153,15 +195,75 @@ pub fn merge_csvs(paths: &[impl AsRef<Path>]) -> Result<Vec<Value>> {
|
||||
.and_then(|s| s.to_str())
|
||||
.unwrap_or("unknown")
|
||||
.to_string();
|
||||
let records = parse_csv(path)?;
|
||||
let bytes = fs.read(path)?;
|
||||
let (headers, records, warnings) = parse_csv_bytes(&bytes)
|
||||
.with_context(|| format!("Failed to parse CSV file: {}", path.display()))?;
|
||||
let at = merged.source_for(&stem);
|
||||
let columns = &mut merged.sources[at].columns;
|
||||
for header in headers {
|
||||
if header != FILE_CATEGORY && !columns.contains(&header) {
|
||||
columns.push(header);
|
||||
}
|
||||
}
|
||||
merged
|
||||
.warnings
|
||||
.extend(warnings.iter().map(|w| format!("{}: {w}", path.display())));
|
||||
for mut record in records {
|
||||
if let Value::Object(ref mut map) = record {
|
||||
map.insert("File".to_string(), Value::String(stem.clone()));
|
||||
map.insert(FILE_CATEGORY.to_string(), Value::String(stem.clone()));
|
||||
}
|
||||
all_records.push(record);
|
||||
merged.records.push(record);
|
||||
}
|
||||
}
|
||||
Ok(all_records)
|
||||
Ok(merged)
|
||||
}
|
||||
|
||||
impl Merged {
|
||||
fn source_for(&mut self, item: &str) -> usize {
|
||||
match self.sources.iter().position(|s| s.item == item) {
|
||||
Some(at) => at,
|
||||
None => {
|
||||
self.sources.push(SourceFile {
|
||||
item: item.to_string(),
|
||||
columns: vec![],
|
||||
});
|
||||
self.sources.len() - 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One `improvise import` argument. Whether a path names a folder is the
|
||||
/// host's question, so the caller classifies and [`expand_import_paths`] lists.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ImportPath {
|
||||
File(PathBuf),
|
||||
Folder(PathBuf),
|
||||
}
|
||||
|
||||
/// The files an import reads: each argument in turn, a folder standing for its
|
||||
/// `*.csv` documents in name order. See `context/code/formula-and-io.md`.
|
||||
pub fn expand_import_paths(fs: &dyn Storage, args: &[ImportPath]) -> Result<Vec<PathBuf>> {
|
||||
let mut paths = Vec::new();
|
||||
for arg in args {
|
||||
match arg {
|
||||
ImportPath::File(path) => paths.push(path.clone()),
|
||||
ImportPath::Folder(dir) => paths.extend(csv_documents_in(fs, dir)?),
|
||||
}
|
||||
}
|
||||
Ok(paths)
|
||||
}
|
||||
|
||||
/// A folder's `*.csv` documents, sorted by `Storage::list`'s contract. A folder
|
||||
/// with none is refused by name rather than imported as nothing.
|
||||
fn csv_documents_in(fs: &dyn Storage, dir: &Path) -> Result<Vec<PathBuf>> {
|
||||
let csvs: Vec<PathBuf> = fs
|
||||
.list(dir)?
|
||||
.into_iter()
|
||||
.filter(|p| csv_path_p(p))
|
||||
.collect();
|
||||
ensure!(!csvs.is_empty(), "no CSV files in folder {}", dir.display());
|
||||
Ok(csvs)
|
||||
}
|
||||
|
||||
fn parse_csv_field(field: &str) -> Value {
|
||||
@@ -302,6 +404,118 @@ mod tests {
|
||||
assert_eq!(records[0]["Name"], Value::String("A".to_string()));
|
||||
}
|
||||
|
||||
// ── Folder arguments (improvise-lpzu) ─────────────────────────────
|
||||
|
||||
use crate::storage::MemoryFs;
|
||||
|
||||
fn folder(files: &[(&str, &str)]) -> MemoryFs {
|
||||
let fs = MemoryFs::new();
|
||||
for (path, body) in files {
|
||||
fs.write(Path::new(path), body.as_bytes()).unwrap();
|
||||
}
|
||||
fs
|
||||
}
|
||||
|
||||
fn names(paths: &[PathBuf]) -> Vec<String> {
|
||||
paths.iter().map(|p| p.display().to_string()).collect()
|
||||
}
|
||||
|
||||
/// Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn a_folder_expands_to_its_csv_files_in_name_order_and_skips_the_rest() {
|
||||
let fs = folder(&[
|
||||
("maat/coupling.csv", "entity\n"),
|
||||
("maat/notes.txt", "not data"),
|
||||
("maat/age.CSV", "entity\n"),
|
||||
("maat/out.improv", "v2025-04-09\n"),
|
||||
("maat/abs-churn.csv", "date\n"),
|
||||
("elsewhere/other.csv", "x\n"),
|
||||
]);
|
||||
let paths = expand_import_paths(&fs, &[ImportPath::Folder("maat".into())]).unwrap();
|
||||
assert_eq!(
|
||||
names(&paths),
|
||||
["maat/abs-churn.csv", "maat/age.CSV", "maat/coupling.csv"]
|
||||
);
|
||||
}
|
||||
|
||||
/// Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn a_folder_with_no_csv_files_is_refused_by_name() {
|
||||
let only_text = folder(&[("maat/notes.txt", "x"), ("maat/out.improv", "y")]);
|
||||
let empty = MemoryFs::new();
|
||||
for fs in [&only_text, &empty] {
|
||||
let err = expand_import_paths(fs, &[ImportPath::Folder("maat".into())])
|
||||
.expect_err("a folder with nothing to import is refused");
|
||||
assert_eq!(err.to_string(), "no CSV files in folder maat");
|
||||
}
|
||||
}
|
||||
|
||||
/// Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn files_and_folders_expand_in_argument_order() {
|
||||
let fs = folder(&[
|
||||
("b/two.csv", "x\n"),
|
||||
("b/one.csv", "x\n"),
|
||||
("a.csv", "x\n"),
|
||||
("z.csv", "x\n"),
|
||||
]);
|
||||
let args = [
|
||||
ImportPath::File("z.csv".into()),
|
||||
ImportPath::Folder("b".into()),
|
||||
ImportPath::File("a.csv".into()),
|
||||
];
|
||||
let paths = expand_import_paths(&fs, &args).unwrap();
|
||||
assert_eq!(names(&paths), ["z.csv", "b/one.csv", "b/two.csv", "a.csv"]);
|
||||
}
|
||||
|
||||
/// Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn files_with_different_headers_merge_without_inventing_the_missing_columns() {
|
||||
let fs = folder(&[
|
||||
(
|
||||
"m/coupling.csv",
|
||||
"entity,coupled,degree\nsrc/a.rs,src/b.rs,80\n",
|
||||
),
|
||||
("m/age.csv", "entity,age-months\nsrc/a.rs,2\n"),
|
||||
]);
|
||||
let merged = merge_csvs_from(&fs, &["m/coupling.csv", "m/age.csv"]).unwrap();
|
||||
assert!(merged.warnings.is_empty(), "{:?}", merged.warnings);
|
||||
let source = |item: &str, columns: &[&str]| SourceFile {
|
||||
item: item.to_string(),
|
||||
columns: columns.iter().map(|c| c.to_string()).collect(),
|
||||
};
|
||||
assert_eq!(
|
||||
merged.sources,
|
||||
[
|
||||
source("coupling", &["entity", "coupled", "degree"]),
|
||||
source("age", &["entity", "age-months"]),
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
merged.records,
|
||||
[
|
||||
serde_json::json!({"entity": "src/a.rs", "coupled": "src/b.rs", "degree": 80, "File": "coupling"}),
|
||||
serde_json::json!({"entity": "src/a.rs", "age-months": 2, "File": "age"}),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/// improvise-8jk6.12 (3): two files with one stem are one item, and its
|
||||
/// columns are both files'. Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn two_files_with_one_stem_union_their_columns() {
|
||||
let fs = folder(&[
|
||||
("a/x.csv", "entity,one\ne,1\n"),
|
||||
("b/x.csv", "entity,two\ne,2\n"),
|
||||
]);
|
||||
let merged = merge_csvs_from(&fs, &["a/x.csv", "b/x.csv"]).unwrap();
|
||||
let expected = SourceFile {
|
||||
item: "x".to_string(),
|
||||
columns: ["entity", "one", "two"].map(String::from).to_vec(),
|
||||
};
|
||||
assert_eq!(merged.sources, [expected]);
|
||||
}
|
||||
|
||||
// ── RFC 4180 edge cases ───────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -7,10 +7,11 @@ use super::analyzer::{
|
||||
DateComponent, DateShape, FieldKind, FieldProposal, analyze_records, extract_array_at_path,
|
||||
extract_date_component, find_array_paths,
|
||||
};
|
||||
use super::csv_parser::{FILE_CATEGORY, Merged, SourceFile};
|
||||
use crate::formula::parse_formula;
|
||||
use crate::model::MAX_CATEGORIES;
|
||||
use crate::model::cell::{CellKey, CellValue};
|
||||
use crate::view::Axis;
|
||||
use crate::view::{Axis, VIRTUAL_VIEW_NAMES, View, ViewStep, records_steps};
|
||||
use crate::workbook::Workbook;
|
||||
|
||||
// ── Pipeline (no UI state) ────────────────────────────────────────────────────
|
||||
@@ -29,6 +30,9 @@ pub struct ImportPipeline {
|
||||
pub formulas: Vec<String>,
|
||||
/// What to do with records that share every coordinate (improvise-cgqt).
|
||||
pub on_duplicate: DuplicatePolicy,
|
||||
/// The files a CSV merge read, each of which gets its own view. Empty for
|
||||
/// a single file or a JSON import (improvise-lpzu).
|
||||
pub sources: Vec<SourceFile>,
|
||||
}
|
||||
|
||||
/// Several source records that land on one coordinate tuple (improvise-cgqt).
|
||||
@@ -64,6 +68,7 @@ impl ImportPipeline {
|
||||
model_name: "Imported Model".to_string(),
|
||||
formulas: vec![],
|
||||
on_duplicate: DuplicatePolicy::Refuse,
|
||||
sources: vec![],
|
||||
};
|
||||
|
||||
// Auto-select if root is an array or there is exactly one candidate path.
|
||||
@@ -77,6 +82,32 @@ impl ImportPipeline {
|
||||
pipeline
|
||||
}
|
||||
|
||||
/// A pipeline over a CSV merge, carrying its sources for the per-file views.
|
||||
pub fn from_merge(merged: Merged) -> Self {
|
||||
let mut pipeline = Self::new(Value::Array(merged.records));
|
||||
pipeline.seed_sources(merged.sources);
|
||||
pipeline
|
||||
}
|
||||
|
||||
/// Every file keeps its `File` item and its columns, rows or none: a
|
||||
/// header-only file is still a file (improvise-lpzu).
|
||||
fn seed_sources(&mut self, sources: Vec<SourceFile>) {
|
||||
let items: Vec<String> = sources.iter().map(|s| s.item.clone()).collect();
|
||||
match self.proposals.iter_mut().find(|p| p.field == FILE_CATEGORY) {
|
||||
Some(file) => file.distinct_values = items,
|
||||
None => self
|
||||
.proposals
|
||||
.push(unseen(FILE_CATEGORY, FieldKind::Category, items)),
|
||||
}
|
||||
for column in sources.iter().flat_map(|s| &s.columns) {
|
||||
if !self.proposals.iter().any(|p| &p.field == column) {
|
||||
self.proposals
|
||||
.push(unseen(column, FieldKind::Label, vec![]));
|
||||
}
|
||||
}
|
||||
self.sources = sources;
|
||||
}
|
||||
|
||||
pub fn select_path(&mut self, path: &str) {
|
||||
self.selected_path = path.to_string();
|
||||
if let Some(arr) = extract_array_at_path(&self.raw, path) {
|
||||
@@ -272,9 +303,6 @@ impl ImportPipeline {
|
||||
wb.add_item("_Measure", &m.field)?;
|
||||
}
|
||||
|
||||
// Per-category tally of records dropped because that category had no
|
||||
// value; a record is charged to the first category it is missing.
|
||||
let mut skipped_per_category = vec![0usize; categories.len()];
|
||||
let mut skipped_non_objects = 0usize;
|
||||
// Per-extraction tally of values the detected format could not parse. A
|
||||
// derived coordinate that is merely absent is improvise-yss5.
|
||||
@@ -292,12 +320,6 @@ impl ImportPipeline {
|
||||
for (at, placement) in placements.into_iter().enumerate() {
|
||||
match placement {
|
||||
Placement::NotObject => skipped_non_objects += 1,
|
||||
Placement::Missing { cat_idx, partial } => {
|
||||
for (cat, item) in &partial {
|
||||
wb.add_item(cat, item)?;
|
||||
}
|
||||
skipped_per_category[cat_idx] += 1;
|
||||
}
|
||||
Placement::Placed {
|
||||
mut coords,
|
||||
measures,
|
||||
@@ -358,12 +380,13 @@ impl ImportPipeline {
|
||||
}
|
||||
}
|
||||
|
||||
let mut warnings = skip_warnings(
|
||||
self.records.len(),
|
||||
categories,
|
||||
&skipped_per_category,
|
||||
skipped_non_objects,
|
||||
);
|
||||
for view in source_views(&wb, &self.sources, &plan) {
|
||||
wb.views.insert(view.name.clone(), view);
|
||||
}
|
||||
|
||||
let mut warnings: Vec<String> = skip_warnings(self.records.len(), skipped_non_objects)
|
||||
.into_iter()
|
||||
.collect();
|
||||
let (split, whole) = unparsed_dates.split_at(date_extractions.len());
|
||||
warnings.extend(unparsed_date_warnings(date_extractions, split));
|
||||
warnings.extend(
|
||||
@@ -386,6 +409,88 @@ impl ImportPipeline {
|
||||
}
|
||||
}
|
||||
|
||||
/// A proposal for a column no record has a value in, which is what an
|
||||
/// all-null column already gets from the analyzer (improvise-l1f5).
|
||||
fn unseen(field: &str, kind: FieldKind, distinct_values: Vec<String>) -> FieldProposal {
|
||||
FieldProposal {
|
||||
field: field.to_string(),
|
||||
kind,
|
||||
distinct_values,
|
||||
accepted: true,
|
||||
date_format: None,
|
||||
date_components: vec![],
|
||||
date_shape: DateShape::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// One view per merged file, in file order, while `File` is a category. The
|
||||
/// shape and why Filter: `context/code/formula-and-io.md` (improvise-lpzu).
|
||||
fn source_views(wb: &Workbook, sources: &[SourceFile], plan: &Plan) -> Vec<View> {
|
||||
let file_is_a_category = plan.categories.iter().any(|c| c.field == FILE_CATEGORY);
|
||||
let mut views: Vec<View> = vec![];
|
||||
for source in sources.iter().filter(|_| file_is_a_category) {
|
||||
let taken = |name: &str| {
|
||||
wb.views.contains_key(name)
|
||||
|| VIRTUAL_VIEW_NAMES.contains(&name)
|
||||
|| views.iter().any(|v| v.name == name)
|
||||
};
|
||||
let name = std::iter::once(source.item.clone())
|
||||
.chain((2usize..).map(|n| format!("{} {n}", source.item)))
|
||||
.find(|name| !taken(name))
|
||||
.expect("a free name exists, there being finitely many views");
|
||||
views.push(source_view(wb, source, plan, name));
|
||||
}
|
||||
views
|
||||
}
|
||||
|
||||
/// Filtered to the file; its own categories seated as a fresh import seats
|
||||
/// them, every other one on None; other files' measures hidden.
|
||||
fn source_view(wb: &Workbook, source: &SourceFile, plan: &Plan, name: String) -> View {
|
||||
let has = |field: &str| source.columns.iter().any(|c| c == field);
|
||||
let own = |cat: &str| {
|
||||
has(cat)
|
||||
|| plan
|
||||
.date_extractions
|
||||
.iter()
|
||||
.any(|(field, _, _, derived)| derived == cat && has(field))
|
||||
};
|
||||
let mut view = View::new(name);
|
||||
for cat in wb.model.categories.values() {
|
||||
match cat.kind.is_virtual() || own(&cat.name) {
|
||||
true => view.on_category_added(&cat.name, cat.kind),
|
||||
false => {
|
||||
view.category_axes.insert(cat.name.clone(), Axis::None);
|
||||
}
|
||||
}
|
||||
}
|
||||
view.set_axis("_Measure", Axis::Page);
|
||||
view.set_axis(FILE_CATEGORY, Axis::Filter);
|
||||
view.set_page_selection(FILE_CATEGORY, &source.item);
|
||||
let (mine, theirs): (Vec<&FieldProposal>, Vec<&FieldProposal>) =
|
||||
plan.measures.iter().partition(|m| has(&m.field));
|
||||
for measure in theirs {
|
||||
view.hide_item("_Measure", &measure.field);
|
||||
}
|
||||
match mine.first() {
|
||||
Some(first) => view.set_page_selection("_Measure", &first.field),
|
||||
None => into_records(&mut view, wb, &source.item),
|
||||
}
|
||||
view
|
||||
}
|
||||
|
||||
/// A file with no measure has no measure to page to: its view is its records,
|
||||
/// still filtered to the file (improvise-8jk6.13).
|
||||
fn into_records(view: &mut View, wb: &Workbook, item: &str) {
|
||||
for step in records_steps(&wb.model) {
|
||||
match step {
|
||||
ViewStep::Axis { category, axis } => view.set_axis(&category, axis),
|
||||
ViewStep::PageSelection { category, item } => view.set_page_selection(&category, &item),
|
||||
}
|
||||
}
|
||||
view.set_axis(FILE_CATEGORY, Axis::Filter);
|
||||
view.set_page_selection(FILE_CATEGORY, item);
|
||||
}
|
||||
|
||||
/// The accepted fields by role, and the date parts derived from them.
|
||||
struct Plan<'a> {
|
||||
categories: Vec<&'a FieldProposal>,
|
||||
@@ -439,11 +544,6 @@ enum Placement {
|
||||
unparsed: Vec<usize>,
|
||||
chains: Vec<DateChain>,
|
||||
},
|
||||
/// No value for category `cat_idx`; `partial` is what preceded it.
|
||||
Missing {
|
||||
cat_idx: usize,
|
||||
partial: Vec<(String, String)>,
|
||||
},
|
||||
NotObject,
|
||||
}
|
||||
|
||||
@@ -453,18 +553,16 @@ impl Plan<'_> {
|
||||
return Placement::NotObject;
|
||||
};
|
||||
let mut coords: Vec<(String, String)> = vec![];
|
||||
// Charged only once this record survives the walk: a row dropped
|
||||
// for a LATER missing category never lands (context/test-evidence).
|
||||
let mut unparsed: Vec<usize> = vec![];
|
||||
let mut chains: Vec<DateChain> = vec![];
|
||||
|
||||
for (cat_idx, cat_proposal) in self.categories.iter().enumerate() {
|
||||
let Some(v) = map.get(&cat_proposal.field).and_then(scalar_name) else {
|
||||
return Placement::Missing {
|
||||
cat_idx,
|
||||
partial: coords,
|
||||
};
|
||||
};
|
||||
// No value is no coordinate: records are sparse (improvise-8jk6.2).
|
||||
let named = |field: &str| map.get(field).and_then(scalar_name);
|
||||
for (cat_proposal, v) in self
|
||||
.categories
|
||||
.iter()
|
||||
.filter_map(|p| Some((p, named(&p.field)?)))
|
||||
{
|
||||
let hierarchy = self
|
||||
.date_hierarchies
|
||||
.iter()
|
||||
@@ -496,13 +594,13 @@ impl Plan<'_> {
|
||||
}
|
||||
}
|
||||
|
||||
// Attach label values as coords (missing labels become "").
|
||||
for lab in &self.labels {
|
||||
let val = map
|
||||
.get(&lab.field)
|
||||
.and_then(scalar_name)
|
||||
.unwrap_or_default();
|
||||
coords.push((lab.field.clone(), val));
|
||||
// Labels likewise: a blank label is no coordinate, never an item "".
|
||||
for (lab, v) in self
|
||||
.labels
|
||||
.iter()
|
||||
.filter_map(|lab| Some((lab, named(&lab.field)?)))
|
||||
{
|
||||
coords.push((lab.field.clone(), v));
|
||||
}
|
||||
|
||||
let measures = self
|
||||
@@ -647,37 +745,12 @@ fn unparsed_date_warnings(
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Human-readable warnings for records that could not be placed in the grid.
|
||||
/// Empty when every record was imported.
|
||||
fn skip_warnings(
|
||||
total: usize,
|
||||
categories: &[&FieldProposal],
|
||||
skipped_per_category: &[usize],
|
||||
skipped_non_objects: usize,
|
||||
) -> Vec<String> {
|
||||
let mut warnings = vec![];
|
||||
|
||||
let missing_total: usize = skipped_per_category.iter().sum();
|
||||
if missing_total > 0 {
|
||||
let details: Vec<String> = categories
|
||||
.iter()
|
||||
.zip(skipped_per_category)
|
||||
.filter(|(_, n)| **n > 0)
|
||||
.map(|(p, n)| format!("{} ({n})", p.field))
|
||||
.collect();
|
||||
warnings.push(format!(
|
||||
"Skipped {missing_total} of {total} records missing category values: {}",
|
||||
details.join(", ")
|
||||
));
|
||||
}
|
||||
|
||||
if skipped_non_objects > 0 {
|
||||
warnings.push(format!(
|
||||
"Skipped {skipped_non_objects} of {total} records that were not JSON objects"
|
||||
));
|
||||
}
|
||||
|
||||
warnings
|
||||
/// The warning for records that could not be placed in the grid. A record
|
||||
/// missing a category value is placed, so only a non-object is skipped.
|
||||
fn skip_warnings(total: usize, skipped_non_objects: usize) -> Option<String> {
|
||||
(skipped_non_objects > 0).then(|| {
|
||||
format!("Skipped {skipped_non_objects} of {total} records that were not JSON objects")
|
||||
})
|
||||
}
|
||||
|
||||
// ── Wizard (UI state wrapped around the pipeline) ─────────────────────────────
|
||||
@@ -734,8 +807,12 @@ pub struct ImportWizard {
|
||||
|
||||
impl ImportWizard {
|
||||
pub fn new(raw: Value) -> Self {
|
||||
let pipeline = ImportPipeline::new(raw);
|
||||
Self::from_pipeline(ImportPipeline::new(raw))
|
||||
}
|
||||
|
||||
/// A wizard over a pipeline the caller built, such as a CSV merge that
|
||||
/// carries its sources (improvise-lpzu).
|
||||
pub fn from_pipeline(pipeline: ImportPipeline) -> Self {
|
||||
let step = if pipeline.needs_path_selection() {
|
||||
WizardStep::SelectArrayPath
|
||||
} else if pipeline.records.is_empty() {
|
||||
@@ -1233,6 +1310,7 @@ impl ImportWizard {
|
||||
mod tests {
|
||||
use super::{ImportPipeline, collision_warning};
|
||||
use crate::import::analyzer::FieldKind;
|
||||
use crate::view::Axis;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
#[test]
|
||||
@@ -1467,10 +1545,9 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn a_row_dropped_for_a_missing_category_is_not_charged_to_the_date_tally() {
|
||||
// `Date` is walked before `Region`, so a bad date is seen first and the
|
||||
// record is only afterwards dropped for the missing `Region`.
|
||||
fn a_row_kept_without_a_category_is_charged_to_the_date_tally() {
|
||||
let mut p = extracting_month_and_year(json!([
|
||||
{"Date": "01/15/2025", "Region": "East", "Amount": 100.0},
|
||||
{"Date": "2025-02-05", "Amount": 200.0},
|
||||
@@ -1489,23 +1566,23 @@ mod tests {
|
||||
})
|
||||
.map(|p| p.field.as_str())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
categories,
|
||||
vec!["Date", "Region"],
|
||||
"the fixture only discriminates while Date is walked FIRST"
|
||||
);
|
||||
assert_eq!(categories, vec!["Date", "Region"]);
|
||||
|
||||
let (_, warnings) = p.build_model_with_warnings().unwrap();
|
||||
let (wb, warnings) = p.build_model_with_warnings().unwrap();
|
||||
assert!(
|
||||
warnings.iter().all(|w| !w.contains("Date_Month")),
|
||||
"the row never landed, so no derived column lost it: {warnings:?}"
|
||||
stored(&wb).contains(&"Date=2025-02-05, _Measure=Amount = 200".to_string()),
|
||||
"the row lands without Region or a derived part: {:?}",
|
||||
stored(&wb)
|
||||
);
|
||||
assert_eq!(
|
||||
warnings.len(),
|
||||
1,
|
||||
"only the skipped-record warning: {warnings:?}"
|
||||
assert_eq!(warnings.len(), 2, "{warnings:?}");
|
||||
assert!(
|
||||
warnings[0].contains("'Date_Month' missing from 1 row(s)"),
|
||||
"{warnings:?}"
|
||||
);
|
||||
assert!(
|
||||
warnings[1].contains("'Date_Year' missing from 1 row(s)"),
|
||||
"{warnings:?}"
|
||||
);
|
||||
assert!(warnings[0].contains("Region"), "{}", warnings[0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2128,14 +2205,15 @@ mod tests {
|
||||
"null must not become an item: {items:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
warnings.len(),
|
||||
1,
|
||||
"the dropped record is reported: {warnings:?}"
|
||||
stored(&wb),
|
||||
[
|
||||
"_Measure=revenue, region=East = 100",
|
||||
"_Measure=revenue = 200"
|
||||
]
|
||||
);
|
||||
assert!(
|
||||
warnings[0].contains("region"),
|
||||
"warning names the field: {}",
|
||||
warnings[0]
|
||||
warnings.is_empty(),
|
||||
"the record is kept, not skipped: {warnings:?}"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2183,9 +2261,15 @@ mod tests {
|
||||
.collect();
|
||||
assert!(!items.contains(&""), "no empty-named item: {items:?}");
|
||||
assert_eq!(
|
||||
warnings.len(),
|
||||
1,
|
||||
"the dropped record is reported: {warnings:?}"
|
||||
stored(&wb),
|
||||
[
|
||||
"_Measure=revenue, region=East = 100",
|
||||
"_Measure=revenue = 200"
|
||||
]
|
||||
);
|
||||
assert!(
|
||||
warnings.is_empty(),
|
||||
"the record is kept, not skipped: {warnings:?}"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2213,57 +2297,439 @@ mod tests {
|
||||
assert!(warnings.is_empty(), "nothing dropped: {warnings:?}");
|
||||
}
|
||||
|
||||
/// improvise-8jk6.2: a record without the field keeps its cell. It was
|
||||
/// skipped until records became sparse.
|
||||
#[test]
|
||||
fn build_model_record_with_missing_category_value_skipped() {
|
||||
// If a record is missing a category field, it should be skipped
|
||||
fn build_model_record_with_missing_category_field_is_kept() {
|
||||
let raw = json!([
|
||||
{"region": "East", "revenue": 100.0},
|
||||
{"revenue": 200.0}, // missing "region"
|
||||
]);
|
||||
let p = ImportPipeline::new(raw);
|
||||
let wb = p.build_model().unwrap();
|
||||
// Only one cell should exist (the East record)
|
||||
use crate::model::cell::CellKey;
|
||||
let k = CellKey::new(vec![
|
||||
("_Measure".to_string(), "revenue".to_string()),
|
||||
("region".to_string(), "East".to_string()),
|
||||
]);
|
||||
assert!(wb.model.get_cell(&k).is_some());
|
||||
let wb = ImportPipeline::new(raw).build_model().unwrap();
|
||||
assert_eq!(
|
||||
stored(&wb),
|
||||
[
|
||||
"_Measure=revenue, region=East = 100",
|
||||
"_Measure=revenue = 200"
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/// improvise-5j9: skipping a record for a missing category value was
|
||||
/// silent. The warning must name the count, the total and the field(s).
|
||||
/// improvise-8jk6.2: a blank category value used to drop the whole record.
|
||||
/// Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn build_model_warns_about_records_skipped_for_missing_category() {
|
||||
fn a_record_with_no_value_for_a_category_is_kept_without_that_coordinate() {
|
||||
let raw = json!([
|
||||
{"region": "East", "revenue": 100.0},
|
||||
{"revenue": 200.0}, // missing "region"
|
||||
{"revenue": 300.0}, // missing "region"
|
||||
{"Payee": "Acme", "Settlement Date": "2026-01-05", "Amount": 10.0},
|
||||
{"Payee": "Bolt", "Settlement Date": null, "Amount": 20.0},
|
||||
{"Payee": "Acme", "Settlement Date": "", "Amount": 40.0},
|
||||
]);
|
||||
let p = ImportPipeline::new(raw);
|
||||
let (_, warnings) = p.build_model_with_warnings().unwrap();
|
||||
let mut p = ImportPipeline::new(raw);
|
||||
force_category(&mut p, "Settlement Date");
|
||||
let (wb, warnings) = p.build_model_with_warnings().unwrap();
|
||||
|
||||
assert_eq!(
|
||||
warnings.len(),
|
||||
1,
|
||||
"one skipped-record warning: {warnings:?}"
|
||||
stored(&wb),
|
||||
[
|
||||
"Payee=Acme, Settlement Date=2026-01-05, _Measure=Amount = 10",
|
||||
"Payee=Bolt, _Measure=Amount = 20",
|
||||
"Payee=Acme, _Measure=Amount = 40",
|
||||
]
|
||||
);
|
||||
let items = &wb.model.category("Settlement Date").unwrap().items;
|
||||
assert_eq!(items.keys().collect::<Vec<_>>(), ["2026-01-05"]);
|
||||
assert!(warnings.is_empty(), "{warnings:?}");
|
||||
}
|
||||
|
||||
/// CSV files as a folder import merges them, sources and all.
|
||||
fn merged_pipeline(files: &[(&str, &str)]) -> ImportPipeline {
|
||||
use crate::import::csv_parser::merge_csvs_from;
|
||||
use crate::storage::{MemoryFs, Storage};
|
||||
let fs = MemoryFs::new();
|
||||
for (path, body) in files {
|
||||
fs.write(std::path::Path::new(path), body.as_bytes())
|
||||
.unwrap();
|
||||
}
|
||||
let paths: Vec<&str> = files.iter().map(|(p, _)| *p).collect();
|
||||
let merged = merge_csvs_from(&fs, &paths).unwrap();
|
||||
assert!(merged.warnings.is_empty(), "{:?}", merged.warnings);
|
||||
ImportPipeline::from_merge(merged)
|
||||
}
|
||||
|
||||
/// code-maat's age and coupling reports.
|
||||
const CODE_MAAT: [(&str, &str); 2] = [
|
||||
(
|
||||
"maat/age.csv",
|
||||
"entity,age-months\nsrc/a.rs,2\nsrc/c.rs,7\n",
|
||||
),
|
||||
(
|
||||
"maat/coupling.csv",
|
||||
"entity,coupled,degree,average-revs\nsrc/a.rs,src/b.rs,80,5\nsrc/b.rs,src/c.rs,40,3\n",
|
||||
),
|
||||
];
|
||||
|
||||
fn code_maat_records() -> Value {
|
||||
Value::Array(merged_pipeline(&CODE_MAAT).records)
|
||||
}
|
||||
|
||||
/// Every non-empty cell a view's grid draws, row by row.
|
||||
fn drawn(wb: &crate::workbook::Workbook, view: &str) -> Vec<f64> {
|
||||
use crate::view::GridLayout;
|
||||
let layout = GridLayout::with_frozen_records(&wb.model, &wb.views[view], None);
|
||||
(0..layout.row_count())
|
||||
.flat_map(|r| (0..layout.col_count()).map(move |c| (r, c)))
|
||||
.filter_map(|(r, c)| layout.evaluate(&wb.model, &layout.cell_key(r, c)?))
|
||||
.filter_map(|v| v.as_f64())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// improvise-8jk6.2: where a record with no `Date` shows up, axis by axis.
|
||||
/// Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn a_record_with_no_coordinate_is_drawn_only_where_that_category_is_off_the_grid() {
|
||||
use crate::model::cell::CellKey;
|
||||
use crate::persistence::{format_md, parse_md};
|
||||
use crate::view::{GridLayout, drill_snapshot};
|
||||
let mut p = ImportPipeline::new(json!([
|
||||
{"Payee": "Acme", "Date": "d1", "Amount": 10.0},
|
||||
{"Payee": "Bolt", "Amount": 20.0},
|
||||
]));
|
||||
force_category(&mut p, "Date");
|
||||
let (mut wb, _) = p.build_model_with_warnings().unwrap();
|
||||
let mut seat = |date: Axis, payee: Axis| {
|
||||
let view = wb.active_view_mut();
|
||||
view.set_axis("_Measure", Axis::None);
|
||||
view.set_axis("Payee", payee);
|
||||
view.set_axis("Date", date);
|
||||
(drawn(&wb, "Default"), wb.clone())
|
||||
};
|
||||
assert_eq!(
|
||||
seat(Axis::Column, Axis::Row).0,
|
||||
[10.0],
|
||||
"no column holds Bolt"
|
||||
);
|
||||
assert_eq!(seat(Axis::Page, Axis::Row).0, [10.0], "no page holds Bolt");
|
||||
assert_eq!(
|
||||
seat(Axis::None, Axis::Row).0,
|
||||
[10.0, 20.0],
|
||||
"None counts it"
|
||||
);
|
||||
|
||||
let (_, mut records) = seat(Axis::None, Axis::None);
|
||||
records.active_view_mut().set_axis("_Index", Axis::Row);
|
||||
records.active_view_mut().set_axis("_Dim", Axis::Column);
|
||||
let layout = GridLayout::with_frozen_records(&records.model, records.active_view(), None);
|
||||
let row: Vec<String> = (0..layout.col_count())
|
||||
.map(|c| layout.records_display(1, c).unwrap_or_default())
|
||||
.collect();
|
||||
assert_eq!(row, ["", "Bolt", "", "20", ""], "a blank Date cell");
|
||||
|
||||
let text = format_md(&records);
|
||||
assert!(
|
||||
warnings[0].contains('2'),
|
||||
"warning counts skipped rows: {}",
|
||||
warnings[0]
|
||||
text.contains("\nPayee=Bolt, _Measure=Amount = 20\n"),
|
||||
"{text}"
|
||||
);
|
||||
assert!(
|
||||
warnings[0].contains('3'),
|
||||
"warning names the total: {}",
|
||||
warnings[0]
|
||||
assert_eq!(format_md(&parse_md(&text).unwrap()), text);
|
||||
|
||||
let drill = |cat: &str, item: &str| {
|
||||
let key = CellKey::new(vec![(cat.to_string(), item.to_string())]);
|
||||
drill_snapshot(&records.model, &key).len()
|
||||
};
|
||||
assert_eq!((drill("Payee", "Bolt"), drill("Date", "d1")), (1, 1));
|
||||
}
|
||||
|
||||
/// improvise-lpzu, the Codex P2 on PR #314: a header-only file keeps its
|
||||
/// identity. Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn a_header_only_file_keeps_its_file_item_its_columns_and_its_view() {
|
||||
use crate::view::GridLayout;
|
||||
let (wb, _) = merged_pipeline(&[
|
||||
("maat/age.csv", "entity,age-months\nsrc/a.rs,2\n"),
|
||||
("maat/coupling.csv", "entity,coupled,degree,average-revs\n"),
|
||||
("maat/rev.csv", "entity,n-revs\nsrc/a.rs,9\n"),
|
||||
])
|
||||
.build_model_with_warnings()
|
||||
.unwrap();
|
||||
let files: Vec<&String> = wb.model.category("File").unwrap().items.keys().collect();
|
||||
assert_eq!(files, ["age", "coupling", "rev"]);
|
||||
for column in ["coupled", "degree", "average-revs"] {
|
||||
assert!(wb.model.category(column).is_some(), "{column} is missing");
|
||||
}
|
||||
let view = &wb.views["coupling"];
|
||||
assert_eq!(view.page_selection("File"), Some("coupling"));
|
||||
let layout = GridLayout::with_frozen_records(&wb.model, view, None);
|
||||
assert!(layout.is_records_mode() && layout.row_count() == 0);
|
||||
assert_eq!(drawn(&wb, "age"), [2.0]);
|
||||
}
|
||||
|
||||
/// improvise-8jk6.13: a view never pages onto a measure it hides.
|
||||
/// Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn a_file_with_no_measure_gets_a_records_view_not_another_files_measure() {
|
||||
use crate::view::GridLayout;
|
||||
let (wb, _) = merged_pipeline(&[
|
||||
("t/a.csv", "entity,x\ne1,1\n"),
|
||||
("t/tags.csv", "entity,tag\ne1,red\n"),
|
||||
])
|
||||
.build_model_with_warnings()
|
||||
.unwrap();
|
||||
let tags = &wb.views["tags"];
|
||||
assert_eq!(tags.page_selection("_Measure"), None);
|
||||
assert_ne!(tags.axis_of("_Measure"), Axis::Page, "{tags:?}");
|
||||
let layout = GridLayout::with_frozen_records(&wb.model, tags, None);
|
||||
assert!(layout.is_records_mode(), "{tags:?}");
|
||||
assert_eq!(
|
||||
layout.row_count(),
|
||||
0,
|
||||
"a record with no measure has no cell"
|
||||
);
|
||||
assert!(
|
||||
warnings[0].contains("region"),
|
||||
"warning names the missing field: {}",
|
||||
warnings[0]
|
||||
assert!(!wb.views["a"].is_records_layout());
|
||||
}
|
||||
|
||||
/// improvise-8jk6.12 (1): a date part derived from the file's own column
|
||||
/// is the file's. Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn a_files_view_seats_the_date_parts_of_its_own_date_column() {
|
||||
use crate::import::analyzer::DateComponent;
|
||||
let mut p = merged_pipeline(&[
|
||||
("m/abs-churn.csv", "date,added\n2026-01-02,10\n"),
|
||||
("m/age.csv", "entity,age-months\nsrc/a.rs,2\n"),
|
||||
]);
|
||||
let date = p.proposals.iter_mut().find(|f| f.field == "date").unwrap();
|
||||
assert_eq!(date.kind, FieldKind::TimeCategory);
|
||||
date.date_components = vec![DateComponent::Month];
|
||||
date.date_shape = crate::import::analyzer::DateShape::Split;
|
||||
let (wb, _) = p.build_model_with_warnings().unwrap();
|
||||
assert_ne!(wb.views["abs-churn"].axis_of("date_Month"), Axis::None);
|
||||
assert_eq!(wb.views["age"].axis_of("date_Month"), Axis::None);
|
||||
}
|
||||
|
||||
/// improvise-8jk6.3 under improvise-8jk6.12: the default shape puts the date
|
||||
/// parts IN the file's own column, so that column is what its view seats.
|
||||
#[test]
|
||||
fn a_files_view_seats_its_own_hierarchical_date_column() {
|
||||
use crate::import::analyzer::DateComponent;
|
||||
let mut p = merged_pipeline(&[
|
||||
(
|
||||
"m/abs-churn.csv",
|
||||
"date,added
|
||||
2026-01-02,10
|
||||
",
|
||||
),
|
||||
(
|
||||
"m/age.csv",
|
||||
"entity,age-months
|
||||
src/a.rs,2
|
||||
",
|
||||
),
|
||||
]);
|
||||
let date = p.proposals.iter_mut().find(|f| f.field == "date").unwrap();
|
||||
date.date_components = vec![DateComponent::Year, DateComponent::Month];
|
||||
let (wb, _) = p.build_model_with_warnings().unwrap();
|
||||
assert!(wb.model.category("date_Month").is_none(), "one category");
|
||||
assert_ne!(wb.views["abs-churn"].axis_of("date"), Axis::None);
|
||||
assert_eq!(wb.views["age"].axis_of("date"), Axis::None);
|
||||
}
|
||||
|
||||
/// improvise-8jk6.12 (2): two files of one batch never share a view name.
|
||||
/// Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn two_files_whose_names_collide_in_one_batch_keep_a_view_each() {
|
||||
let (wb, _) = merged_pipeline(&[
|
||||
("d/Default.csv", "Region,Revenue\nEast,1\n"),
|
||||
("d/Default 2.csv", "Region,Revenue\nEast,2\n"),
|
||||
])
|
||||
.build_model_with_warnings()
|
||||
.unwrap();
|
||||
let filters: Vec<(&str, Option<&str>)> = wb
|
||||
.views
|
||||
.values()
|
||||
.map(|v| (v.name.as_str(), v.page_selection("File")))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
filters,
|
||||
[
|
||||
("Default", None),
|
||||
("Default 2", Some("Default")),
|
||||
("Default 2 2", Some("Default 2")),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/// improvise-8jk6.12 (4): a whitespace-only value is no coordinate, never
|
||||
/// an item. Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn a_whitespace_only_category_value_is_no_coordinate() {
|
||||
let (wb, _) = ImportPipeline::new(json!([
|
||||
{"region": "East", "revenue": 1.0},
|
||||
{"region": " ", "revenue": 2.0},
|
||||
]))
|
||||
.build_model_with_warnings()
|
||||
.unwrap();
|
||||
let items: Vec<&String> = wb.model.category("region").unwrap().items.keys().collect();
|
||||
assert_eq!(items, ["East"]);
|
||||
assert_eq!(
|
||||
stored(&wb),
|
||||
["_Measure=revenue, region=East = 1", "_Measure=revenue = 2"]
|
||||
);
|
||||
}
|
||||
|
||||
/// improvise-lpzu: each merged file gets a view of its own records and
|
||||
/// columns. Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn a_merge_gives_each_file_a_view_projected_to_its_columns() {
|
||||
let (wb, _) = merged_pipeline(&CODE_MAAT)
|
||||
.build_model_with_warnings()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
wb.views.keys().collect::<Vec<_>>(),
|
||||
["Default", "age", "coupling"]
|
||||
);
|
||||
assert_eq!(wb.active_view, "Default");
|
||||
|
||||
let age = &wb.views["age"];
|
||||
assert_eq!(age.axis_of("File"), Axis::Filter);
|
||||
assert_eq!(age.page_selection("File"), Some("age"));
|
||||
assert_eq!(age.axis_of("entity"), Axis::Row);
|
||||
assert_eq!(age.axis_of("coupled"), Axis::None);
|
||||
assert_eq!(age.axis_of("_Measure"), Axis::Page);
|
||||
assert_eq!(age.page_selection("_Measure"), Some("age-months"));
|
||||
assert!(age.is_hidden("_Measure", "degree") && age.is_hidden("_Measure", "average-revs"));
|
||||
assert!(!age.is_hidden("_Measure", "age-months"));
|
||||
assert_eq!(drawn(&wb, "age"), [2.0, 7.0]);
|
||||
|
||||
let coupling = &wb.views["coupling"];
|
||||
assert_eq!(coupling.page_selection("File"), Some("coupling"));
|
||||
assert_eq!(coupling.axis_of("entity"), Axis::Row);
|
||||
assert_eq!(coupling.axis_of("coupled"), Axis::Column);
|
||||
assert_eq!(coupling.page_selection("_Measure"), Some("average-revs"));
|
||||
assert!(coupling.is_hidden("_Measure", "age-months"));
|
||||
assert!(!coupling.is_hidden("_Measure", "average-revs"));
|
||||
assert_eq!(drawn(&wb, "coupling"), [5.0, 3.0]);
|
||||
}
|
||||
|
||||
/// improvise-lpzu: two files of one schema are told apart by the filter,
|
||||
/// not by their columns. Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn a_files_view_holds_only_that_files_records() {
|
||||
let (wb, _) = merged_pipeline(&[
|
||||
("s/q1.csv", "Region,Revenue\nEast,10\nWest,1\n"),
|
||||
("s/q2.csv", "Region,Revenue\nEast,20\nWest,2\n"),
|
||||
])
|
||||
.build_model_with_warnings()
|
||||
.unwrap();
|
||||
assert_eq!(drawn(&wb, "q1"), [10.0, 1.0]);
|
||||
assert_eq!(drawn(&wb, "q2"), [20.0, 2.0]);
|
||||
}
|
||||
|
||||
/// improvise-lpzu: a view name is taken from the file only while it is
|
||||
/// free. Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn a_files_view_name_skips_names_already_held() {
|
||||
let (wb, _) = merged_pipeline(&[
|
||||
("d/Default.csv", "Region,Revenue\nEast,1\n"),
|
||||
("d/_Drill.csv", "Region,Revenue\nEast,2\n"),
|
||||
])
|
||||
.build_model_with_warnings()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
wb.views.keys().collect::<Vec<_>>(),
|
||||
["Default", "Default 2", "_Drill 2"]
|
||||
);
|
||||
assert_eq!(
|
||||
wb.views["Default 2"].page_selection("File"),
|
||||
Some("Default")
|
||||
);
|
||||
assert_eq!(wb.views["_Drill 2"].page_selection("File"), Some("_Drill"));
|
||||
}
|
||||
|
||||
/// improvise-lpzu: the per-file views persist, and in the same order on
|
||||
/// every import. Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn per_file_views_survive_a_round_trip_and_repeat_byte_for_byte() {
|
||||
use crate::persistence::{format_md, parse_md};
|
||||
let build = || {
|
||||
let (wb, _) = merged_pipeline(&CODE_MAAT)
|
||||
.build_model_with_warnings()
|
||||
.unwrap();
|
||||
format_md(&wb)
|
||||
};
|
||||
let text = build();
|
||||
assert_eq!(build(), text);
|
||||
let reread = parse_md(&text).unwrap();
|
||||
assert_eq!(format_md(&reread), text);
|
||||
assert_eq!(
|
||||
reread.views.keys().collect::<Vec<_>>(),
|
||||
["Default", "age", "coupling"]
|
||||
);
|
||||
assert_eq!(reread.views["age"].axis_of("File"), Axis::Filter);
|
||||
assert_eq!(reread.views["age"].page_selection("File"), Some("age"));
|
||||
assert_eq!(drawn(&reread, "coupling"), [5.0, 3.0]);
|
||||
}
|
||||
|
||||
/// improvise-lpzu: a column a file lacks is absent from its records, never
|
||||
/// 0, "" or a skip. Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn files_with_different_headers_import_as_one_sparse_model() {
|
||||
let p = ImportPipeline::new(code_maat_records());
|
||||
let kinds: Vec<(&str, FieldKind)> = p
|
||||
.proposals
|
||||
.iter()
|
||||
.map(|f| (f.field.as_str(), f.kind.clone()))
|
||||
.collect();
|
||||
for (field, kind) in [
|
||||
("entity", FieldKind::Category),
|
||||
("coupled", FieldKind::Category),
|
||||
("File", FieldKind::Category),
|
||||
("age-months", FieldKind::Measure),
|
||||
("degree", FieldKind::Measure),
|
||||
("average-revs", FieldKind::Measure),
|
||||
] {
|
||||
assert!(kinds.contains(&(field, kind.clone())), "{field}: {kinds:?}");
|
||||
}
|
||||
|
||||
let (wb, warnings) = p.build_model_with_warnings().unwrap();
|
||||
let mut cells = stored(&wb);
|
||||
cells.sort();
|
||||
assert_eq!(
|
||||
cells,
|
||||
[
|
||||
"File=age, _Measure=age-months, entity=src/a.rs = 2",
|
||||
"File=age, _Measure=age-months, entity=src/c.rs = 7",
|
||||
"File=coupling, _Measure=average-revs, coupled=src/b.rs, entity=src/a.rs = 5",
|
||||
"File=coupling, _Measure=average-revs, coupled=src/c.rs, entity=src/b.rs = 3",
|
||||
"File=coupling, _Measure=degree, coupled=src/b.rs, entity=src/a.rs = 80",
|
||||
"File=coupling, _Measure=degree, coupled=src/c.rs, entity=src/b.rs = 40",
|
||||
]
|
||||
);
|
||||
for cat in wb.model.regular_category_names() {
|
||||
let items = &wb.model.category(cat).unwrap().items;
|
||||
assert!(!items.contains_key(""), "{cat} has an empty item");
|
||||
}
|
||||
assert!(warnings.is_empty(), "{warnings:?}");
|
||||
}
|
||||
|
||||
/// improvise-lpzu: the label half of the rule — absent, not an item named
|
||||
/// "". Evidence: `context/test-evidence/formula-and-io.md`.
|
||||
#[test]
|
||||
fn a_label_column_a_record_lacks_is_no_coordinate_not_an_empty_item() {
|
||||
let mut p = ImportPipeline::new(code_maat_records());
|
||||
p.proposals
|
||||
.iter_mut()
|
||||
.find(|f| f.field == "coupled")
|
||||
.unwrap()
|
||||
.kind = FieldKind::Label;
|
||||
let (wb, _) = p.build_model_with_warnings().unwrap();
|
||||
let mut cells = stored(&wb);
|
||||
cells.sort();
|
||||
assert_eq!(
|
||||
cells[0],
|
||||
"File=age, _Measure=age-months, entity=src/a.rs = 2"
|
||||
);
|
||||
let items = &wb.model.category("coupled").unwrap().items;
|
||||
assert!(!items.contains_key(""), "{:?}", items.keys());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_model_warns_nothing_when_every_record_is_complete() {
|
||||
let raw = json!([
|
||||
|
||||
Reference in New Issue
Block a user