fiddlerwoaroof/improvise
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(import): a date column imports as one hierarchical category, the flat split a choice (improvise-8jk6.3)

Browse Source 
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VdfEGXA7aJ6ftMsCc9PJze
This commit is contained in:
Claude (agent-hier)
2026-09-24 22:15:14 -07:00
committed by Edward Langley
parent e09ec75226
commit 195f3b44b8
12 changed files with 550 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/improvise-io/src/import/analyzer.rs
+27
View File
@@ -22,6 +22,26 @@ pub enum DateComponent {
Quarter,
}
impl DateComponent {
/// The components as a hierarchy nests them, coarsest first.
pub const COARSE_TO_FINE: [DateComponent; 3] = [
DateComponent::Year,
DateComponent::Quarter,
DateComponent::Month,
];
}
/// How a date column's components become categories (improvise-8jk6.3).
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum DateShape {
/// ONE category named after the column: its items are the finest chosen
/// component, grouped by the coarser ones. The user's default.
#[default]
Hierarchy,
/// The column's raw values, plus one flat category per chosen component.
Split,
}
#[derive(Debug, Clone)]
pub struct FieldProposal {
pub field: String,
@@ -32,6 +52,8 @@ pub struct FieldProposal {
pub date_format: Option<String>,
/// Which date components to extract as new categories.
pub date_components: Vec<DateComponent>,
/// Whether those components nest in one category or split into several.
pub date_shape: DateShape,
}
impl FieldProposal {
@@ -132,6 +154,7 @@ pub fn analyze_records(records: &[Value]) -> Vec<FieldProposal> {
accepted: true,
date_format: None,
date_components: vec![],
date_shape: DateShape::default(),
};
}
@@ -159,6 +182,7 @@ pub fn analyze_records(records: &[Value]) -> Vec<FieldProposal> {
accepted: true,
date_format,
date_components: vec![],
date_shape: DateShape::default(),
};
}
@@ -170,6 +194,7 @@ pub fn analyze_records(records: &[Value]) -> Vec<FieldProposal> {
accepted: true,
date_format: None,
date_components: vec![],
date_shape: DateShape::default(),
};
}
@@ -180,6 +205,7 @@ pub fn analyze_records(records: &[Value]) -> Vec<FieldProposal> {
accepted: true,
date_format: None,
date_components: vec![],
date_shape: DateShape::default(),
};
}
@@ -191,6 +217,7 @@ pub fn analyze_records(records: &[Value]) -> Vec<FieldProposal> {
accepted: true,
date_format: None,
date_components: vec![],
date_shape: DateShape::default(),
}
})
.collect()
crates/improvise-io/src/import/wizard.rs
+292 -9
View File
@@ -4,7 +4,7 @@ use anyhow::{Result, anyhow};
use serde_json::Value;
use super::analyzer::{
DateComponent, FieldKind, FieldProposal, analyze_records, extract_array_at_path,
DateComponent, DateShape, FieldKind, FieldProposal, analyze_records, extract_array_at_path,
extract_date_component, find_array_paths,
};
use crate::formula::parse_formula;
@@ -116,7 +116,10 @@ impl ImportPipeline {
.filter(|p| p.accepted)
.map(|p| match p.kind {
FieldKind::Category => 1,
FieldKind::TimeCategory if p.date_format.is_some() => 1 + p.date_components.len(),
FieldKind::TimeCategory if p.date_format.is_some() => match p.date_shape {
DateShape::Hierarchy => 1,
DateShape::Split => 1 + p.date_components.len(),
},
FieldKind::TimeCategory => 1,
FieldKind::Measure | FieldKind::Label => 0,
})
@@ -177,6 +180,7 @@ impl ImportPipeline {
&& p.kind == FieldKind::TimeCategory
&& p.date_format.is_some()
&& !p.date_components.is_empty()
&& p.date_shape == DateShape::Split
})
.flat_map(|p| {
let fmt = p.date_format.as_deref().unwrap();
@@ -191,11 +195,17 @@ impl ImportPipeline {
})
})
.collect();
let categories = accepted(&[FieldKind::Category, FieldKind::TimeCategory]);
let date_hierarchies = categories
.iter()
.filter_map(|p| DateHierarchy::of(p))
.collect();
Plan {
categories: accepted(&[FieldKind::Category, FieldKind::TimeCategory]),
categories,
measures: accepted(&[FieldKind::Measure]),
labels: accepted(&[FieldKind::Label]),
date_extractions,
date_hierarchies,
}
}
@@ -222,11 +232,16 @@ impl ImportPipeline {
for cat_proposal in categories {
wb.add_category(&cat_proposal.field)?;
for val in cat_proposal
.distinct_values
// A hierarchical date column's items are its components, added below.
let raw_items = match plan
.date_hierarchies
.iter()
.filter(|v| !v.trim().is_empty())
.any(|h| h.field == cat_proposal.field)
{
true => &[][..],
false => &cat_proposal.distinct_values[..],
};
for val in raw_items.iter().filter(|v| !v.trim().is_empty()) {
wb.add_item(&cat_proposal.field, val)?;
}
}
@@ -263,7 +278,7 @@ impl ImportPipeline {
let mut skipped_non_objects = 0usize;
// Per-extraction tally of values the detected format could not parse. A
// derived coordinate that is merely absent is improvise-yss5.
let mut unparsed_dates = vec![0usize; date_extractions.len()];
let mut unparsed_dates = vec![0usize; date_extractions.len() + plan.date_hierarchies.len()];
// Two rows sharing every coordinate are one cell; last-row-wins lost the
// earlier one (improvise-2cw9). The map only deduplicates — gotcha 18.
let mut cells: Vec<PendingCell> = vec![];
@@ -287,7 +302,11 @@ impl ImportPipeline {
mut coords,
measures,
unparsed,
chains,
} => {
for chain in &chains {
add_date_chain(&mut wb, chain)?;
}
for (cat, item) in &coords {
wb.add_item(cat, item)?;
}
@@ -345,7 +364,21 @@ impl ImportPipeline {
&skipped_per_category,
skipped_non_objects,
);
warnings.extend(unparsed_date_warnings(date_extractions, &unparsed_dates));
let (split, whole) = unparsed_dates.split_at(date_extractions.len());
warnings.extend(unparsed_date_warnings(date_extractions, split));
warnings.extend(
plan.date_hierarchies
.iter()
.zip(whole)
.filter(|(_, n)| **n > 0)
.map(|(h, n)| {
format!(
"'{}' kept its raw value in {n} row(s), ungrouped; it did not match \
the date format '{}'",
h.field, h.format
)
}),
);
warnings.extend(collision_warning(&summed));
warnings.extend(occurrence.map(|name| separated_note(name, &collisions)));
@@ -359,6 +392,40 @@ struct Plan<'a> {
measures: Vec<&'a FieldProposal>,
labels: Vec<&'a FieldProposal>,
date_extractions: Vec<(&'a str, &'a str, DateComponent, String)>,
date_hierarchies: Vec<DateHierarchy<'a>>,
}
/// A date column imported as ONE category: its items are the finest chosen
/// component, grouped by the coarser ones (improvise-8jk6.3, core.md §19).
struct DateHierarchy<'a> {
field: &'a str,
format: &'a str,
/// The chosen components, coarsest first.
levels: Vec<DateComponent>,
}
impl<'a> DateHierarchy<'a> {
fn of(p: &'a FieldProposal) -> Option<Self> {
let levels: Vec<DateComponent> = DateComponent::COARSE_TO_FINE
.into_iter()
.filter(|c| p.date_components.contains(c))
.collect();
let hierarchical =
p.kind == FieldKind::TimeCategory && p.date_shape == DateShape::Hierarchy;
(hierarchical && !levels.is_empty()).then_some(Self {
field: &p.field,
format: p.date_format.as_deref()?,
levels,
})
}
/// Every level's name for one value, root first; the last is the item.
fn chain(&self, value: &str) -> Option<Vec<String>> {
self.levels
.iter()
.map(|c| extract_date_component(value, self.format, *c))
.collect()
}
}
/// Where one source record lands. Worked out once, for the build and for the
@@ -370,6 +437,7 @@ enum Placement {
coords: Vec<(String, String)>,
measures: Vec<(String, f64)>,
unparsed: Vec<usize>,
chains: Vec<DateChain>,
},
/// No value for category `cat_idx`; `partial` is what preceded it.
Missing {
@@ -388,6 +456,7 @@ impl Plan<'_> {
// Charged only once this record survives the walk: a row dropped
// for a LATER missing category never lands (context/test-evidence).
let mut unparsed: Vec<usize> = vec![];
let mut chains: Vec<DateChain> = vec![];
for (cat_idx, cat_proposal) in self.categories.iter().enumerate() {
let Some(v) = map.get(&cat_proposal.field).and_then(scalar_name) else {
@@ -396,7 +465,24 @@ impl Plan<'_> {
partial: coords,
};
};
coords.push((cat_proposal.field.clone(), v.clone()));
let hierarchy = self
.date_hierarchies
.iter()
.enumerate()
.find(|(_, h)| h.field == cat_proposal.field);
let item = match hierarchy.map(|(at, h)| (at, h.chain(&v))) {
Some((_, Some(chain))) => {
let item = chain.last().cloned().unwrap_or_default();
chains.push((cat_proposal.field.clone(), chain));
item
}
Some((at, None)) => {
unparsed.push(self.date_extractions.len() + at);
v.clone()
}
None => v.clone(),
};
coords.push((cat_proposal.field.clone(), item));
// Extract date components from this field's value
for (at, (field, fmt, comp, derived_name)) in self.date_extractions.iter().enumerate() {
@@ -428,10 +514,33 @@ impl Plan<'_> {
coords,
measures,
unparsed,
chains,
}
}
}
/// One hierarchical date coordinate: its category and its chain, root first.
type DateChain = (String, Vec<String>);
/// Register a date item under its groups, and each group under its parent.
fn add_date_chain(wb: &mut Workbook, (cat, chain): &DateChain) -> Result<()> {
let Some((item, groups)) = chain.split_last() else {
return Ok(());
};
match groups.last() {
Some(group) => wb.add_item_in_group(cat, item, group)?,
None => wb.add_item(cat, item)?,
};
let category = wb
.model
.category_mut(cat)
.ok_or_else(|| anyhow!("category '{cat}' vanished mid-import"))?;
for pair in groups.windows(2) {
category.add_group(crate::model::category::Group::new(&pair[1]).with_parent(&pair[0]));
}
Ok(())
}
/// A tuple, and each record on it: its 1-based position and the measures it writes.
type OnTuple<'a> = (&'a Vec<(String, String)>, Vec<(usize, Vec<&'a str>)>);
@@ -1026,6 +1135,18 @@ impl ImportWizard {
}
}
/// Switch the field at the cursor between ONE hierarchical category and the
/// flat split into one category per component (improvise-8jk6.3).
pub fn toggle_date_shape(&mut self) {
if let Some((pi, _)) = self.date_config_at_cursor() {
let proposal = &mut self.pipeline.proposals[pi];
proposal.date_shape = match proposal.date_shape {
DateShape::Hierarchy => DateShape::Split,
DateShape::Split => DateShape::Hierarchy,
};
}
}
// ── Formula editing ────────────────────────────────────────────────────────
/// Buffer for typing a new formula in the DefineFormulas step.
@@ -1327,6 +1448,7 @@ mod tests {
prop.date_format = Some("%m/%d/%Y".to_string());
prop.date_components.push(DateComponent::Month);
prop.date_components.push(DateComponent::Year);
prop.date_shape = crate::import::analyzer::DateShape::Split;
}
}
p
@@ -1431,6 +1553,7 @@ mod tests {
for prop in &mut p.proposals {
if prop.field == "Date" && prop.kind == FieldKind::TimeCategory {
prop.date_components.push(DateComponent::Month);
prop.date_shape = crate::import::analyzer::DateShape::Split;
}
}
let wb = p.build_model().unwrap();
@@ -2197,6 +2320,7 @@ mod tests {
for prop in &mut p.proposals {
if prop.field == "Date" {
prop.date_components.push(DateComponent::Month);
prop.date_shape = crate::import::analyzer::DateShape::Split;
}
}
let wb = p.build_model().unwrap();
@@ -2709,6 +2833,7 @@ mod tests {
w.advance();
assert_eq!(w.step, WizardStep::ConfigureDates, "setup");
w.toggle_date_component();
w.toggle_date_shape();
w.advance();
assert_eq!(w.step, WizardStep::ResolveDuplicates, "setup");
w.move_cursor(1);
@@ -2728,3 +2853,161 @@ mod tests {
);
}
}
/// improvise-8jk6.3: a date column imports as ONE hierarchical category.
#[cfg(test)]
mod date_hierarchy {
use super::{DuplicatePolicy, ImportPipeline};
use crate::import::analyzer::{DateComponent, DateShape, FieldKind};
use crate::model::cell::CellKey;
use crate::view::GridLayout;
use serde_json::json;
/// Two days of one month, a month in the next quarter, and January of the
/// NEXT year, so a month must stay unique across years.
fn records() -> serde_json::Value {
json!([
{"Date": "01/15/2024", "Amount": 1.0},
{"Date": "01/20/2024", "Amount": 10.0},
{"Date": "04/02/2024", "Amount": 100.0},
{"Date": "01/03/2025", "Amount": 1000.0},
])
}
fn pipeline(components: &[DateComponent], shape: DateShape) -> ImportPipeline {
let mut p = ImportPipeline::new(records());
p.on_duplicate = DuplicatePolicy::Sum;
let date = p.proposals.iter_mut().find(|p| p.field == "Date").unwrap();
assert_eq!(date.kind, FieldKind::TimeCategory, "precondition");
date.date_components = components.to_vec();
date.date_shape = shape;
p
}
fn cell(month: &str) -> CellKey {
CellKey::new(vec![
("Date".into(), month.into()),
("_Measure".into(), "Amount".into()),
])
}
/// The default shape: ONE category named after the column, its items the
/// months, grouped into quarters grouped into years.
#[test]
fn a_date_column_imports_as_one_category_of_months_in_quarters_in_years() {
use DateComponent::*;
let p = pipeline(&[Month, Year, Quarter], DateShape::default());
assert_eq!(p.proposed_category_count(), 1, "one slot, not four");
let wb = p.build_model().unwrap();
let names: Vec<&str> = wb.model.regular_category_names();
assert_eq!(
names,
vec!["Date"],
"no Date_Year, Date_Quarter or Date_Month"
);
let date = wb.model.category("Date").unwrap();
let items: Vec<&str> = date.items.keys().map(String::as_str).collect();
assert_eq!(items, vec!["2024-01", "2024-04", "2025-01"]);
assert_eq!(date.item_chain("2024-01"), vec!["2024", "2024-Q1"]);
assert_eq!(date.item_chain("2025-01"), vec!["2025", "2025-Q1"]);
assert_eq!(
wb.model.get_cell(&cell("2024-01")).and_then(|v| v.as_f64()),
Some(11.0),
"two days of one month are one cell"
);
assert_eq!(
wb.model.get_cell(&cell("2025-01")).and_then(|v| v.as_f64()),
Some(1000.0),
"January 2025 is not January 2024"
);
}
/// Chosen levels only: without Quarter, months group straight into years.
#[test]
fn a_level_left_unchosen_is_skipped_not_emptied() {
use DateComponent::*;
let wb = pipeline(&[Year, Month], DateShape::Hierarchy)
.build_model()
.unwrap();
let date = wb.model.category("Date").unwrap();
assert_eq!(date.item_chain("2024-04"), vec!["2024"]);
let wb = pipeline(&[Quarter], DateShape::Hierarchy)
.build_model()
.unwrap();
let date = wb.model.category("Date").unwrap();
let items: Vec<&str> = date.items.keys().map(String::as_str).collect();
assert_eq!(
items,
vec!["2024-Q1", "2024-Q2", "2025-Q1"],
"flat quarters"
);
assert!(date.item_chain("2024-Q1").is_empty());
}
/// The user's ruling on improvise-felo.6: the hierarchy opens folded, one
/// row per year.
#[test]
fn an_imported_date_hierarchy_opens_folded_to_its_years() {
use DateComponent::*;
let wb = pipeline(&[Year, Quarter, Month], DateShape::Hierarchy)
.build_model()
.unwrap();
let layout = GridLayout::with_frozen_records(&wb.model, wb.active_view(), None);
let rows: Vec<String> = (0..layout.row_count())
.map(|r| layout.row_label(r))
.collect();
assert_eq!(rows, vec!["2024", "2025"]);
let v = layout.evaluate(&wb.model, &layout.cell_key(0, 0).unwrap());
assert_eq!(
v.and_then(|v| v.as_f64()),
Some(111.0),
"2024 folds its months"
);
}
/// The flat split stays available as a choice, and is what it was.
#[test]
fn the_split_shape_keeps_the_raw_dates_and_one_category_per_component() {
use DateComponent::*;
let p = pipeline(&[Year, Month], DateShape::Split);
assert_eq!(p.proposed_category_count(), 3);
let wb = p.build_model().unwrap();
let mut names: Vec<&str> = wb.model.regular_category_names();
names.sort_unstable();
assert_eq!(names, vec!["Date", "Date_Month", "Date_Year"]);
assert!(
wb.model
.category("Date")
.unwrap()
.items
.contains_key("01/15/2024")
);
}
/// A value the detected format cannot read keeps its raw text as an
/// ungrouped item, and the build says so rather than dropping the record.
#[test]
fn an_unparsed_date_keeps_its_value_and_is_warned_about() {
use DateComponent::*;
let mut p = pipeline(&[Year, Month], DateShape::Hierarchy);
let date = p.proposals.iter_mut().find(|p| p.field == "Date").unwrap();
date.date_format = Some("%Y-%m-%d".into());
let (wb, warnings) = p.build_model_with_warnings().unwrap();
assert!(wb.model.category("Date_Year").is_none(), "one category");
let date = wb.model.category("Date").unwrap();
assert!(
date.items.contains_key("01/15/2024"),
"the raw value is kept"
);
assert!(
date.item_chain("01/15/2024").is_empty(),
"and it is ungrouped"
);
assert!(
warnings
.iter()
.any(|w| w.contains("'Date' kept its raw value in 4 row(s)")),
"{warnings:?}"
);
}
}