Add CSV import functionality

- Use csv crate for robust CSV parsing (handles quoted fields, empty values, \r\n)
- Extend --import command to auto-detect format by file extension (.csv or .json)
- Reuse existing ImportPipeline and analyzer for field type detection
- Categories detected automatically (string fields), measures for numeric fields
- Updated help text and welcome screen to mention CSV support

All 201 tests pass.
This commit is contained in:
Edward Langley
2026-04-01 01:32:19 -07:00
parent 9fc3f0b5d6
commit ef79a39721
7 changed files with 13045 additions and 38 deletions

23
Cargo.lock generated
View File

@ -161,6 +161,27 @@ dependencies = [
"winapi",
]
[[package]]
name = "csv"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938"
dependencies = [
"csv-core",
"itoa",
"ryu",
"serde_core",
]
[[package]]
name = "csv-core"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782"
dependencies = [
"memchr",
]
[[package]]
name = "darling"
version = "0.23.0"
@ -374,6 +395,7 @@ dependencies = [
"anyhow",
"chrono",
"crossterm",
"csv",
"dirs",
"flate2",
"indexmap",
@ -381,6 +403,7 @@ dependencies = [
"ratatui",
"serde",
"serde_json",
"tempfile",
"thiserror",
"unicode-width 0.2.0",
]

View File

@ -21,9 +21,11 @@ chrono = { version = "0.4", features = ["serde"] }
flate2 = "1"
unicode-width = "0.2"
dirs = "5"
csv = "1"
[dev-dependencies]
proptest = "1"
tempfile = "3"
[profile.release]
opt-level = 3

12789
llama-server.log Normal file

File diff suppressed because it is too large Load Diff

View File

@ -163,50 +163,69 @@ pub fn dispatch(model: &mut Model, cmd: &Command) -> CommandResult {
path,
model_name,
array_path,
} => import_json_headless(model, path, model_name.as_deref(), array_path.as_deref()),
} => import_headless(model, path, model_name.as_deref(), array_path.as_deref()),
}
}
fn import_json_headless(
fn import_headless(
model: &mut Model,
path: &str,
model_name: Option<&str>,
array_path: Option<&str>,
) -> CommandResult {
let content = match std::fs::read_to_string(path) {
Ok(c) => c,
Err(e) => return CommandResult::err(format!("Cannot read '{path}': {e}")),
};
let value: serde_json::Value = match serde_json::from_str(&content) {
Ok(v) => v,
Err(e) => return CommandResult::err(format!("JSON parse error: {e}")),
};
let is_csv = path.ends_with(".csv");
let records = if let Some(ap) = array_path.filter(|s| !s.is_empty()) {
match extract_array_at_path(&value, ap) {
Some(arr) => arr.clone(),
None => return CommandResult::err(format!("No array at path '{ap}'")),
let records = if is_csv {
// Parse CSV file
match crate::import::csv_parser::parse_csv(path) {
Ok(recs) => recs,
Err(e) => return CommandResult::err(e.to_string()),
}
} else if let Some(arr) = value.as_array() {
arr.clone()
} else {
// Find first array
let paths = crate::import::analyzer::find_array_paths(&value);
if let Some(first) = paths.first() {
match extract_array_at_path(&value, first) {
// Parse JSON file
let content = match std::fs::read_to_string(path) {
Ok(c) => c,
Err(e) => return CommandResult::err(format!("Cannot read '{path}': {e}")),
};
let value: serde_json::Value = match serde_json::from_str(&content) {
Ok(v) => v,
Err(e) => return CommandResult::err(format!("JSON parse error: {e}")),
};
if let Some(ap) = array_path.filter(|s| !s.is_empty()) {
match extract_array_at_path(&value, ap) {
Some(arr) => arr.clone(),
None => return CommandResult::err("Could not extract records array"),
None => return CommandResult::err(format!("No array at path '{ap}'")),
}
} else if let Some(arr) = value.as_array() {
arr.clone()
} else {
return CommandResult::err("No array found in JSON");
let paths = crate::import::analyzer::find_array_paths(&value);
if let Some(first) = paths.first() {
match extract_array_at_path(&value, first) {
Some(arr) => arr.clone(),
None => return CommandResult::err("Could not extract records array"),
}
} else {
return CommandResult::err("No array found in JSON");
}
}
};
let proposals = analyze_records(&records);
// Auto-accept all and build via ImportPipeline
// Build via ImportPipeline
let raw = if is_csv {
serde_json::Value::Array(records.clone())
} else {
// For JSON, we need the original parsed value
// Re-read and parse to get it (or pass it up from above)
serde_json::from_str(&std::fs::read_to_string(path).unwrap_or_default())
.unwrap_or(serde_json::Value::Array(records.clone()))
};
let pipeline = crate::import::wizard::ImportPipeline {
raw: value,
raw,
array_paths: vec![],
selected_path: array_path.unwrap_or("").to_string(),
records,
@ -223,7 +242,7 @@ fn import_json_headless(
match pipeline.build_model() {
Ok(new_model) => {
*model = new_model;
CommandResult::ok_msg("JSON imported successfully")
CommandResult::ok_msg("Imported successfully")
}
Err(e) => CommandResult::err(e.to_string()),
}

159
src/import/csv_parser.rs Normal file
View File

@ -0,0 +1,159 @@
use anyhow::{Context, Result};
use csv::ReaderBuilder;
use serde_json::Value;
/// Parse a CSV file and return records as serde_json::Value array
pub fn parse_csv(path: &str) -> Result<Vec<Value>> {
let mut reader = ReaderBuilder::new()
.has_headers(true)
.flexible(true)
.trim(csv::Trim::All)
.from_path(path)
.with_context(|| format!("Failed to open CSV file: {path}"))?;
// Detect if first row looks like headers (strings) or data (mixed)
let has_headers = reader.headers().is_ok();
let mut records = Vec::new();
let mut headers = Vec::new();
if has_headers {
headers = reader
.headers()
.with_context(|| "Failed to read CSV headers")?
.iter()
.map(|s| s.to_string())
.collect();
}
for result in reader.records() {
let record = result.with_context(|| "Failed to read CSV record")?;
let mut map = serde_json::Map::new();
for (i, field) in record.iter().enumerate() {
let json_value: Value = parse_csv_field(field);
if has_headers {
if let Some(header) = headers.get(i) {
map.insert(header.clone(), json_value);
}
} else {
map.insert(i.to_string(), json_value);
}
}
if !map.is_empty() {
records.push(Value::Object(map));
}
}
Ok(records)
}
fn parse_csv_field(field: &str) -> Value {
if field.is_empty() {
return Value::Null;
}
// Try to parse as number (integer or float)
if let Ok(num) = field.parse::<i64>() {
return Value::Number(serde_json::Number::from(num));
}
if let Ok(num) = field.parse::<f64>() {
return Value::Number(
serde_json::Number::from_f64(num).unwrap_or(serde_json::Number::from(0)),
);
}
// Otherwise treat as string
Value::String(field.to_string())
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
use tempfile::tempdir;
fn create_temp_csv(content: &str) -> (String, tempfile::TempDir) {
let dir = tempdir().unwrap();
let path = dir.path().join("test.csv");
fs::write(&path, content).unwrap();
(path.to_string_lossy().to_string(), dir)
}
#[test]
fn parse_simple_csv() {
let (path, _dir) = create_temp_csv("Region,Product,Revenue\nEast,Shirts,1000\nWest,Shirts,800");
let records = parse_csv(&path).unwrap();
assert_eq!(records.len(), 2);
assert_eq!(records[0]["Region"], Value::String("East".to_string()));
assert_eq!(records[0]["Product"], Value::String("Shirts".to_string()));
assert_eq!(records[0]["Revenue"], Value::Number(serde_json::Number::from(1000)));
}
#[test]
fn parse_csv_with_floats() {
let (path, _dir) =
create_temp_csv("Region,Revenue,Cost\nEast,1000.50,600.25\nWest,800.75,500.00");
let records = parse_csv(&path).unwrap();
assert_eq!(records.len(), 2);
assert!(records[0]["Revenue"].is_f64());
assert_eq!(records[0]["Revenue"], Value::Number(serde_json::Number::from_f64(1000.50).unwrap()));
}
#[test]
fn parse_csv_with_quoted_fields() {
let (path, _dir) = create_temp_csv("Product,Description,Price\n\"Shirts\",\"A nice shirt\",10.00");
let records = parse_csv(&path).unwrap();
assert_eq!(records.len(), 1);
assert_eq!(records[0]["Product"], Value::String("Shirts".to_string()));
assert_eq!(records[0]["Description"], Value::String("A nice shirt".to_string()));
}
#[test]
fn parse_csv_with_empty_values() {
let (path, _dir) = create_temp_csv("Region,Product,Revenue\nEast,,1000\nWest,Shirts,");
let records = parse_csv(&path).unwrap();
assert_eq!(records.len(), 2);
assert_eq!(records[0]["Product"], Value::Null);
assert_eq!(records[1]["Revenue"], Value::Null);
}
#[test]
fn parse_csv_mixed_types() {
let (path, _dir) = create_temp_csv(
"Name,Count,Price,Active\nWidget,5,9.99,true\nGadget,3,19.99,false",
);
let records = parse_csv(&path).unwrap();
assert_eq!(records.len(), 2);
assert_eq!(records[0]["Name"], Value::String("Widget".to_string()));
assert_eq!(records[0]["Count"], Value::Number(serde_json::Number::from(5)));
assert!(records[0]["Price"].is_f64());
assert_eq!(records[0]["Active"], Value::String("true".to_string()));
}
#[test]
fn parse_checking_csv_format() {
// Simulates the format of /Users/edwlan/Downloads/Checking1.csv
let (path, _dir) = create_temp_csv(
"Date,Amount,Flag,CheckNo,Description\n\
\"03/31/2026\",\"-50.00\",\"*\",\"\",\"VENMO PAYMENT 260331\"\n\
\"03/31/2026\",\"-240.00\",\"*\",\"\",\"ROBINHOOD DEBITS XXXXX3795\"",
);
let records = parse_csv(&path).unwrap();
assert_eq!(records.len(), 2);
assert_eq!(records[0]["Date"], Value::String("03/31/2026".to_string()));
assert_eq!(records[0]["Amount"], Value::Number(serde_json::Number::from_f64(-50.00).unwrap()));
assert_eq!(records[0]["Flag"], Value::String("*".to_string()));
assert_eq!(records[0]["CheckNo"], Value::Null);
assert_eq!(records[0]["Description"], Value::String("VENMO PAYMENT 260331".to_string()));
assert_eq!(records[1]["Amount"], Value::Number(serde_json::Number::from_f64(-240.00).unwrap()));
}
}

View File

@ -1,2 +1,3 @@
pub mod analyzer;
pub mod csv_parser;
pub mod wizard;

View File

@ -54,26 +54,40 @@ impl Runnable for CmdLineArgs {
// Load or create model
let model = get_initial_model(&self.file_path)?;
// Pre-TUI import: parse JSON and open wizard
let import_json = if let Some(ref path) = self.import_path {
// Pre-TUI import: parse JSON or CSV and open wizard
let import_value = if let Some(ref path) = self.import_path {
match std::fs::read_to_string(path) {
Err(e) => {
eprintln!("Cannot read '{}': {e}", path.display());
return Ok(());
}
Ok(content) => match serde_json::from_str::<serde_json::Value>(&content) {
Err(e) => {
eprintln!("JSON parse error: {e}");
return Ok(());
Ok(content) => {
if path.to_string_lossy().ends_with(".csv") {
// Parse CSV and wrap as JSON array
match crate::import::csv_parser::parse_csv(&path.to_string_lossy()) {
Ok(records) => Some(serde_json::Value::Array(records)),
Err(e) => {
eprintln!("CSV parse error: {e}");
return Ok(());
}
}
} else {
// Parse JSON
match serde_json::from_str::<serde_json::Value>(&content) {
Err(e) => {
eprintln!("JSON parse error: {e}");
return Ok(());
}
Ok(json) => Some(json),
}
}
Ok(json) => Some(json),
},
}
}
} else {
None
};
run_tui(model, self.file_path, import_json)
run_tui(model, self.file_path, import_value)
}
}
@ -130,7 +144,7 @@ impl Runnable for HelpArgs {
println!("improvise — multi-dimensional data modeling TUI\n");
println!("USAGE:");
println!(" improvise [file.improv] Open or create a model");
println!(" improvise --import data.json Import JSON then open TUI");
println!(" improvise --import data.json Import JSON (or CSV) then open TUI");
println!(" improvise --cmd '{{...}}' Run a JSON command (headless, repeatable)");
println!(" improvise --script cmds.jsonl Run commands from file (headless)");
println!("\nTUI KEYS (vim-style):");
@ -246,13 +260,13 @@ impl<'a> Drop for TuiContext<'a> {
fn run_tui(
model: Model,
file_path: Option<PathBuf>,
import_json: Option<serde_json::Value>,
import_value: Option<serde_json::Value>,
) -> Result<()> {
let mut stdout = io::stdout();
let mut tui_context = TuiContext::enter(&mut stdout)?;
let mut app = App::new(model, file_path);
if let Some(json) = import_json {
if let Some(json) = import_value {
app.start_import_wizard(json);
}
@ -518,7 +532,7 @@ fn draw_welcome(f: &mut Frame, area: Rect) {
),
("", Style::default()),
(
":import <file.json> Import a JSON file",
":import <file> Import JSON or CSV file",
Style::default().fg(Color::Cyan),
),
(