Add CSV import functionality
- Use csv crate for robust CSV parsing (handles quoted fields, empty values, \r\n) - Extend --import command to auto-detect format by file extension (.csv or .json) - Reuse existing ImportPipeline and analyzer for field type detection - Categories detected automatically (string fields), measures for numeric fields - Updated help text and welcome screen to mention CSV support All 201 tests pass.
This commit is contained in:
23
Cargo.lock
generated
23
Cargo.lock
generated
@ -161,6 +161,27 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "csv"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938"
|
||||
dependencies = [
|
||||
"csv-core",
|
||||
"itoa",
|
||||
"ryu",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "csv-core"
|
||||
version = "0.1.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling"
|
||||
version = "0.23.0"
|
||||
@ -374,6 +395,7 @@ dependencies = [
|
||||
"anyhow",
|
||||
"chrono",
|
||||
"crossterm",
|
||||
"csv",
|
||||
"dirs",
|
||||
"flate2",
|
||||
"indexmap",
|
||||
@ -381,6 +403,7 @@ dependencies = [
|
||||
"ratatui",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"unicode-width 0.2.0",
|
||||
]
|
||||
|
||||
@ -21,9 +21,11 @@ chrono = { version = "0.4", features = ["serde"] }
|
||||
flate2 = "1"
|
||||
unicode-width = "0.2"
|
||||
dirs = "5"
|
||||
csv = "1"
|
||||
|
||||
[dev-dependencies]
|
||||
proptest = "1"
|
||||
tempfile = "3"
|
||||
|
||||
[profile.release]
|
||||
opt-level = 3
|
||||
|
||||
12789
llama-server.log
Normal file
12789
llama-server.log
Normal file
File diff suppressed because it is too large
Load Diff
@ -163,50 +163,69 @@ pub fn dispatch(model: &mut Model, cmd: &Command) -> CommandResult {
|
||||
path,
|
||||
model_name,
|
||||
array_path,
|
||||
} => import_json_headless(model, path, model_name.as_deref(), array_path.as_deref()),
|
||||
} => import_headless(model, path, model_name.as_deref(), array_path.as_deref()),
|
||||
}
|
||||
}
|
||||
|
||||
fn import_json_headless(
|
||||
fn import_headless(
|
||||
model: &mut Model,
|
||||
path: &str,
|
||||
model_name: Option<&str>,
|
||||
array_path: Option<&str>,
|
||||
) -> CommandResult {
|
||||
let content = match std::fs::read_to_string(path) {
|
||||
Ok(c) => c,
|
||||
Err(e) => return CommandResult::err(format!("Cannot read '{path}': {e}")),
|
||||
};
|
||||
let value: serde_json::Value = match serde_json::from_str(&content) {
|
||||
Ok(v) => v,
|
||||
Err(e) => return CommandResult::err(format!("JSON parse error: {e}")),
|
||||
};
|
||||
let is_csv = path.ends_with(".csv");
|
||||
|
||||
let records = if let Some(ap) = array_path.filter(|s| !s.is_empty()) {
|
||||
match extract_array_at_path(&value, ap) {
|
||||
Some(arr) => arr.clone(),
|
||||
None => return CommandResult::err(format!("No array at path '{ap}'")),
|
||||
let records = if is_csv {
|
||||
// Parse CSV file
|
||||
match crate::import::csv_parser::parse_csv(path) {
|
||||
Ok(recs) => recs,
|
||||
Err(e) => return CommandResult::err(e.to_string()),
|
||||
}
|
||||
} else if let Some(arr) = value.as_array() {
|
||||
arr.clone()
|
||||
} else {
|
||||
// Find first array
|
||||
let paths = crate::import::analyzer::find_array_paths(&value);
|
||||
if let Some(first) = paths.first() {
|
||||
match extract_array_at_path(&value, first) {
|
||||
// Parse JSON file
|
||||
let content = match std::fs::read_to_string(path) {
|
||||
Ok(c) => c,
|
||||
Err(e) => return CommandResult::err(format!("Cannot read '{path}': {e}")),
|
||||
};
|
||||
let value: serde_json::Value = match serde_json::from_str(&content) {
|
||||
Ok(v) => v,
|
||||
Err(e) => return CommandResult::err(format!("JSON parse error: {e}")),
|
||||
};
|
||||
|
||||
if let Some(ap) = array_path.filter(|s| !s.is_empty()) {
|
||||
match extract_array_at_path(&value, ap) {
|
||||
Some(arr) => arr.clone(),
|
||||
None => return CommandResult::err("Could not extract records array"),
|
||||
None => return CommandResult::err(format!("No array at path '{ap}'")),
|
||||
}
|
||||
} else if let Some(arr) = value.as_array() {
|
||||
arr.clone()
|
||||
} else {
|
||||
return CommandResult::err("No array found in JSON");
|
||||
let paths = crate::import::analyzer::find_array_paths(&value);
|
||||
if let Some(first) = paths.first() {
|
||||
match extract_array_at_path(&value, first) {
|
||||
Some(arr) => arr.clone(),
|
||||
None => return CommandResult::err("Could not extract records array"),
|
||||
}
|
||||
} else {
|
||||
return CommandResult::err("No array found in JSON");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let proposals = analyze_records(&records);
|
||||
|
||||
// Auto-accept all and build via ImportPipeline
|
||||
// Build via ImportPipeline
|
||||
let raw = if is_csv {
|
||||
serde_json::Value::Array(records.clone())
|
||||
} else {
|
||||
// For JSON, we need the original parsed value
|
||||
// Re-read and parse to get it (or pass it up from above)
|
||||
serde_json::from_str(&std::fs::read_to_string(path).unwrap_or_default())
|
||||
.unwrap_or(serde_json::Value::Array(records.clone()))
|
||||
};
|
||||
|
||||
let pipeline = crate::import::wizard::ImportPipeline {
|
||||
raw: value,
|
||||
raw,
|
||||
array_paths: vec![],
|
||||
selected_path: array_path.unwrap_or("").to_string(),
|
||||
records,
|
||||
@ -223,7 +242,7 @@ fn import_json_headless(
|
||||
match pipeline.build_model() {
|
||||
Ok(new_model) => {
|
||||
*model = new_model;
|
||||
CommandResult::ok_msg("JSON imported successfully")
|
||||
CommandResult::ok_msg("Imported successfully")
|
||||
}
|
||||
Err(e) => CommandResult::err(e.to_string()),
|
||||
}
|
||||
|
||||
159
src/import/csv_parser.rs
Normal file
159
src/import/csv_parser.rs
Normal file
@ -0,0 +1,159 @@
|
||||
use anyhow::{Context, Result};
|
||||
use csv::ReaderBuilder;
|
||||
use serde_json::Value;
|
||||
|
||||
/// Parse a CSV file and return records as serde_json::Value array
|
||||
pub fn parse_csv(path: &str) -> Result<Vec<Value>> {
|
||||
let mut reader = ReaderBuilder::new()
|
||||
.has_headers(true)
|
||||
.flexible(true)
|
||||
.trim(csv::Trim::All)
|
||||
.from_path(path)
|
||||
.with_context(|| format!("Failed to open CSV file: {path}"))?;
|
||||
|
||||
// Detect if first row looks like headers (strings) or data (mixed)
|
||||
let has_headers = reader.headers().is_ok();
|
||||
|
||||
let mut records = Vec::new();
|
||||
let mut headers = Vec::new();
|
||||
|
||||
if has_headers {
|
||||
headers = reader
|
||||
.headers()
|
||||
.with_context(|| "Failed to read CSV headers")?
|
||||
.iter()
|
||||
.map(|s| s.to_string())
|
||||
.collect();
|
||||
}
|
||||
|
||||
for result in reader.records() {
|
||||
let record = result.with_context(|| "Failed to read CSV record")?;
|
||||
let mut map = serde_json::Map::new();
|
||||
|
||||
for (i, field) in record.iter().enumerate() {
|
||||
let json_value: Value = parse_csv_field(field);
|
||||
if has_headers {
|
||||
if let Some(header) = headers.get(i) {
|
||||
map.insert(header.clone(), json_value);
|
||||
}
|
||||
} else {
|
||||
map.insert(i.to_string(), json_value);
|
||||
}
|
||||
}
|
||||
|
||||
if !map.is_empty() {
|
||||
records.push(Value::Object(map));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(records)
|
||||
}
|
||||
|
||||
fn parse_csv_field(field: &str) -> Value {
|
||||
if field.is_empty() {
|
||||
return Value::Null;
|
||||
}
|
||||
|
||||
// Try to parse as number (integer or float)
|
||||
if let Ok(num) = field.parse::<i64>() {
|
||||
return Value::Number(serde_json::Number::from(num));
|
||||
}
|
||||
|
||||
if let Ok(num) = field.parse::<f64>() {
|
||||
return Value::Number(
|
||||
serde_json::Number::from_f64(num).unwrap_or(serde_json::Number::from(0)),
|
||||
);
|
||||
}
|
||||
|
||||
// Otherwise treat as string
|
||||
Value::String(field.to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs;
|
||||
use tempfile::tempdir;
|
||||
|
||||
fn create_temp_csv(content: &str) -> (String, tempfile::TempDir) {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.csv");
|
||||
fs::write(&path, content).unwrap();
|
||||
(path.to_string_lossy().to_string(), dir)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_simple_csv() {
|
||||
let (path, _dir) = create_temp_csv("Region,Product,Revenue\nEast,Shirts,1000\nWest,Shirts,800");
|
||||
let records = parse_csv(&path).unwrap();
|
||||
|
||||
assert_eq!(records.len(), 2);
|
||||
assert_eq!(records[0]["Region"], Value::String("East".to_string()));
|
||||
assert_eq!(records[0]["Product"], Value::String("Shirts".to_string()));
|
||||
assert_eq!(records[0]["Revenue"], Value::Number(serde_json::Number::from(1000)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_csv_with_floats() {
|
||||
let (path, _dir) =
|
||||
create_temp_csv("Region,Revenue,Cost\nEast,1000.50,600.25\nWest,800.75,500.00");
|
||||
let records = parse_csv(&path).unwrap();
|
||||
|
||||
assert_eq!(records.len(), 2);
|
||||
assert!(records[0]["Revenue"].is_f64());
|
||||
assert_eq!(records[0]["Revenue"], Value::Number(serde_json::Number::from_f64(1000.50).unwrap()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_csv_with_quoted_fields() {
|
||||
let (path, _dir) = create_temp_csv("Product,Description,Price\n\"Shirts\",\"A nice shirt\",10.00");
|
||||
let records = parse_csv(&path).unwrap();
|
||||
|
||||
assert_eq!(records.len(), 1);
|
||||
assert_eq!(records[0]["Product"], Value::String("Shirts".to_string()));
|
||||
assert_eq!(records[0]["Description"], Value::String("A nice shirt".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_csv_with_empty_values() {
|
||||
let (path, _dir) = create_temp_csv("Region,Product,Revenue\nEast,,1000\nWest,Shirts,");
|
||||
let records = parse_csv(&path).unwrap();
|
||||
|
||||
assert_eq!(records.len(), 2);
|
||||
assert_eq!(records[0]["Product"], Value::Null);
|
||||
assert_eq!(records[1]["Revenue"], Value::Null);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_csv_mixed_types() {
|
||||
let (path, _dir) = create_temp_csv(
|
||||
"Name,Count,Price,Active\nWidget,5,9.99,true\nGadget,3,19.99,false",
|
||||
);
|
||||
let records = parse_csv(&path).unwrap();
|
||||
|
||||
assert_eq!(records.len(), 2);
|
||||
assert_eq!(records[0]["Name"], Value::String("Widget".to_string()));
|
||||
assert_eq!(records[0]["Count"], Value::Number(serde_json::Number::from(5)));
|
||||
assert!(records[0]["Price"].is_f64());
|
||||
assert_eq!(records[0]["Active"], Value::String("true".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_checking_csv_format() {
|
||||
// Simulates the format of /Users/edwlan/Downloads/Checking1.csv
|
||||
let (path, _dir) = create_temp_csv(
|
||||
"Date,Amount,Flag,CheckNo,Description\n\
|
||||
\"03/31/2026\",\"-50.00\",\"*\",\"\",\"VENMO PAYMENT 260331\"\n\
|
||||
\"03/31/2026\",\"-240.00\",\"*\",\"\",\"ROBINHOOD DEBITS XXXXX3795\"",
|
||||
);
|
||||
let records = parse_csv(&path).unwrap();
|
||||
|
||||
assert_eq!(records.len(), 2);
|
||||
assert_eq!(records[0]["Date"], Value::String("03/31/2026".to_string()));
|
||||
assert_eq!(records[0]["Amount"], Value::Number(serde_json::Number::from_f64(-50.00).unwrap()));
|
||||
assert_eq!(records[0]["Flag"], Value::String("*".to_string()));
|
||||
assert_eq!(records[0]["CheckNo"], Value::Null);
|
||||
assert_eq!(records[0]["Description"], Value::String("VENMO PAYMENT 260331".to_string()));
|
||||
assert_eq!(records[1]["Amount"], Value::Number(serde_json::Number::from_f64(-240.00).unwrap()));
|
||||
}
|
||||
}
|
||||
@ -1,2 +1,3 @@
|
||||
pub mod analyzer;
|
||||
pub mod csv_parser;
|
||||
pub mod wizard;
|
||||
|
||||
40
src/main.rs
40
src/main.rs
@ -54,26 +54,40 @@ impl Runnable for CmdLineArgs {
|
||||
// Load or create model
|
||||
let model = get_initial_model(&self.file_path)?;
|
||||
|
||||
// Pre-TUI import: parse JSON and open wizard
|
||||
let import_json = if let Some(ref path) = self.import_path {
|
||||
// Pre-TUI import: parse JSON or CSV and open wizard
|
||||
let import_value = if let Some(ref path) = self.import_path {
|
||||
match std::fs::read_to_string(path) {
|
||||
Err(e) => {
|
||||
eprintln!("Cannot read '{}': {e}", path.display());
|
||||
return Ok(());
|
||||
}
|
||||
Ok(content) => match serde_json::from_str::<serde_json::Value>(&content) {
|
||||
Err(e) => {
|
||||
eprintln!("JSON parse error: {e}");
|
||||
return Ok(());
|
||||
Ok(content) => {
|
||||
if path.to_string_lossy().ends_with(".csv") {
|
||||
// Parse CSV and wrap as JSON array
|
||||
match crate::import::csv_parser::parse_csv(&path.to_string_lossy()) {
|
||||
Ok(records) => Some(serde_json::Value::Array(records)),
|
||||
Err(e) => {
|
||||
eprintln!("CSV parse error: {e}");
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Parse JSON
|
||||
match serde_json::from_str::<serde_json::Value>(&content) {
|
||||
Err(e) => {
|
||||
eprintln!("JSON parse error: {e}");
|
||||
return Ok(());
|
||||
}
|
||||
Ok(json) => Some(json),
|
||||
}
|
||||
}
|
||||
Ok(json) => Some(json),
|
||||
},
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
run_tui(model, self.file_path, import_json)
|
||||
run_tui(model, self.file_path, import_value)
|
||||
}
|
||||
}
|
||||
|
||||
@ -130,7 +144,7 @@ impl Runnable for HelpArgs {
|
||||
println!("improvise — multi-dimensional data modeling TUI\n");
|
||||
println!("USAGE:");
|
||||
println!(" improvise [file.improv] Open or create a model");
|
||||
println!(" improvise --import data.json Import JSON then open TUI");
|
||||
println!(" improvise --import data.json Import JSON (or CSV) then open TUI");
|
||||
println!(" improvise --cmd '{{...}}' Run a JSON command (headless, repeatable)");
|
||||
println!(" improvise --script cmds.jsonl Run commands from file (headless)");
|
||||
println!("\nTUI KEYS (vim-style):");
|
||||
@ -246,13 +260,13 @@ impl<'a> Drop for TuiContext<'a> {
|
||||
fn run_tui(
|
||||
model: Model,
|
||||
file_path: Option<PathBuf>,
|
||||
import_json: Option<serde_json::Value>,
|
||||
import_value: Option<serde_json::Value>,
|
||||
) -> Result<()> {
|
||||
let mut stdout = io::stdout();
|
||||
let mut tui_context = TuiContext::enter(&mut stdout)?;
|
||||
let mut app = App::new(model, file_path);
|
||||
|
||||
if let Some(json) = import_json {
|
||||
if let Some(json) = import_value {
|
||||
app.start_import_wizard(json);
|
||||
}
|
||||
|
||||
@ -518,7 +532,7 @@ fn draw_welcome(f: &mut Frame, area: Rect) {
|
||||
),
|
||||
("", Style::default()),
|
||||
(
|
||||
":import <file.json> Import a JSON file",
|
||||
":import <file> Import JSON or CSV file",
|
||||
Style::default().fg(Color::Cyan),
|
||||
),
|
||||
(
|
||||
|
||||
Reference in New Issue
Block a user