Add CSV import functionality #1
23
Cargo.lock
generated
23
Cargo.lock
generated
@ -161,6 +161,27 @@ dependencies = [
|
|||||||
"winapi",
|
"winapi",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "csv"
|
||||||
|
version = "1.4.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938"
|
||||||
|
dependencies = [
|
||||||
|
"csv-core",
|
||||||
|
"itoa",
|
||||||
|
"ryu",
|
||||||
|
"serde_core",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "csv-core"
|
||||||
|
version = "0.1.13"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782"
|
||||||
|
dependencies = [
|
||||||
|
"memchr",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "darling"
|
name = "darling"
|
||||||
version = "0.23.0"
|
version = "0.23.0"
|
||||||
@ -374,6 +395,7 @@ dependencies = [
|
|||||||
"anyhow",
|
"anyhow",
|
||||||
"chrono",
|
"chrono",
|
||||||
"crossterm",
|
"crossterm",
|
||||||
|
"csv",
|
||||||
"dirs",
|
"dirs",
|
||||||
"flate2",
|
"flate2",
|
||||||
"indexmap",
|
"indexmap",
|
||||||
@ -381,6 +403,7 @@ dependencies = [
|
|||||||
"ratatui",
|
"ratatui",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
"tempfile",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
"unicode-width 0.2.0",
|
"unicode-width 0.2.0",
|
||||||
]
|
]
|
||||||
|
|||||||
@ -21,9 +21,11 @@ chrono = { version = "0.4", features = ["serde"] }
|
|||||||
flate2 = "1"
|
flate2 = "1"
|
||||||
unicode-width = "0.2"
|
unicode-width = "0.2"
|
||||||
dirs = "5"
|
dirs = "5"
|
||||||
|
csv = "1"
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
proptest = "1"
|
proptest = "1"
|
||||||
|
tempfile = "3"
|
||||||
|
|
||||||
[profile.release]
|
[profile.release]
|
||||||
opt-level = 3
|
opt-level = 3
|
||||||
|
|||||||
12789
llama-server.log
Normal file
12789
llama-server.log
Normal file
File diff suppressed because it is too large
Load Diff
@ -163,16 +163,26 @@ pub fn dispatch(model: &mut Model, cmd: &Command) -> CommandResult {
|
|||||||
path,
|
path,
|
||||||
model_name,
|
model_name,
|
||||||
array_path,
|
array_path,
|
||||||
} => import_json_headless(model, path, model_name.as_deref(), array_path.as_deref()),
|
} => import_headless(model, path, model_name.as_deref(), array_path.as_deref()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn import_json_headless(
|
fn import_headless(
|
||||||
model: &mut Model,
|
model: &mut Model,
|
||||||
path: &str,
|
path: &str,
|
||||||
model_name: Option<&str>,
|
model_name: Option<&str>,
|
||||||
array_path: Option<&str>,
|
array_path: Option<&str>,
|
||||||
) -> CommandResult {
|
) -> CommandResult {
|
||||||
|
let is_csv = path.ends_with(".csv");
|
||||||
|
|
||||||
|
let records = if is_csv {
|
||||||
|
// Parse CSV file
|
||||||
|
match crate::import::csv_parser::parse_csv(path) {
|
||||||
|
Ok(recs) => recs,
|
||||||
|
Err(e) => return CommandResult::err(e.to_string()),
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Parse JSON file
|
||||||
let content = match std::fs::read_to_string(path) {
|
let content = match std::fs::read_to_string(path) {
|
||||||
Ok(c) => c,
|
Ok(c) => c,
|
||||||
Err(e) => return CommandResult::err(format!("Cannot read '{path}': {e}")),
|
Err(e) => return CommandResult::err(format!("Cannot read '{path}': {e}")),
|
||||||
@ -182,7 +192,7 @@ fn import_json_headless(
|
|||||||
Err(e) => return CommandResult::err(format!("JSON parse error: {e}")),
|
Err(e) => return CommandResult::err(format!("JSON parse error: {e}")),
|
||||||
};
|
};
|
||||||
|
|
||||||
let records = if let Some(ap) = array_path.filter(|s| !s.is_empty()) {
|
if let Some(ap) = array_path.filter(|s| !s.is_empty()) {
|
||||||
match extract_array_at_path(&value, ap) {
|
match extract_array_at_path(&value, ap) {
|
||||||
Some(arr) => arr.clone(),
|
Some(arr) => arr.clone(),
|
||||||
None => return CommandResult::err(format!("No array at path '{ap}'")),
|
None => return CommandResult::err(format!("No array at path '{ap}'")),
|
||||||
@ -190,7 +200,6 @@ fn import_json_headless(
|
|||||||
} else if let Some(arr) = value.as_array() {
|
} else if let Some(arr) = value.as_array() {
|
||||||
arr.clone()
|
arr.clone()
|
||||||
} else {
|
} else {
|
||||||
// Find first array
|
|
||||||
let paths = crate::import::analyzer::find_array_paths(&value);
|
let paths = crate::import::analyzer::find_array_paths(&value);
|
||||||
if let Some(first) = paths.first() {
|
if let Some(first) = paths.first() {
|
||||||
match extract_array_at_path(&value, first) {
|
match extract_array_at_path(&value, first) {
|
||||||
@ -200,13 +209,23 @@ fn import_json_headless(
|
|||||||
} else {
|
} else {
|
||||||
return CommandResult::err("No array found in JSON");
|
return CommandResult::err("No array found in JSON");
|
||||||
}
|
}
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let proposals = analyze_records(&records);
|
let proposals = analyze_records(&records);
|
||||||
|
|
||||||
// Auto-accept all and build via ImportPipeline
|
// Build via ImportPipeline
|
||||||
|
let raw = if is_csv {
|
||||||
|
serde_json::Value::Array(records.clone())
|
||||||
|
} else {
|
||||||
|
// For JSON, we need the original parsed value
|
||||||
|
// Re-read and parse to get it (or pass it up from above)
|
||||||
|
serde_json::from_str(&std::fs::read_to_string(path).unwrap_or_default())
|
||||||
|
.unwrap_or(serde_json::Value::Array(records.clone()))
|
||||||
|
};
|
||||||
|
|
||||||
let pipeline = crate::import::wizard::ImportPipeline {
|
let pipeline = crate::import::wizard::ImportPipeline {
|
||||||
raw: value,
|
raw,
|
||||||
array_paths: vec![],
|
array_paths: vec![],
|
||||||
selected_path: array_path.unwrap_or("").to_string(),
|
selected_path: array_path.unwrap_or("").to_string(),
|
||||||
records,
|
records,
|
||||||
@ -223,7 +242,7 @@ fn import_json_headless(
|
|||||||
match pipeline.build_model() {
|
match pipeline.build_model() {
|
||||||
Ok(new_model) => {
|
Ok(new_model) => {
|
||||||
*model = new_model;
|
*model = new_model;
|
||||||
CommandResult::ok_msg("JSON imported successfully")
|
CommandResult::ok_msg("Imported successfully")
|
||||||
}
|
}
|
||||||
Err(e) => CommandResult::err(e.to_string()),
|
Err(e) => CommandResult::err(e.to_string()),
|
||||||
}
|
}
|
||||||
|
|||||||
159
src/import/csv_parser.rs
Normal file
159
src/import/csv_parser.rs
Normal file
@ -0,0 +1,159 @@
|
|||||||
|
use anyhow::{Context, Result};
|
||||||
|
use csv::ReaderBuilder;
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
/// Parse a CSV file and return records as serde_json::Value array
|
||||||
|
pub fn parse_csv(path: &str) -> Result<Vec<Value>> {
|
||||||
|
let mut reader = ReaderBuilder::new()
|
||||||
|
.has_headers(true)
|
||||||
|
.flexible(true)
|
||||||
|
.trim(csv::Trim::All)
|
||||||
|
.from_path(path)
|
||||||
|
.with_context(|| format!("Failed to open CSV file: {path}"))?;
|
||||||
|
|
||||||
|
// Detect if first row looks like headers (strings) or data (mixed)
|
||||||
|
let has_headers = reader.headers().is_ok();
|
||||||
|
|
||||||
|
let mut records = Vec::new();
|
||||||
|
let mut headers = Vec::new();
|
||||||
|
|
||||||
|
if has_headers {
|
||||||
|
headers = reader
|
||||||
|
.headers()
|
||||||
|
.with_context(|| "Failed to read CSV headers")?
|
||||||
|
.iter()
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
.collect();
|
||||||
|
}
|
||||||
|
|
||||||
|
for result in reader.records() {
|
||||||
|
let record = result.with_context(|| "Failed to read CSV record")?;
|
||||||
|
let mut map = serde_json::Map::new();
|
||||||
|
|
||||||
|
for (i, field) in record.iter().enumerate() {
|
||||||
|
let json_value: Value = parse_csv_field(field);
|
||||||
|
if has_headers {
|
||||||
|
if let Some(header) = headers.get(i) {
|
||||||
|
map.insert(header.clone(), json_value);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
map.insert(i.to_string(), json_value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !map.is_empty() {
|
||||||
|
records.push(Value::Object(map));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(records)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_csv_field(field: &str) -> Value {
|
||||||
|
if field.is_empty() {
|
||||||
|
return Value::Null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try to parse as number (integer or float)
|
||||||
|
if let Ok(num) = field.parse::<i64>() {
|
||||||
|
return Value::Number(serde_json::Number::from(num));
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Ok(num) = field.parse::<f64>() {
|
||||||
|
return Value::Number(
|
||||||
|
serde_json::Number::from_f64(num).unwrap_or(serde_json::Number::from(0)),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Otherwise treat as string
|
||||||
|
Value::String(field.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use std::fs;
|
||||||
|
use tempfile::tempdir;
|
||||||
|
|
||||||
|
fn create_temp_csv(content: &str) -> (String, tempfile::TempDir) {
|
||||||
|
let dir = tempdir().unwrap();
|
||||||
|
let path = dir.path().join("test.csv");
|
||||||
|
fs::write(&path, content).unwrap();
|
||||||
|
(path.to_string_lossy().to_string(), dir)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_simple_csv() {
|
||||||
|
let (path, _dir) = create_temp_csv("Region,Product,Revenue\nEast,Shirts,1000\nWest,Shirts,800");
|
||||||
|
let records = parse_csv(&path).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(records.len(), 2);
|
||||||
|
assert_eq!(records[0]["Region"], Value::String("East".to_string()));
|
||||||
|
assert_eq!(records[0]["Product"], Value::String("Shirts".to_string()));
|
||||||
|
assert_eq!(records[0]["Revenue"], Value::Number(serde_json::Number::from(1000)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_csv_with_floats() {
|
||||||
|
let (path, _dir) =
|
||||||
|
create_temp_csv("Region,Revenue,Cost\nEast,1000.50,600.25\nWest,800.75,500.00");
|
||||||
|
let records = parse_csv(&path).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(records.len(), 2);
|
||||||
|
assert!(records[0]["Revenue"].is_f64());
|
||||||
|
assert_eq!(records[0]["Revenue"], Value::Number(serde_json::Number::from_f64(1000.50).unwrap()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_csv_with_quoted_fields() {
|
||||||
|
let (path, _dir) = create_temp_csv("Product,Description,Price\n\"Shirts\",\"A nice shirt\",10.00");
|
||||||
|
let records = parse_csv(&path).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(records.len(), 1);
|
||||||
|
assert_eq!(records[0]["Product"], Value::String("Shirts".to_string()));
|
||||||
|
assert_eq!(records[0]["Description"], Value::String("A nice shirt".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_csv_with_empty_values() {
|
||||||
|
let (path, _dir) = create_temp_csv("Region,Product,Revenue\nEast,,1000\nWest,Shirts,");
|
||||||
|
let records = parse_csv(&path).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(records.len(), 2);
|
||||||
|
assert_eq!(records[0]["Product"], Value::Null);
|
||||||
|
assert_eq!(records[1]["Revenue"], Value::Null);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_csv_mixed_types() {
|
||||||
|
let (path, _dir) = create_temp_csv(
|
||||||
|
"Name,Count,Price,Active\nWidget,5,9.99,true\nGadget,3,19.99,false",
|
||||||
|
);
|
||||||
|
let records = parse_csv(&path).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(records.len(), 2);
|
||||||
|
assert_eq!(records[0]["Name"], Value::String("Widget".to_string()));
|
||||||
|
assert_eq!(records[0]["Count"], Value::Number(serde_json::Number::from(5)));
|
||||||
|
assert!(records[0]["Price"].is_f64());
|
||||||
|
assert_eq!(records[0]["Active"], Value::String("true".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_checking_csv_format() {
|
||||||
|
// Simulates the format of /Users/edwlan/Downloads/Checking1.csv
|
||||||
|
let (path, _dir) = create_temp_csv(
|
||||||
|
"Date,Amount,Flag,CheckNo,Description\n\
|
||||||
|
\"03/31/2026\",\"-50.00\",\"*\",\"\",\"VENMO PAYMENT 260331\"\n\
|
||||||
|
\"03/31/2026\",\"-240.00\",\"*\",\"\",\"ROBINHOOD DEBITS XXXXX3795\"",
|
||||||
|
);
|
||||||
|
let records = parse_csv(&path).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(records.len(), 2);
|
||||||
|
assert_eq!(records[0]["Date"], Value::String("03/31/2026".to_string()));
|
||||||
|
assert_eq!(records[0]["Amount"], Value::Number(serde_json::Number::from_f64(-50.00).unwrap()));
|
||||||
|
assert_eq!(records[0]["Flag"], Value::String("*".to_string()));
|
||||||
|
assert_eq!(records[0]["CheckNo"], Value::Null);
|
||||||
|
assert_eq!(records[0]["Description"], Value::String("VENMO PAYMENT 260331".to_string()));
|
||||||
|
assert_eq!(records[1]["Amount"], Value::Number(serde_json::Number::from_f64(-240.00).unwrap()));
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,2 +1,3 @@
|
|||||||
pub mod analyzer;
|
pub mod analyzer;
|
||||||
|
pub mod csv_parser;
|
||||||
pub mod wizard;
|
pub mod wizard;
|
||||||
|
|||||||
32
src/main.rs
32
src/main.rs
@ -54,26 +54,40 @@ impl Runnable for CmdLineArgs {
|
|||||||
// Load or create model
|
// Load or create model
|
||||||
let model = get_initial_model(&self.file_path)?;
|
let model = get_initial_model(&self.file_path)?;
|
||||||
|
|
||||||
// Pre-TUI import: parse JSON and open wizard
|
// Pre-TUI import: parse JSON or CSV and open wizard
|
||||||
let import_json = if let Some(ref path) = self.import_path {
|
let import_value = if let Some(ref path) = self.import_path {
|
||||||
match std::fs::read_to_string(path) {
|
match std::fs::read_to_string(path) {
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
eprintln!("Cannot read '{}': {e}", path.display());
|
eprintln!("Cannot read '{}': {e}", path.display());
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
Ok(content) => match serde_json::from_str::<serde_json::Value>(&content) {
|
Ok(content) => {
|
||||||
|
if path.to_string_lossy().ends_with(".csv") {
|
||||||
|
// Parse CSV and wrap as JSON array
|
||||||
|
match crate::import::csv_parser::parse_csv(&path.to_string_lossy()) {
|
||||||
|
Ok(records) => Some(serde_json::Value::Array(records)),
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("CSV parse error: {e}");
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Parse JSON
|
||||||
|
match serde_json::from_str::<serde_json::Value>(&content) {
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
eprintln!("JSON parse error: {e}");
|
eprintln!("JSON parse error: {e}");
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
Ok(json) => Some(json),
|
Ok(json) => Some(json),
|
||||||
},
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
|
||||||
run_tui(model, self.file_path, import_json)
|
run_tui(model, self.file_path, import_value)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -130,7 +144,7 @@ impl Runnable for HelpArgs {
|
|||||||
println!("improvise — multi-dimensional data modeling TUI\n");
|
println!("improvise — multi-dimensional data modeling TUI\n");
|
||||||
println!("USAGE:");
|
println!("USAGE:");
|
||||||
println!(" improvise [file.improv] Open or create a model");
|
println!(" improvise [file.improv] Open or create a model");
|
||||||
println!(" improvise --import data.json Import JSON then open TUI");
|
println!(" improvise --import data.json Import JSON (or CSV) then open TUI");
|
||||||
println!(" improvise --cmd '{{...}}' Run a JSON command (headless, repeatable)");
|
println!(" improvise --cmd '{{...}}' Run a JSON command (headless, repeatable)");
|
||||||
println!(" improvise --script cmds.jsonl Run commands from file (headless)");
|
println!(" improvise --script cmds.jsonl Run commands from file (headless)");
|
||||||
println!("\nTUI KEYS (vim-style):");
|
println!("\nTUI KEYS (vim-style):");
|
||||||
@ -246,13 +260,13 @@ impl<'a> Drop for TuiContext<'a> {
|
|||||||
fn run_tui(
|
fn run_tui(
|
||||||
model: Model,
|
model: Model,
|
||||||
file_path: Option<PathBuf>,
|
file_path: Option<PathBuf>,
|
||||||
import_json: Option<serde_json::Value>,
|
import_value: Option<serde_json::Value>,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let mut stdout = io::stdout();
|
let mut stdout = io::stdout();
|
||||||
let mut tui_context = TuiContext::enter(&mut stdout)?;
|
let mut tui_context = TuiContext::enter(&mut stdout)?;
|
||||||
let mut app = App::new(model, file_path);
|
let mut app = App::new(model, file_path);
|
||||||
|
|
||||||
if let Some(json) = import_json {
|
if let Some(json) = import_value {
|
||||||
app.start_import_wizard(json);
|
app.start_import_wizard(json);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -518,7 +532,7 @@ fn draw_welcome(f: &mut Frame, area: Rect) {
|
|||||||
),
|
),
|
||||||
("", Style::default()),
|
("", Style::default()),
|
||||||
(
|
(
|
||||||
":import <file.json> Import a JSON file",
|
":import <file> Import JSON or CSV file",
|
||||||
Style::default().fg(Color::Cyan),
|
Style::default().fg(Color::Cyan),
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
|
|||||||
Reference in New Issue
Block a user