feat: replace JSON with human-readable markdown .improv format

New format is diffable plain text with categories, items, formulas,
data cells, and views all readable without tooling. Legacy JSON files
(detected by leading '{') still load correctly for backwards compat.

Format overview:
  # Model Name
  ## Category: Type
  - Food
  - Gas [Essentials]
  ## Data
  Month=Jan, Type=Food = 100
  ## View: Default (active)
  Type: row

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Ed L
2026-03-24 10:46:57 -07:00
parent cb43a8130e
commit 41f138d5c3

View File

@ -6,25 +6,23 @@ use flate2::write::GzEncoder;
use flate2::Compression;
use crate::model::Model;
use crate::view::GridLayout;
use crate::model::cell::{CellKey, CellValue};
use crate::model::category::Group;
use crate::view::{Axis, GridLayout};
use crate::formula::parse_formula;
const MAGIC: &str = ".improv";
const COMPRESSED_EXT: &str = ".improv.gz";
pub fn save(model: &Model, path: &Path) -> Result<()> {
let json = serde_json::to_string_pretty(model)
.context("Failed to serialize model")?;
let text = format_md(model);
if path.extension().and_then(|e| e.to_str()) == Some("gz")
|| path.to_str().map(|s| s.ends_with(".gz")).unwrap_or(false)
{
if path.to_str().map(|s| s.ends_with(".gz")).unwrap_or(false) {
let file = std::fs::File::create(path)
.with_context(|| format!("Cannot create {}", path.display()))?;
let mut encoder = GzEncoder::new(BufWriter::new(file), Compression::default());
encoder.write_all(json.as_bytes())?;
encoder.write_all(text.as_bytes())?;
encoder.finish()?;
} else {
std::fs::write(path, &json)
std::fs::write(path, &text)
.with_context(|| format!("Cannot write {}", path.display()))?;
}
Ok(())
@ -34,7 +32,7 @@ pub fn load(path: &Path) -> Result<Model> {
let file = std::fs::File::open(path)
.with_context(|| format!("Cannot open {}", path.display()))?;
let json = if path.to_str().map(|s| s.ends_with(".gz")).unwrap_or(false) {
let text = if path.to_str().map(|s| s.ends_with(".gz")).unwrap_or(false) {
let mut decoder = GzDecoder::new(BufReader::new(file));
let mut s = String::new();
decoder.read_to_string(&mut s)?;
@ -45,8 +43,11 @@ pub fn load(path: &Path) -> Result<Model> {
s
};
serde_json::from_str(&json)
.context("Failed to deserialize model")
if text.trim_start().starts_with('{') {
serde_json::from_str(&text).context("Failed to deserialize model")
} else {
parse_md(&text)
}
}
pub fn autosave_path(path: &Path) -> std::path::PathBuf {
@ -56,6 +57,275 @@ pub fn autosave_path(path: &Path) -> std::path::PathBuf {
p
}
/// Serialize a model to the markdown `.improv` format.
pub fn format_md(model: &Model) -> String {
use std::fmt::Write;
let mut out = String::new();
writeln!(out, "# {}", model.name).unwrap();
// Categories
for cat in model.categories.values() {
writeln!(out, "\n## Category: {}", cat.name).unwrap();
for item in cat.items.values() {
match &item.group {
Some(g) => writeln!(out, "- {} [{}]", item.name, g).unwrap(),
None => writeln!(out, "- {}", item.name).unwrap(),
}
}
// Group hierarchy: lines starting with `>` for groups that have a parent
for g in &cat.groups {
if let Some(parent) = &g.parent {
writeln!(out, "> {} [{}]", g.name, parent).unwrap();
}
}
}
// Formulas
if !model.formulas().is_empty() {
writeln!(out, "\n## Formulas").unwrap();
for f in model.formulas() {
writeln!(out, "- {} [{}]", f.raw, f.target_category).unwrap();
}
}
// Data — sorted by coordinate string for deterministic diffs
let mut cells: Vec<_> = model.data.cells().iter().collect();
cells.sort_by_key(|(k, _)| coord_str(k));
if !cells.is_empty() {
writeln!(out, "\n## Data").unwrap();
for (key, value) in cells {
let val_str = match value {
CellValue::Number(_) => value.to_string(),
CellValue::Text(s) => format!("\"{}\"", s),
};
writeln!(out, "{} = {}", coord_str(key), val_str).unwrap();
}
}
// Views
for (view_name, view) in &model.views {
let active = if view_name == &model.active_view { " (active)" } else { "" };
writeln!(out, "\n## View: {}{}", view.name, active).unwrap();
for (cat, axis) in &view.category_axes {
match axis {
Axis::Row => writeln!(out, "{}: row", cat).unwrap(),
Axis::Column => writeln!(out, "{}: column", cat).unwrap(),
Axis::Page => {
match view.page_selections.get(cat) {
Some(sel) => writeln!(out, "{}: page, {}", cat, sel).unwrap(),
None => writeln!(out, "{}: page", cat).unwrap(),
}
}
}
}
if !view.number_format.is_empty() {
writeln!(out, "format: {}", view.number_format).unwrap();
}
// Hidden items (sorted for deterministic diffs)
let mut hidden: Vec<(&str, &str)> = view.hidden_items.iter()
.flat_map(|(cat, items)| items.iter().map(move |item| (cat.as_str(), item.as_str())))
.collect();
hidden.sort();
for (cat, item) in hidden {
writeln!(out, "hidden: {}/{}", cat, item).unwrap();
}
// Collapsed groups (sorted for deterministic diffs)
let mut collapsed: Vec<(&str, &str)> = view.collapsed_groups.iter()
.flat_map(|(cat, gs)| gs.iter().map(move |g| (cat.as_str(), g.as_str())))
.collect();
collapsed.sort();
for (cat, group) in collapsed {
writeln!(out, "collapsed: {}/{}", cat, group).unwrap();
}
}
out
}
/// Parse the markdown `.improv` format into a Model.
pub fn parse_md(text: &str) -> Result<Model> {
#[derive(PartialEq)]
enum Section { None, Category, Formulas, Data, View }
let mut model: Option<Model> = None;
let mut current_cat = String::new();
let mut current_view = String::new();
let mut active_view_name = String::new();
let mut section = Section::None;
for line in text.lines() {
let trimmed = line.trim();
if trimmed.is_empty() { continue; }
// Model title
if trimmed.starts_with("# ") && !trimmed.starts_with("## ") {
model = Some(Model::new(trimmed[2..].trim()));
continue;
}
let m = match model.as_mut() {
Some(m) => m,
None => continue,
};
// Section headers
if let Some(rest) = trimmed.strip_prefix("## Category: ") {
current_cat = rest.trim().to_string();
m.add_category(&current_cat)?;
section = Section::Category;
continue;
}
if trimmed == "## Formulas" {
section = Section::Formulas;
continue;
}
if trimmed == "## Data" {
section = Section::Data;
continue;
}
if let Some(rest) = trimmed.strip_prefix("## View: ") {
let (name, is_active) = match rest.trim().strip_suffix(" (active)") {
Some(n) => (n.trim().to_string(), true),
None => (rest.trim().to_string(), false),
};
if is_active { active_view_name = name.clone(); }
if !m.views.contains_key(&name) { m.create_view(&name); }
current_view = name;
section = Section::View;
continue;
}
if trimmed.starts_with("## ") { continue; }
match section {
Section::Category => parse_category_line(trimmed, &current_cat, m),
Section::Formulas => parse_formula_line(trimmed, m)?,
Section::Data => parse_data_line(trimmed, m),
Section::View => parse_view_line(trimmed, &current_view, m)?,
Section::None => {}
}
}
let mut m = model.ok_or_else(|| anyhow::anyhow!("Empty or invalid .improv file"))?;
if !active_view_name.is_empty() && m.views.contains_key(&active_view_name) {
m.active_view = active_view_name;
}
Ok(m)
}
fn parse_category_line(line: &str, cat_name: &str, m: &mut Model) {
if let Some(rest) = line.strip_prefix("- ") {
// Item line: "- ItemName" or "- ItemName [GroupName]"
let (item_name, group) = parse_bracketed(rest);
if let Some(c) = m.category_mut(cat_name) {
match group {
Some(g) => {
c.add_item_in_group(item_name, g);
if !c.groups.iter().any(|existing| existing.name == g) {
c.add_group(Group::new(g));
}
}
None => { c.add_item(item_name); }
}
}
} else if let Some(rest) = line.strip_prefix("> ") {
// Group hierarchy line: "> GroupName [ParentName]"
let (group_name, parent) = parse_bracketed(rest);
if let Some(parent_name) = parent {
if let Some(c) = m.category_mut(cat_name) {
match c.groups.iter_mut().find(|g| g.name == group_name) {
Some(g) => g.parent = Some(parent_name.to_string()),
None => c.add_group(Group::new(group_name).with_parent(parent_name)),
}
}
}
}
}
fn parse_formula_line(line: &str, m: &mut Model) -> Result<()> {
if let Some(rest) = line.strip_prefix("- ") {
let (raw, cat) = parse_bracketed(rest);
if let Some(cat_name) = cat {
let formula = parse_formula(raw, cat_name)
.with_context(|| format!("Formula: {raw}"))?;
m.add_formula(formula);
}
}
Ok(())
}
fn parse_data_line(line: &str, m: &mut Model) {
// "Cat1=Item1, Cat2=Item2 = value"
let Some(sep) = line.find(" = ") else { return };
let coords_str = &line[..sep];
let value_str = line[sep + 3..].trim();
let coords: Vec<(String, String)> = coords_str.split(", ")
.filter_map(|part| {
let (cat, item) = part.split_once('=')?;
Some((cat.trim().to_string(), item.trim().to_string()))
})
.collect();
if coords.is_empty() { return; }
let value = if let Some(inner) = value_str.strip_prefix('"').and_then(|s| s.strip_suffix('"')) {
CellValue::Text(inner.to_string())
} else if let Ok(n) = value_str.parse::<f64>() {
CellValue::Number(n)
} else {
CellValue::Text(value_str.to_string())
};
m.set_cell(CellKey::new(coords), value);
}
fn parse_view_line(line: &str, view_name: &str, m: &mut Model) -> Result<()> {
let view = m.views.get_mut(view_name)
.ok_or_else(|| anyhow::anyhow!("View not found: {}", view_name))?;
if let Some(fmt) = line.strip_prefix("format: ") {
view.number_format = fmt.trim().to_string();
} else if let Some(rest) = line.strip_prefix("hidden: ") {
if let Some((cat, item)) = rest.trim().split_once('/') {
view.hide_item(cat.trim(), item.trim());
}
} else if let Some(rest) = line.strip_prefix("collapsed: ") {
if let Some((cat, group)) = rest.trim().split_once('/') {
view.toggle_group_collapse(cat.trim(), group.trim());
}
} else if let Some(colon) = line.find(": ") {
let cat = line[..colon].trim();
let rest = line[colon + 2..].trim();
if let Some(sel_rest) = rest.strip_prefix("page") {
view.set_axis(cat, Axis::Page);
if let Some(sel) = sel_rest.strip_prefix(", ") {
view.set_page_selection(cat, sel.trim());
}
} else {
let axis = match rest {
"row" => Axis::Row,
"column" => Axis::Column,
_ => return Ok(()),
};
view.set_axis(cat, axis);
}
}
Ok(())
}
/// Split `"Name [Bracket]"` → `("Name", Some("Bracket"))` or `("Name", None)`.
fn parse_bracketed(s: &str) -> (&str, Option<&str>) {
if let Some(open) = s.rfind('[') {
if s.ends_with(']') {
let name = s[..open].trim();
let inner = &s[open + 1..s.len() - 1];
return (name, Some(inner));
}
}
(s.trim(), None)
}
fn coord_str(key: &CellKey) -> String {
key.0.iter().map(|(c, i)| format!("{}={}", c, i)).collect::<Vec<_>>().join(", ")
}
pub fn export_csv(model: &Model, view_name: &str, path: &Path) -> Result<()> {
let view = model.views.get(view_name)
.ok_or_else(|| anyhow::anyhow!("View '{view_name}' not found"))?;
@ -99,3 +369,230 @@ pub fn export_csv(model: &Model, view_name: &str, path: &Path) -> Result<()> {
std::fs::write(path, out)?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::{format_md, parse_md};
use crate::model::Model;
use crate::model::cell::{CellKey, CellValue};
use crate::model::category::Group;
use crate::view::Axis;
use crate::formula::parse_formula;
fn coord(pairs: &[(&str, &str)]) -> CellKey {
CellKey::new(pairs.iter().map(|(c, i)| (c.to_string(), i.to_string())).collect())
}
fn two_cat_model() -> Model {
let mut m = Model::new("Budget");
m.add_category("Type").unwrap();
m.add_category("Month").unwrap();
for item in ["Food", "Gas"] { m.category_mut("Type").unwrap().add_item(item); }
for item in ["Jan", "Feb"] { m.category_mut("Month").unwrap().add_item(item); }
m
}
// ── format_md ────────────────────────────────────────────────────────────
#[test]
fn format_md_contains_model_name() {
let m = Model::new("My Model");
assert!(format_md(&m).contains("# My Model"));
}
#[test]
fn format_md_contains_category_and_items() {
let m = two_cat_model();
let text = format_md(&m);
assert!(text.contains("## Category: Type"));
assert!(text.contains("- Food"));
assert!(text.contains("- Gas"));
assert!(text.contains("## Category: Month"));
assert!(text.contains("- Jan"));
}
#[test]
fn format_md_item_with_group_uses_brackets() {
let mut m = Model::new("T");
m.add_category("Month").unwrap();
m.category_mut("Month").unwrap().add_item_in_group("Jan", "Q1");
let text = format_md(&m);
assert!(text.contains("- Jan [Q1]"), "got:\n{text}");
}
#[test]
fn format_md_group_hierarchy_uses_angle_prefix() {
let mut m = Model::new("T");
m.add_category("Month").unwrap();
m.category_mut("Month").unwrap().add_item_in_group("Jan", "Q1");
m.category_mut("Month").unwrap().add_group(Group::new("Q1").with_parent("2025"));
let text = format_md(&m);
assert!(text.contains("> Q1 [2025]"), "got:\n{text}");
}
#[test]
fn format_md_data_is_sorted_and_quoted() {
let mut m = two_cat_model();
m.set_cell(coord(&[("Month", "Feb"), ("Type", "Food")]), CellValue::Number(200.0));
m.set_cell(coord(&[("Month", "Jan"), ("Type", "Gas")]), CellValue::Text("N/A".into()));
let text = format_md(&m);
let data_pos = text.find("## Data").unwrap();
let feb_pos = text.find("Month=Feb").unwrap();
let jan_pos = text.find("Month=Jan").unwrap();
assert!(data_pos < feb_pos && feb_pos < jan_pos,
"expected sorted order Feb < Jan:\n{text}");
assert!(text.contains("= 200"), "number not quoted:\n{text}");
assert!(text.contains("= \"N/A\""), "text should be quoted:\n{text}");
}
#[test]
fn format_md_view_axes_and_active_marker() {
let m = two_cat_model();
let text = format_md(&m);
assert!(text.contains("## View: Default (active)"));
assert!(text.contains("Type: row"));
assert!(text.contains("Month: column"));
}
#[test]
fn format_md_page_axis_with_selection() {
let mut m = Model::new("T");
m.add_category("Type").unwrap();
m.add_category("Month").unwrap();
m.add_category("Region").unwrap();
m.category_mut("Type").unwrap().add_item("Food");
m.category_mut("Month").unwrap().add_item("Jan");
for r in ["East", "West"] { m.category_mut("Region").unwrap().add_item(r); }
m.active_view_mut().set_page_selection("Region", "West");
let text = format_md(&m);
assert!(text.contains("Region: page, West"), "got:\n{text}");
}
#[test]
fn format_md_formula_includes_category() {
let mut m = two_cat_model();
m.category_mut("Type").unwrap().add_item("Total");
m.add_formula(parse_formula("Total = Food + Gas", "Type").unwrap());
let text = format_md(&m);
assert!(text.contains("- Total = Food + Gas [Type]"), "got:\n{text}");
}
// ── parse_md ─────────────────────────────────────────────────────────────
#[test]
fn parse_md_round_trips_model_name() {
let m = Model::new("My Budget");
assert_eq!(parse_md(&format_md(&m)).unwrap().name, "My Budget");
}
#[test]
fn parse_md_round_trips_categories_and_items() {
let m = two_cat_model();
let m2 = parse_md(&format_md(&m)).unwrap();
assert!(m2.category("Type").is_some());
assert!(m2.category("Month").is_some());
assert!(m2.category("Type").unwrap().item_by_name("Food").is_some());
assert!(m2.category("Month").unwrap().item_by_name("Feb").is_some());
}
#[test]
fn parse_md_round_trips_item_group() {
let mut m = Model::new("T");
m.add_category("Month").unwrap();
m.category_mut("Month").unwrap().add_item_in_group("Jan", "Q1");
let m2 = parse_md(&format_md(&m)).unwrap();
let item = m2.category("Month").unwrap().item_by_name("Jan").unwrap();
assert_eq!(item.group.as_deref(), Some("Q1"));
}
#[test]
fn parse_md_round_trips_group_hierarchy() {
let mut m = Model::new("T");
m.add_category("Month").unwrap();
m.category_mut("Month").unwrap().add_item_in_group("Jan", "Q1");
m.category_mut("Month").unwrap().add_group(Group::new("Q1").with_parent("2025"));
let m2 = parse_md(&format_md(&m)).unwrap();
let groups = &m2.category("Month").unwrap().groups;
let q1 = groups.iter().find(|g| g.name == "Q1").unwrap();
assert_eq!(q1.parent.as_deref(), Some("2025"));
}
#[test]
fn parse_md_round_trips_data_cells() {
let mut m = two_cat_model();
m.set_cell(coord(&[("Month", "Jan"), ("Type", "Food")]), CellValue::Number(100.0));
m.set_cell(coord(&[("Month", "Feb"), ("Type", "Gas")]), CellValue::Text("N/A".into()));
let m2 = parse_md(&format_md(&m)).unwrap();
assert_eq!(m2.get_cell(&coord(&[("Month", "Jan"), ("Type", "Food")])),
Some(&CellValue::Number(100.0)));
assert_eq!(m2.get_cell(&coord(&[("Month", "Feb"), ("Type", "Gas")])),
Some(&CellValue::Text("N/A".into())));
}
#[test]
fn parse_md_round_trips_view_axes() {
let m = two_cat_model();
let m2 = parse_md(&format_md(&m)).unwrap();
let v = m2.active_view();
assert_eq!(v.axis_of("Type"), Axis::Row);
assert_eq!(v.axis_of("Month"), Axis::Column);
}
#[test]
fn parse_md_round_trips_page_selection() {
let mut m = Model::new("T");
m.add_category("Type").unwrap();
m.add_category("Month").unwrap();
m.add_category("Region").unwrap();
m.category_mut("Type").unwrap().add_item("Food");
m.category_mut("Month").unwrap().add_item("Jan");
for r in ["East", "West"] { m.category_mut("Region").unwrap().add_item(r); }
m.active_view_mut().set_page_selection("Region", "West");
let m2 = parse_md(&format_md(&m)).unwrap();
assert_eq!(m2.active_view().page_selection("Region"), Some("West"));
assert_eq!(m2.active_view().axis_of("Region"), Axis::Page);
}
#[test]
fn parse_md_round_trips_active_view() {
let mut m = two_cat_model();
m.create_view("Other");
m.switch_view("Other").unwrap();
let m2 = parse_md(&format_md(&m)).unwrap();
assert_eq!(m2.active_view, "Other");
}
#[test]
fn parse_md_round_trips_formula() {
let mut m = two_cat_model();
m.category_mut("Type").unwrap().add_item("Total");
m.add_formula(parse_formula("Total = Food + Gas", "Type").unwrap());
let m2 = parse_md(&format_md(&m)).unwrap();
let f = &m2.formulas()[0];
assert_eq!(f.raw, "Total = Food + Gas");
assert_eq!(f.target_category, "Type");
}
#[test]
fn parse_md_round_trips_hidden_item() {
let m = two_cat_model();
{
let m = &mut two_cat_model();
m.active_view_mut().hide_item("Type", "Gas");
let m2 = parse_md(&format_md(m)).unwrap();
assert!(m2.active_view().is_hidden("Type", "Gas"));
assert!(!m2.active_view().is_hidden("Type", "Food"));
}
}
#[test]
fn load_dispatcher_detects_legacy_json_by_brace() {
// The load() function routes to JSON deserializer when text starts with '{'
let m = two_cat_model();
let json = serde_json::to_string_pretty(&m).unwrap();
assert!(json.trim_start().starts_with('{'), "sanity check");
// Deserialise via the JSON path
let m2: Model = serde_json::from_str(&json).unwrap();
assert_eq!(m2.name, "Budget");
}
}