powerio/format/

mod.rs

//! Readers and writers for supported case formats, all meeting at [`Network`].
//!
//! Each format module owns its reader and/or writer: MATPOWER `.m`,
//! PowerModels JSON, PSS/E `.raw`, PowerWorld `.aux`, egret `ModelData` JSON,
//! pandapower JSON, PyPSA CSV folders, PSLF `.epc`, GO Challenge 3 JSON, and
//! Surge JSON. PowerWorld `.pwb` cases, GO Challenge 3 JSON canonical output,
//! and PowerWorld `.pwd` displays are read only. Case input and
//! output formats meet here, so adding a writable format is one module plus
//! one hub registration.
//! [`parse_file`] reads Network cases, detecting the format from its extension;
//! [`parse_display_file`] reads display artifacts such as PowerWorld `.pwd`.
//! [`write_as`] serializes a `Network` to text targets. Directory formats,
//! such as PyPSA CSV folders, use explicit filesystem helpers. Non-finite
//! numeric values, such as MATPOWER `Inf`/`NaN` angle limits, are written as
//! JSON `null`.
//!
//! # Fidelity behavior
//!
//! Conversion is two-tier:
//!
//! - **Same format writes return the original text.** A reader keeps its source
//!   text (see [`Network`]), so writing back to the same format returns every
//!   field, comment, and numeric token.
//! - **Cross-format keeps maximal fidelity with itemized loss.** Whatever the
//!   target format cannot represent is reported in the [`Conversion`] `warnings`,
//!   never dropped silently. On the read side, readers itemize what they ignore
//!   in [`Parsed`] `warnings`.

use std::collections::{BTreeSet, HashMap};
use std::fmt;
use std::str::FromStr;
use std::sync::Arc;

use serde_json::{Map, Value};

use crate::gen_cost::{GenCostPatch, MissingGenCostPolicy};
use crate::network::{Branch, BranchRatingSet, Bus, BusId, BusType, Network, SourceFormat};
use crate::{Error, Result};
use routing::{Detection, SourceFormat as DetectedFormat, TransmissionFormat};

mod egret;
mod goc3;
mod matpower;
mod pandapower;
mod powermodels;
pub mod powerworld;
mod pslf;
mod psse;
mod pypsa;
pub mod routing;
mod surge;

pub use egret::{parse_egret_json, write_egret_json};
#[doc(hidden)]
pub use goc3::bridge as goc3_bridge;
pub use goc3::parse_goc3_json;
pub use matpower::{parse_matpower, parse_matpower_file, write_matpower};
pub use pandapower::{parse_pandapower_json, write_pandapower_json};
pub use powermodels::{parse_powermodels_json, write_powermodels_json};
pub use powerworld::{PwdDisplay, PwdSubstation, parse_powerworld, write_powerworld};
pub use pslf::{parse_pslf, write_pslf};
pub use psse::{parse_psse, write_psse, write_psse_rev};
pub use pypsa::{PypsaCsvOutputs, read_pypsa_csv_folder, write_pypsa_csv_folder};
pub use surge::{parse_surge_json, write_surge_json};

/// A target interchange format. See [`write_as`].
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum TargetFormat {
    /// PowerModels.jl network data JSON.
    PowerModelsJson,
    /// egret `ModelData` JSON.
    EgretJson,
    /// PSS/E `.raw` at the given revision. `rev` selects the record layout the
    /// writer emits (33, 34, or 35); 33 is the historical default. The reader
    /// takes the revision from the file header, so this only affects writes.
    Psse { rev: u32 },
    /// PowerWorld auxiliary `.aux`.
    PowerWorld,
    /// pandapower `pandapowerNet` JSON.
    PandapowerJson,
    /// MATPOWER `.m` (round-trip; byte-exact when the case kept its source).
    Matpower,
    /// The canonical PowerIO snapshot: [`Network`] serialized as JSON, validated
    /// on read. Lossless for every model field; the retained source text is the
    /// one exclusion (see [`Network::to_json`]).
    PowerioJson,
    /// GE PSLF `.epc` (round-trip; byte-exact when the case kept its source).
    Pslf,
    /// ARPA-E GO Challenge 3 JSON input data. This is read only except for
    /// same format source echo when the parsed network still carries its source.
    Goc3Json,
    /// Surge native JSON network document.
    SurgeJson,
}

impl TargetFormat {
    /// Conventional file extension for this format (no leading dot).
    #[must_use]
    pub fn extension(self) -> &'static str {
        match self {
            TargetFormat::PowerModelsJson
            | TargetFormat::EgretJson
            | TargetFormat::PandapowerJson
            | TargetFormat::PowerioJson
            | TargetFormat::Goc3Json
            | TargetFormat::SurgeJson => "json",
            TargetFormat::Psse { .. } => "raw",
            TargetFormat::PowerWorld => "aux",
            TargetFormat::Matpower => "m",
            TargetFormat::Pslf => "epc",
        }
    }

    /// Human-readable format name for diagnostics.
    #[must_use]
    pub fn label(self) -> &'static str {
        match self {
            TargetFormat::PowerModelsJson => "PowerModels JSON",
            TargetFormat::EgretJson => "egret JSON",
            TargetFormat::Psse { .. } => "PSS/E .raw",
            TargetFormat::PowerWorld => "PowerWorld .aux",
            TargetFormat::PandapowerJson => "pandapower JSON",
            TargetFormat::Matpower => "MATPOWER .m",
            TargetFormat::PowerioJson => "PowerIO JSON",
            TargetFormat::Pslf => "PSLF .epc",
            TargetFormat::Goc3Json => "GO Challenge 3 JSON",
            TargetFormat::SurgeJson => "Surge JSON",
        }
    }

    /// Canonical API token for this format.
    #[must_use]
    pub fn token(self) -> &'static str {
        match self {
            TargetFormat::PowerModelsJson => "powermodels-json",
            TargetFormat::EgretJson => "egret-json",
            TargetFormat::Psse { rev: 34 } => "psse34",
            TargetFormat::Psse { rev: 35 } => "psse35",
            TargetFormat::Psse { .. } => "psse",
            TargetFormat::PowerWorld => "powerworld",
            TargetFormat::PandapowerJson => "pandapower-json",
            TargetFormat::Matpower => "matpower",
            TargetFormat::PowerioJson => "powerio-json",
            TargetFormat::Pslf => "pslf",
            TargetFormat::Goc3Json => "goc3-json",
            TargetFormat::SurgeJson => "surge-json",
        }
    }
}

impl fmt::Display for TargetFormat {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.write_str(self.token())
    }
}

impl FromStr for TargetFormat {
    type Err = Error;

    fn from_str(name: &str) -> Result<Self> {
        target_format_from_name(name).ok_or_else(|| Error::UnknownFormat(name.to_string()))
    }
}

/// A display artifact format. These files are not power network cases and do
/// not parse to [`Network`].
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum DisplayFormat {
    /// PowerWorld oneline display `.pwd`.
    PowerWorld,
}

impl DisplayFormat {
    /// Conventional file extension for this display format (no leading dot).
    #[must_use]
    pub fn extension(self) -> &'static str {
        match self {
            DisplayFormat::PowerWorld => "pwd",
        }
    }

    /// Human-readable format name for diagnostics.
    #[must_use]
    pub fn label(self) -> &'static str {
        match self {
            DisplayFormat::PowerWorld => "PowerWorld .pwd",
        }
    }

    /// Canonical API token for this format.
    #[must_use]
    pub fn token(self) -> &'static str {
        match self {
            DisplayFormat::PowerWorld => "powerworld-display",
        }
    }
}

impl fmt::Display for DisplayFormat {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.write_str(self.token())
    }
}

impl FromStr for DisplayFormat {
    type Err = Error;

    fn from_str(name: &str) -> Result<Self> {
        display_format_from_name(name).ok_or_else(|| Error::UnknownFormat(name.to_string()))
    }
}

/// Map a display format name to a [`DisplayFormat`], or `None` if unrecognized.
/// Accepts `pwd`, `powerworld-pwd`, and `powerworld-display`.
#[must_use]
pub fn display_format_from_name(name: &str) -> Option<DisplayFormat> {
    Some(match name.to_ascii_lowercase().as_str() {
        "pwd" | "powerworld-pwd" | "powerworld-display" => DisplayFormat::PowerWorld,
        _ => return None,
    })
}

/// Map a format name (with the common aliases) to a [`TargetFormat`], or `None`
/// if unrecognized. Accepts `matpower`/`m`, `powermodels-json`/`powermodels`/`pm`,
/// `egret-json`/`egret`, `pandapower-json`/`pandapower`/`pp`, `psse`/`raw`,
/// `powerworld`/`aux`, `powerio-json`/`powerio`/`json` (the canonical snapshot;
/// plain `json` means this one, the foreign JSON dialects are namespaced),
/// `pslf`/`epc`, `goc3-json`/`goc3`, and `surge-json`/`surge`.
/// Case-insensitive. The one place the bindings (Python, C ABI) share, so a new
/// text format means one new arm here, not three. PyPSA CSV folders, GridFM
/// datasets, and PowerWorld `.pwb` are directory or read only inputs with no
/// text target; they are routed by [`crate::format::routing`].
///
/// The `powermodelsjson`/`egretjson`/`pandapowerjson` aliases let a
/// [`SourceFormat`]'s string form (`{:?}` lowercased, e.g. `"PowerModelsJson"`)
/// round-trip back to a target, so `net.to_format(other.source_format)` works
/// for every format.
#[must_use]
pub fn target_format_from_name(name: &str) -> Option<TargetFormat> {
    Some(match routing::transmission_format_from_name(name)? {
        TransmissionFormat::Matpower => TargetFormat::Matpower,
        TransmissionFormat::PowerModelsJson => TargetFormat::PowerModelsJson,
        TransmissionFormat::EgretJson => TargetFormat::EgretJson,
        TransmissionFormat::Psse => TargetFormat::Psse { rev: 33 },
        TransmissionFormat::Psse34 => TargetFormat::Psse { rev: 34 },
        TransmissionFormat::Psse35 => TargetFormat::Psse { rev: 35 },
        TransmissionFormat::PowerWorld => TargetFormat::PowerWorld,
        TransmissionFormat::PandapowerJson => TargetFormat::PandapowerJson,
        TransmissionFormat::PowerioJson => TargetFormat::PowerioJson,
        TransmissionFormat::Pslf => TargetFormat::Pslf,
        TransmissionFormat::Goc3Json => TargetFormat::Goc3Json,
        TransmissionFormat::SurgeJson => TargetFormat::SurgeJson,
        TransmissionFormat::PypsaCsv | TransmissionFormat::Pwb | TransmissionFormat::Gridfm => {
            return None;
        }
    })
}

/// Output of a display parse. v0.2.2 supports PowerWorld `.pwd`; future display
/// formats can add variants without changing the parse entry point.
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub enum DisplayData {
    /// PowerWorld oneline display data.
    PowerWorld(PwdDisplay),
}

impl DisplayData {
    /// The display format represented by this value.
    #[must_use]
    pub fn format(&self) -> DisplayFormat {
        match self {
            DisplayData::PowerWorld(_) => DisplayFormat::PowerWorld,
        }
    }
}

fn display_file_guidance() -> Error {
    Error::UnknownFormat(
        "a PowerWorld .pwd is display data, not a Network case; \
         use parse_display_file(path, None)"
            .into(),
    )
}

/// Parse display bytes in the named display `format`.
///
/// # Errors
/// [`Error::UnknownFormat`] if `format` is not a display format; otherwise the
/// reader's own [`Error`] on malformed input.
pub fn parse_display_bytes(bytes: &[u8], format: &str) -> Result<DisplayData> {
    let fmt =
        display_format_from_name(format).ok_or_else(|| Error::UnknownFormat(format.to_string()))?;
    match fmt {
        DisplayFormat::PowerWorld => Ok(DisplayData::PowerWorld(powerworld::parse_pwd_display(
            bytes,
        )?)),
    }
}

/// Parse the display file at `path`, choosing the reader from `from` or, when
/// `None`, from the extension. v0.2.2 infers PowerWorld `.pwd`.
///
/// # Errors
/// [`Error::UnknownFormat`] if `from` is unrecognized or the extension cannot
/// be mapped; [`Error::Io`] if the file cannot be read; the reader's own
/// [`Error`] on malformed input.
pub fn parse_display_file(
    path: impl AsRef<std::path::Path>,
    from: Option<&str>,
) -> Result<DisplayData> {
    let path = path.as_ref();
    let fmt = match from {
        Some(f) => {
            display_format_from_name(f).ok_or_else(|| Error::UnknownFormat(f.to_string()))?
        }
        None => match path
            .extension()
            .and_then(|e| e.to_str())
            .map(str::to_ascii_lowercase)
            .as_deref()
        {
            Some("pwd") => DisplayFormat::PowerWorld,
            other => {
                return Err(Error::UnknownFormat(format!(
                    "cannot infer display format from file extension {other:?}; \
                     pass an explicit display format"
                )));
            }
        },
    };
    let bytes = std::fs::read(path)?;
    match fmt {
        DisplayFormat::PowerWorld => Ok(DisplayData::PowerWorld(powerworld::parse_pwd_display(
            &bytes,
        )?)),
    }
}

/// Whether a format name means a PyPSA CSV folder. PyPSA folders are directory
/// inputs, not text targets, so they have no [`TargetFormat`] arm; this is the
/// companion alias matcher to [`target_format_from_name`] and the one place the
/// PyPSA aliases live.
fn is_pypsa_csv_name(name: &str) -> bool {
    matches!(
        name.to_ascii_lowercase().replace(['-', '_'], "").as_str(),
        "pypsacsv" | "pypsa"
    )
}

/// Whether a source format name means PSLF EPC.
fn is_pslf_name(name: &str) -> bool {
    matches!(
        name.to_ascii_lowercase().replace(['-', '_'], "").as_str(),
        "pslf" | "epc" | "pslfepc"
    )
}

/// Parse the case file at `path`, choosing the reader from `from` (the
/// [`target_format_from_name`] names plus `pypsa-csv`/`pypsa`, `pwb`, `pslf`,
/// and `epc`) or, when `None`, from the path: a directory containing
/// `network.csv` parses as a PyPSA CSV folder (any other directory fails:
/// [`Error::UnknownFormat`] when its name maps to no extension, the I/O error
/// otherwise), and a file maps by extension (`m`/`json`/`raw`/`aux`/`pwb`/`epc`),
/// case insensitively (issue #97: `.RAW` is as common as `.raw` in the wild). A
/// `.json` file is classified by top level shape markers: pandapower
/// (`"_class": "pandapowerNet"`), egret (`elements` and `system`), GO Challenge
/// 3 (`network` plus `time_series_input`/`reliability`), Surge JSON
/// (`format: "surge-json"`), powerio-json (`buses` plus network keys), and
/// PowerModels JSON (`baseMVA`, `branch`, `gen`, or `gencost`). JSON matching
/// distribution markers, ambiguous markers, or no known markers returns
/// [`Error::UnknownFormat`]. Pass `from` to force a
/// transmission format. PowerWorld `.pwb` is a binary read only format with no
/// retained source; PSLF `.epc` is text and has a writer. Returns [`Parsed`]:
/// the network plus the reader's fidelity warnings.
///
/// The one path-based parser the CLI and the Python/C/Julia bindings share (each
/// exposes the same `parse_file(path, from)` shape), so adding a source format is
/// one edit here, not one per binding. For in-memory text use [`parse_str`].
///
/// # Errors
/// [`Error::UnknownFormat`] if `from` is unrecognized or the extension can't be
/// mapped; [`Error::Io`] if the file can't be read; the reader's own [`Error`]
/// on malformed input.
pub fn parse_file(path: impl AsRef<std::path::Path>, from: Option<&str>) -> Result<Parsed> {
    let path = path.as_ref();
    // PyPSA CSV folders are directories, not files; dispatch them before any
    // extension logic. `from` accepts the pypsa aliases, and a bare directory
    // with a `network.csv` auto-detects.
    if from.is_some_and(is_pypsa_csv_name)
        || (from.is_none() && path.is_dir() && path.join("network.csv").is_file())
    {
        return pypsa::read_pypsa_csv_folder(path);
    }
    // PowerWorld `.pwb` is binary and read only; dispatch it before the text
    // read. `from` accepts "pwb" for files with a different extension.
    let ext = path
        .extension()
        .and_then(|e| e.to_str())
        .map(str::to_ascii_lowercase);
    if from.is_some_and(|f| f.eq_ignore_ascii_case("pwb"))
        || (from.is_none() && ext.as_deref() == Some("pwb"))
    {
        let bytes = std::fs::read(path)?;
        let stem = path.file_stem().and_then(|s| s.to_str());
        // The binary reader is total (no fidelity warnings); wrap its network
        // in the shared [`Parsed`] shape.
        let network = powerworld::parse_pwb(&bytes, stem)?;
        return Ok(Parsed {
            network,
            warnings: Vec::new(),
        });
    }
    if from.is_some_and(is_pslf_name) || (from.is_none() && ext.as_deref() == Some("epc")) {
        let text = std::fs::read_to_string(path)?;
        let stem = path.file_stem().and_then(|s| s.to_str());
        let mut warnings = Vec::new();
        let network = pslf::parse_pslf_source(Arc::new(text), stem, &mut warnings)?;
        reject_empty_case(&network, "PSLF .epc")?;
        return Ok(Parsed { network, warnings });
    }
    // Settle the format before touching the file: an unmapped or binary
    // extension must surface as UnknownFormat, not as the UTF-8 read error
    // the text formats' loader would hit first. `.pwd` gets its own arm
    // because the display sibling ships next to every case file in the wild
    // and carries no case data.
    if from.is_none() && ext.as_deref() == Some("pwd") {
        return Err(display_file_guidance());
    }
    let fmt_hint = match from {
        Some(f) => {
            if display_format_from_name(f).is_some() {
                return Err(display_file_guidance());
            }
            Some(target_format_from_name(f).ok_or_else(|| Error::UnknownFormat(f.to_string()))?)
        }
        None => {
            // Everything but `.json` (sniffed below) resolves without the text.
            match ext.as_deref() {
                Some("m") => Some(TargetFormat::Matpower),
                Some("raw") => Some(TargetFormat::Psse { rev: 33 }),
                Some("aux") => Some(TargetFormat::PowerWorld),
                Some("json") => None,
                other => {
                    return Err(Error::UnknownFormat(format!(
                        "cannot infer from file extension {other:?}; \
                         pass an explicit source format"
                    )));
                }
            }
        }
    };
    // Read the file once into an owned buffer; the reader moves it straight into
    // the retained source (byte-exact round-trip) with no copy. Sniffing a
    // `.json` borrows the text before the move.
    let text = std::fs::read_to_string(path)?;
    let fmt = match fmt_hint {
        Some(fmt) => fmt,
        None => sniff_json(&text)?,
    };
    // The file stem is the name hint for formats that don't carry their own name.
    let stem = path.file_stem().and_then(|s| s.to_str());
    read_source(Arc::new(text), fmt, stem)
}

/// Read an owned `source` buffer as `fmt`, using `name_hint` (e.g. the file
/// stem) when the format carries no name of its own. The single format→reader
/// map: [`parse_file`] and [`parse_str`] both funnel through it, so every format
/// is dispatched the same way. Each reader takes the owned `Arc` so
/// it moves the buffer straight into the retained source (no copy) and is free
/// to specialize its parse internally. Owns the [`Parsed`] warnings vector;
/// readers that report fidelity loss append to it.
fn read_source(source: Arc<String>, fmt: TargetFormat, name_hint: Option<&str>) -> Result<Parsed> {
    let mut warnings = Vec::new();
    let net = match fmt {
        TargetFormat::Matpower => matpower::parse_matpower_source(source, name_hint),
        TargetFormat::PowerModelsJson => {
            powermodels::parse_powermodels_json_source(source, name_hint, &mut warnings)
        }
        TargetFormat::Psse { .. } => psse::parse_psse_source(source, name_hint, &mut warnings),
        TargetFormat::PowerWorld => {
            powerworld::parse_powerworld_source(source, name_hint, &mut warnings)
        }
        TargetFormat::EgretJson => egret::parse_egret_source(source, name_hint),
        TargetFormat::PandapowerJson => {
            pandapower::parse_pandapower_source(source, name_hint, &mut warnings)
        }
        // The canonical snapshot: validated deserialization of the model itself.
        // It carries its own name and source_format, so the hint doesn't apply.
        TargetFormat::PowerioJson => Network::from_json(&source),
        // PSLF read normally enters through the `is_pslf_name`/`.epc` fast path in
        // parse_file / parse_str; this arm keeps the funnel total.
        TargetFormat::Pslf => pslf::parse_pslf_source(source, name_hint, &mut warnings),
        TargetFormat::Goc3Json => goc3::parse_goc3_source(source, name_hint, &mut warnings),
        TargetFormat::SurgeJson => surge::parse_surge_source(source, name_hint, &mut warnings),
    }?;
    reject_empty_case(&net, fmt.label())?;
    Ok(Parsed {
        network: net,
        warnings,
    })
}

/// A case with no buses is content-free for every consumer. Most readers
/// already reject it on a missing required table, but a JSON carrying only
/// `baseMVA` would otherwise parse to a hollow network; reject it in the
/// [`read_source`] funnel so every parse path (file and in-memory) is guarded,
/// and in the PyPSA folder reader, which bypasses the funnel.
pub(crate) fn reject_empty_case(net: &Network, format: &'static str) -> Result<()> {
    if net.buses.is_empty() {
        return Err(Error::FormatRead {
            format,
            message: "case has no buses".into(),
        });
    }
    Ok(())
}

/// The JSON formats share the `.json` extension, so an explicit source format
/// isn't always given. Classification lives here so the CLI and bindings use
/// the same top level markers as the Rust parsers.
fn sniff_json(text: &str) -> Result<TargetFormat> {
    match routing::classify_json_text(text) {
        Detection::Known(DetectedFormat::Transmission(format)) => transmission_json_target(format),
        Detection::Known(DetectedFormat::Distribution(format)) => {
            Err(Error::UnknownFormat(format!(
                "JSON looks like distribution `{}`; use the distribution parser or pass an explicit transmission format",
                format.name()
            )))
        }
        Detection::Ambiguous => Err(Error::UnknownFormat(
            "ambiguous JSON markers; pass an explicit source format".into(),
        )),
        Detection::Unknown => Err(Error::UnknownFormat(
            "cannot infer JSON format; pass an explicit source format".into(),
        )),
    }
}

fn transmission_json_target(format: TransmissionFormat) -> Result<TargetFormat> {
    match format {
        TransmissionFormat::PowerModelsJson => Ok(TargetFormat::PowerModelsJson),
        TransmissionFormat::EgretJson => Ok(TargetFormat::EgretJson),
        TransmissionFormat::PandapowerJson => Ok(TargetFormat::PandapowerJson),
        TransmissionFormat::PowerioJson => Ok(TargetFormat::PowerioJson),
        TransmissionFormat::Goc3Json => Ok(TargetFormat::Goc3Json),
        TransmissionFormat::SurgeJson => Ok(TargetFormat::SurgeJson),
        other => Err(Error::UnknownFormat(format!(
            "JSON classifier returned non-JSON transmission format `{}`",
            other.name()
        ))),
    }
}

/// Parse in-memory case `text` of the named `format` (see
/// [`target_format_from_name`]). Returns [`Parsed`]: the network plus the
/// reader's fidelity warnings.
///
/// # Errors
/// [`Error::UnknownFormat`] if `format` is unrecognized; the reader's own
/// [`Error`] on malformed input.
pub fn parse_str(text: &str, format: &str) -> Result<Parsed> {
    if is_pslf_name(format) {
        let mut warnings = Vec::new();
        let network = pslf::parse_pslf_source(Arc::new(text.to_owned()), None, &mut warnings)?;
        reject_empty_case(&network, "PSLF .epc")?;
        return Ok(Parsed { network, warnings });
    }
    let fmt =
        target_format_from_name(format).ok_or_else(|| Error::UnknownFormat(format.to_string()))?;
    read_source(Arc::new(text.to_owned()), fmt, None)
}

/// Output of a parse: the network plus the reader's fidelity warnings,
/// tables and columns the model cannot carry, reported instead of dropped
/// silently. Empty for readers that don't report read warnings (currently
/// readers that do not need to reduce any source fields).
///
/// `#[non_exhaustive]`: a returns-only type, so downstream code reads it but
/// never constructs it, leaving room to add parse metadata without a breaking
/// change.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct Parsed {
    pub network: Network,
    pub warnings: Vec<String>,
}

/// Output of a conversion: the serialized text plus any fidelity warnings:
/// data the target can't represent, defaults synthesized, or blocks mapped best
/// effort. An empty `warnings` means a faithful conversion. For [`convert_file`]
/// and [`convert_str`], `warnings` carries the read side ([`Parsed`] warnings)
/// too, ahead of the write side.
///
/// `#[non_exhaustive]`: a returns-only type, so downstream code reads it but
/// never constructs it, leaving room to add fidelity metadata without a breaking
/// change.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct Conversion {
    pub text: String,
    pub warnings: Vec<String>,
}

/// Optional write-time policies layered on top of the neutral [`Network`].
///
/// The default is a no-op and preserves the old `write_as` / `convert_*`
/// behavior. Non-default options work on a cloned network and never mutate the
/// caller's case.
#[derive(Debug, Clone, Default)]
pub struct WriteOptions {
    pub missing_gen_cost: MissingGenCostPolicy,
    pub gen_cost_patches: Vec<GenCostPatch>,
}

impl WriteOptions {
    #[must_use]
    pub fn is_default(&self) -> bool {
        self.missing_gen_cost.is_preserve() && self.gen_cost_patches.is_empty()
    }
}

/// Convert a [`Network`] to `format`. Writing back to the source format returns
/// the retained source text; otherwise the network is serialized into the target.
///
/// # Errors
/// Only a `PowerioJson` serialization failure (none arise from this model
/// today). A non-finite value is not an error: readers legitimately produce
/// `Inf` limits and the bindings materialize every network through the
/// snapshot, so it is written as `null` with a fidelity warning naming the
/// field: that output serves the one-way transports but does not read back
/// (the validating reader rejects the `null`).
pub fn write_as(net: &Network, format: TargetFormat) -> Result<Conversion> {
    if is_echo(net, format) {
        if let Some(src) = &net.source {
            return Ok(Conversion {
                text: src.to_string(),
                warnings: Vec::new(),
            });
        }
    }
    let mut conv = match format {
        TargetFormat::PowerModelsJson => write_powermodels_json(net),
        TargetFormat::EgretJson => write_egret_json(net),
        TargetFormat::Psse { rev } => write_psse_rev(net, rev),
        TargetFormat::PowerWorld => write_powerworld(net),
        TargetFormat::PandapowerJson => write_pandapower_json(net),
        // From another source (or no retained source): canonical MATPOWER from
        // the folded model, which itemizes what it can't carry (HVDC, gen caps,
        // extras, a partial-cost case).
        TargetFormat::Matpower => matpower::write_matpower_conversion(net),
        // The snapshot serializes the model itself, so the usual target
        // passes don't apply (warn_normalized_tap would even be FALSE here:
        // the snapshot preserves the line/transformer labels it warns about);
        // return before them. The one fidelity loss the snapshot can suffer
        // is JSON's missing Inf/NaN: serde writes them as `null`, which
        // `from_json` rejects on the way back, so warn, naming every field.
        TargetFormat::PowerioJson => {
            return net.to_json().map(|text| Conversion {
                text,
                warnings: net
                    .non_finite_fields()
                    .into_iter()
                    .map(|path| {
                        format!(
                            "{path} is not finite; JSON has no Inf/NaN, so it is written as \
                             null and this snapshot will not read back as powerio-json"
                        )
                    })
                    .collect(),
            });
        }
        TargetFormat::Pslf => write_pslf(net),
        TargetFormat::SurgeJson => write_surge_json(net),
        TargetFormat::Goc3Json => {
            return Err(Error::WriteUnsupported {
                format: "goc3-json",
            });
        }
    };
    warn_normalized_tap(net, format, &mut conv);
    warn_missing_reference(net, format, &mut conv);
    warn_dropped_frequency(net, format, &mut conv);
    warn_psse_downgrade(net, format, &mut conv);
    warn_dropped_transformer_charging(net, format, &mut conv);
    Ok(conv)
}

/// Convert a [`Network`] with write-time cost policies. The old [`write_as`]
/// behavior is preserved when `options` is default.
pub fn write_as_with_options(
    net: &Network,
    format: TargetFormat,
    options: &WriteOptions,
) -> Result<Conversion> {
    if options.is_default() {
        return write_as(net, format);
    }

    let mut working = net.clone();
    let report =
        working.apply_gen_cost_policy(&options.gen_cost_patches, options.missing_gen_cost)?;
    let mut policy_warnings = Vec::new();
    if report.patched > 0 {
        policy_warnings.push(format!(
            "generator cost patch applied to {} generator(s)",
            report.patched
        ));
    }
    if report.synthesized > 0 {
        policy_warnings.push(match options.missing_gen_cost {
            MissingGenCostPolicy::Fill {
                c2,
                c1,
                c0,
                startup,
                shutdown,
            } => format!(
                "generator cost synthesized for {} generator(s): model 2, ncost 3, \
                 coeffs [{c2}, {c1}, {c0}], startup {startup}, shutdown {shutdown}",
                report.synthesized
            ),
            _ => unreachable!("only Fill synthesizes costs"),
        });
    }
    if report.patched > 0 || report.synthesized > 0 {
        working.source = None;
    }

    let mut conv = write_as(&working, format)?;
    policy_warnings.append(&mut conv.warnings);
    conv.warnings = policy_warnings;
    Ok(conv)
}

/// Allocate a circuit id for an element keyed by `key` — a bus for loads/shunts,
/// or a `(from, to)` pair for branches: reuse the source-supplied `preferred` id
/// when it is still free on this key, else the lowest free positional id. Keeps
/// parallel devices distinct so the `(key, id)` uniqueness rule the PSS/E and
/// PSLF records require holds even when the source supplies colliding ids.
pub(super) fn allocate_circuit_id<K: Ord + Clone>(
    preferred: Option<&str>,
    key: K,
    used: &mut std::collections::BTreeMap<K, std::collections::BTreeSet<String>>,
) -> String {
    let taken = used.entry(key).or_default();
    if let Some(id) = preferred {
        if taken.insert(id.to_owned()) {
            return id.to_owned();
        }
    }
    let mut n = 1u32;
    loop {
        let candidate = n.to_string();
        if taken.insert(candidate.clone()) {
            return candidate;
        }
        n += 1;
    }
}

/// Warn when a PSS/E source is re-serialized at an older revision than its own.
/// `parse_file` maps every `.raw` to revision 33 and the `psse`/`raw` aliases
/// resolve to 33, so writing a v34/v35 source through the default target skips
/// the echo path (revisions differ) and re-emits the v33 layout, dropping the
/// modern records (12 named ratings, load DG/LOADTYPE columns, the system-wide
/// block) and any unmodeled section the echo would have preserved. Name the
/// downgrade instead of performing it silently.
fn warn_psse_downgrade(net: &Network, format: TargetFormat, conv: &mut Conversion) {
    if let (TargetFormat::Psse { rev }, SourceFormat::Psse, Some(src)) =
        (format, net.source_format, net.source.as_ref())
    {
        let src_rev = psse::header_rev(src);
        if src_rev > rev {
            conv.warnings.push(format!(
                "PSS/E source is revision {src_rev} but the write target is revision {rev}; \
                 the older layout drops fields the source carried (write to psse{src_rev} to keep them)"
            ));
        }
    }
}

/// Warn when a non-default system frequency writes to a format with no frequency
/// field. PSS/E (`BASFRQ`) and pandapower (`f_hz`) carry it; MATPOWER,
/// PowerModels, egret, and PowerWorld have nowhere to put it, so a 50 Hz case
/// would silently read back as the 60 Hz default. Report the loss instead.
fn warn_dropped_frequency(net: &Network, format: TargetFormat, conv: &mut Conversion) {
    let carries_frequency = matches!(
        format,
        TargetFormat::Psse { .. } | TargetFormat::PandapowerJson
    );
    if carries_frequency {
        return;
    }
    if (net.base_frequency - crate::network::DEFAULT_BASE_FREQUENCY).abs() > 1e-9 {
        conv.warnings.push(format!(
            "system base frequency {} Hz dropped: {} has no frequency field (reads back as {} Hz)",
            net.base_frequency,
            format.label(),
            crate::network::DEFAULT_BASE_FREQUENCY
        ));
    }
}

/// Warn when a transformer carries line charging and the target's
/// transformer record has no susceptance column to hold it. The PSLF `.epc`
/// transformer record is the one such target; PSS/E writes representable
/// magnetizing admittance and the MATPOWER shaped writers keep the legacy total
/// projection on the branch row, so neither drops it.
fn warn_dropped_transformer_charging(net: &Network, format: TargetFormat, conv: &mut Conversion) {
    if !matches!(format, TargetFormat::Pslf) {
        return;
    }
    let n = net
        .branches
        .iter()
        .filter(|b| b.is_transformer() && b.legacy_total_charging_b() != 0.0)
        .count();
    if n > 0 {
        conv.warnings.push(format!(
            "{n} transformer(s) carry line charging that the PSLF .epc transformer \
             record cannot represent; the charging was dropped"
        ));
    }
}

pub(super) fn branch_rating_set_drop_warning(
    target: &str,
    branch_index: usize,
    branch: &Branch,
    rating: &BranchRatingSet,
) -> String {
    format!(
        "branch {} ({} to {}) rating set {}={} MVA dropped: {} has no field for branch rating sets beyond rate_a, rate_b, and rate_c",
        branch_index + 1,
        branch.from,
        branch.to,
        rating.name,
        rating.rate_mva,
        target
    )
}

pub(super) fn warn_extra_branch_rating_sets(
    target: &str,
    net: &Network,
    warnings: &mut Vec<String>,
) {
    for (branch_index, branch) in net.branches.iter().enumerate() {
        for rating in &branch.rating_sets {
            warnings.push(branch_rating_set_drop_warning(
                target,
                branch_index,
                branch,
                rating,
            ));
        }
    }
}

/// Convert a case file to `to`, optionally forcing the source format with
/// `from`.
///
/// This is the canonical file-conversion helper shared by the bindings. It
/// parses `path` once, writes the resulting [`Network`] to `to`, and returns the
/// converted text plus any fidelity warnings, read side first. An echo (writing
/// back to the source format) returns the retained text with no warnings.
///
/// # Errors
/// As [`parse_file`].
pub fn convert_file(
    path: impl AsRef<std::path::Path>,
    to: TargetFormat,
    from: Option<&str>,
) -> Result<Conversion> {
    let parsed = parse_file(path, from)?;
    let mut conv = write_as(&parsed.network, to)?;
    if !is_echo(&parsed.network, to) {
        conv.warnings.splice(0..0, parsed.warnings);
    }
    Ok(conv)
}

/// Convert a case file with write-time cost policies.
pub fn convert_file_with_options(
    path: impl AsRef<std::path::Path>,
    to: TargetFormat,
    from: Option<&str>,
    options: &WriteOptions,
) -> Result<Conversion> {
    let parsed = parse_file(path, from)?;
    let mut conv = write_as_with_options(&parsed.network, to, options)?;
    if !is_echo(&parsed.network, to) || !options.is_default() {
        conv.warnings.splice(0..0, parsed.warnings);
    }
    Ok(conv)
}

/// Convert in-memory case `text` of the named `format` (see
/// [`target_format_from_name`]) to `to`.
///
/// The in-memory sibling of [`convert_file`], shared by the bindings: parses
/// `text` once and writes the resulting [`Network`] to `to`, with no file
/// staging in between. Warnings are read side first, as in [`convert_file`].
///
/// # Errors
/// As [`parse_str`].
pub fn convert_str(text: &str, to: TargetFormat, format: &str) -> Result<Conversion> {
    let parsed = parse_str(text, format)?;
    let mut conv = write_as(&parsed.network, to)?;
    if !is_echo(&parsed.network, to) {
        conv.warnings.splice(0..0, parsed.warnings);
    }
    Ok(conv)
}

/// Convert in-memory case text with write-time cost policies.
pub fn convert_str_with_options(
    text: &str,
    to: TargetFormat,
    format: &str,
    options: &WriteOptions,
) -> Result<Conversion> {
    let parsed = parse_str(text, format)?;
    let mut conv = write_as_with_options(&parsed.network, to, options)?;
    if !is_echo(&parsed.network, to) || !options.is_default() {
        conv.warnings.splice(0..0, parsed.warnings);
    }
    Ok(conv)
}

/// Write `net` into the directory `out_dir` as the named directory-shaped
/// format: the directory sibling of [`write_as`], sharing its name-dispatch
/// role for the bindings. PyPSA CSV (`pypsa-csv`/`pypsa`) is the one such
/// format today; a text format name is rejected by name, pointing at
/// [`write_as`]. Returns the write's fidelity warnings.
///
/// # Errors
/// [`Error::UnknownFormat`] for a non-directory format name; the writer's own
/// [`Error`] otherwise.
pub fn write_dir(
    net: &Network,
    to: &str,
    out_dir: impl AsRef<std::path::Path>,
) -> Result<Vec<String>> {
    if is_pypsa_csv_name(to) {
        return write_pypsa_csv_folder(net, out_dir.as_ref()).map(|o| o.warnings);
    }
    Err(Error::UnknownFormat(format!(
        "{to} is not a directory format (directory targets: pypsa-csv/pypsa); \
         text formats serialize through write_as / to_format"
    )))
}

/// Warn when a network with no reference (slack) bus converts to a format
/// whose solvers require one. PowerWorld `.pwb` is the one source that
/// systematically lacks the designation (the binary does not store it), so
/// the silent case would be common; `to_normalized` synthesizes a slack at
/// the largest pmax in service generator bus for consumers that need one.
fn warn_missing_reference(net: &Network, format: TargetFormat, conv: &mut Conversion) {
    let needs_ref = matches!(
        format,
        TargetFormat::Matpower
            | TargetFormat::Psse { .. }
            | TargetFormat::PowerModelsJson
            | TargetFormat::PandapowerJson
            | TargetFormat::Pslf
            | TargetFormat::SurgeJson
    );
    if needs_ref {
        conv.warnings.extend(missing_reference_warning(net));
    }
}

/// The slackless-network warning itself, shared with the PyPSA folder writer
/// (which produces `PypsaCsvOutputs`, not a [`Conversion`], so it cannot go
/// through [`warn_missing_reference`]).
pub(super) fn missing_reference_warning(net: &Network) -> Option<String> {
    (!net.buses.iter().any(|b| b.kind == BusType::Ref)).then(|| {
        "no reference (slack) bus in the source network; power flow tools \
         reject such cases; to_normalized synthesizes a slack at the \
         largest pmax in service generator bus"
            .to_string()
    })
}

/// A normalized network has its tap canonicalized to `1.0` on every line (the
/// `0 → 1` rule), but [`Branch::is_transformer`](crate::network::Branch::is_transformer),
/// the test these writers use to split lines from transformers, keys off
/// `tap != 0`. So a normalized line is written into the transformer section/type.
/// The power flow is identical (a unity-ratio, zero-shift transformer equals a
/// line), but the label is not, so report the fidelity loss rather than relabel
/// it silently. MATPOWER has no separate transformer representation (just a `TAP`
/// column), so it is exempt.
// `tap == 1.0` / `shift == 0.0` are exact by construction: normalization sets a
// line's tap from `effective_tap()` (the literal `1.0`) and its shift from
// `0.0 * DEG_TO_RAD` (exactly `0.0`), so an epsilon compare would be wrong here.
#[allow(clippy::float_cmp)]
fn warn_normalized_tap(net: &Network, format: TargetFormat, conv: &mut Conversion) {
    if matches!(format, TargetFormat::Matpower) {
        return;
    }
    conv.warnings.extend(normalized_tap_warning(net));
}

/// The normalized-label warning itself, shared with the PyPSA folder writer.
// `tap == 1.0` / `shift == 0.0` are exact by construction (see
// `warn_normalized_tap`), so an epsilon compare would be wrong here.
#[allow(clippy::float_cmp)]
pub(super) fn normalized_tap_warning(net: &Network) -> Option<String> {
    if !net.is_normalized() {
        return None;
    }
    // After normalization a line (raw tap 0) and a unity-ratio transformer (raw
    // tap 1) both read as tap 1.0 / shift 0.0, so they cannot be told apart. Count
    // them together as the branches whose line/transformer label is now ambiguous.
    let ambiguous = net
        .branches
        .iter()
        .filter(|b| b.tap == 1.0 && b.shift == 0.0)
        .count();
    (ambiguous > 0).then(|| {
        format!(
            "normalized network: {ambiguous} branch(es) have unit tap and no phase \
             shift, so the line/transformer label is not preserved (the power flow \
             is identical)"
        )
    })
}

/// True when `value` is set and deviates from `reference`: the shared test for
/// "does this rating column carry information the target cannot" used by the
/// rate_b/rate_c drop warnings.
fn nonzero_differs(value: f64, reference: f64) -> bool {
    value.abs() > f64::EPSILON && (value - reference).abs() > f64::EPSILON
}

/// Set a bus's kind through the `bus_pos` index, leaving Isolated buses alone.
/// Shared by the readers that derive bus kinds from generator/slack tables.
pub(crate) fn set_bus_kind(
    buses: &mut [Bus],
    bus_pos: &HashMap<BusId, usize>,
    bus: BusId,
    kind: BusType,
) {
    if let Some(&idx) = bus_pos.get(&bus) {
        if buses[idx].kind != BusType::Isolated {
            buses[idx].kind = kind;
        }
    }
}

/// `base_kv` of a bus through the `bus_pos` index; 0.0 for an unknown bus.
pub(crate) fn bus_kv(buses: &[Bus], bus_pos: &HashMap<BusId, usize>, bus: BusId) -> f64 {
    bus_pos
        .get(&bus)
        .and_then(|&i| buses.get(i))
        .map_or(0.0, |b| b.base_kv)
}

/// Replace characters that would corrupt a quoted or delimited field with
/// `replacement`, so a free-form name can't shift or truncate the record it sits
/// in. `forbidden` lists the destination's quote, delimiter, and comment chars.
/// Returns the value borrowed unchanged when it holds none of them, so the common
/// clean-name path allocates nothing.
///
/// Each text writer calls this at its quoting seam and warns when the result
/// differs from the input (the substitution silently alters operator-facing
/// names): the PSS/E single-quoted bus name and the PowerWorld double-quoted bus
/// name both interpolate a `Network` name straight into a quoted field, where an
/// embedded quote (or, for PSS/E, the `/` inline-comment delimiter) would shift
/// every later column of the record.
pub(crate) fn sanitize_quoted<'a>(
    value: &'a str,
    forbidden: &[char],
    replacement: char,
) -> std::borrow::Cow<'a, str> {
    if value.contains(forbidden) {
        value
            .chars()
            .map(|c| {
                if forbidden.contains(&c) {
                    replacement
                } else {
                    c
                }
            })
            .collect::<String>()
            .into()
    } else {
        std::borrow::Cow::Borrowed(value)
    }
}

/// Impedance base `v_kv² / base_mva`; 1.0 when either base is missing, so a
/// per-unit ↔ ohm conversion on it is the identity.
pub(crate) fn zbase(v_kv: f64, base_mva: f64) -> f64 {
    if v_kv > 0.0 && base_mva > 0.0 {
        v_kv * v_kv / base_mva
    } else {
        1.0
    }
}

/// Whether writing `net` to `target` echoes the retained source text: the
/// target is the source format and the source is still attached. An echo
/// reproduces the input byte for byte, so read fidelity warnings don't apply.
fn is_echo(net: &Network, target: TargetFormat) -> bool {
    let Some(src) = &net.source else { return false };
    if !same_format(target, net.source_format) {
        return false;
    }
    // A PSS/E source echoes only when the requested revision equals the source's
    // own; any other revision must go through write_psse_rev so the caller gets
    // the layout it asked for instead of the original bytes.
    if let TargetFormat::Psse { rev } = target {
        return psse::header_rev(src) == rev;
    }
    true
}

/// Whether a write target is the same format the network was read from.
fn same_format(target: TargetFormat, source: SourceFormat) -> bool {
    matches!(
        (target, source),
        (TargetFormat::Matpower, SourceFormat::Matpower)
            | (TargetFormat::PowerModelsJson, SourceFormat::PowerModelsJson)
            | (TargetFormat::EgretJson, SourceFormat::EgretJson)
            | (TargetFormat::Psse { .. }, SourceFormat::Psse)
            | (TargetFormat::PowerWorld, SourceFormat::PowerWorld)
            | (TargetFormat::PandapowerJson, SourceFormat::PandapowerJson)
            | (TargetFormat::Pslf, SourceFormat::Pslf)
            | (TargetFormat::Goc3Json, SourceFormat::Goc3Json)
            | (TargetFormat::SurgeJson, SourceFormat::SurgeJson)
    )
}

/// JSON number for a finite `f64`; `Value::Null` for `NaN`/`±Inf`.
pub(crate) fn jnum(x: f64) -> Value {
    serde_json::Number::from_f64(x).map_or(Value::Null, Value::Number)
}

/// Serialize a built JSON tree into a [`Conversion`], appending one warning that
/// names every field where a non-finite `f64` was written as `null` (JSON has no
/// `±Inf`/`NaN`). Shared by the JSON writers.
pub(crate) fn finish(root: Map<String, Value>, mut warnings: Vec<String>) -> Conversion {
    let value = Value::Object(root);
    let mut nulls = BTreeSet::new();
    collect_null_keys(&value, &mut nulls);
    if !nulls.is_empty() {
        warnings.push(format!(
            "non-finite numeric values written as JSON null in field(s): {}",
            nulls.into_iter().collect::<Vec<_>>().join(", ")
        ));
    }
    let text = serde_json::to_string_pretty(&value).expect("a serde_json::Value always serializes");
    Conversion { text, warnings }
}

/// Collect the names of object keys whose value is `null`, anywhere in the tree.
fn collect_null_keys(value: &Value, out: &mut BTreeSet<String>) {
    match value {
        Value::Object(map) => {
            for (key, val) in map {
                if val.is_null() {
                    out.insert(key.clone());
                } else {
                    collect_null_keys(val, out);
                }
            }
        }
        Value::Array(items) => items.iter().for_each(|v| collect_null_keys(v, out)),
        _ => {}
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::network::SourceFormat;

    #[test]
    fn source_format_strings_round_trip_to_a_target() {
        // The bindings expose `source_format` as its `{:?}` form, and
        // `to_format` routes that string back through `target_format_from_name`.
        // Every writable source format must resolve, including PowerModelsJson /
        // EgretJson, whose camel-case names need the `powermodelsjson` /
        // `egretjson` aliases (issue #75).
        for (sf, want) in [
            (SourceFormat::Matpower, TargetFormat::Matpower),
            (SourceFormat::PowerModelsJson, TargetFormat::PowerModelsJson),
            (SourceFormat::EgretJson, TargetFormat::EgretJson),
            (SourceFormat::Psse, TargetFormat::Psse { rev: 33 }),
            (SourceFormat::PowerWorld, TargetFormat::PowerWorld),
            (SourceFormat::PandapowerJson, TargetFormat::PandapowerJson),
            (SourceFormat::Pslf, TargetFormat::Pslf),
            (SourceFormat::Goc3Json, TargetFormat::Goc3Json),
            (SourceFormat::SurgeJson, TargetFormat::SurgeJson),
        ] {
            let token = format!("{sf:?}");
            assert_eq!(
                target_format_from_name(&token),
                Some(want),
                "source_format {token:?} did not round-trip"
            );
        }
        // The derived/in-memory source formats have no writer target, and
        // neither does the read only .pwb binary.
        for sf in [
            SourceFormat::InMemory,
            SourceFormat::Normalized,
            SourceFormat::Gridfm,
            SourceFormat::PypsaCsv,
            SourceFormat::PowerWorldBinary,
        ] {
            assert_eq!(target_format_from_name(&format!("{sf:?}")), None);
        }
    }
}