powerio_dist/dss/
lex.rs

1//! Tokenizer matching OpenDSS's TParser (Parser/ParserDel.cpp).
2//!
3//! A command line is a sequence of parameters, positional or `name=value`.
4//! Delimiters are `,` and `=` plus space and tab; a token opening with one of
5//! `( " ' [ {` runs to the matching closer and keeps delimiters inside;
6//! `!` and `//` start a comment that eats the rest of the line. A token
7//! beginning with `@` is replaced by the named parser variable, keeping any
8//! `.node` suffix. Quoted tokens parse as RPN when read as numbers; vector
9//! values re-tokenize their content with `|` terminating a matrix row.
10
11use std::collections::BTreeMap;
12
13use super::rpn::{self, RpnCalc};
14
15/// Parser variables (`var @x=...`), looked up case insensitively with the
16/// leading `@` included in the key.
17pub type VarMap = BTreeMap<String, String>;
18
19const BEGIN_QUOTE: &[u8] = b"(\"'[{";
20const END_QUOTE: &[u8] = b")\"']}";
21
22/// What ended the last token.
23#[derive(Clone, Copy, PartialEq, Debug)]
24enum Delim {
25    Whitespace,
26    Char(u8),
27    Comment,
28}
29
30/// One parameter from a command line.
31#[derive(Clone, Debug, PartialEq)]
32pub struct Param {
33    /// Property name to the left of `=`; `None` for a positional value.
34    pub name: Option<String>,
35    pub value: Value,
36}
37
38/// A raw value token. `quoted` records that the token came from a quote pair,
39/// which switches numeric interpretation to RPN.
40#[derive(Clone, Debug, PartialEq, Default)]
41pub struct Value {
42    pub text: String,
43    pub quoted: bool,
44}
45
46/// A `bus1=name.1.2.0` bus reference: name plus ordered node numbers.
47#[derive(Clone, Debug, PartialEq)]
48pub struct BusSpec {
49    pub name: String,
50    /// Node numbers as written; `0` is ground. Unparseable nodes become -1,
51    /// matching the reference parser's error marker.
52    pub nodes: Vec<i32>,
53}
54
55#[derive(Debug, thiserror::Error, PartialEq)]
56pub enum ValueError {
57    #[error("`{0}` is not a number")]
58    NotANumber(String),
59    #[error("bad RPN token `{token}` in `{expr}`")]
60    BadRpn { expr: String, token: String },
61}
62
63pub struct Scanner<'a> {
64    buf: &'a [u8],
65    pos: usize,
66    last_delim: Delim,
67    /// Extra delimiter, the matrix row terminator `|` during vector parsing.
68    row_term: bool,
69    vars: Option<&'a VarMap>,
70}
71
72impl<'a> Scanner<'a> {
73    pub fn new(line: &'a str, vars: Option<&'a VarMap>) -> Self {
74        let mut s = Scanner {
75            buf: line.as_bytes(),
76            pos: 0,
77            last_delim: Delim::Whitespace,
78            row_term: false,
79            vars,
80        };
81        s.skip_whitespace();
82        s
83    }
84
85    /// The var table this scanner substitutes `@name` tokens against, if
86    /// any. Lets a caller that recurses mid line (a `Redirect`) reinstall
87    /// the same ambient table for the nested text to see and extend.
88    pub(crate) fn vars(&self) -> Option<&'a VarMap> {
89        self.vars
90    }
91
92    fn skip_whitespace(&mut self) {
93        while self.pos < self.buf.len() && matches!(self.buf[self.pos], b' ' | b'\t') {
94            self.pos += 1;
95        }
96    }
97
98    fn is_delim_char(&self, b: u8) -> bool {
99        b == b',' || b == b'=' || (self.row_term && b == b'|')
100    }
101
102    fn at_comment(&self) -> bool {
103        match self.buf.get(self.pos) {
104            Some(b'!') => true,
105            Some(b'/') => self.buf.get(self.pos + 1) == Some(&b'/'),
106            _ => false,
107        }
108    }
109
110    /// TParser::GetToken. Returns `None` at end of line; an empty token can
111    /// occur mid stream (e.g. between consecutive commas), as in the
112    /// reference.
113    fn get_token(&mut self) -> Option<(String, bool)> {
114        if self.pos >= self.buf.len() {
115            return None;
116        }
117        self.last_delim = Delim::Whitespace;
118        let mut quoted = false;
119        let text;
120
121        let open = self.buf[self.pos];
122        if let Some(qi) = BEGIN_QUOTE.iter().position(|&q| q == open) {
123            let close = END_QUOTE[qi];
124            self.pos += 1;
125            let start = self.pos;
126            while self.pos < self.buf.len() && self.buf[self.pos] != close {
127                self.pos += 1;
128            }
129            text = String::from_utf8_lossy(&self.buf[start..self.pos]).into_owned();
130            if self.pos < self.buf.len() {
131                self.pos += 1; // past the closer
132            }
133            quoted = true;
134        } else {
135            let start = self.pos;
136            while self.pos < self.buf.len() {
137                if self.at_comment() {
138                    self.last_delim = Delim::Comment;
139                    break;
140                }
141                let b = self.buf[self.pos];
142                if self.is_delim_char(b) {
143                    self.last_delim = Delim::Char(b);
144                    break;
145                }
146                if matches!(b, b' ' | b'\t') {
147                    self.last_delim = Delim::Whitespace;
148                    break;
149                }
150                self.pos += 1;
151            }
152            text = String::from_utf8_lossy(&self.buf[start..self.pos]).into_owned();
153        }
154
155        if self.last_delim == Delim::Comment {
156            self.pos = self.buf.len();
157            return Some((text, quoted));
158        }
159
160        // Move past one terminating delimiter, eating whitespace around it,
161        // so `a = b` and `a=b` scan identically.
162        if self.last_delim == Delim::Whitespace {
163            self.skip_whitespace();
164        }
165        if self.pos < self.buf.len() {
166            if self.at_comment() {
167                self.pos = self.buf.len();
168                return Some((text, quoted));
169            }
170            let b = self.buf[self.pos];
171            if self.is_delim_char(b) {
172                self.last_delim = Delim::Char(b);
173                self.pos += 1;
174            }
175        }
176        self.skip_whitespace();
177        Some((text, quoted))
178    }
179
180    /// TParser::CheckforVar: a token starting with `@` is replaced by its
181    /// variable value, keeping a `.node.node` suffix (`^` also cuts the
182    /// name). A value stored as `{...}` unwraps and becomes a quoted token.
183    fn substitute(&self, token: String, quoted: bool) -> (String, bool) {
184        if token.len() < 2 || !token.starts_with('@') {
185            return (token, quoted);
186        }
187        let Some(vars) = self.vars else {
188            return (token, quoted);
189        };
190        let cut = token.find(['.', '^']).unwrap_or(token.len());
191        let (name, suffix) = token.split_at(cut);
192        let key = name.to_ascii_lowercase();
193        let Some(value) = vars.get(&key) else {
194            return (token, quoted);
195        };
196        if let Some(inner) = value.strip_prefix('{').and_then(|v| v.strip_suffix('}')) {
197            (format!("{inner}{suffix}"), true)
198        } else {
199            (format!("{value}{suffix}"), quoted)
200        }
201    }
202
203    /// TParser::GetNextParam: one positional or `name=value` parameter.
204    /// Variable substitution applies to the value, never the name.
205    pub fn next_param(&mut self) -> Option<Param> {
206        let (tok, quoted) = self.get_token()?;
207        let (name, raw) = if self.last_delim == Delim::Char(b'=') {
208            (Some(tok), self.get_token().unwrap_or_default())
209        } else {
210            (None, (tok, quoted))
211        };
212        let (text, quoted) = self.substitute(raw.0, raw.1);
213        Some(Param {
214            name,
215            value: Value { text, quoted },
216        })
217    }
218
219    /// Remaining unscanned text, trimmed; the argument tail for commands that
220    /// take free text.
221    pub fn remainder(&self) -> &str {
222        std::str::from_utf8(&self.buf[self.pos.min(self.buf.len())..])
223            .unwrap_or_default()
224            .trim()
225    }
226}
227
228impl Value {
229    pub fn new(text: impl Into<String>) -> Self {
230        Value {
231            text: text.into(),
232            quoted: false,
233        }
234    }
235
236    /// TParser::MakeDouble_: quoted tokens evaluate as RPN, bare tokens must
237    /// be plain numbers. An empty value is 0, as in the reference.
238    pub fn to_f64(&self, vars: Option<&VarMap>) -> Result<f64, ValueError> {
239        if self.text.is_empty() {
240            return Ok(0.0);
241        }
242        if self.quoted {
243            return self.eval_rpn(vars);
244        }
245        rpn::parse_number(&self.text).ok_or_else(|| ValueError::NotANumber(self.text.clone()))
246    }
247
248    /// TParser::MakeInteger_: parse as a double and round.
249    pub fn to_i64(&self, vars: Option<&VarMap>) -> Result<i64, ValueError> {
250        self.to_f64(vars).map(|v| v.round() as i64)
251    }
252
253    fn eval_rpn(&self, vars: Option<&VarMap>) -> Result<f64, ValueError> {
254        let mut calc = RpnCalc::new();
255        let mut scan = Scanner::new(&self.text, vars);
256        while let Some((tok, _)) = scan.get_token() {
257            if tok.is_empty() {
258                continue;
259            }
260            let (tok, _) = scan.substitute(tok, false);
261            if !calc.apply(&tok) {
262                return Err(ValueError::BadRpn {
263                    expr: self.text.clone(),
264                    token: tok,
265                });
266            }
267        }
268        Ok(calc.x())
269    }
270
271    /// TParser::ParseAsVector over the whole value: numbers separated by
272    /// whitespace or commas. `|` row terminators split a matrix value into
273    /// rows; a plain vector is one row.
274    pub fn to_rows(&self, vars: Option<&VarMap>) -> Result<Vec<Vec<f64>>, ValueError> {
275        let mut rows = Vec::new();
276        let mut row = Vec::new();
277        let mut scan = Scanner::new(&self.text, vars);
278        scan.row_term = true;
279        while let Some((tok, quoted)) = scan.get_token() {
280            if !tok.is_empty() {
281                let (text, quoted) = scan.substitute(tok, quoted);
282                row.push(Value { text, quoted }.to_f64(vars)?);
283            }
284            if scan.last_delim == Delim::Char(b'|') {
285                rows.push(std::mem::take(&mut row));
286            }
287        }
288        if !row.is_empty() || rows.is_empty() {
289            rows.push(row);
290        }
291        Ok(rows)
292    }
293
294    /// A flat numeric vector (kVs, taps, ZIPV, ...).
295    pub fn to_vector(&self, vars: Option<&VarMap>) -> Result<Vec<f64>, ValueError> {
296        Ok(self.to_rows(vars)?.into_iter().flatten().collect())
297    }
298
299    /// A list of string items (`buses=(b1, b2)`, `conns=(wye delta)`).
300    pub fn to_string_list(&self, vars: Option<&VarMap>) -> Vec<String> {
301        let mut out = Vec::new();
302        let mut scan = Scanner::new(&self.text, vars);
303        while let Some((tok, quoted)) = scan.get_token() {
304            if !tok.is_empty() {
305                out.push(scan.substitute(tok, quoted).0);
306            }
307        }
308        out
309    }
310
311    /// TParser::ParseAsBusName: `name.1.2.0` into name and node list.
312    pub fn to_bus_spec(&self) -> BusSpec {
313        let text = self.text.trim();
314        match text.split_once('.') {
315            None => BusSpec {
316                name: text.to_string(),
317                nodes: Vec::new(),
318            },
319            Some((name, rest)) => BusSpec {
320                name: name.trim().to_string(),
321                nodes: rest
322                    .split('.')
323                    .map(|n| n.trim().parse::<i32>().unwrap_or(-1))
324                    .collect(),
325            },
326        }
327    }
328
329    /// OpenDSS boolean: leading `y`/`t`/`1` is true, anything else false.
330    pub fn to_bool(&self) -> bool {
331        matches!(
332            self.text.bytes().next().map(|b| b.to_ascii_lowercase()),
333            Some(b'y' | b't' | b'1')
334        )
335    }
336}
337
338#[cfg(test)]
339mod tests {
340    use super::*;
341
342    fn params(line: &str) -> Vec<(Option<String>, String, bool)> {
343        let mut scan = Scanner::new(line, None);
344        let mut out = Vec::new();
345        while let Some(p) = scan.next_param() {
346            out.push((p.name, p.value.text, p.value.quoted));
347        }
348        out
349    }
350
351    #[test]
352    fn positional_and_named() {
353        let p = params("Line.l1 bus1=a bus2=b 0.3");
354        assert_eq!(p[0], (None, "Line.l1".into(), false));
355        assert_eq!(p[1], (Some("bus1".into()), "a".into(), false));
356        assert_eq!(p[2], (Some("bus2".into()), "b".into(), false));
357        assert_eq!(p[3], (None, "0.3".into(), false));
358    }
359
360    #[test]
361    fn spaces_around_equals() {
362        assert_eq!(params("a = b"), params("a=b"));
363        assert_eq!(params("a =b"), params("a= b"));
364    }
365
366    #[test]
367    fn comma_separates() {
368        let p = params("conns=(wye, delta)");
369        assert_eq!(p[0], (Some("conns".into()), "wye, delta".into(), true));
370    }
371
372    #[test]
373    fn quote_pairs() {
374        for (open, close) in [('(', ')'), ('"', '"'), ('\'', '\''), ('[', ']'), ('{', '}')] {
375            let line = format!("x={open}1 2 3{close}");
376            let p = params(&line);
377            assert_eq!(p[0], (Some("x".into()), "1 2 3".into(), true), "{open}");
378        }
379    }
380
381    #[test]
382    fn comments_stop_the_line() {
383        assert_eq!(params("a=1 ! trailing").len(), 1);
384        assert_eq!(params("a=1 // trailing").len(), 1);
385        assert_eq!(params("a=1!glued").len(), 1);
386        assert_eq!(params("! whole line").first().unwrap().1, "");
387    }
388
389    #[test]
390    fn quoted_values_keep_comment_markers() {
391        let p = params(r#"a="x // y" b='x ! y' c={x // y}"#);
392        assert_eq!(p[0], (Some("a".into()), "x // y".into(), true));
393        assert_eq!(p[1], (Some("b".into()), "x ! y".into(), true));
394        assert_eq!(p[2], (Some("c".into()), "x // y".into(), true));
395    }
396
397    #[test]
398    fn slash_alone_is_not_a_comment() {
399        let p = params("x=a/b");
400        assert_eq!(p[0], (Some("x".into()), "a/b".into(), false));
401    }
402
403    #[test]
404    fn rpn_value() {
405        let v = Value {
406            text: "8 1000 /".into(),
407            quoted: true,
408        };
409        assert_eq!(v.to_f64(None), Ok(0.008));
410        let bare = Value::new("3.5");
411        assert_eq!(bare.to_f64(None), Ok(3.5));
412        let bad = Value::new("abc");
413        assert!(bad.to_f64(None).is_err());
414    }
415
416    #[test]
417    fn quoted_single_number_is_rpn() {
418        let v = Value {
419            text: "42".into(),
420            quoted: true,
421        };
422        assert_eq!(v.to_f64(None), Ok(42.0));
423    }
424
425    #[test]
426    fn matrix_rows() {
427        let v = Value {
428            text: "0.088 | 0.031 0.090 | 0.030 0.031 0.088".into(),
429            quoted: true,
430        };
431        let rows = v.to_rows(None).unwrap();
432        assert_eq!(rows.len(), 3);
433        assert_eq!(rows[0], vec![0.088]);
434        assert_eq!(rows[2], vec![0.030, 0.031, 0.088]);
435    }
436
437    #[test]
438    fn vector_with_commas() {
439        let v = Value {
440            text: "7.2, 0.24".into(),
441            quoted: true,
442        };
443        assert_eq!(v.to_vector(None).unwrap(), vec![7.2, 0.24]);
444    }
445
446    #[test]
447    fn rpn_inside_vector() {
448        let v = Value {
449            text: "1 \"8 1000 /\"".into(),
450            quoted: true,
451        };
452        assert_eq!(v.to_vector(None).unwrap(), vec![1.0, 0.008]);
453    }
454
455    #[test]
456    fn bus_dotting() {
457        let b = Value::new("632.1.2.3.0").to_bus_spec();
458        assert_eq!(b.name, "632");
459        assert_eq!(b.nodes, vec![1, 2, 3, 0]);
460        let plain = Value::new("sourcebus").to_bus_spec();
461        assert_eq!(plain.name, "sourcebus");
462        assert_eq!(plain.nodes, [] as [i32; 0]);
463        let bad = Value::new("b.1.x").to_bus_spec();
464        assert_eq!(bad.nodes, vec![1, -1]);
465    }
466
467    #[test]
468    fn var_substitution() {
469        let mut vars = VarMap::new();
470        vars.insert("@kv".into(), "12.47".into());
471        vars.insert("@bus".into(), "632".into());
472        vars.insert("@expr".into(), "{2 3 *}".into());
473        let mut scan = Scanner::new("kv=@kv bus1=@bus.1.2 x=@expr y=@undef", Some(&vars));
474        let p1 = scan.next_param().unwrap();
475        assert_eq!(p1.value.text, "12.47");
476        let p2 = scan.next_param().unwrap();
477        assert_eq!(p2.value.text, "632.1.2");
478        let p3 = scan.next_param().unwrap();
479        assert_eq!(p3.value.text, "2 3 *");
480        assert!(p3.value.quoted);
481        assert_eq!(p3.value.to_f64(Some(&vars)), Ok(6.0));
482        let p4 = scan.next_param().unwrap();
483        assert_eq!(p4.value.text, "@undef");
484    }
485
486    #[test]
487    fn string_list() {
488        let v = Value {
489            text: "b1, b2".into(),
490            quoted: true,
491        };
492        assert_eq!(v.to_string_list(None), vec!["b1", "b2"]);
493    }
494
495    #[test]
496    fn booleans() {
497        assert!(Value::new("yes").to_bool());
498        assert!(Value::new("Y").to_bool());
499        assert!(Value::new("true").to_bool());
500        assert!(Value::new("1").to_bool());
501        assert!(!Value::new("no").to_bool());
502        assert!(!Value::new("false").to_bool());
503        assert!(!Value::new("").to_bool());
504    }
505}
powerio_dist/dss/lex.rs

powerio_dist/dss/
lex.rs
