1use crate::format::routing::{Detection, JsonClass, classify_json_bytes};
9use crate::{
10 ArtifactPath, Error, FormatId, ParseOptions, PioModule, PioValue, Result, Source, SourceTree,
11 SourceTreeLimits,
12};
13pub use powerio_archive::ArchiveLimits;
14pub use powerio_archive::session::Cancellation;
15mod manifest;
16mod nested;
17pub use manifest::{
18 CollectionCase, CollectionManifest, CollectionOrigin, CollectionReaderOptions, SelectionOrigin,
19 SourceBundle, SourceBundleScope,
20};
21use nested::{Session, Work};
22use serde::{Deserialize, Serialize};
23use std::path::Path;
24use std::sync::Arc;
25
26pub mod codes {
28 powerio_core::diagnostic_codes! {
29 SELECTION = "REQUEST.COLLECTION.SELECTION", Error,
30 "the collection selection is invalid or incomplete", category = Request;
31 IO = "READ.COLLECTION.IO", Error,
32 "collection input could not be read", category = Data;
33 LIMIT = "READ.COLLECTION.LIMIT", Error,
34 "collection acquisition or inspection exceeds its budget", category = Data;
35 }
36}
37fn selection_error(message: impl Into<String>) -> Error {
38 Error::new(&codes::SELECTION, message)
39}
40
41#[derive(Clone, Copy, Debug)]
58#[non_exhaustive]
59pub struct CollectionLimits {
60 pub tree: SourceTreeLimits,
61 pub archive: ArchiveLimits,
62 pub probe_bytes: u64,
64 pub inspection_bytes: u64,
66 pub max_zip_layers: u32,
68 pub work_bytes: u64,
71}
72impl Default for CollectionLimits {
73 fn default() -> Self {
74 Self {
75 tree: SourceTreeLimits::new(4096, 64 << 20, 64).expect("valid defaults"),
76 archive: ArchiveLimits::default(),
77 probe_bytes: 1 << 20,
78 inspection_bytes: 8 << 20,
79 max_zip_layers: 2,
80 work_bytes: 256 << 20,
81 }
82 }
83}
84
85#[derive(Clone, Debug, Serialize)]
87#[non_exhaustive]
88pub struct CollectionEntry {
89 pub path: String,
90 pub bytes: u64,
91}
92
93#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
95#[serde(rename_all = "kebab-case")]
96#[non_exhaustive]
97pub enum CollectionOperation {
98 #[default]
99 Parse,
100 Deserialize,
101}
102
103#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
108#[serde(deny_unknown_fields)]
109#[non_exhaustive]
110pub struct CollectionSelection {
111 pub root: Option<String>,
112 pub primary: Option<String>,
113 #[serde(default)]
115 pub members: Option<Vec<String>>,
116 pub format: Option<String>,
117 #[serde(default)]
118 pub operation: CollectionOperation,
119}
120fn canonical_token(name: &str) -> &str {
122 crate::resolve_format(name).map_or(name, |format| format.token)
123}
124
125impl CollectionSelection {
126 #[must_use]
127 pub fn file(primary: impl Into<String>) -> Self {
128 Self {
129 primary: Some(primary.into()),
130 ..Self::default()
131 }
132 }
133 pub(super) fn format_token(&self) -> Option<&str> {
134 self.format.as_deref().map(canonical_token)
135 }
136
137 pub(super) fn has_companions(&self) -> bool {
138 match self.format_token() {
139 Some("dss" | "gridlabd" | "gridlabd-arras") => true,
140 Some(_) => false,
141 None => self.primary.as_deref().is_some_and(|name| {
142 name.rsplit_once('.').is_some_and(|(_, ext)| {
143 ext.eq_ignore_ascii_case("dss") || ext.eq_ignore_ascii_case("glm")
144 })
145 }),
146 }
147 }
148
149 pub(super) fn has_sincal_tables(&self, bytes: &[u8]) -> bool {
152 matches!(
153 self.format_token(),
154 Some("sincal" | "sincal-balanced" | "sincal-multiconductor")
155 ) && bytes.starts_with(b"\0\x01\0\0Standard Jet DB\0")
156 }
157
158 #[must_use]
159 pub fn format(mut self, format: impl Into<String>) -> Self {
160 self.format = Some(format.into());
161 self
162 }
163}
164
165#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
167#[serde(rename_all = "kebab-case")]
168#[non_exhaustive]
169pub enum CandidateStatus {
170 Selectable,
171 NeedsOptions,
172 NeedsFormat,
173 ProbeLimit,
174 UnsupportedBuild,
175 Incomplete,
176}
177#[derive(Clone, Debug, Serialize)]
178#[non_exhaustive]
179pub struct CollectionCandidate {
180 pub selection: CollectionSelection,
181 pub status: CandidateStatus,
182}
183
184#[derive(Debug)]
187#[non_exhaustive]
188pub struct CollectionResult {
189 pub module: PioModule<PioValue>,
190 pub dependencies: Vec<String>,
191 pub incomplete: bool,
193}
194
195#[derive(Clone, Debug)]
209pub struct Collection {
210 tree: SourceTree,
211 inventory: Arc<[CollectionEntry]>,
212 limits: CollectionLimits,
213 name: String,
214 original: Option<Arc<[u8]>>,
215 session: Session,
216 zip_layers: u32,
217 cancel: Cancellation,
218 containers: Vec<String>,
219 origin_name: String,
220}
221impl Collection {
222 pub fn from_files(
224 name: impl Into<String>,
225 files: impl IntoIterator<Item = (ArtifactPath, Arc<[u8]>)>,
226 limits: CollectionLimits,
227 ) -> Result<Self> {
228 if limits.max_zip_layers == 0 || limits.work_bytes == 0 {
229 return Err(selection_error(
230 "collection session limits must be positive",
231 ));
232 }
233 let name = name.into();
234 let tree = SourceTree::new(&name, files, limits.tree)?;
235 let source = tree.source(None, None, limits.tree)?;
236 let inventory: Arc<[CollectionEntry]> = source
237 .entry_names()?
238 .into_iter()
239 .map(|path| {
240 let bytes = source.buffer(&path)?.bytes().len() as u64;
241 Ok(CollectionEntry {
242 path: path.to_string(),
243 bytes,
244 })
245 })
246 .collect::<Result<Vec<_>>>()?
247 .into();
248 let session = Session::new(&inventory);
249 Ok(Self {
250 origin_name: name.clone(),
251 session,
252 zip_layers: 0,
253 cancel: Cancellation::default(),
254 containers: Vec::new(),
255 tree,
256 inventory,
257 limits,
258 name,
259 original: None,
260 })
261 }
262
263 pub fn from_zip(
266 name: impl Into<String>,
267 bytes: Arc<[u8]>,
268 limits: CollectionLimits,
269 ) -> Result<Self> {
270 Self::from_zip_cancellable(name, bytes, limits, Cancellation::default())
271 }
272
273 pub fn from_zip_cancellable(
275 name: impl Into<String>,
276 bytes: Arc<[u8]>,
277 limits: CollectionLimits,
278 cancel: Cancellation,
279 ) -> Result<Self> {
280 cancel.check()?;
281 let mut archive = powerio_archive::Archive::open(Arc::clone(&bytes), limits.archive)?;
282 let archive_entries = archive.entries().len();
283 let directories: Vec<_> = archive
284 .entries()
285 .filter(|e| e.is_directory())
286 .map(|e| e.path().clone())
287 .collect();
288 let entries = archive
289 .entries()
290 .filter(|e| !e.is_directory())
291 .map(|e| (e.path().clone(), e.size()))
292 .collect::<Vec<_>>();
293 if entries.len() > limits.tree.max_files()
294 || entries
295 .iter()
296 .try_fold(0u64, |n, (_, size)| n.checked_add(*size))
297 .is_none_or(|n| n > limits.tree.max_bytes())
298 {
299 return Err(Error::new(
300 &codes::LIMIT,
301 "ZIP exceeds collection tree limits",
302 ));
303 }
304 let expanded: u64 = entries.iter().map(|(_, size)| size).sum();
305 if (bytes.len() as u64).saturating_add(expanded) > limits.work_bytes {
306 return Err(Error::new(
307 &codes::LIMIT,
308 "ZIP exceeds collection work limit",
309 ));
310 }
311 for path in directories {
312 archive.read_cancellable(&path, 0, &cancel)?;
313 }
314 let files = entries
315 .into_iter()
316 .map(|(path, _)| {
317 archive
318 .read_cancellable(&path, limits.tree.max_bytes(), &cancel)
319 .map(|bytes| (path, bytes))
320 })
321 .collect::<Result<Vec<_>>>()?;
322 let mut collection = Self::from_files(name, files, limits)?;
323 collection.zip_layers = 1;
324 collection.cancel = cancel;
325 collection.session.initialize_archive(archive_entries)?;
326 collection.original = Some(bytes);
327 Ok(collection)
328 }
329
330 pub fn from_directory(path: impl AsRef<Path>, limits: CollectionLimits) -> Result<Self> {
333 let source = Source::open(path.as_ref())?;
334 if !source.is_directory() {
335 return Err(selection_error("a collection folder must be a directory"));
336 }
337 let names = source.entry_names()?;
338 if names.len() > limits.tree.max_files() {
339 return Err(Error::new(&codes::LIMIT, "collection file count exceeded"));
340 }
341 let mut remaining = limits.tree.max_bytes();
342 let mut files = Vec::new();
343 for name in names {
344 if name.as_str().split('/').count() > limits.tree.max_depth() {
345 return Err(Error::new(&codes::LIMIT, "collection path depth exceeded"));
346 }
347 let buffer = source.root_buffer_with_limit(name.as_str(), remaining)?;
348 remaining -= buffer.bytes().len() as u64;
349 files.push((name, buffer.shared_bytes()));
350 }
351 Self::from_files(source.name(), files, limits)
352 }
353 #[must_use]
354 pub fn name(&self) -> &str {
355 &self.name
356 }
357 #[must_use]
358 pub fn entries(&self) -> &[CollectionEntry] {
359 &self.inventory
360 }
361 #[must_use]
362 pub fn original_zip(&self) -> Option<Arc<[u8]>> {
363 self.original.clone()
364 }
365
366 pub fn inspect(&self) -> Result<Vec<CollectionCandidate>> {
369 self.cancel.check()?;
370 let mut work = Work::new(self.limits.work_bytes);
371 work.charge(self.entries().iter().map(|e| e.path.len() as u64 + 1).sum())?;
372 let source = self.tree.source(None, None, self.limits.tree)?;
373 let mut remaining = self.limits.inspection_bytes;
374 let mut candidates = Vec::new();
375 for entry in self.entries() {
376 self.cancel.check()?;
377 let extension = entry
378 .path
379 .rsplit('.')
380 .next()
381 .unwrap_or_default()
382 .to_ascii_lowercase();
383 let mut selection = CollectionSelection::file(&entry.path);
384 let mut status = CandidateStatus::Selectable;
385 selection.format = match extension.as_str() {
386 "dss" => Some("dss"),
387 "glm" => Some("gridlabd"),
388 "m" => Some("matpower"),
389 "raw" => Some("psse"),
390 "rawx" => Some("psse-rawx"),
391 "aux" => Some("powerworld"),
392 "epc" => Some("pslf"),
393 "uct" | "ucte" => Some("ucte"),
394 "cdf" => Some("ieee-cdf"),
395 "pwb" => Some("pwb"),
396 "xiidm" => Some("xiidm"),
397 "con" => Some("psse-con"),
398 "sub" => Some("psse-sub"),
399 "mon" => Some("psse-mon"),
400 "json" | "jiidm" => {
401 if entry.bytes > self.limits.probe_bytes || entry.bytes > remaining {
402 status = CandidateStatus::ProbeLimit;
403 None
404 } else {
405 work.charge(entry.bytes)?;
406 remaining -= entry.bytes;
407 let bytes = source.root_buffer(&entry.path)?;
408 match classify_json_bytes(bytes.content_bytes()) {
409 JsonClass::Module => {
410 selection.operation = CollectionOperation::Deserialize;
411 None
412 }
413 JsonClass::Case(Detection::Known(format)) => Some(format.name()),
414 JsonClass::Case(_) => {
415 status = CandidateStatus::NeedsFormat;
416 None
417 }
418 }
419 }
420 }
421 "xml" | "sinx" | "zip" => {
422 status = CandidateStatus::NeedsOptions;
423 None
424 }
425 _ => continue,
426 }
427 .map(str::to_owned);
428 candidates.push(CollectionCandidate { selection, status });
429 }
430 candidates.extend(self.dataset_candidates());
431 Ok(candidates)
432 }
433
434 pub fn source(&self, selection: &CollectionSelection) -> Result<Source> {
437 let root = selection
438 .root
439 .as_deref()
440 .map(ArtifactPath::new)
441 .transpose()?;
442 if selection.members.is_some() {
443 return self.group_source(selection, root.as_ref());
444 }
445 let Some(primary) = selection.primary.as_deref() else {
446 return self.dataset_source(selection, root.as_ref());
447 };
448 let primary = ArtifactPath::new(primary)?;
449 let scoped = self
450 .tree
451 .source(root.as_ref(), Some(&primary), self.limits.tree)?;
452 let format = selection.format_token();
453 if primary.as_str().to_ascii_lowercase().ends_with(".zip") && format != Some("cgmes") {
454 return Err(selection_error(
455 "transport ZIP members require explicit nested inspection; CGMES ZIPs require format cgmes",
456 ));
457 }
458 let mut source = if selection.has_companions()
459 || selection.has_sincal_tables(scoped.primary_buffer()?.content_bytes())
460 {
461 scoped
462 } else {
463 let bytes = scoped.primary_buffer()?.shared_bytes();
464 Source::from_memory(primary.as_str(), bytes)?
465 };
466 if let Some(format) = format {
467 source = source.with_format(FormatId::new(format)?);
468 }
469 Ok(source)
470 }
471
472 pub fn parse(
475 &self,
476 selection: &CollectionSelection,
477 options: &ParseOptions,
478 ) -> Result<CollectionResult> {
479 if options.acquisition_root.is_some() {
480 return Err(selection_error(
481 "collection sources cannot acquire host files",
482 ));
483 }
484 if options
485 .format
486 .as_ref()
487 .is_some_and(|f| Some(canonical_token(f.as_str())) != selection.format_token())
488 {
489 return Err(selection_error(
490 "set the format on the collection selection",
491 ));
492 }
493 self.cancel.check()?;
494 self.charge_selection(selection)?;
495 let source = self.source(selection)?;
496 let module = match selection.operation {
497 CollectionOperation::Parse => crate::parse_with_options(&source, options)?,
498 CollectionOperation::Deserialize => {
499 if selection.format.is_some() {
500 return Err(selection_error("deserialize does not accept a grid format"));
501 }
502 crate::deserialize(&source)?
503 }
504 };
505 let unresolved = [
506 &powerio_dist::diagnostics::codes::READ_DSS_INCLUDE_LOAD_FAILED,
507 &powerio_dist::diagnostics::codes::READ_DSS_INCLUDE_DEPTH_LIMIT,
508 ];
509 let incomplete = module
510 .diagnostics()
511 .iter()
512 .any(|d| unresolved.iter().any(|info| d.code() == info.code));
513 Ok(CollectionResult {
514 module,
515 dependencies: source
516 .dependencies()
517 .iter()
518 .map(|b| b.name().to_owned())
519 .collect(),
520 incomplete,
521 })
522 }
523}
524
525impl Collection {
526 fn dataset_candidates(&self) -> Vec<CollectionCandidate> {
527 let names: std::collections::BTreeSet<_> =
528 self.entries().iter().map(|e| e.path.as_str()).collect();
529 self.entries()
530 .iter()
531 .filter_map(|entry| {
532 let (root, leaf) = entry
533 .path
534 .rsplit_once('/')
535 .map_or((None, entry.path.as_str()), |(r, l)| {
536 (Some(r.to_owned()), l)
537 });
538 let format = match leaf {
539 "network.csv" => "pypsa-csv",
540 "bus_data.parquet" => "gridfm",
541 _ => return None,
542 };
543 let prefix = root.as_ref().map_or_else(String::new, |r| format!("{r}/"));
544 let required: &[&str] = if format == "gridfm" {
545 &["gen_data.parquet", "branch_data.parquet"]
546 } else {
547 &["buses.csv"]
548 };
549 let status = if format == "gridfm" && !cfg!(feature = "gridfm") {
550 CandidateStatus::UnsupportedBuild
551 } else if required
552 .iter()
553 .any(|leaf| !names.contains(format!("{prefix}{leaf}").as_str()))
554 {
555 CandidateStatus::Incomplete
556 } else {
557 CandidateStatus::Selectable
558 };
559 Some(CollectionCandidate {
560 selection: CollectionSelection {
561 root,
562 primary: None,
563 members: None,
564 format: Some(format.into()),
565 operation: CollectionOperation::Parse,
566 },
567 status,
568 })
569 })
570 .collect()
571 }
572
573 fn dataset_source(
574 &self,
575 selection: &CollectionSelection,
576 root: Option<&ArtifactPath>,
577 ) -> Result<Source> {
578 if selection.operation != CollectionOperation::Parse {
579 return Err(selection_error("deserialize requires a primary file"));
580 }
581 let format = selection.format_token().ok_or_else(|| {
582 selection_error("a directory selection requires an explicit dataset format")
583 })?;
584 let marker = match format {
585 "pypsa-csv" => "network.csv",
586 "gridfm" if cfg!(feature = "gridfm") => "bus_data.parquet",
587 "gridfm" => return Err(selection_error("GridFM support is disabled in this build")),
588 _ => {
589 return Err(selection_error(
590 "select an explicit primary or a PyPSA/GridFM dataset root",
591 ));
592 }
593 };
594 let source = self.tree.source(root, None, self.limits.tree)?;
595 source.root_buffer(marker)?;
596 let nested: Vec<_> = source
599 .entry_names()?
600 .into_iter()
601 .filter_map(|p| {
602 let (parent, leaf) = p.as_str().rsplit_once('/')?;
603 matches!(leaf, "network.csv" | "bus_data.parquet").then(|| format!("{parent}/"))
604 })
605 .collect();
606 if nested.is_empty() {
607 return Ok(source.with_format(FormatId::new(format)?));
608 }
609 let files = source
610 .entry_names()?
611 .into_iter()
612 .filter(|p| !nested.iter().any(|prefix| p.as_str().starts_with(prefix)))
613 .map(|p| source.buffer(&p).map(|b| (p, b.shared_bytes())))
614 .collect::<Result<Vec<_>>>()?;
615 Ok(SourceTree::new(&self.name, files, self.limits.tree)?
616 .source(None, None, self.limits.tree)?
617 .with_format(FormatId::new(format)?))
618 }
619}
620
621impl CollectionSelection {
622 #[must_use]
624 pub fn group(format: impl Into<String>, members: Vec<String>) -> Self {
625 Self {
626 format: Some(format.into()),
627 members: Some(members),
628 ..Self::default()
629 }
630 }
631}
632impl Collection {
633 fn group_source(
634 &self,
635 selection: &CollectionSelection,
636 root: Option<&ArtifactPath>,
637 ) -> Result<Source> {
638 if selection.primary.is_some() || selection.operation != CollectionOperation::Parse {
639 return Err(selection_error(
640 "document groups require parse and no primary",
641 ));
642 }
643 let format = selection
644 .format_token()
645 .ok_or_else(|| selection_error("document groups require an explicit format"))?;
646 if !matches!(format, "cgmes" | "cim-dist" | "goc3-json") {
647 return Err(selection_error(
648 "document groups support cgmes, cim-dist, or goc3-json",
649 ));
650 }
651 let members = selection.members.as_ref().expect("checked group");
652 if members.is_empty() || members.len() > self.limits.tree.max_files() {
653 return Err(selection_error(
654 "document group must have a bounded nonempty member list",
655 ));
656 }
657 let scope = self.tree.source(root, None, self.limits.tree)?;
658 let mut files = Vec::new();
659 for name in members {
660 let path = ArtifactPath::new(name)?;
661 if format == "cim-dist" && name.contains('/') {
662 return Err(selection_error(
663 "CIM group members must be top-level XML files relative to the selected root",
664 ));
665 }
666 let extension = name.rsplit('.').next().unwrap_or_default();
667 if !match format {
668 "cgmes" => {
669 extension.eq_ignore_ascii_case("xml") || extension.eq_ignore_ascii_case("zip")
670 }
671 "cim-dist" => extension.eq_ignore_ascii_case("xml"),
672 _ => extension.eq_ignore_ascii_case("json"),
673 } {
674 return Err(selection_error(
675 "group member suffix does not match the explicit format",
676 ));
677 }
678 let bytes = scope.root_buffer(name)?.shared_bytes();
679 files.push((path, bytes));
680 }
681 Ok(SourceTree::new(&self.name, files, self.limits.tree)?
682 .source(None, None, self.limits.tree)?
683 .with_format(FormatId::new(format)?))
684 }
685}
686
687impl Collection {
688 pub fn open(path: impl AsRef<Path>, limits: CollectionLimits) -> Result<Self> {
691 use std::io::Read;
692 let path = path.as_ref();
693 if path.is_dir() {
694 return Self::from_directory(path, limits);
695 }
696 let file = std::fs::File::open(path).map_err(|e| Error::new(&codes::IO, e.to_string()))?;
697 let mut bytes = Vec::new();
698 file.take(limits.archive.max_input_bytes().saturating_add(1))
699 .read_to_end(&mut bytes)
700 .map_err(|e| Error::new(&codes::IO, e.to_string()))?;
701 Self::from_zip(path.to_string_lossy(), bytes.into(), limits)
702 }
703}