1use std::collections::BTreeMap;
2use std::ffi::{OsStr, OsString};
3use std::fmt;
4use std::io::Read;
5use std::path::{Component, Path, PathBuf};
6use std::sync::{Arc, Mutex, MutexGuard};
7
8use crate::validation::{MAX_FORMAT_ID_BYTES, valid_nonempty_text};
9use crate::{Error, SourceId};
10
11const MAX_REFERENCED_FILES: usize = 4_096;
14
15const DEFAULT_SOURCE_BYTES: u64 = 1 << 30;
17const MAX_REFERENCED_BYTES: u64 = DEFAULT_SOURCE_BYTES;
18const DEFAULT_PRIMARY_BYTES: u64 = DEFAULT_SOURCE_BYTES;
19
20const PRIMARY_LIMIT_VAR: &str = "POWERIO_MAX_PRIMARY_BYTES";
21const REFERENCED_LIMIT_VAR: &str = "POWERIO_MAX_REFERENCED_BYTES";
22
23#[derive(Clone, Copy, Debug, PartialEq, Eq)]
26struct SourceLimits {
27 primary: u64,
28 referenced: u64,
29}
30
31impl SourceLimits {
32 fn from_lookup(lookup: impl Fn(&'static str) -> Option<OsString>) -> Result<Self, Error> {
36 Ok(Self {
37 primary: parse_primary_limit(lookup(PRIMARY_LIMIT_VAR).as_deref())?,
38 referenced: parse_referenced_limit(lookup(REFERENCED_LIMIT_VAR).as_deref())?,
39 })
40 }
41}
42
43fn referenced_byte_limit() -> Result<u64, Error> {
44 parse_referenced_limit(std::env::var_os(REFERENCED_LIMIT_VAR).as_deref())
45}
46
47fn parse_referenced_limit(value: Option<&OsStr>) -> Result<u64, Error> {
48 parse_byte_limit(REFERENCED_LIMIT_VAR, value, MAX_REFERENCED_BYTES)
49}
50
51fn parse_primary_limit(value: Option<&OsStr>) -> Result<u64, Error> {
52 parse_byte_limit(PRIMARY_LIMIT_VAR, value, DEFAULT_PRIMARY_BYTES)
53}
54
55fn parse_byte_limit(name: &str, value: Option<&OsStr>, default: u64) -> Result<u64, Error> {
56 let Some(value) = value else {
57 return Ok(default);
58 };
59 let limit = value
60 .to_str()
61 .filter(|value| !value.is_empty() && value.bytes().all(|b| b.is_ascii_digit()))
62 .and_then(|value| value.parse::<u64>().ok())
63 .filter(|&limit| limit > 0 && isize::try_from(limit).is_ok());
64 limit.ok_or_else(|| Error::new(&crate::codes::REQUEST_SOURCE_INVALID_LIMIT,
65 format!("{name} must be a positive decimal byte count within this platform's allocation limit")))
66}
67
68#[derive(Clone, Copy)]
69enum ReadBudget {
70 Primary(u64),
71 Referenced { remaining: u64, total: u64 },
72}
73
74impl ReadBudget {
75 fn bytes(self) -> u64 {
76 match self {
77 Self::Primary(n) => n,
78 Self::Referenced { remaining, .. } => remaining,
79 }
80 }
81 fn exceeded(self, name: &str) -> Error {
82 match self {
83 Self::Primary(limit) => Error::new(
84 &crate::codes::READ_IO_PRIMARY_BUDGET,
85 format!("primary source `{name}` exceeds its {limit} byte limit"),
86 ),
87 Self::Referenced { total, .. } => Error::new(
88 &crate::codes::READ_IO_REFERENCE_BUDGET,
89 format!(
90 "referenced file `{name}` would take this source past its {total} byte acquisition budget"
91 ),
92 ),
93 }
94 }
95}
96
97const MAX_REFERENCED_DEPTH: usize = 64;
99
100const UTF8_BOM: [u8; 3] = [0xEF, 0xBB, 0xBF];
101
102#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
104pub struct FormatId(Box<str>);
105
106impl FormatId {
107 pub fn new(id: impl Into<String>) -> Result<Self, Error> {
109 let id = id.into();
110 if !valid_format_id(&id) {
111 return Err(Error::new(
112 &crate::codes::REQUEST_FORMAT_INVALID_ID,
113 "a format ID must be bounded lower case ASCII segments separated by single hyphens",
114 ));
115 }
116 Ok(Self(id.into_boxed_str()))
117 }
118
119 #[must_use]
120 pub fn as_str(&self) -> &str {
121 &self.0
122 }
123}
124
125impl fmt::Display for FormatId {
126 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
127 formatter.write_str(&self.0)
128 }
129}
130
131fn valid_format_id(id: &str) -> bool {
132 if id.is_empty() || id.len() > MAX_FORMAT_ID_BYTES {
133 return false;
134 }
135 let bytes = id.as_bytes();
136 if !bytes[0].is_ascii_lowercase() || bytes.last() == Some(&b'-') {
137 return false;
138 }
139 let mut previous_hyphen = false;
140 for byte in bytes {
141 if *byte == b'-' {
142 if previous_hyphen {
143 return false;
144 }
145 previous_hyphen = true;
146 } else if byte.is_ascii_lowercase() || byte.is_ascii_digit() {
147 previous_hyphen = false;
148 } else {
149 return false;
150 }
151 }
152 true
153}
154
155#[derive(Debug)]
156struct SourceBufferData {
157 id: SourceId,
158 name: Box<str>,
159 bytes: Arc<[u8]>,
160 directory: Box<[Box<str>]>,
164}
165
166#[derive(Clone, Debug)]
168pub struct SourceBuffer(Arc<SourceBufferData>);
169
170impl SourceBuffer {
171 fn new(
172 id: SourceId,
173 name: impl Into<String>,
174 bytes: Arc<[u8]>,
175 directory: Vec<String>,
176 ) -> Self {
177 Self(Arc::new(SourceBufferData {
178 id,
179 name: name.into().into_boxed_str(),
180 bytes,
181 directory: directory.into_iter().map(String::into_boxed_str).collect(),
182 }))
183 }
184
185 #[must_use]
186 pub fn id(&self) -> &SourceId {
187 &self.0.id
188 }
189
190 #[must_use]
191 pub fn name(&self) -> &str {
192 &self.0.name
193 }
194
195 #[must_use]
198 pub fn bytes(&self) -> &[u8] {
199 &self.0.bytes
200 }
201
202 #[must_use]
203 pub fn shared_bytes(&self) -> Arc<[u8]> {
204 Arc::clone(&self.0.bytes)
205 }
206
207 #[must_use]
209 pub fn has_utf8_bom(&self) -> bool {
210 self.0.bytes.starts_with(&UTF8_BOM)
211 }
212
213 #[must_use]
217 pub fn content_bytes(&self) -> &[u8] {
218 let bytes: &[u8] = &self.0.bytes;
219 if bytes.starts_with(&UTF8_BOM) {
220 &bytes[UTF8_BOM.len()..]
221 } else {
222 bytes
223 }
224 }
225
226 pub fn directory_segments(&self) -> impl Iterator<Item = &str> {
231 self.0.directory.iter().map(AsRef::as_ref)
232 }
233}
234
235#[derive(Debug)]
248struct FileAcquisition {
249 root_display: PathBuf,
250 max_bytes: u64,
251 selected: bool,
254 state: Mutex<AcquisitionState>,
255}
256
257#[derive(Debug, Default)]
258struct AcquisitionState {
259 root: Option<platform::RootHandle>,
260 cache: BTreeMap<String, SourceBuffer>,
261 listed: Option<Vec<crate::ArtifactPath>>,
266 files: usize,
267 bytes: u64,
268}
269
270impl AcquisitionState {
271 fn pinned_root(&mut self, root_display: &Path) -> Result<&platform::RootHandle, Error> {
274 if self.root.is_none() {
275 let root = platform::open_root(root_display)
276 .map_err(|cause| open_error(&crate::codes::READ_IO_OPEN, root_display, cause))?;
277 self.root = Some(root);
278 }
279 Ok(self.root.as_ref().expect("pinned above"))
280 }
281}
282
283#[derive(Debug)]
284enum SourceProvider {
285 Memory {
286 primary: Option<SourceBuffer>,
287 named: BTreeMap<String, SourceBuffer>,
288 },
289 File {
290 primary: SourceBuffer,
291 acquisition: FileAcquisition,
292 },
293 Directory {
294 acquisition: FileAcquisition,
295 },
296}
297
298pub const PRIMARY_SOURCE_ID: &str = "/input";
306
307#[derive(Clone)]
314pub struct Source {
315 name: Arc<str>,
316 provider: Arc<SourceProvider>,
317 declared_format: Option<FormatId>,
318}
319
320impl Source {
321 pub fn open(path: impl Into<PathBuf>) -> Result<Self, Error> {
331 let path: PathBuf = path.into();
332 Self::open_with_limits(&path, std::env::var_os)
333 }
334
335 fn open_with_limits(
336 path: &Path,
337 lookup: impl Fn(&'static str) -> Option<OsString>,
338 ) -> Result<Self, Error> {
339 if path.as_os_str().is_empty() {
340 return Err(Error::new(
341 &crate::codes::REQUEST_SOURCE_INVALID_PATH,
342 "source path cannot be empty",
343 ));
344 }
345 let limits = SourceLimits::from_lookup(lookup)?;
347 let name: Arc<str> = path.to_string_lossy().into_owned().into();
348
349 let file = match platform::open_no_follow(path) {
354 Ok(file) => file,
355 Err(error) if platform::is_symlink_refusal(&error) => {
356 return Err(Error::new(
357 &crate::codes::REQUEST_SOURCE_SYMLINK_REFUSED,
358 format!("source `{}` is a symbolic link", path.display()),
359 ));
360 }
361 Err(error) if platform::is_directory_open_failure(&error, path) => {
362 return Self::open_directory(name, path, limits.referenced);
363 }
364 Err(cause) => return Err(open_error(&crate::codes::READ_IO_OPEN, path, cause)),
365 };
366 let metadata = file
367 .metadata()
368 .map_err(|cause| open_error(&crate::codes::READ_IO_METADATA, path, cause))?;
369 if metadata.is_dir() {
370 drop(file);
371 return Self::open_directory(name, path, limits.referenced);
372 }
373 let bytes = read_open_file(file, &name, ReadBudget::Primary(limits.primary))?;
374 let root_display = canonical_parent(path)?;
375 let primary = SourceBuffer::new(
376 SourceId::new(PRIMARY_SOURCE_ID)?,
377 name.to_string(),
378 bytes,
379 Vec::new(),
380 );
381 Ok(Self {
382 name,
383 provider: Arc::new(SourceProvider::File {
384 primary,
385 acquisition: FileAcquisition {
386 root_display,
387 max_bytes: limits.referenced,
388 selected: false,
389 state: Mutex::new(AcquisitionState::default()),
390 },
391 }),
392 declared_format: None,
393 })
394 }
395
396 fn open_directory(name: Arc<str>, path: &Path, max_bytes: u64) -> Result<Self, Error> {
397 let root_display = std::fs::canonicalize(path)
398 .map_err(|cause| open_error(&crate::codes::READ_IO_METADATA, path, cause))?;
399 Ok(Self {
400 name,
401 provider: Arc::new(SourceProvider::Directory {
402 acquisition: FileAcquisition {
403 root_display,
404 max_bytes,
405 selected: false,
406 state: Mutex::new(AcquisitionState::default()),
407 },
408 }),
409 declared_format: None,
410 })
411 }
412
413 pub fn from_memory(
417 name: impl Into<String>,
418 bytes: impl Into<Arc<[u8]>>,
419 ) -> Result<Self, Error> {
420 let name = name.into();
421 if !valid_nonempty_text(&name) {
422 return Err(Error::new(
423 &crate::codes::REQUEST_SOURCE_INVALID_NAME,
424 "an in-memory source requires a nonempty bounded name",
425 ));
426 }
427 let primary = SourceBuffer::new(
428 SourceId::new(PRIMARY_SOURCE_ID)?,
429 name.clone(),
430 bytes.into(),
431 Vec::new(),
432 );
433 Ok(Self {
434 name: name.into(),
435 provider: Arc::new(SourceProvider::Memory {
436 primary: Some(primary),
437 named: BTreeMap::new(),
438 }),
439 declared_format: None,
440 })
441 }
442
443 pub fn from_memory_tree(
450 name: impl Into<String>,
451 files: impl IntoIterator<Item = (crate::ArtifactPath, Arc<[u8]>)>,
452 primary: Option<crate::ArtifactPath>,
453 ) -> Result<Self, Error> {
454 let name = name.into();
455 if !valid_nonempty_text(&name) {
456 return Err(Error::new(
457 &crate::codes::REQUEST_SOURCE_INVALID_NAME,
458 "an in-memory source requires a nonempty bounded name",
459 ));
460 }
461 let mut named = BTreeMap::new();
462 let mut total_bytes = 0u64;
463 for (path, bytes) in files {
464 let segments = resolve_segments(&[], path.as_str())?;
465 let key = segments.join("/");
466 total_bytes = total_bytes.saturating_add(bytes.len() as u64);
467 if named.len() >= MAX_REFERENCED_FILES || total_bytes > MAX_REFERENCED_BYTES {
468 return Err(Error::new(
469 &crate::codes::READ_IO_REFERENCE_BUDGET,
470 "an in-memory tree exceeds 4096 files or 64 MiB",
471 ));
472 }
473 let id = if primary.as_ref() == Some(&path) {
474 PRIMARY_SOURCE_ID
475 } else {
476 &key
477 };
478 let directory = segments[..segments.len() - 1].to_vec();
479 let buffer = SourceBuffer::new(SourceId::new(id)?, key.clone(), bytes, directory);
480 if named.insert(key.clone(), buffer).is_some() {
481 return Err(Error::new(
482 &crate::codes::REQUEST_SOURCE_INVALID_PATH,
483 format!("duplicate in-memory file `{key}`"),
484 ));
485 }
486 }
487 let primary = primary
488 .map(|path| {
489 named.get(path.as_str()).cloned().ok_or_else(|| {
490 Error::new(
491 &crate::codes::REQUEST_SOURCE_UNKNOWN_BUFFER,
492 format!("primary file `{}` was not supplied", path.as_str()),
493 )
494 })
495 })
496 .transpose()?;
497 let name = primary.as_ref().map_or(name.as_str(), SourceBuffer::name);
498 Ok(Self {
499 name: name.into(),
500 provider: Arc::new(SourceProvider::Memory { primary, named }),
501 declared_format: None,
502 })
503 }
504
505 pub fn with_named_buffer(
510 self,
511 name: impl Into<String>,
512 bytes: impl Into<Arc<[u8]>>,
513 ) -> Result<Self, Error> {
514 let name = name.into();
515 let segments = resolve_segments(&[], &name)?;
516 let key = segments.join("/");
517 let mut provider = Arc::try_unwrap(self.provider).map_err(|_| {
518 Error::new(
519 &crate::codes::REQUEST_SOURCE_INVALID_NAME,
520 "named buffers are supplied while constructing a source, before it is shared",
521 )
522 })?;
523 let SourceProvider::Memory { named, .. } = &mut provider else {
524 return Err(Error::new(
525 &crate::codes::REQUEST_SOURCE_INVALID_NAME,
526 "named buffers belong to in-memory sources; a file source acquires referenced files beneath its root",
527 ));
528 };
529 let directory = segments[..segments.len() - 1].to_vec();
530 let buffer = SourceBuffer::new(SourceId::new(&key)?, key.clone(), bytes.into(), directory);
531 named.insert(key, buffer);
532 Ok(Self {
533 name: self.name,
534 provider: Arc::new(provider),
535 declared_format: self.declared_format,
536 })
537 }
538
539 pub fn with_acquisition_root(self, root: impl Into<PathBuf>) -> Result<Self, Error> {
543 let requested = root.into();
544 let SourceProvider::File {
545 primary,
546 acquisition,
547 } = &*self.provider
548 else {
549 return Err(Error::new(
550 &crate::codes::REQUEST_SOURCE_INVALID_PATH,
551 "an acquisition root applies to a file source",
552 ));
553 };
554 let canonical = std::fs::canonicalize(&requested)
555 .map_err(|cause| open_error(&crate::codes::READ_IO_METADATA, &requested, cause))?;
556 let Ok(remainder) = acquisition.root_display.strip_prefix(&canonical) else {
557 return Err(Error::new(
558 &crate::codes::REQUEST_SOURCE_INVALID_PATH,
559 format!(
560 "the case file directory {} is outside the requested acquisition root {}",
561 acquisition.root_display.display(),
562 canonical.display()
563 ),
564 ));
565 };
566 let mut directory = Vec::new();
567 for component in remainder.components() {
568 let Component::Normal(segment) = component else {
569 return Err(Error::new(
570 &crate::codes::REQUEST_SOURCE_INVALID_PATH,
571 "the acquisition root does not resolve to a plain prefix of the case directory",
572 ));
573 };
574 let Some(segment) = segment.to_str().filter(|text| plain_segment(text)) else {
575 return Err(Error::new(
576 &crate::codes::REQUEST_SOURCE_INVALID_PATH,
577 "the acquisition root does not resolve to a plain prefix of the case directory",
578 ));
579 };
580 directory.push(segment.to_owned());
581 }
582 let primary = SourceBuffer::new(
583 primary.id().clone(),
584 primary.name().to_owned(),
585 primary.shared_bytes(),
586 directory,
587 );
588 Ok(Self {
589 name: self.name,
590 provider: Arc::new(SourceProvider::File {
591 primary,
592 acquisition: FileAcquisition {
593 root_display: canonical,
594 max_bytes: acquisition.max_bytes,
595 selected: true,
596 state: Mutex::new(AcquisitionState::default()),
597 },
598 }),
599 declared_format: self.declared_format,
600 })
601 }
602
603 #[must_use]
605 pub fn with_format(mut self, format: FormatId) -> Self {
606 self.declared_format = Some(format);
607 self
608 }
609
610 #[must_use]
611 pub fn name(&self) -> &str {
612 &self.name
613 }
614
615 #[must_use]
616 pub const fn format(&self) -> Option<&FormatId> {
617 self.declared_format.as_ref()
618 }
619
620 #[must_use]
621 pub fn is_directory(&self) -> bool {
622 matches!(
623 &*self.provider,
624 SourceProvider::Directory { .. } | SourceProvider::Memory { primary: None, .. }
625 )
626 }
627
628 pub fn acquisition_byte_limit(&self) -> Result<u64, Error> {
632 match &*self.provider {
633 SourceProvider::File { acquisition, .. }
634 | SourceProvider::Directory { acquisition } => Ok(acquisition.max_bytes),
635 SourceProvider::Memory { .. } => referenced_byte_limit(),
636 }
637 }
638
639 pub fn primary_buffer(&self) -> Result<SourceBuffer, Error> {
641 match &*self.provider {
642 SourceProvider::Memory {
643 primary: Some(primary),
644 ..
645 }
646 | SourceProvider::File { primary, .. } => Ok(primary.clone()),
647 SourceProvider::Directory { .. } | SourceProvider::Memory { primary: None, .. } => {
648 Err(Error::new(
649 &crate::codes::REQUEST_SOURCE_DIRECTORY_REQUIRED,
650 "a directory source has no implicit primary buffer",
651 )
652 .with_source(self.clone()))
653 }
654 }
655 }
656
657 pub fn buffer(&self, name: &crate::ArtifactPath) -> Result<SourceBuffer, Error> {
660 if matches!(
661 &*self.provider,
662 SourceProvider::Memory { primary: None, .. }
663 ) {
664 return self.root_buffer(name.as_str());
665 }
666 let SourceProvider::Directory { acquisition } = &*self.provider else {
667 return Err(Error::new(
668 &crate::codes::REQUEST_SOURCE_DIRECTORY_REQUIRED,
669 "named child buffers require a directory source",
670 )
671 .with_source(self.clone()));
672 };
673 let segments = resolve_segments(&[], name.as_str())
674 .map_err(|error| error.with_source(self.clone()))?;
675 acquisition
676 .acquire(&segments)
677 .map_err(|error| error.with_source(self.clone()))
678 }
679
680 pub fn root_buffer(&self, name: &str) -> Result<SourceBuffer, Error> {
685 match &*self.provider {
686 SourceProvider::Memory { named, .. } => {
687 let segments = resolve_segments(&[], name)?;
688 let key = segments.join("/");
689 named.get(&key).cloned().ok_or_else(|| {
690 Error::new(
691 &crate::codes::REQUEST_SOURCE_UNKNOWN_BUFFER,
692 format!(
693 "referenced buffer `{key}` was not supplied to this in-memory source"
694 ),
695 )
696 .with_source(self.clone())
697 })
698 }
699 SourceProvider::File { acquisition, .. }
700 | SourceProvider::Directory { acquisition } => {
701 let segments = resolve_segments(&[], name)?;
702 acquisition
703 .acquire(&segments)
704 .map_err(|error| error.with_source(self.clone()))
705 }
706 }
707 }
708
709 #[must_use]
715 pub fn selected_acquisition_root(&self) -> Option<&Path> {
716 match &*self.provider {
717 SourceProvider::Memory { .. } | SourceProvider::Directory { .. } => None,
718 SourceProvider::File { acquisition, .. } => acquisition
719 .selected
720 .then_some(acquisition.root_display.as_path()),
721 }
722 }
723
724 pub fn referenced_buffer(
729 &self,
730 referrer: &SourceBuffer,
731 name: &str,
732 ) -> Result<SourceBuffer, Error> {
733 let referrer_directory: Vec<&str> = referrer.directory_segments().collect();
734 match &*self.provider {
735 SourceProvider::Memory { named, .. } => {
736 let segments = resolve_segments(&referrer_directory, name)?;
737 let key = segments.join("/");
738 named.get(&key).cloned().ok_or_else(|| {
739 Error::new(
740 &crate::codes::REQUEST_SOURCE_UNKNOWN_BUFFER,
741 format!(
742 "referenced buffer `{key}` was not supplied to this in-memory source"
743 ),
744 )
745 .with_source(self.clone())
746 })
747 }
748 SourceProvider::File { acquisition, .. }
749 | SourceProvider::Directory { acquisition } => {
750 let segments = match absolute_to_root_relative(&acquisition.root_display, name) {
751 Some(root_relative) => root_relative?,
752 None => resolve_segments(&referrer_directory, name)?,
753 };
754 acquisition
755 .acquire(&segments)
756 .map_err(|error| error.with_source(self.clone()))
757 }
758 }
759 }
760
761 #[allow(clippy::too_many_lines)] pub fn entry_names(&self) -> Result<Vec<crate::ArtifactPath>, Error> {
768 match &*self.provider {
769 SourceProvider::Memory { named, .. } => named
770 .keys()
771 .map(|name| crate::ArtifactPath::new(name.clone()))
772 .collect(),
773 SourceProvider::File { .. } => Err(Error::new(
774 &crate::codes::REQUEST_SOURCE_DIRECTORY_REQUIRED,
775 "entry listing requires a directory source",
776 )
777 .with_source(self.clone())),
778 SourceProvider::Directory { acquisition } => {
779 struct Frame<Handle> {
793 prefix: Vec<String>,
794 directory: Handle,
795 subdirectories: Vec<String>,
797 }
798
799 let mut state = acquisition.lock();
800 if let Some(listed) = &state.listed {
801 return Ok(listed.clone());
802 }
803 let root = state.pinned_root(&acquisition.root_display)?;
804 let budget_refusal = || {
805 Error::new(
806 &crate::codes::READ_IO_REFERENCE_BUDGET,
807 format!(
808 "the source directory holds more than {MAX_REFERENCED_FILES} entries"
809 ),
810 )
811 };
812 let root_handle = root.duplicate_handle().map_err(|cause| {
813 Error::new(
814 &crate::codes::READ_IO_METADATA,
815 "cannot list the source directory root",
816 )
817 .with_cause(cause)
818 })?;
819 let mut names = Vec::new();
820 let mut undescended = 0usize;
823 let mut frames = Vec::new();
824 let mut arriving = Some((Vec::<String>::new(), root_handle));
825 while let Some((prefix, directory)) = arriving.take() {
826 let allowance = MAX_REFERENCED_FILES
827 .checked_sub(names.len() + undescended)
828 .filter(|allowance| *allowance > 0)
829 .ok_or_else(budget_refusal)?;
830 let entries =
831 platform::list_entries(&directory, allowance).map_err(|cause| {
832 if platform::is_entry_budget(&cause) {
833 budget_refusal()
834 } else {
835 listing_error(&prefix.join("/"), cause)
836 }
837 })?;
838 let mut subdirectories = Vec::new();
839 for (name, is_directory) in entries {
840 if names.len() + undescended + subdirectories.len() >= MAX_REFERENCED_FILES
841 {
842 return Err(budget_refusal());
843 }
844 if is_directory {
845 if prefix.len() + 1 >= MAX_REFERENCED_DEPTH {
846 return Err(Error::new(
847 &crate::codes::READ_IO_REFERENCE_BUDGET,
848 format!(
849 "the source directory nests more than {MAX_REFERENCED_DEPTH} levels deep"
850 ),
851 ));
852 }
853 subdirectories.push(name);
854 } else {
855 let mut child = prefix.clone();
856 child.push(name);
857 names.push(crate::ArtifactPath::new(child.join("/"))?);
858 }
859 }
860 undescended += subdirectories.len();
861 frames.push(Frame {
862 prefix,
863 directory,
864 subdirectories,
865 });
866
867 while let Some(frame) = frames.last_mut() {
872 let Some(name) = frame.subdirectories.pop() else {
873 frames.pop();
874 continue;
875 };
876 undescended -= 1;
877 let mut child = frame.prefix.clone();
878 let handle = platform::open_child_directory(&frame.directory, &name)
879 .map_err(|cause| {
880 child.push(name.clone());
881 listing_error(&child.join("/"), cause)
882 })?;
883 child.push(name);
884 arriving = Some((child, handle));
885 break;
886 }
887 }
888 names.sort();
889 state.listed = Some(names.clone());
890 Ok(names)
891 }
892 }
893 }
894
895 #[must_use]
897 pub fn acquired_buffers(&self) -> Vec<SourceBuffer> {
898 match &*self.provider {
899 SourceProvider::Memory { primary, named } => {
900 let mut buffers: Vec<_> = primary.iter().cloned().collect();
901 buffers.extend(
902 named
903 .values()
904 .filter(|buffer| {
905 primary
906 .as_ref()
907 .is_none_or(|entry| entry.id() != buffer.id())
908 })
909 .cloned(),
910 );
911 buffers
912 }
913 SourceProvider::File {
914 primary,
915 acquisition,
916 } => {
917 let mut buffers = vec![primary.clone()];
918 buffers.extend(acquisition.lock().cache.values().cloned());
919 buffers
920 }
921 SourceProvider::Directory { acquisition } => {
922 acquisition.lock().cache.values().cloned().collect()
923 }
924 }
925 }
926}
927
928impl fmt::Debug for Source {
929 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
930 formatter
931 .debug_struct("Source")
932 .field("name", &self.name)
933 .field("is_directory", &self.is_directory())
934 .field("declared_format", &self.declared_format)
935 .field("acquired_buffer_count", &self.acquired_buffers().len())
936 .finish_non_exhaustive()
937 }
938}
939
940impl FileAcquisition {
941 fn lock(&self) -> MutexGuard<'_, AcquisitionState> {
942 self.state
943 .lock()
944 .unwrap_or_else(std::sync::PoisonError::into_inner)
945 }
946
947 fn acquire(&self, segments: &[String]) -> Result<SourceBuffer, Error> {
954 let key = segments.join("/");
955 let mut state = self.lock();
956 if let Some(buffer) = state.cache.get(&key) {
957 return Ok(buffer.clone());
958 }
959 if state.files >= MAX_REFERENCED_FILES {
960 return Err(Error::new(
961 &crate::codes::READ_IO_REFERENCE_BUDGET,
962 format!("this source already acquired {MAX_REFERENCED_FILES} referenced files"),
963 ));
964 }
965 let remaining = self.max_bytes.saturating_sub(state.bytes);
966 let root = state.pinned_root(&self.root_display)?;
967 let file = root.open_beneath(segments).map_err(|error| {
968 if platform::is_symlink_refusal(&error) {
969 Error::new(
970 &crate::codes::REQUEST_SOURCE_SYMLINK_REFUSED,
971 format!("referenced file `{key}` crosses a symbolic link"),
972 )
973 } else {
974 Error::new(
975 &crate::codes::READ_IO_OPEN,
976 format!("cannot open referenced file `{key}`"),
977 )
978 .with_cause(error)
979 }
980 })?;
981 let bytes = read_open_file(
982 file,
983 &key,
984 ReadBudget::Referenced {
985 remaining,
986 total: self.max_bytes,
987 },
988 )?;
989 let directory = segments[..segments.len() - 1].to_vec();
990 let buffer = SourceBuffer::new(SourceId::new(&key)?, key.clone(), bytes, directory);
991 state.files += 1;
992 state.bytes += buffer.bytes().len() as u64;
993 state.cache.insert(key, buffer.clone());
994 Ok(buffer)
995 }
996}
997
998fn plain_segment(segment: &str) -> bool {
1004 if segment.is_empty()
1005 || segment == "."
1006 || segment == ".."
1007 || segment.contains(['/', '\\', '\0', ':'])
1008 {
1009 return false;
1010 }
1011 let mut components = Path::new(segment).components();
1012 matches!(components.next(), Some(Component::Normal(text)) if text == std::ffi::OsStr::new(segment))
1013 && components.next().is_none()
1014}
1015
1016fn resolve_segments(referrer_directory: &[&str], name: &str) -> Result<Vec<String>, Error> {
1023 if name.is_empty()
1024 || name.len() > crate::validation::MAX_ARTIFACT_PATH_BYTES
1025 || name.contains('\0')
1026 || name.starts_with(['/', '\\'])
1027 {
1028 return Err(Error::new(
1029 &crate::codes::REQUEST_SOURCE_INVALID_PATH,
1030 "a referenced name must be a nonempty bounded relative path",
1031 ));
1032 }
1033 let mut segments: Vec<String> = referrer_directory
1034 .iter()
1035 .map(|segment| (*segment).to_owned())
1036 .collect();
1037 for raw in name.split(['/', '\\']) {
1038 match raw {
1039 "" | "." => {}
1040 ".." => {
1041 if segments.pop().is_none() {
1042 return Err(Error::new(
1043 &crate::codes::REQUEST_SOURCE_ESCAPES_ROOT,
1044 format!("referenced name `{name}` resolves outside the acquisition root"),
1045 ));
1046 }
1047 }
1048 segment => {
1049 if segment.len() > crate::validation::MAX_ARTIFACT_PATH_BYTES
1050 || !plain_segment(segment)
1051 {
1052 return Err(Error::new(
1053 &crate::codes::REQUEST_SOURCE_INVALID_PATH,
1054 format!("referenced name `{name}` is not a portable relative path"),
1055 ));
1056 }
1057 segments.push(segment.to_owned());
1058 }
1059 }
1060 }
1061 if segments.is_empty() {
1062 return Err(Error::new(
1063 &crate::codes::REQUEST_SOURCE_INVALID_PATH,
1064 format!("referenced name `{name}` does not name a file"),
1065 ));
1066 }
1067 if segments.len() > MAX_REFERENCED_DEPTH {
1068 return Err(Error::new(
1069 &crate::codes::READ_IO_REFERENCE_BUDGET,
1070 format!("referenced name `{name}` nests more than {MAX_REFERENCED_DEPTH} levels deep"),
1071 ));
1072 }
1073 Ok(segments)
1074}
1075
1076fn absolute_to_root_relative(root: &Path, name: &str) -> Option<Result<Vec<String>, Error>> {
1082 let path = Path::new(name);
1083 if !path.is_absolute() {
1084 return None;
1085 }
1086 let Ok(remainder) = path.strip_prefix(root) else {
1087 return Some(Err(Error::new(
1088 &crate::codes::REQUEST_SOURCE_ESCAPES_ROOT,
1089 format!("referenced name `{name}` resolves outside the acquisition root"),
1090 )));
1091 };
1092 let mut segments = Vec::new();
1093 for component in remainder.components() {
1094 let text = match component {
1095 Component::Normal(text) => text.to_str(),
1096 _ => None,
1097 };
1098 let Some(text) = text.filter(|text| plain_segment(text)) else {
1099 return Some(Err(Error::new(
1100 &crate::codes::REQUEST_SOURCE_INVALID_PATH,
1101 format!("referenced name `{name}` is not a portable path beneath the root"),
1102 )));
1103 };
1104 segments.push(text.to_owned());
1105 }
1106 if segments.is_empty() {
1107 return Some(Err(Error::new(
1108 &crate::codes::REQUEST_SOURCE_INVALID_PATH,
1109 format!("referenced name `{name}` does not name a file"),
1110 )));
1111 }
1112 if segments.len() > MAX_REFERENCED_DEPTH {
1113 return Some(Err(Error::new(
1114 &crate::codes::READ_IO_REFERENCE_BUDGET,
1115 format!("referenced name `{name}` nests more than {MAX_REFERENCED_DEPTH} levels deep"),
1116 )));
1117 }
1118 Some(Ok(segments))
1119}
1120
1121fn canonical_parent(path: &Path) -> Result<PathBuf, Error> {
1122 let parent = match path.parent() {
1123 Some(parent) if !parent.as_os_str().is_empty() => parent,
1124 _ => Path::new("."),
1125 };
1126 std::fs::canonicalize(parent)
1127 .map_err(|cause| open_error(&crate::codes::READ_IO_METADATA, parent, cause))
1128}
1129
1130fn open_error(info: &'static crate::DiagnosticInfo, path: &Path, cause: std::io::Error) -> Error {
1131 Error::new(info, format!("cannot open source `{}`", path.display())).with_cause(cause)
1132}
1133
1134fn listing_error(display: &str, cause: std::io::Error) -> Error {
1137 if platform::is_symlink_refusal(&cause) {
1138 Error::new(
1139 &crate::codes::REQUEST_SOURCE_SYMLINK_REFUSED,
1140 format!("source directory `{display}` crosses a symbolic link"),
1141 )
1142 } else {
1143 Error::new(
1144 &crate::codes::READ_IO_METADATA,
1145 format!("cannot list source directory `{display}`"),
1146 )
1147 .with_cause(cause)
1148 }
1149}
1150
1151fn read_open_file(file: std::fs::File, name: &str, budget: ReadBudget) -> Result<Arc<[u8]>, Error> {
1158 let max_bytes = budget.bytes();
1159 let metadata = file.metadata().map_err(|cause| {
1160 Error::new(
1161 &crate::codes::READ_IO_METADATA,
1162 format!("cannot inspect source buffer `{name}`"),
1163 )
1164 .with_cause(cause)
1165 })?;
1166 if !metadata.is_file() {
1167 return Err(Error::new(
1168 &crate::codes::REQUEST_SOURCE_NOT_A_FILE,
1169 format!("source buffer `{name}` is not a regular file"),
1170 ));
1171 }
1172 let declared_length = metadata.len();
1173 if declared_length > max_bytes {
1174 return Err(budget.exceeded(name));
1175 }
1176 let capacity = usize::try_from(declared_length).map_err(|cause| {
1177 Error::new(
1178 &crate::codes::READ_IO_ALLOCATION_REFUSED,
1179 format!("source buffer `{name}` is too large for this platform"),
1180 )
1181 .with_cause(cause)
1182 })?;
1183 let mut bytes = Vec::new();
1184 bytes.try_reserve_exact(capacity).map_err(|cause| {
1185 Error::new(
1186 &crate::codes::READ_IO_ALLOCATION_REFUSED,
1187 format!("cannot reserve {declared_length} bytes for source buffer `{name}`"),
1188 )
1189 .with_cause(cause)
1190 })?;
1191 let read_limit = declared_length
1192 .checked_add(1)
1193 .ok_or_else(|| {
1194 Error::new(
1195 &crate::codes::READ_IO_ALLOCATION_REFUSED,
1196 format!("source buffer `{name}` is too large to read safely"),
1197 )
1198 })?
1199 .min(max_bytes.saturating_add(1));
1200 let mut file = file;
1201 file.by_ref()
1202 .take(read_limit)
1203 .read_to_end(&mut bytes)
1204 .map_err(|cause| {
1205 Error::new(
1206 &crate::codes::READ_IO_READ,
1207 format!("cannot read source buffer `{name}`"),
1208 )
1209 .with_cause(cause)
1210 })?;
1211 if bytes.len() != capacity {
1212 return Err(Error::new(
1213 &crate::codes::READ_IO_SOURCE_CHANGED,
1214 format!("source buffer `{name}` changed length while it was read"),
1215 ));
1216 }
1217 Ok(bytes.into())
1218}
1219
1220#[cfg(unix)]
1221mod platform {
1222 use std::ffi::CString;
1232 use std::fs::File;
1233 use std::os::fd::{AsRawFd, FromRawFd, OwnedFd};
1234 use std::os::unix::ffi::OsStrExt;
1235 use std::path::Path;
1236
1237 #[derive(Debug)]
1238 pub(super) struct RootHandle(OwnedFd);
1239
1240 pub(super) fn open_no_follow(path: &Path) -> std::io::Result<File> {
1243 let path = c_string(path.as_os_str().as_bytes())?;
1244 let fd = unsafe {
1248 libc::open(
1249 path.as_ptr(),
1250 libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_CLOEXEC | libc::O_NONBLOCK,
1251 )
1252 };
1253 if fd < 0 {
1254 return Err(std::io::Error::last_os_error());
1255 }
1256 let file = unsafe { File::from_raw_fd(fd) };
1258 clear_nonblock(&file)?;
1259 Ok(file)
1260 }
1261
1262 pub(super) fn open_root(path: &Path) -> std::io::Result<RootHandle> {
1267 let path = c_string(path.as_os_str().as_bytes())?;
1268 let fd = unsafe {
1270 libc::open(
1271 path.as_ptr(),
1272 libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_CLOEXEC | libc::O_NONBLOCK,
1273 )
1274 };
1275 if fd < 0 {
1276 return Err(std::io::Error::last_os_error());
1277 }
1278 let file = unsafe { File::from_raw_fd(fd) };
1280 if !file.metadata()?.is_dir() {
1281 return Err(std::io::Error::from(std::io::ErrorKind::NotADirectory));
1282 }
1283 clear_nonblock(&file)?;
1284 Ok(RootHandle(file.into()))
1285 }
1286
1287 impl RootHandle {
1288 pub(super) fn open_beneath(&self, segments: &[String]) -> std::io::Result<File> {
1289 let mut directory: Option<OwnedFd> = None;
1290 let (file_segment, directories) =
1291 segments.split_last().expect("resolution yields a file");
1292 for segment in directories {
1293 let next = self.open_at(
1298 directory.as_ref(),
1299 segment,
1300 libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_CLOEXEC | libc::O_NONBLOCK,
1301 )?;
1302 let next = File::from(next);
1303 if !next.metadata()?.is_dir() {
1304 return Err(std::io::Error::from(std::io::ErrorKind::NotADirectory));
1305 }
1306 directory = Some(next.into());
1307 }
1308 let fd = self.open_at(
1309 directory.as_ref(),
1310 file_segment,
1311 libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_CLOEXEC | libc::O_NONBLOCK,
1312 )?;
1313 let file = File::from(fd);
1314 clear_nonblock(&file)?;
1315 Ok(file)
1316 }
1317
1318 pub(super) fn duplicate_handle(&self) -> std::io::Result<DirectoryHandle> {
1326 let segment = c_string(b".")?;
1327 let fd = unsafe {
1330 libc::openat(
1331 self.0.as_raw_fd(),
1332 segment.as_ptr(),
1333 libc::O_RDONLY | libc::O_CLOEXEC | libc::O_DIRECTORY,
1334 )
1335 };
1336 if fd < 0 {
1337 return Err(std::io::Error::last_os_error());
1338 }
1339 Ok(unsafe { OwnedFd::from_raw_fd(fd) })
1341 }
1342
1343 fn open_at(
1344 &self,
1345 directory: Option<&OwnedFd>,
1346 segment: &str,
1347 flags: libc::c_int,
1348 ) -> std::io::Result<OwnedFd> {
1349 if !super::plain_segment(segment) {
1350 return Err(std::io::Error::from(std::io::ErrorKind::InvalidInput));
1351 }
1352 let at = directory.map_or_else(|| self.0.as_raw_fd(), AsRawFd::as_raw_fd);
1353 let segment = c_string(segment.as_bytes())?;
1354 let fd = unsafe { libc::openat(at, segment.as_ptr(), flags) };
1358 if fd < 0 {
1359 return Err(std::io::Error::last_os_error());
1360 }
1361 Ok(unsafe { OwnedFd::from_raw_fd(fd) })
1363 }
1364 }
1365
1366 pub(super) type DirectoryHandle = OwnedFd;
1369
1370 pub(super) fn open_child_directory(
1374 parent: &DirectoryHandle,
1375 name: &str,
1376 ) -> std::io::Result<DirectoryHandle> {
1377 if !super::plain_segment(name) {
1378 return Err(std::io::Error::from(std::io::ErrorKind::InvalidInput));
1379 }
1380 let segment = c_string(name.as_bytes())?;
1381 let fd = unsafe {
1385 libc::openat(
1386 parent.as_raw_fd(),
1387 segment.as_ptr(),
1388 libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_CLOEXEC | libc::O_NONBLOCK,
1389 )
1390 };
1391 if fd < 0 {
1392 return Err(std::io::Error::last_os_error());
1393 }
1394 let file = unsafe { File::from_raw_fd(fd) };
1396 if !file.metadata()?.is_dir() {
1397 return Err(std::io::Error::from(std::io::ErrorKind::NotADirectory));
1398 }
1399 Ok(file.into())
1400 }
1401
1402 const ENTRY_BUDGET_MARKER: &str = "directory entry allowance exhausted";
1403
1404 fn entry_budget_error() -> std::io::Error {
1405 std::io::Error::other(ENTRY_BUDGET_MARKER)
1406 }
1407
1408 pub(super) fn is_entry_budget(error: &std::io::Error) -> bool {
1411 error.kind() == std::io::ErrorKind::Other && error.to_string().contains(ENTRY_BUDGET_MARKER)
1412 }
1413
1414 pub(super) fn list_entries(
1421 directory: &DirectoryHandle,
1422 max: usize,
1423 ) -> std::io::Result<Vec<(String, bool)>> {
1424 use std::os::fd::IntoRawFd;
1425
1426 let raw = directory.try_clone()?.into_raw_fd();
1429 let stream = unsafe { libc::fdopendir(raw) };
1432 if stream.is_null() {
1433 let error = std::io::Error::last_os_error();
1434 unsafe { libc::close(raw) };
1437 return Err(error);
1438 }
1439 let mut entries = Vec::new();
1440 loop {
1441 errno_clear();
1443 let entry = unsafe { libc::readdir(stream) };
1444 if entry.is_null() {
1445 let error = std::io::Error::last_os_error();
1446 unsafe { libc::closedir(stream) };
1449 if error.raw_os_error().is_some_and(|code| code != 0) {
1450 return Err(error);
1451 }
1452 break;
1453 }
1454 let name_bytes = unsafe {
1459 std::ffi::CStr::from_ptr((&raw const (*entry).d_name).cast::<libc::c_char>())
1460 };
1461 let Ok(name) = name_bytes.to_str() else {
1462 continue;
1463 };
1464 if name == "." || name == ".." {
1465 continue;
1466 }
1467 if entries.len() == max {
1468 unsafe { libc::closedir(stream) };
1470 return Err(entry_budget_error());
1471 }
1472 let kind = unsafe { (*entry).d_type };
1474 let is_directory = match kind {
1475 libc::DT_DIR => true,
1476 libc::DT_UNKNOWN => {
1477 let mut stat: libc::stat = unsafe { std::mem::zeroed() };
1480 let segment =
1481 c_string(name.as_bytes()).expect("directory entry names carry no NUL");
1482 let status = unsafe {
1486 libc::fstatat(
1487 libc::dirfd(stream),
1488 segment.as_ptr(),
1489 &raw mut stat,
1490 libc::AT_SYMLINK_NOFOLLOW,
1491 )
1492 };
1493 status == 0 && stat.st_mode & libc::S_IFMT == libc::S_IFDIR
1494 }
1495 _ => false,
1496 };
1497 entries.push((name.to_owned(), is_directory));
1498 }
1499 Ok(entries)
1500 }
1501
1502 fn errno_clear() {
1503 errno::set_errno(errno::Errno(0));
1504 }
1505
1506 fn clear_nonblock(file: &File) -> std::io::Result<()> {
1508 let fd = file.as_raw_fd();
1509 let flags = unsafe { libc::fcntl(fd, libc::F_GETFL) };
1512 if flags < 0 {
1513 return Err(std::io::Error::last_os_error());
1514 }
1515 let status = unsafe { libc::fcntl(fd, libc::F_SETFL, flags & !libc::O_NONBLOCK) };
1517 if status < 0 {
1518 return Err(std::io::Error::last_os_error());
1519 }
1520 Ok(())
1521 }
1522
1523 pub(super) fn is_symlink_refusal(error: &std::io::Error) -> bool {
1524 matches!(error.raw_os_error(), Some(libc::ELOOP | libc::EMLINK))
1525 }
1526
1527 pub(super) fn is_directory_open_failure(error: &std::io::Error, path: &Path) -> bool {
1528 let _ = path;
1532 error.raw_os_error() == Some(libc::EISDIR)
1533 }
1534
1535 fn c_string(bytes: &[u8]) -> std::io::Result<CString> {
1536 CString::new(bytes).map_err(|_| std::io::Error::from(std::io::ErrorKind::InvalidInput))
1537 }
1538
1539 #[cfg(test)]
1540 mod tests {
1541 use super::*;
1542
1543 #[test]
1544 fn open_root_refuses_a_symbolic_link_to_a_directory() {
1545 let base = std::env::temp_dir().join(format!(
1546 "powerio-core-open-root-{}-{}",
1547 std::process::id(),
1548 std::time::SystemTime::now()
1549 .duration_since(std::time::UNIX_EPOCH)
1550 .unwrap()
1551 .as_nanos()
1552 ));
1553 std::fs::create_dir_all(&base).unwrap();
1554 assert!(open_root(&base).is_ok());
1555 let link = base.join("link");
1556 std::os::unix::fs::symlink(&base, &link).unwrap();
1557 let error = open_root(&link).unwrap_err();
1558 assert!(
1559 is_symlink_refusal(&error) || error.kind() == std::io::ErrorKind::NotADirectory,
1560 "{error:?}"
1561 );
1562 std::fs::remove_dir_all(&base).unwrap();
1563 }
1564 }
1565}
1566
1567#[cfg(not(unix))]
1568mod platform {
1569 use std::fs::File;
1577 use std::path::{Path, PathBuf};
1578
1579 #[derive(Debug)]
1580 pub(super) struct RootHandle {
1581 root: PathBuf,
1582 }
1583
1584 pub(super) fn open_no_follow(path: &Path) -> std::io::Result<File> {
1585 let file = open_reparse_refused(path)?;
1586 Ok(file)
1587 }
1588
1589 pub(super) fn open_root(path: &Path) -> std::io::Result<RootHandle> {
1590 drop(open_directory_pinned(path)?);
1594 Ok(RootHandle {
1595 root: path.to_path_buf(),
1596 })
1597 }
1598
1599 impl RootHandle {
1600 pub(super) fn open_beneath(&self, segments: &[String]) -> std::io::Result<File> {
1601 let mut path = self.root.clone();
1602 let (file_segment, directories) =
1603 segments.split_last().expect("resolution yields a file");
1604 let mut held = Vec::with_capacity(directories.len() + 1);
1608 held.push(open_directory_pinned(&self.root)?);
1609 for segment in directories {
1610 push_plain_segment(&mut path, segment)?;
1611 held.push(open_directory_pinned(&path)?);
1612 }
1613 push_plain_segment(&mut path, file_segment)?;
1614 let file = open_reparse_refused(&path)?;
1615 drop(held);
1616 Ok(file)
1617 }
1618
1619 pub(super) fn duplicate_handle(&self) -> std::io::Result<DirectoryHandle> {
1622 let handle = open_directory_pinned(&self.root)?;
1623 Ok(DirectoryHandle {
1624 path: self.root.clone(),
1625 _handle: handle,
1626 })
1627 }
1628 }
1629
1630 #[derive(Debug)]
1634 pub(super) struct DirectoryHandle {
1635 path: PathBuf,
1636 _handle: File,
1637 }
1638
1639 pub(super) fn open_child_directory(
1643 parent: &DirectoryHandle,
1644 name: &str,
1645 ) -> std::io::Result<DirectoryHandle> {
1646 let mut path = parent.path.clone();
1647 push_plain_segment(&mut path, name)?;
1648 let handle = open_directory_pinned(&path)?;
1649 Ok(DirectoryHandle {
1650 path,
1651 _handle: handle,
1652 })
1653 }
1654
1655 const ENTRY_BUDGET_MARKER: &str = "directory entry allowance exhausted";
1656
1657 fn entry_budget_error() -> std::io::Error {
1658 std::io::Error::other(ENTRY_BUDGET_MARKER)
1659 }
1660
1661 pub(super) fn is_entry_budget(error: &std::io::Error) -> bool {
1664 error.kind() == std::io::ErrorKind::Other && error.to_string().contains(ENTRY_BUDGET_MARKER)
1665 }
1666
1667 pub(super) fn list_entries(
1672 directory: &DirectoryHandle,
1673 max: usize,
1674 ) -> std::io::Result<Vec<(String, bool)>> {
1675 let mut entries = Vec::new();
1676 for entry in std::fs::read_dir(&directory.path)? {
1677 let entry = entry?;
1678 let Ok(name) = entry.file_name().into_string() else {
1679 continue;
1680 };
1681 if entries.len() == max {
1682 return Err(entry_budget_error());
1683 }
1684 let is_directory = entry.file_type()?.is_dir();
1685 entries.push((name, is_directory));
1686 }
1687 Ok(entries)
1688 }
1689
1690 fn push_plain_segment(path: &mut PathBuf, segment: &str) -> std::io::Result<()> {
1693 if !super::plain_segment(segment) {
1694 return Err(std::io::Error::from(std::io::ErrorKind::InvalidInput));
1695 }
1696 path.push(segment);
1697 Ok(())
1698 }
1699
1700 #[cfg(windows)]
1706 fn open_directory_pinned(path: &Path) -> std::io::Result<File> {
1707 use std::os::windows::fs::{MetadataExt, OpenOptionsExt};
1708
1709 const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000;
1710 const FILE_FLAG_BACKUP_SEMANTICS: u32 = 0x0200_0000;
1711 const FILE_ATTRIBUTE_REPARSE_POINT: u32 = 0x0000_0400;
1712 const FILE_ATTRIBUTE_DIRECTORY: u32 = 0x0000_0010;
1713 const FILE_SHARE_READ: u32 = 0x0000_0001;
1714 const FILE_SHARE_WRITE: u32 = 0x0000_0002;
1715
1716 let file = std::fs::OpenOptions::new()
1717 .read(true)
1718 .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE)
1719 .custom_flags(FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS)
1720 .open(path)?;
1721 let attributes = file.metadata()?.file_attributes();
1722 if attributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 {
1723 return Err(symlink_error());
1724 }
1725 if attributes & FILE_ATTRIBUTE_DIRECTORY == 0 {
1726 return Err(std::io::Error::from(std::io::ErrorKind::NotADirectory));
1727 }
1728 Ok(file)
1729 }
1730
1731 #[cfg(not(windows))]
1732 fn open_directory_pinned(path: &Path) -> std::io::Result<File> {
1733 let metadata = std::fs::symlink_metadata(path)?;
1734 if metadata.file_type().is_symlink() {
1735 return Err(symlink_error());
1736 }
1737 if !metadata.is_dir() {
1738 return Err(std::io::Error::from(std::io::ErrorKind::NotADirectory));
1739 }
1740 File::open(path)
1741 }
1742
1743 #[cfg(windows)]
1744 fn open_reparse_refused(path: &Path) -> std::io::Result<File> {
1745 use std::os::windows::fs::{MetadataExt, OpenOptionsExt};
1746
1747 const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000;
1748 const FILE_ATTRIBUTE_REPARSE_POINT: u32 = 0x0000_0400;
1749
1750 let file = std::fs::OpenOptions::new()
1751 .read(true)
1752 .custom_flags(FILE_FLAG_OPEN_REPARSE_POINT)
1753 .open(path)?;
1754 let attributes = file.metadata()?.file_attributes();
1755 if attributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 {
1756 return Err(symlink_error());
1757 }
1758 Ok(file)
1759 }
1760
1761 #[cfg(not(windows))]
1762 fn open_reparse_refused(path: &Path) -> std::io::Result<File> {
1763 let metadata = std::fs::symlink_metadata(path)?;
1764 if metadata.file_type().is_symlink() {
1765 return Err(symlink_error());
1766 }
1767 File::open(path)
1768 }
1769
1770 fn symlink_error() -> std::io::Error {
1771 std::io::Error::new(std::io::ErrorKind::InvalidData, "symbolic link refused")
1772 }
1773
1774 pub(super) fn is_symlink_refusal(error: &std::io::Error) -> bool {
1775 error.kind() == std::io::ErrorKind::InvalidData
1776 && error.to_string().contains("symbolic link refused")
1777 }
1778
1779 pub(super) fn is_directory_open_failure(error: &std::io::Error, path: &Path) -> bool {
1780 let _ = error;
1781 std::fs::symlink_metadata(path).is_ok_and(|metadata| metadata.is_dir())
1782 }
1783}
1784
1785pub const MEMORY_SOURCE_NAME: &str = "<memory>";
1789
1790pub trait IntoSource {
1801 fn into_source(self) -> Result<Source, Error>;
1807}
1808
1809impl IntoSource for Source {
1810 fn into_source(self) -> Result<Source, Error> {
1811 Ok(self)
1812 }
1813}
1814
1815impl IntoSource for &Source {
1816 fn into_source(self) -> Result<Source, Error> {
1817 Ok(self.clone())
1818 }
1819}
1820
1821macro_rules! into_source_by_name {
1823 ($($input:ty),* $(,)?) => {
1824 $(
1825 impl IntoSource for $input {
1826 fn into_source(self) -> Result<Source, Error> {
1827 Source::open(PathBuf::from(self))
1828 }
1829 }
1830 )*
1831 };
1832}
1833
1834into_source_by_name!(&str, &String, String, &Path, &PathBuf, PathBuf);
1835
1836macro_rules! into_source_by_content {
1838 ($($input:ty => $bytes:expr),* $(,)?) => {
1839 $(
1840 impl IntoSource for $input {
1841 fn into_source(self) -> Result<Source, Error> {
1842 let bytes: Arc<[u8]> = $bytes(self);
1843 Source::from_memory(MEMORY_SOURCE_NAME, bytes)
1844 }
1845 }
1846 )*
1847 };
1848}
1849
1850into_source_by_content!(
1851 &[u8] => Arc::<[u8]>::from,
1852 Vec<u8> => Arc::<[u8]>::from,
1853 Arc<[u8]> => std::convert::identity,
1854);
1855
1856impl<const N: usize> IntoSource for &[u8; N] {
1857 fn into_source(self) -> Result<Source, Error> {
1858 Source::from_memory(MEMORY_SOURCE_NAME, Arc::<[u8]>::from(self.as_slice()))
1859 }
1860}
1861
1862#[cfg(test)]
1863mod tests {
1864 use std::time::{SystemTime, UNIX_EPOCH};
1865
1866 use super::*;
1867 use crate::ArtifactPath;
1868
1869 static PROCESS_RESOURCE_TESTS: Mutex<()> = Mutex::new(());
1874
1875 fn process_resource_guard() -> MutexGuard<'static, ()> {
1876 PROCESS_RESOURCE_TESTS
1877 .lock()
1878 .unwrap_or_else(std::sync::PoisonError::into_inner)
1879 }
1880
1881 struct CountingAllocator;
1884
1885 static ALLOCATED_BYTES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
1886
1887 thread_local! {
1888 static MEASURING: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
1889 }
1890
1891 fn measured_bytes<T>(work: impl FnOnce() -> T) -> (T, usize) {
1892 let _ = MEASURING.try_with(|flag| flag.set(true));
1893 let before = ALLOCATED_BYTES.load(std::sync::atomic::Ordering::Relaxed);
1894 let value = work();
1895 let after = ALLOCATED_BYTES.load(std::sync::atomic::Ordering::Relaxed);
1896 let _ = MEASURING.try_with(|flag| flag.set(false));
1897 (value, after.saturating_sub(before))
1898 }
1899
1900 unsafe impl std::alloc::GlobalAlloc for CountingAllocator {
1903 unsafe fn alloc(&self, layout: std::alloc::Layout) -> *mut u8 {
1904 if MEASURING.try_with(std::cell::Cell::get).unwrap_or(false) {
1905 ALLOCATED_BYTES.fetch_add(layout.size(), std::sync::atomic::Ordering::Relaxed);
1906 }
1907 unsafe { std::alloc::System.alloc(layout) }
1908 }
1909
1910 unsafe fn dealloc(&self, ptr: *mut u8, layout: std::alloc::Layout) {
1911 unsafe { std::alloc::System.dealloc(ptr, layout) }
1912 }
1913
1914 unsafe fn realloc(
1915 &self,
1916 ptr: *mut u8,
1917 layout: std::alloc::Layout,
1918 new_size: usize,
1919 ) -> *mut u8 {
1920 if MEASURING.try_with(std::cell::Cell::get).unwrap_or(false) {
1921 ALLOCATED_BYTES.fetch_add(new_size, std::sync::atomic::Ordering::Relaxed);
1922 }
1923 unsafe { std::alloc::System.realloc(ptr, layout, new_size) }
1924 }
1925 }
1926
1927 #[global_allocator]
1928 static COUNTING: CountingAllocator = CountingAllocator;
1929
1930 fn test_root(name: &str) -> PathBuf {
1931 let nonce = SystemTime::now()
1932 .duration_since(UNIX_EPOCH)
1933 .unwrap()
1934 .as_nanos();
1935 std::env::temp_dir().join(format!(
1936 "powerio-core-{name}-{}-{nonce}",
1937 std::process::id()
1938 ))
1939 }
1940
1941 #[test]
1942 fn format_ids_use_the_exact_open_grammar() {
1943 for id in ["matpower", "psse-raw", "doe-go-3", "x1"] {
1944 assert_eq!(FormatId::new(id).unwrap().as_str(), id);
1945 }
1946 for id in ["", "1matpower", "MATPOWER", "psse--raw", "psse-", "p_sse"] {
1947 assert!(FormatId::new(id).is_err(), "{id}");
1948 }
1949 assert!(FormatId::new("a".repeat(MAX_FORMAT_ID_BYTES + 1)).is_err());
1950 }
1951
1952 #[test]
1953 fn memory_sources_retain_arbitrary_binary_bytes_without_copying() {
1954 let bytes: Arc<[u8]> = vec![0, 255, 0, 128].into();
1955 let pointer = bytes.as_ptr();
1956 let source = Source::from_memory("input.bin", Arc::clone(&bytes))
1957 .unwrap()
1958 .with_format(FormatId::new("pwb").unwrap());
1959 let buffer = source.primary_buffer().unwrap();
1960 assert_eq!(buffer.bytes(), [0, 255, 0, 128]);
1961 assert_eq!(buffer.bytes().as_ptr(), pointer);
1962 assert_eq!(source.format().unwrap().as_str(), "pwb");
1963 assert!(Source::from_memory("", Vec::new()).is_err());
1964 assert!(Source::from_memory("x\0y", Vec::new()).is_err());
1965 }
1966
1967 #[test]
1968 fn a_bom_is_retained_and_skipped_for_the_parser_without_a_second_buffer() {
1969 let bytes: Vec<u8> = [0xEF, 0xBB, 0xBF, b'm', b'p', b'c'].to_vec();
1970 let source = Source::from_memory("case.m", bytes).unwrap();
1971 let buffer = source.primary_buffer().unwrap();
1972 assert!(buffer.has_utf8_bom());
1973 assert_eq!(buffer.bytes().len(), 6);
1974 assert_eq!(buffer.content_bytes(), b"mpc");
1975 assert_eq!(
1977 buffer.content_bytes().as_ptr(),
1978 buffer.bytes()[3..].as_ptr()
1979 );
1980
1981 let plain = Source::from_memory("case.m", b"mpc".to_vec()).unwrap();
1982 let plain = plain.primary_buffer().unwrap();
1983 assert!(!plain.has_utf8_bom());
1984 assert_eq!(plain.content_bytes(), plain.bytes());
1985 }
1986
1987 #[test]
1988 fn a_memory_source_resolves_named_buffers_and_never_the_filesystem() {
1989 let source = Source::from_memory("master.dss", b"redirect sub/feeder.dss".to_vec())
1990 .unwrap()
1991 .with_named_buffer("sub/feeder.dss", b"feeder".to_vec())
1992 .unwrap();
1993 let primary = source.primary_buffer().unwrap();
1994 let feeder = source
1995 .referenced_buffer(&primary, "sub/feeder.dss")
1996 .unwrap();
1997 assert_eq!(feeder.bytes(), b"feeder");
1998
1999 let sibling = source
2002 .referenced_buffer(&feeder, "../sub/feeder.dss")
2003 .unwrap();
2004 assert_eq!(sibling.bytes(), b"feeder");
2005
2006 assert!(source.referenced_buffer(&primary, "missing.dss").is_err());
2007 let escape = source.referenced_buffer(&primary, "../outside.dss");
2008 assert_eq!(
2009 escape.unwrap_err().category(),
2010 crate::ErrorCategory::Request
2011 );
2012 }
2013
2014 #[test]
2015 fn a_file_source_acquires_referenced_files_beneath_its_containing_directory() {
2016 let root = test_root("file-refs");
2017 std::fs::create_dir_all(root.join("sub")).unwrap();
2018 std::fs::write(root.join("master.dss"), b"master").unwrap();
2019 std::fs::write(root.join("sub/feeder.dss"), b"feeder").unwrap();
2020 std::fs::write(root.join("sub/coords.csv"), b"coords").unwrap();
2021
2022 let source = Source::open(root.join("master.dss")).unwrap();
2023 let primary = source.primary_buffer().unwrap();
2024 assert_eq!(primary.bytes(), b"master");
2025
2026 let feeder = source
2027 .referenced_buffer(&primary, "sub/feeder.dss")
2028 .unwrap();
2029 assert_eq!(feeder.bytes(), b"feeder");
2030 let coords = source.referenced_buffer(&feeder, "coords.csv").unwrap();
2033 assert_eq!(coords.bytes(), b"coords");
2034 let master_again = source.referenced_buffer(&feeder, "../master.dss").unwrap();
2036 assert_eq!(master_again.bytes(), b"master");
2037 let escape = source.referenced_buffer(&primary, "../escape.dss");
2038 assert_eq!(
2039 escape.unwrap_err().category(),
2040 crate::ErrorCategory::Request
2041 );
2042
2043 let again = source
2045 .referenced_buffer(&primary, "sub/feeder.dss")
2046 .unwrap();
2047 assert_eq!(again.bytes().as_ptr(), feeder.bytes().as_ptr());
2048 assert_eq!(source.acquired_buffers().len(), 4);
2049 std::fs::remove_dir_all(root).unwrap();
2050 }
2051
2052 #[test]
2053 fn an_explicitly_wider_root_admits_shared_files_and_still_confines() {
2054 let root = test_root("wider-root");
2055 std::fs::create_dir_all(root.join("cases")).unwrap();
2056 std::fs::create_dir_all(root.join("shared")).unwrap();
2057 std::fs::write(root.join("cases/master.dss"), b"master").unwrap();
2058 std::fs::write(root.join("shared/wires.dss"), b"wires").unwrap();
2059
2060 let narrow = Source::open(root.join("cases/master.dss")).unwrap();
2062 let primary = narrow.primary_buffer().unwrap();
2063 assert!(
2064 narrow
2065 .referenced_buffer(&primary, "../shared/wires.dss")
2066 .is_err()
2067 );
2068
2069 let wide = Source::open(root.join("cases/master.dss"))
2071 .unwrap()
2072 .with_acquisition_root(&root)
2073 .unwrap();
2074 let primary = wide.primary_buffer().unwrap();
2075 let wires = wide
2076 .referenced_buffer(&primary, "../shared/wires.dss")
2077 .unwrap();
2078 assert_eq!(wires.bytes(), b"wires");
2079 assert!(
2080 wide.referenced_buffer(&primary, "../../etc/passwd")
2081 .is_err()
2082 );
2083
2084 let outside = test_root("wider-root-outside");
2086 std::fs::create_dir_all(&outside).unwrap();
2087 assert!(
2088 Source::open(root.join("cases/master.dss"))
2089 .unwrap()
2090 .with_acquisition_root(&outside)
2091 .is_err()
2092 );
2093 std::fs::remove_dir_all(outside).ok();
2094 std::fs::remove_dir_all(root).unwrap();
2095 }
2096
2097 #[test]
2098 fn directory_buffers_are_lazy_cached_and_binary_safe() {
2099 let root = test_root("directory");
2100 std::fs::create_dir_all(root.join("nested")).unwrap();
2101 std::fs::write(root.join("nested/data.bin"), [0, 255, 7]).unwrap();
2102 let source = Source::open(&root).unwrap();
2103 assert!(source.is_directory());
2104 assert!(source.acquired_buffers().is_empty());
2105 let name = ArtifactPath::new("nested/data.bin").unwrap();
2106 let first = source.buffer(&name).unwrap();
2107 let second = source.buffer(&name).unwrap();
2108 assert_eq!(first.bytes(), [0, 255, 7]);
2109 assert_eq!(first.bytes().as_ptr(), second.bytes().as_ptr());
2110 assert_eq!(source.acquired_buffers().len(), 1);
2111 std::fs::remove_dir_all(root).unwrap();
2112 }
2113
2114 #[cfg(unix)]
2115 #[test]
2116 fn source_acquisition_refuses_root_and_child_symlinks() {
2117 use std::os::unix::fs::symlink;
2118
2119 let root = test_root("symlink");
2120 std::fs::create_dir_all(&root).unwrap();
2121 std::fs::write(root.join("real.bin"), b"real").unwrap();
2122 symlink(root.join("real.bin"), root.join("link.bin")).unwrap();
2123 let source = Source::open(&root).unwrap();
2124 let error = source
2125 .buffer(&ArtifactPath::new("link.bin").unwrap())
2126 .unwrap_err();
2127 assert_eq!(error.category(), crate::ErrorCategory::Request);
2128
2129 std::fs::create_dir_all(root.join("real-dir")).unwrap();
2131 std::fs::write(root.join("real-dir/inner.bin"), b"inner").unwrap();
2132 symlink(root.join("real-dir"), root.join("link-dir")).unwrap();
2133 let error = source
2134 .buffer(&ArtifactPath::new("link-dir/inner.bin").unwrap())
2135 .unwrap_err();
2136 assert_eq!(error.category(), crate::ErrorCategory::Request);
2137
2138 let root_link = root.with_extension("link");
2139 symlink(&root, &root_link).unwrap();
2140 assert!(Source::open(&root_link).is_err());
2141 std::fs::remove_file(root_link).unwrap();
2142 std::fs::remove_dir_all(root).unwrap();
2143 }
2144
2145 #[cfg(unix)]
2146 #[test]
2147 fn a_named_pipe_is_refused_promptly_and_siblings_still_acquire() {
2148 use std::os::unix::ffi::OsStrExt;
2149
2150 let root = test_root("fifo");
2151 std::fs::create_dir_all(&root).unwrap();
2152 std::fs::write(root.join("real.csv"), b"real").unwrap();
2153 let fifo = root.join("pipe.dat");
2154 let c_path = std::ffi::CString::new(fifo.as_os_str().as_bytes()).unwrap();
2155 assert_eq!(unsafe { libc::mkfifo(c_path.as_ptr(), 0o644) }, 0);
2158
2159 let (sender, receiver) = std::sync::mpsc::channel();
2163 let opened_root = root.clone();
2164 let worker = std::thread::spawn(move || {
2165 let open_error = Source::open(opened_root.join("pipe.dat")).map(|_| ());
2166 let directory = Source::open(&opened_root).unwrap();
2167 let buffer_error = directory
2168 .buffer(&ArtifactPath::new("pipe.dat").unwrap())
2169 .map(|_| ());
2170 let sibling = directory
2171 .buffer(&ArtifactPath::new("real.csv").unwrap())
2172 .map(|buffer| buffer.bytes().to_vec());
2173 sender.send((open_error, buffer_error, sibling)).unwrap();
2174 });
2175 let (open_error, buffer_error, sibling) = receiver
2176 .recv_timeout(std::time::Duration::from_secs(10))
2177 .expect("acquisition on a writerless pipe completes promptly");
2178 worker.join().unwrap();
2179 assert_eq!(
2180 open_error.unwrap_err().category(),
2181 crate::ErrorCategory::Request
2182 );
2183 assert_eq!(
2184 buffer_error.unwrap_err().category(),
2185 crate::ErrorCategory::Request
2186 );
2187 assert_eq!(sibling.unwrap(), b"real");
2188 std::fs::remove_dir_all(root).unwrap();
2189 }
2190
2191 #[test]
2192 fn default_budget_accepts_a_primary_and_reference_larger_than_64_mib() {
2193 assert_eq!(DEFAULT_SOURCE_BYTES, 1_073_741_824);
2194 let root = test_root("large-default-budget");
2195 std::fs::create_dir_all(&root).unwrap();
2196 let path = root.join("large.dat");
2197 let size = (64 << 20) + 1;
2198 std::fs::File::create(&path).unwrap().set_len(size).unwrap();
2199 let source = Source::open(&path).unwrap();
2200 assert_eq!(source.primary_buffer().unwrap().bytes().len() as u64, size);
2201 drop(source);
2202 let directory = Source::open(&root).unwrap();
2203 let name = crate::ArtifactPath::new("large.dat").unwrap();
2204 assert_eq!(directory.buffer(&name).unwrap().bytes().len() as u64, size);
2205 std::fs::remove_dir_all(root).unwrap();
2206 }
2207
2208 #[test]
2209 fn primary_limits_validate_configuration_and_bound_allocation() {
2210 use std::ffi::OsStr;
2211 assert_eq!(parse_primary_limit(None).unwrap(), DEFAULT_PRIMARY_BYTES);
2212 assert_eq!(parse_primary_limit(Some(OsStr::new("32"))).unwrap(), 32);
2213 for value in ["", "0", "-1", "+1", " 32", "3KiB", "18446744073709551615"] {
2214 assert!(parse_primary_limit(Some(OsStr::new(value))).is_err());
2215 }
2216 let root = test_root("primary-budget");
2217 std::fs::create_dir_all(&root).unwrap();
2218 let path = root.join("input");
2219 std::fs::write(&path, b"abcd").unwrap();
2220 for limit in [4, 5] {
2221 let bytes = read_open_file(
2222 std::fs::File::open(&path).unwrap(),
2223 "input",
2224 ReadBudget::Primary(limit),
2225 )
2226 .unwrap();
2227 assert_eq!(&*bytes, b"abcd");
2228 }
2229 assert!(
2230 read_open_file(
2231 std::fs::File::open(&path).unwrap(),
2232 "input",
2233 ReadBudget::Primary(3)
2234 )
2235 .is_err()
2236 );
2237 std::fs::File::create(&path)
2238 .unwrap()
2239 .set_len(DEFAULT_PRIMARY_BYTES * 4)
2240 .unwrap();
2241 let (error, allocated) = measured_bytes(|| {
2242 read_open_file(
2243 std::fs::File::open(&path).unwrap(),
2244 "input",
2245 ReadBudget::Primary(DEFAULT_PRIMARY_BYTES),
2246 )
2247 .unwrap_err()
2248 });
2249 assert!(error.to_string().contains("primary source"));
2250 assert!(allocated < 4096);
2251 std::fs::remove_dir_all(root).unwrap();
2252 }
2253
2254 #[test]
2255 fn open_validates_both_limits_before_reading_files_or_directories() {
2256 fn lookup(
2259 primary: Option<&'static str>,
2260 referenced: Option<&'static str>,
2261 ) -> impl Fn(&'static str) -> Option<OsString> {
2262 move |name| match name {
2263 PRIMARY_LIMIT_VAR => primary.map(OsString::from),
2264 REFERENCED_LIMIT_VAR => referenced.map(OsString::from),
2265 other => panic!("unexpected variable {other}"),
2266 }
2267 }
2268 fn code(result: Result<Source, Error>) -> &'static str {
2269 result
2270 .err()
2271 .and_then(|e| e.info())
2272 .expect("registered")
2273 .code
2274 }
2275 let invalid = crate::codes::REQUEST_SOURCE_INVALID_LIMIT.code;
2276
2277 assert_eq!(
2278 SourceLimits::from_lookup(lookup(None, None)).unwrap(),
2279 SourceLimits {
2280 primary: DEFAULT_PRIMARY_BYTES,
2281 referenced: MAX_REFERENCED_BYTES,
2282 }
2283 );
2284 let root = test_root("limit-validation");
2285 std::fs::create_dir_all(&root).unwrap();
2286 let file = root.join("input");
2287 std::fs::write(&file, b"abcd").unwrap();
2288
2289 for (primary, referenced) in [(Some("0"), None), (None, Some("-1"))] {
2291 let opened = Source::open_with_limits(&root, lookup(primary, referenced));
2292 assert_eq!(code(opened), invalid);
2293 }
2294 let opened = Source::open_with_limits(&file, lookup(Some("3"), Some("1KiB")));
2297 assert_eq!(code(opened), invalid);
2298 let opened = Source::open_with_limits(&file, lookup(Some("3"), Some("9")));
2299 assert_eq!(code(opened), crate::codes::READ_IO_PRIMARY_BUDGET.code);
2300
2301 let source = Source::open_with_limits(&file, lookup(Some("4"), Some("7"))).unwrap();
2302 assert_eq!(source.primary_buffer().unwrap().bytes(), b"abcd");
2303 assert_eq!(source.acquisition_byte_limit().unwrap(), 7);
2304 let directory = Source::open_with_limits(&root, lookup(Some("1"), Some("5"))).unwrap();
2305 assert_eq!(directory.acquisition_byte_limit().unwrap(), 5);
2306 std::fs::remove_dir_all(root).unwrap();
2307 }
2308
2309 #[test]
2310 fn referenced_limits_are_aggregate_and_cached_reads_are_free() {
2311 use std::ffi::OsStr;
2312 assert_eq!(parse_byte_limit("limit", None, 8).unwrap(), 8);
2313 assert_eq!(
2314 parse_byte_limit("limit", Some(OsStr::new("12")), 8).unwrap(),
2315 12
2316 );
2317 for value in ["", "0", "-1", "+1", " 8", "8MiB", "18446744073709551615"] {
2318 assert!(parse_byte_limit("limit", Some(OsStr::new(value)), 8).is_err());
2319 }
2320 let root = test_root("small-referenced-budget");
2321 std::fs::create_dir_all(&root).unwrap();
2322 std::fs::write(root.join("a.xml"), b"abcd").unwrap();
2323 std::fs::write(root.join("b.xml"), b"efgh").unwrap();
2324 std::fs::write(root.join("c.xml"), b"i").unwrap();
2325 let mut source = Source::open(&root).unwrap();
2326 let SourceProvider::Directory { acquisition } = Arc::get_mut(&mut source.provider).unwrap()
2327 else {
2328 panic!("directory provider");
2329 };
2330 acquisition.max_bytes = 8;
2331 let name = |s: &str| crate::ArtifactPath::new(s).unwrap();
2332 source.buffer(&name("a.xml")).unwrap();
2333 source.clone().buffer(&name("a.xml")).unwrap();
2334 source.buffer(&name("b.xml")).unwrap();
2335 let error = source.buffer(&name("c.xml")).unwrap_err();
2336 assert!(error.to_string().contains("8 byte acquisition budget"));
2337 std::fs::remove_dir_all(root).unwrap();
2338 }
2339
2340 #[test]
2341 fn an_over_budget_referenced_file_is_refused_before_allocation() {
2342 let root = test_root("budget");
2343 std::fs::create_dir_all(&root).unwrap();
2344 std::fs::write(root.join("master.dss"), b"master").unwrap();
2345 let big = std::fs::File::create(root.join("big.dat")).unwrap();
2349 big.set_len(MAX_REFERENCED_BYTES * 4).unwrap();
2350 drop(big);
2351
2352 let source = Source::open(root.join("master.dss")).unwrap();
2353 let primary = source.primary_buffer().unwrap();
2354 let (error, allocated) =
2359 measured_bytes(|| source.referenced_buffer(&primary, "big.dat").unwrap_err());
2360 assert!(error.to_string().contains("acquisition budget"), "{error}");
2361 assert!(
2362 (allocated as u64) < MAX_REFERENCED_BYTES / 16,
2363 "the refused acquisition allocated {allocated} bytes"
2364 );
2365 std::fs::remove_dir_all(root).unwrap();
2366 }
2367
2368 #[cfg(unix)]
2369 #[test]
2370 fn racing_entry_listing_never_names_files_outside_the_root() {
2371 use std::os::unix::fs::symlink;
2372
2373 let root = test_root("race-list");
2374 std::fs::create_dir_all(root.join("sub")).unwrap();
2375 std::fs::write(root.join("sub/inside.txt"), b"inside").unwrap();
2376 let outside = test_root("race-list-outside");
2377 std::fs::create_dir_all(&outside).unwrap();
2378 std::fs::write(outside.join("outside-only.txt"), b"outside").unwrap();
2379
2380 let source = Source::open(&root).unwrap();
2381 let stop = std::sync::atomic::AtomicBool::new(false);
2382 std::thread::scope(|scope| {
2383 let flipper = scope.spawn(|| {
2384 while !stop.load(std::sync::atomic::Ordering::Relaxed) {
2385 let _ = std::fs::remove_dir_all(root.join("sub"));
2386 let _ = symlink(&outside, root.join("sub"));
2387 let _ = std::fs::remove_file(root.join("sub"));
2388 let _ = std::fs::create_dir(root.join("sub"));
2389 let _ = std::fs::write(root.join("sub/inside.txt"), b"inside");
2390 }
2391 });
2392 for _ in 0..50 {
2393 if let Ok(names) = source.entry_names() {
2396 assert!(
2397 names
2398 .iter()
2399 .all(|name| !name.as_str().contains("outside-only")),
2400 "{names:?}"
2401 );
2402 }
2403 }
2404 stop.store(true, std::sync::atomic::Ordering::Relaxed);
2405 flipper.join().unwrap();
2406 });
2407 std::fs::remove_dir_all(&root).ok();
2408 std::fs::remove_dir_all(&outside).ok();
2409 }
2410
2411 #[test]
2412 fn referenced_names_must_be_portable_relative_paths() {
2413 let root = test_root("portable-names");
2414 std::fs::create_dir_all(root.join("sub")).unwrap();
2415 std::fs::write(root.join("master.dss"), b"master").unwrap();
2416 std::fs::write(root.join("sub/feeder.dss"), b"feeder").unwrap();
2417 let source = Source::open(root.join("master.dss")).unwrap();
2418 let primary = source.primary_buffer().unwrap();
2419
2420 for name in ["..\\escape.dss", "\\escape.dss", "C:\\escape.dss", "C:x"] {
2423 let error = source.referenced_buffer(&primary, name).unwrap_err();
2424 assert_eq!(
2425 error.category(),
2426 crate::ErrorCategory::Request,
2427 "{name}: {error}"
2428 );
2429 }
2430 assert!(source.root_buffer("..\\master.dss").is_err());
2431 assert!(source.root_buffer("\\master.dss").is_err());
2432
2433 let feeder = source
2435 .referenced_buffer(&primary, "sub/feeder.dss")
2436 .unwrap();
2437 assert_eq!(feeder.bytes(), b"feeder");
2438
2439 let absolute = root.canonicalize().unwrap().join("sub").join("feeder.dss");
2445 let again = source
2446 .referenced_buffer(&primary, absolute.to_str().unwrap())
2447 .unwrap();
2448 assert_eq!(again.bytes().as_ptr(), feeder.bytes().as_ptr());
2449 assert!(
2451 source
2452 .acquired_buffers()
2453 .iter()
2454 .all(|buffer| !buffer.name().contains("escape"))
2455 );
2456 std::fs::remove_dir_all(root).unwrap();
2457 }
2458
2459 #[cfg(unix)]
2460 #[test]
2461 fn live_sources_hold_no_directory_descriptors_before_acquisition() {
2462 fn open_descriptor_count() -> usize {
2463 let table = if cfg!(target_os = "macos") {
2464 "/dev/fd"
2465 } else {
2466 "/proc/self/fd"
2467 };
2468 std::fs::read_dir(table).unwrap().count()
2469 }
2470
2471 let _guard = process_resource_guard();
2472 let root = test_root("fd-count");
2473 std::fs::create_dir_all(&root).unwrap();
2474 std::fs::write(root.join("case.m"), b"case").unwrap();
2475 std::fs::write(root.join("ref.csv"), b"ref").unwrap();
2476
2477 let before = open_descriptor_count();
2478 let sources: Vec<Source> = (0..300)
2479 .map(|_| Source::open(root.join("case.m")).unwrap())
2480 .collect();
2481 let held = open_descriptor_count();
2482 assert!(
2483 held <= before + 4,
2484 "{} sources hold {} descriptors over the baseline {}",
2485 sources.len(),
2486 held - before,
2487 before
2488 );
2489
2490 for source in sources.iter().take(32) {
2495 let primary = source.primary_buffer().unwrap();
2496 let first = source.referenced_buffer(&primary, "ref.csv").unwrap();
2497 let second = source.referenced_buffer(&primary, "ref.csv").unwrap();
2498 assert_eq!(first.bytes().as_ptr(), second.bytes().as_ptr());
2499 }
2500 drop(sources);
2501 std::fs::remove_dir_all(root).unwrap();
2502 }
2503
2504 #[test]
2505 fn entry_listing_returns_names_of_every_length_exactly() {
2506 let root = test_root("name-lengths");
2509 std::fs::create_dir_all(&root).unwrap();
2510 let long = "n".repeat(200);
2511 for name in ["a", "medium-name.csv", long.as_str()] {
2512 std::fs::write(root.join(name), b"x").unwrap();
2513 }
2514 let source = Source::open(&root).unwrap();
2515 let mut names: Vec<String> = source
2516 .entry_names()
2517 .unwrap()
2518 .iter()
2519 .map(|name| name.as_str().to_owned())
2520 .collect();
2521 names.sort();
2522 let mut expected = vec!["a".to_owned(), "medium-name.csv".to_owned(), long];
2523 expected.sort();
2524 assert_eq!(names, expected);
2525 std::fs::remove_dir_all(root).unwrap();
2526 }
2527
2528 #[cfg(unix)]
2529 #[test]
2530 fn a_directory_nested_past_the_depth_bound_is_refused_promptly() {
2531 use std::os::fd::{AsRawFd, FromRawFd};
2532
2533 let _guard = process_resource_guard();
2536
2537 let root = test_root("deep-chain");
2538 std::fs::create_dir_all(&root).unwrap();
2539 let name = std::ffi::CString::new("d").unwrap();
2542 let mut level = std::fs::File::open(&root).unwrap();
2543 for _ in 0..(MAX_REFERENCED_DEPTH + 40) {
2544 unsafe {
2548 assert_eq!(libc::mkdirat(level.as_raw_fd(), name.as_ptr(), 0o755), 0);
2549 let fd = libc::openat(
2550 level.as_raw_fd(),
2551 name.as_ptr(),
2552 libc::O_RDONLY | libc::O_CLOEXEC,
2553 );
2554 assert!(fd >= 0);
2555 level = std::fs::File::from_raw_fd(fd);
2556 }
2557 }
2558 drop(level);
2559
2560 let (sender, receiver) = std::sync::mpsc::channel();
2561 let listed_root = root.clone();
2562 let worker = std::thread::spawn(move || {
2563 let source = Source::open(&listed_root).unwrap();
2564 sender.send(source.entry_names().map(|_| ())).unwrap();
2565 });
2566 let outcome = receiver
2567 .recv_timeout(std::time::Duration::from_secs(10))
2568 .expect("the depth refusal returns promptly");
2569 worker.join().unwrap();
2570 let error = outcome.expect_err("a chain past the depth bound refuses");
2571 assert!(error.to_string().contains("levels deep"), "{error}");
2572
2573 let mut fds = vec![std::fs::File::open(&root).unwrap()];
2576 loop {
2577 let last = fds.last().unwrap();
2578 let fd = unsafe {
2580 libc::openat(
2581 last.as_raw_fd(),
2582 name.as_ptr(),
2583 libc::O_RDONLY | libc::O_CLOEXEC,
2584 )
2585 };
2586 if fd < 0 {
2587 break;
2588 }
2589 fds.push(unsafe { std::fs::File::from_raw_fd(fd) });
2591 }
2592 while fds.len() > 1 {
2593 let parent = &fds[fds.len() - 2];
2594 unsafe {
2596 libc::unlinkat(parent.as_raw_fd(), name.as_ptr(), libc::AT_REMOVEDIR);
2597 }
2598 fds.pop();
2599 }
2600 drop(fds);
2601 std::fs::remove_dir_all(&root).unwrap();
2602 }
2603
2604 #[test]
2605 fn a_directory_past_the_entry_budget_is_refused_with_bounded_memory() {
2606 let root = test_root("entry-budget");
2607 std::fs::create_dir_all(&root).unwrap();
2608 let excess = MAX_REFERENCED_FILES * 4;
2612 for index in 0..excess {
2613 std::fs::write(root.join(format!("f{index:05}.csv")), b"").unwrap();
2614 }
2615 let source = Source::open(&root).unwrap();
2616 let (error, allocated) = measured_bytes(|| source.entry_names().unwrap_err());
2617 assert!(error.to_string().contains("entries"), "{error}");
2618 assert!(
2623 allocated < MAX_REFERENCED_FILES * 192,
2624 "the refused listing allocated {allocated} bytes"
2625 );
2626 std::fs::remove_dir_all(root).unwrap();
2627 }
2628
2629 #[cfg(unix)]
2630 #[test]
2631 fn listing_breadth_never_scales_open_descriptors() {
2632 fn open_descriptor_count() -> usize {
2633 let table = if cfg!(target_os = "macos") {
2634 "/dev/fd"
2635 } else {
2636 "/proc/self/fd"
2637 };
2638 std::fs::read_dir(table).unwrap().count()
2639 }
2640
2641 let _guard = process_resource_guard();
2642 let root = test_root("breadth");
2643 let breadth = 400usize;
2647 for index in 0..breadth {
2648 let sub = root.join(format!("s{index:03}"));
2649 std::fs::create_dir_all(&sub).unwrap();
2650 std::fs::write(sub.join("data.csv"), b"x").unwrap();
2651 }
2652 std::fs::create_dir_all(root.join("nested/a/b/c")).unwrap();
2653 std::fs::write(root.join("nested/a/b/c/deep.csv"), b"x").unwrap();
2654
2655 let mut original: libc::rlimit = unsafe { std::mem::zeroed() };
2661 assert_eq!(
2662 unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &raw mut original) },
2664 0
2665 );
2666 let lowered = libc::rlimit {
2667 rlim_cur: 256,
2668 rlim_max: original.rlim_max,
2669 };
2670 assert_eq!(
2672 unsafe { libc::setrlimit(libc::RLIMIT_NOFILE, &raw const lowered) },
2673 0
2674 );
2675
2676 let baseline = open_descriptor_count();
2677 let peak = std::sync::atomic::AtomicUsize::new(0);
2678 let stop = std::sync::atomic::AtomicBool::new(false);
2679 let names = std::thread::scope(|scope| {
2680 let sampler = scope.spawn(|| {
2681 while !stop.load(std::sync::atomic::Ordering::Relaxed) {
2682 let count = open_descriptor_count();
2683 peak.fetch_max(count, std::sync::atomic::Ordering::Relaxed);
2684 }
2685 });
2686 let source = Source::open(&root).unwrap();
2687 let names = source.entry_names().unwrap();
2688 stop.store(true, std::sync::atomic::Ordering::Relaxed);
2689 sampler.join().unwrap();
2690 drop(source);
2691 names
2692 });
2693 assert_eq!(
2695 unsafe { libc::setrlimit(libc::RLIMIT_NOFILE, &raw const original) },
2696 0
2697 );
2698
2699 assert_eq!(names.len(), breadth + 1, "every file listed");
2700 let sampled_peak = peak.load(std::sync::atomic::Ordering::Relaxed);
2701 assert!(
2702 sampled_peak <= baseline + MAX_REFERENCED_DEPTH + 16,
2703 "the walk held {sampled_peak} descriptors over a baseline of {baseline}"
2704 );
2705 std::fs::remove_dir_all(root).unwrap();
2706 }
2707
2708 #[test]
2709 fn a_directory_listing_still_names_windows_reserved_spellings() {
2710 let root = test_root("reserved-listing");
2713 std::fs::create_dir_all(&root).unwrap();
2714 std::fs::write(root.join("aux.dss"), b"content").unwrap();
2715 let source = Source::open(&root).unwrap();
2716 let names = source.entry_names().unwrap();
2717 assert_eq!(names.len(), 1);
2718 assert_eq!(names[0].as_str(), "aux.dss");
2719 std::fs::remove_dir_all(root).unwrap();
2720 }
2721
2722 #[test]
2723 fn a_directory_listing_repeats_and_survives_acquisition() {
2724 let root = test_root("repeat-listing");
2729 std::fs::create_dir_all(&root).unwrap();
2730 std::fs::write(root.join("network.csv"), b"name\nseq\n").unwrap();
2731 std::fs::write(root.join("buses.csv"), b"name\nB1\n").unwrap();
2732 let source = Source::open(&root).unwrap();
2733 let first = source.entry_names().unwrap();
2734 assert_eq!(first.len(), 2);
2735 let name = ArtifactPath::new("network.csv").unwrap();
2736 source.buffer(&name).unwrap();
2737 let second = source.entry_names().unwrap();
2738 assert_eq!(first, second);
2739 std::fs::remove_dir_all(root).unwrap();
2740 }
2741
2742 #[test]
2743 fn concurrent_acquisition_retains_one_buffer_for_one_name() {
2744 let root = test_root("concurrent");
2745 std::fs::create_dir_all(&root).unwrap();
2746 std::fs::write(root.join("shared.csv"), b"shared").unwrap();
2747 let source = Source::open(&root).unwrap();
2748 let name = ArtifactPath::new("shared.csv").unwrap();
2749 let buffers: Vec<_> = std::thread::scope(|scope| {
2750 (0..8)
2751 .map(|_| {
2752 let source = source.clone();
2753 let name = name.clone();
2754 scope.spawn(move || source.buffer(&name).unwrap())
2755 })
2756 .collect::<Vec<_>>()
2757 .into_iter()
2758 .map(|handle| handle.join().unwrap())
2759 .collect()
2760 });
2761 let pointer = buffers[0].bytes().as_ptr();
2762 assert!(
2763 buffers
2764 .iter()
2765 .all(|buffer| buffer.bytes().as_ptr() == pointer)
2766 );
2767 assert_eq!(source.acquired_buffers().len(), 1);
2768 std::fs::remove_dir_all(root).unwrap();
2769 }
2770 #[cfg(unix)]
2771 #[test]
2772 fn a_refused_listing_never_shortens_the_next_one() {
2773 let root = test_root("refused-listing");
2777 std::fs::create_dir_all(&root).unwrap();
2778 for index in 0..(MAX_REFERENCED_FILES + 5) {
2779 std::fs::write(root.join(format!("f{index}.txt")), b"x").unwrap();
2780 }
2781 let source = Source::open(&root).unwrap();
2782 let first = source.entry_names().unwrap_err();
2783 let second = source.entry_names().unwrap_err();
2784 assert_eq!(
2785 first.diagnostics().first().map(|d| d.code().to_owned()),
2786 second.diagnostics().first().map(|d| d.code().to_owned()),
2787 "the refusal repeats rather than shrinking into a partial listing"
2788 );
2789 drop(source);
2790 let _ = std::fs::remove_dir_all(&root);
2791 }
2792}