rfc9162_tiles/
store.rs

1//! Where tiles live: a trait over store paths, a POSIX directory behind it,
2//! and an in-memory store for tests and single-process use.
3//!
4//! Paths are `/`-separated and relative to the store root. A write lands
5//! whole or not at all: the POSIX store writes a temporary file beside the
6//! target, syncs it, renames it over the path and syncs the directory, as
7//! Tessera's POSIX storage does.
8
9use std::collections::BTreeMap;
10use std::fs::{self, File, OpenOptions};
11use std::io::{self, Write};
12use std::path::{Component, Path, PathBuf};
13use std::sync::atomic::{AtomicU64, Ordering};
14use std::sync::{Arc, Mutex};
15
16/// Byte storage keyed by store path.
17pub trait TileStore {
18    /// The bytes at `path`, or nothing when there is no such file.
19    fn get(&self, path: &str) -> io::Result<Option<Vec<u8>>>;
20    /// Writes `bytes` at `path`, whole or not at all, replacing what is there.
21    fn put(&self, path: &str, bytes: &[u8]) -> io::Result<()>;
22    /// Writes `bytes` at `path` only when nothing is there; `false` when
23    /// something is.
24    fn create_new(&self, path: &str, bytes: &[u8]) -> io::Result<bool>;
25    /// Removes `path`; `false` when there was nothing to remove.
26    fn delete(&self, path: &str) -> io::Result<bool>;
27    /// Moves `from` to `to`, replacing what is at `to`.
28    fn rename(&self, from: &str, to: &str) -> io::Result<()>;
29    /// Every file path under `prefix` (a directory path without a trailing
30    /// slash, or the empty string for the whole store), sorted.
31    fn list(&self, prefix: &str) -> io::Result<Vec<String>>;
32    /// Whether anything is stored at `path`.
33    fn exists(&self, path: &str) -> io::Result<bool> {
34        Ok(self.get(path)?.is_some())
35    }
36}
37
38impl<T: TileStore + ?Sized> TileStore for &T {
39    fn get(&self, path: &str) -> io::Result<Option<Vec<u8>>> {
40        (**self).get(path)
41    }
42    fn put(&self, path: &str, bytes: &[u8]) -> io::Result<()> {
43        (**self).put(path, bytes)
44    }
45    fn create_new(&self, path: &str, bytes: &[u8]) -> io::Result<bool> {
46        (**self).create_new(path, bytes)
47    }
48    fn delete(&self, path: &str) -> io::Result<bool> {
49        (**self).delete(path)
50    }
51    fn rename(&self, from: &str, to: &str) -> io::Result<()> {
52        (**self).rename(from, to)
53    }
54    fn list(&self, prefix: &str) -> io::Result<Vec<String>> {
55        (**self).list(prefix)
56    }
57    fn exists(&self, path: &str) -> io::Result<bool> {
58        (**self).exists(path)
59    }
60}
61
62impl<T: TileStore + ?Sized> TileStore for Arc<T> {
63    fn get(&self, path: &str) -> io::Result<Option<Vec<u8>>> {
64        (**self).get(path)
65    }
66    fn put(&self, path: &str, bytes: &[u8]) -> io::Result<()> {
67        (**self).put(path, bytes)
68    }
69    fn create_new(&self, path: &str, bytes: &[u8]) -> io::Result<bool> {
70        (**self).create_new(path, bytes)
71    }
72    fn delete(&self, path: &str) -> io::Result<bool> {
73        (**self).delete(path)
74    }
75    fn rename(&self, from: &str, to: &str) -> io::Result<()> {
76        (**self).rename(from, to)
77    }
78    fn list(&self, prefix: &str) -> io::Result<Vec<String>> {
79        (**self).list(prefix)
80    }
81    fn exists(&self, path: &str) -> io::Result<bool> {
82        (**self).exists(path)
83    }
84}
85
86/// A directory. Every path is resolved under the root; `..`, absolute paths
87/// and empty elements are refused.
88#[derive(Debug, Clone)]
89pub struct PosixStore {
90    root: PathBuf,
91    counter: Arc<AtomicU64>,
92}
93
94impl PosixStore {
95    /// A store rooted at `root`, created when absent.
96    pub fn open(root: impl Into<PathBuf>) -> io::Result<Self> {
97        let root = root.into();
98        fs::create_dir_all(&root)?;
99        fs::create_dir_all(root.join(".tmp"))?;
100        Ok(Self {
101            root,
102            counter: Arc::new(AtomicU64::new(0)),
103        })
104    }
105
106    /// The directory the store lives in.
107    #[must_use]
108    pub fn root(&self) -> &Path {
109        &self.root
110    }
111
112    fn resolve(&self, path: &str) -> io::Result<PathBuf> {
113        if path.is_empty() {
114            return Ok(self.root.clone());
115        }
116        let relative = Path::new(path);
117        if relative
118            .components()
119            .any(|c| !matches!(c, Component::Normal(_)))
120        {
121            return Err(io::Error::new(
122                io::ErrorKind::InvalidInput,
123                format!("store path {path:?} leaves the store"),
124            ));
125        }
126        Ok(self.root.join(relative))
127    }
128
129    fn write_temp(&self, bytes: &[u8]) -> io::Result<PathBuf> {
130        let n = self.counter.fetch_add(1, Ordering::Relaxed);
131        let name = format!("{}-{n}", std::process::id());
132        let temp = self.root.join(".tmp").join(name);
133        let mut file = File::create(&temp)?;
134        file.write_all(bytes)?;
135        file.sync_all()?;
136        Ok(temp)
137    }
138
139    fn sync_dir(dir: &Path) -> io::Result<()> {
140        File::open(dir)?.sync_all()
141    }
142
143    fn walk(dir: &Path, root: &Path, out: &mut Vec<String>) -> io::Result<()> {
144        for entry in fs::read_dir(dir)? {
145            let entry = entry?;
146            let path = entry.path();
147            if entry.file_type()?.is_dir() {
148                Self::walk(&path, root, out)?;
149            } else {
150                let relative = path.strip_prefix(root).map_err(io::Error::other)?;
151                let text = relative
152                    .components()
153                    .map(|c| c.as_os_str().to_string_lossy().into_owned())
154                    .collect::<Vec<_>>()
155                    .join("/");
156                out.push(text);
157            }
158        }
159        Ok(())
160    }
161}
162
163impl TileStore for PosixStore {
164    fn get(&self, path: &str) -> io::Result<Option<Vec<u8>>> {
165        match fs::read(self.resolve(path)?) {
166            Ok(bytes) => Ok(Some(bytes)),
167            Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None),
168            Err(e) => Err(e),
169        }
170    }
171
172    fn put(&self, path: &str, bytes: &[u8]) -> io::Result<()> {
173        let target = self.resolve(path)?;
174        let parent = target
175            .parent()
176            .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "no parent"))?;
177        fs::create_dir_all(parent)?;
178        let temp = self.write_temp(bytes)?;
179        fs::rename(&temp, &target)?;
180        Self::sync_dir(parent)
181    }
182
183    fn create_new(&self, path: &str, bytes: &[u8]) -> io::Result<bool> {
184        let target = self.resolve(path)?;
185        let parent = target
186            .parent()
187            .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "no parent"))?;
188        fs::create_dir_all(parent)?;
189        match OpenOptions::new()
190            .write(true)
191            .create_new(true)
192            .open(&target)
193        {
194            Ok(mut file) => {
195                file.write_all(bytes)?;
196                file.sync_all()?;
197                Self::sync_dir(parent)?;
198                Ok(true)
199            }
200            Err(e) if e.kind() == io::ErrorKind::AlreadyExists => Ok(false),
201            Err(e) => Err(e),
202        }
203    }
204
205    fn delete(&self, path: &str) -> io::Result<bool> {
206        let target = self.resolve(path)?;
207        match fs::remove_file(&target) {
208            Ok(()) => {
209                if let Some(parent) = target.parent() {
210                    Self::sync_dir(parent)?;
211                }
212                Ok(true)
213            }
214            Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false),
215            Err(e) => Err(e),
216        }
217    }
218
219    fn rename(&self, from: &str, to: &str) -> io::Result<()> {
220        let source = self.resolve(from)?;
221        let target = self.resolve(to)?;
222        if let Some(parent) = target.parent() {
223            fs::create_dir_all(parent)?;
224        }
225        fs::rename(&source, &target)?;
226        if let Some(parent) = target.parent() {
227            Self::sync_dir(parent)?;
228        }
229        if let Some(parent) = source.parent() {
230            Self::sync_dir(parent)?;
231        }
232        Ok(())
233    }
234
235    fn list(&self, prefix: &str) -> io::Result<Vec<String>> {
236        let dir = self.resolve(prefix)?;
237        let mut out = Vec::new();
238        match fs::metadata(&dir) {
239            Ok(meta) if meta.is_dir() => Self::walk(&dir, &self.root, &mut out)?,
240            Ok(_) => {}
241            Err(e) if e.kind() == io::ErrorKind::NotFound => {}
242            Err(e) => return Err(e),
243        }
244        if prefix.is_empty() {
245            out.retain(|p| !p.starts_with(".tmp/"));
246        }
247        out.sort();
248        Ok(out)
249    }
250
251    fn exists(&self, path: &str) -> io::Result<bool> {
252        Ok(self.resolve(path)?.is_file())
253    }
254}
255
256/// Files in memory. Clones share the same files, so a writer and a reader can
257/// hold one store.
258#[derive(Debug, Clone, Default)]
259pub struct MemoryStore {
260    files: Arc<Mutex<BTreeMap<String, Vec<u8>>>>,
261}
262
263impl MemoryStore {
264    /// An empty store.
265    #[must_use]
266    pub fn new() -> Self {
267        Self::default()
268    }
269
270    /// Bytes stored, over every file.
271    #[must_use]
272    pub fn total_bytes(&self) -> usize {
273        self.files.lock().unwrap().values().map(Vec::len).sum()
274    }
275
276    /// Bytes stored under `prefix`.
277    #[must_use]
278    pub fn bytes_under(&self, prefix: &str) -> usize {
279        self.files
280            .lock()
281            .unwrap()
282            .iter()
283            .filter(|(k, _)| under(k, prefix))
284            .map(|(_, v)| v.len())
285            .sum()
286    }
287}
288
289fn under(path: &str, prefix: &str) -> bool {
290    prefix.is_empty()
291        || path
292            .strip_prefix(prefix)
293            .is_some_and(|r| r.starts_with('/'))
294}
295
296impl TileStore for MemoryStore {
297    fn get(&self, path: &str) -> io::Result<Option<Vec<u8>>> {
298        Ok(self.files.lock().unwrap().get(path).cloned())
299    }
300
301    fn put(&self, path: &str, bytes: &[u8]) -> io::Result<()> {
302        self.files
303            .lock()
304            .unwrap()
305            .insert(path.to_string(), bytes.to_vec());
306        Ok(())
307    }
308
309    fn create_new(&self, path: &str, bytes: &[u8]) -> io::Result<bool> {
310        let mut files = self.files.lock().unwrap();
311        if files.contains_key(path) {
312            return Ok(false);
313        }
314        files.insert(path.to_string(), bytes.to_vec());
315        Ok(true)
316    }
317
318    fn delete(&self, path: &str) -> io::Result<bool> {
319        Ok(self.files.lock().unwrap().remove(path).is_some())
320    }
321
322    fn rename(&self, from: &str, to: &str) -> io::Result<()> {
323        let mut files = self.files.lock().unwrap();
324        let bytes = files
325            .remove(from)
326            .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, from.to_string()))?;
327        files.insert(to.to_string(), bytes);
328        Ok(())
329    }
330
331    fn list(&self, prefix: &str) -> io::Result<Vec<String>> {
332        Ok(self
333            .files
334            .lock()
335            .unwrap()
336            .keys()
337            .filter(|k| under(k, prefix))
338            .cloned()
339            .collect())
340    }
341}
rfc9162_tiles/store.rs

rfc9162_tiles/
store.rs
