saffron/
storage.rs

1//! This file handles the storage gestion for the state replicator ;
2//! The data are stored on disk in a file, and this file provide functions to
3//! read the whole file (because producing a read proof requires the whole
4//! polynomial) and to update dispersed chunks of data.
5//!
6//! Note: the encoding used for the conversion bytes <-> scalars is the `full`
7//! encoding, meaning that fields elements are encoded over `F::size_in_bytes()`
8//! bytes which is 32 for Pallas & Vesta.
9//! Using the 31 version leads currently to inconsistency when updating if the
10//! diff's new values are greater than what is representable over 31 bytes.
11
12use crate::{commitment::*, diff::Diff, encoding, utils::evals_to_polynomial};
13use ark_ff::PrimeField;
14use ark_poly::{univariate::DensePolynomial, EvaluationDomain, Radix2EvaluationDomain as R2D};
15use kimchi::curve::KimchiCurve;
16use poly_commitment::ipa::SRS;
17use std::{
18    fs::{File, OpenOptions},
19    io::{Read, Seek, SeekFrom, Write},
20};
21
22use crate::SRS_SIZE;
23
24pub struct Data<F: PrimeField> {
25    pub data: Vec<F>,
26}
27
28impl<F: PrimeField> Data<F> {
29    /// Returns the data correpsonding to the provided `bytes`
30    pub fn of_bytes(bytes: &[u8]) -> Data<F> {
31        Data {
32            data: encoding::encode_as_field_elements_full(bytes),
33        }
34    }
35
36    pub fn is_empty(&self) -> bool {
37        self.data.is_empty()
38    }
39
40    /// Returns the length of the data
41    pub fn len(&self) -> usize {
42        self.data.len()
43    }
44
45    /// Returns the polynomial that correspond to the data. If the data is
46    /// bigger than domain's size, the additionnal points will be ignored.
47    /// If the data is smaller, it is padded with zeros
48    pub fn to_polynomial(&self, domain: R2D<F>) -> DensePolynomial<F> {
49        use std::iter;
50        let n = domain.size();
51        let padded_data: Vec<F> = self
52            .data
53            .iter()
54            .cloned()
55            .chain(iter::repeat(F::zero()))
56            .take(n)
57            .collect();
58        evals_to_polynomial(padded_data, domain)
59    }
60
61    /// Commit a `data` of length smaller than `SRS_SIZE`
62    /// If greater data is provided, anything above `SRS_SIZE` is ignored
63    pub fn to_commitment<G: KimchiCurve<ScalarField = F>>(&self, srs: &SRS<G>) -> G {
64        commit_to_field_elems::<G>(srs, &self.data)[0]
65    }
66
67    /// Modifies inplace the provided data with `diff`
68    pub fn apply_inplace(&mut self, diff: &Diff<F>) {
69        let data_slice = std::slice::from_mut(&mut self.data);
70        Diff::apply_inplace(data_slice, diff);
71    }
72
73    /// Returns a new data corresponding to the provided data with `diff` applied
74    pub fn apply(&self, diff: &Diff<F>) -> Data<F> {
75        let mut data = Data {
76            data: self.data.clone(),
77        };
78        data.apply_inplace(diff);
79        data
80    }
81}
82
83/// Creates a file at `path` and fill it with `data`
84/// TODO: For now, we assume the data vector is smaller than SRS_SIZE
85pub fn init<F: PrimeField>(path: &str, data: &Data<F>) -> std::io::Result<()> {
86    // TODO: handle the > SRS_SIZE case
87    assert!(data.len() <= SRS_SIZE);
88    let mut file = File::create(path)?;
89    for x in &data.data {
90        let x_bytes = encoding::decode_full(*x);
91        file.write_all(&x_bytes)?
92    }
93    Ok(())
94}
95
96/// `read(path)` loads the whole content of the file in `path` and stores it as
97/// bytes.
98/// This function raises an error when the path does not exist, or if there is
99/// an issue with reading.
100pub fn read<F: PrimeField>(path: &str) -> std::io::Result<Data<F>> {
101    let mut file = File::open(path)?;
102    let mut buffer = Vec::new();
103    file.read_to_end(&mut buffer)?;
104    // TODO: handle the > SRS_SIZE case (ie Vec<Vec<F>>)
105    Ok(Data::of_bytes(&buffer))
106}
107
108/// Takes a valid diff and update the file accordingly, replacing the old
109/// values by the new ones at the specified indices ; the indices of the diff
110/// are specified by scalars (not by bytes) and the values of the diff are the
111/// new scalar value expected for the new data.
112/// Note that this only update the file, not the commitment
113pub fn update<F: PrimeField>(path: &str, diff: &Diff<F>) -> std::io::Result<()> {
114    let mut file = OpenOptions::new().write(true).open(path)?;
115    let region_offset = diff.region * (SRS_SIZE as u64);
116    let scalar_size = encoding::encoding_size_full::<F>() as u64;
117    for (index, new_value) in diff.addresses.iter().zip(diff.new_values.iter()) {
118        let corresponding_bytes_index = (region_offset + index) * scalar_size;
119        file.seek(SeekFrom::Start(corresponding_bytes_index))?;
120        let new_value_bytes = encoding::decode_full(*new_value);
121        file.write_all(&new_value_bytes)?;
122    }
123    Ok(())
124}
125
126#[cfg(test)]
127mod tests {
128    use crate::{diff::Diff, encoding, storage, storage::Data, Curve, ScalarField, SRS_SIZE};
129    use ark_ff::{One, UniformRand, Zero};
130    use mina_curves::pasta::Fp;
131    use rand::Rng;
132    use std::fs;
133    use tempfile::NamedTempFile;
134
135    #[test]
136    // Test that data commitment stays the same after reading (i.e. data stay
137    // consistent through writing and reading), and test that update is
138    // consistently performed in the file
139    fn test_data_consistency() {
140        let mut rng = o1_utils::tests::make_test_rng(None);
141
142        let srs = poly_commitment::precomputed_srs::get_srs_test();
143
144        // Path of the file that will contain the test data
145        let file = NamedTempFile::new().unwrap();
146        let path = file.path().to_str().unwrap();
147
148        let data_bytes: Vec<u8> = (0..(SRS_SIZE * (encoding::encoding_size_full::<ScalarField>())))
149            .map(|_| rng.gen())
150            .collect();
151        let data = Data::of_bytes(&data_bytes);
152        let data_comm = data.to_commitment(&srs);
153
154        let read_consistency = {
155            let _init_storage_file = storage::init(path, &data);
156            let read_data = storage::read(path).unwrap();
157            let read_data_comm = read_data.to_commitment(&srs);
158
159            // True if read data are the same as initial data
160            Curve::eq(&data_comm, &read_data_comm)
161        };
162
163        let (data_updated, update_consistency) = {
164            let diff = {
165                // The number of updates is proportional to the data length,
166                // but we make sure to have at least one update if the data is
167                // small
168                let nb_updates = std::cmp::max(data.len() / 20, 1);
169                let region = 0;
170                let addresses: Vec<u64> = (0..nb_updates)
171                    .map(|_| (rng.gen_range(0..data.len() as u64)))
172                    .collect();
173                let mut new_values: Vec<ScalarField> =
174                    addresses.iter().map(|_| Fp::rand(&mut rng)).collect();
175                // The first value is replaced by a scalar that would
176                // overflow 31 bytes, so the update is not consistent and the
177                // test fails if this case is not handled
178                new_values[0] = Fp::zero() - Fp::one();
179                Diff {
180                    region,
181                    addresses,
182                    new_values,
183                }
184            };
185
186            let updated_data = data.apply(&diff);
187            let updated_data_comm = updated_data.to_commitment(&srs);
188
189            let _file_update = storage::update(path, &diff);
190
191            let updated_read_data = storage::read(path).unwrap();
192            let updated_read_data_comm = updated_read_data.to_commitment(&srs);
193
194            (
195                Curve::ne(&updated_data_comm, &data_comm),
196                // True if read data from updated file are the same as updated data
197                Curve::eq(&updated_data_comm, &updated_read_data_comm),
198            )
199        };
200
201        let _remove_file = fs::remove_file(path);
202
203        assert!(read_consistency);
204        assert!(data_updated);
205        assert!(update_consistency);
206    }
207}