Merge pull request #31 from QED-it/aurel

Multi-files and example streaming (doc test)

Changelog.md

@@ -1,3 +1,10 @@
+# Version v1.3.1, 2020-10, Rust streaming
+
+Rust:
+- WorkspaceSink creates multiple constraints files instead of a large one.
+- Example code to read in chunked/streamed mode.
+- Consolidate the workspace file discovery into its module.
+
 # Version v1.3.0, 2020-10, Rust streaming
 
 Rust:

rust/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "zkinterface"
-version = "1.3.0"
+version = "1.3.1"
 authors = ["Aurélien Nicolas <[email protected]>"]
 license = "MIT"
 build = "build.rs"

rust/src/cli.rs

@@ -1,14 +1,15 @@
 extern crate serde;
 extern crate serde_json;
 
-use std::fs;
+use std::fs::{File, create_dir_all, remove_file};
 use std::io::{stdin, stdout, Read, Write, copy};
 use std::path::{Path, PathBuf};
 use structopt::StructOpt;
 
 use crate::{Reader, Workspace, Messages, consumers::stats::Stats, Result};
-use std::fs::{File, create_dir_all, remove_file};
-use std::ffi::OsStr;
+use crate::consumers::workspace::{list_workspace_files, has_zkif_extension};
+use crate::consumers::validator::Validator;
+use crate::consumers::simulator::Simulator;
 
 const ABOUT: &str = "
 This is a collection of tools to work with zero-knowledge statements encoded in zkInterface messages.
@@ -38,8 +39,6 @@ Write all the statement files to stdout (to pipe to another program):
 ";
 
 use structopt::clap::AppSettings::*;
-use crate::consumers::validator::Validator;
-use crate::consumers::simulator::Simulator;
 
 #[derive(Debug, StructOpt)]
 #[structopt(
@@ -112,7 +111,7 @@ pub fn cli(options: &Options) -> Result<()> {
 fn load_messages(opts: &Options) -> Result<Reader> {
     let mut reader = Reader::new();
 
-    for path in list_files(opts)? {
+    for path in list_workspace_files(&opts.paths)? {
         if path == Path::new("-") {
             eprintln!("Loading from stdin");
             reader.read_from(stdin())?;
@@ -127,37 +126,7 @@ fn load_messages(opts: &Options) -> Result<Reader> {
 }
 
 fn stream_messages(opts: &Options) -> Result<Workspace> {
-    let paths = list_files(opts)?;
-    Ok(Workspace::new(paths))
-}
-
-fn has_zkif_extension(path: &Path) -> bool {
-    path.extension() == Some(OsStr::new("zkif"))
-}
-
-fn list_files(opts: &Options) -> Result<Vec<PathBuf>> {
-    let mut all_paths = vec![];
-
-    for path in &opts.paths {
-        if has_zkif_extension(path) || path == Path::new("-") {
-            all_paths.push(path.clone());
-        } else {
-            for file in fs::read_dir(path)? {
-                match file {
-                    Ok(file) => {
-                        if has_zkif_extension(&file.path()) {
-                            all_paths.push(file.path());
-                        }
-                    }
-                    Err(err) => {
-                        eprintln!("Warning: {}", err);
-                        continue;
-                    }
-                }
-            }
-        }
-    }
-    Ok(all_paths)
+    Workspace::from_dirs_and_files(&opts.paths)
 }
 
 fn main_example(opts: &Options) -> Result<()> {
@@ -196,7 +165,7 @@ fn main_example(opts: &Options) -> Result<()> {
 }
 
 fn main_cat(opts: &Options) -> Result<()> {
-    for path in list_files(opts)? {
+    for path in list_workspace_files(&opts.paths)? {
         let mut file = File::open(&path)?;
         let mut stdout = stdout();
         copy(&mut file, &mut stdout)?;
@@ -265,14 +234,14 @@ fn main_stats(ws: &Workspace) -> Result<()> {
 }
 
 fn main_clean(opts: &Options) -> Result<()> {
-    let all_files = list_files(opts)?;
+    let all_files = list_workspace_files(&opts.paths)?;
     for file in &all_files {
         eprintln!("Removing {}", file.display());
         match remove_file(file) {
             Err(err) => {
                 eprintln!("Warning: {}", err)
             }
-            _ => {/* OK */}
+            _ => { /* OK */ }
         }
     }
 

rust/src/consumers/workspace.rs

@@ -1,18 +1,63 @@
 use std::path::{PathBuf, Path};
-use std::fs::File;
+use std::fs::{File, read_dir};
 use std::iter;
 use std::io::{Read, stdin};
+use std::ffi::OsStr;
 use crate::consumers::reader::read_buffer;
-use crate::{Message, Messages};
+use crate::{Result, Message, Messages};
 
 
+/// Workspace finds and reads zkInterface messages from a directory.
+/// It supports reading messages one-by-one from large files or from many files.
+/// It supports reading from stdin using dash (-) as a special filename.
+///
+/// # Example
+/// ```
+/// use zkinterface::{Workspace, WorkspaceSink, Sink, Message};
+/// use zkinterface::producers::examples::*;
+/// use std::path::PathBuf;
+///
+/// // Create an example workspace including multiple constraints files.
+/// let dir = PathBuf::from("local/test_workspace");
+/// let mut sink = WorkspaceSink::new(&dir).unwrap();
+/// sink.push_header(example_circuit_header());
+/// sink.push_witness(example_witness());
+/// sink.push_witness(example_witness());
+/// sink.push_constraints(example_constraints());
+/// sink.push_constraints(example_constraints());
+///
+/// // Iterate over the files and observe the messages.
+/// let mut got = vec![];
+///
+/// let ws = Workspace::from_dir(&dir).unwrap();
+/// for msg in ws.iter_messages() {
+///     match msg {
+///         Message::Header(h) => got.push("HEADER"),
+///         Message::Witness(w) => got.push("WITNESS"),
+///         Message::ConstraintSystem(cs) => got.push("CONSTRAINTS"),
+///         _ => {}
+///     }
+/// }
+///
+/// assert_eq!(got, vec!["HEADER", "WITNESS", "WITNESS", "CONSTRAINTS", "CONSTRAINTS"]);
+/// ```
+#[derive(Clone, Default, Debug, Eq, PartialEq)]
 pub struct Workspace {
     paths: Vec<PathBuf>,
     stdin: bool,
 }
 
 impl Workspace {
-    pub fn new(mut paths: Vec<PathBuf>) -> Self {
+    pub fn from_dir(path: &Path) -> Result<Self> {
+        Self::from_dirs_and_files(&[path.to_path_buf()])
+    }
+
+    pub fn from_dirs_and_files(paths: &[PathBuf]) -> Result<Self> {
+        let all_files = list_workspace_files(paths)?;
+        Ok(Self::from_filenames(all_files))
+    }
+
+    pub fn from_filenames(mut paths: Vec<PathBuf>) -> Self {
         if paths == vec![PathBuf::from("-")] {
             Workspace { paths: vec![], stdin: true }
         } else {
@@ -77,4 +122,33 @@ pub fn iterate_stream<'s>(mut stream: impl Read + 's) -> impl Iterator<Item=Vec<
             }
         }
     )
-}
+}
+
+pub fn has_zkif_extension(path: &Path) -> bool {
+    path.extension() == Some(OsStr::new("zkif"))
+}
+
+pub fn list_workspace_files(paths: &[PathBuf]) -> Result<Vec<PathBuf>> {
+    let mut all_paths = vec![];
+
+    for path in paths {
+        if has_zkif_extension(path) || path == Path::new("-") {
+            all_paths.push(path.clone());
+        } else {
+            for file in read_dir(path)? {
+                match file {
+                    Ok(file) => {
+                        if has_zkif_extension(&file.path()) {
+                            all_paths.push(file.path());
+                        }
+                    }
+                    Err(err) => {
+                        eprintln!("Warning: {}", err);
+                        continue;
+                    }
+                }
+            }
+        }
+    }
+    Ok(all_paths)
+}

rust/src/producers/workspace.rs

@@ -72,7 +72,7 @@ fn test_workspace_cleanup() {
     let workspace = PathBuf::from("local/test_workspace_cleanup");
     let _ = remove_dir_all(&workspace);
     let mut sink = WorkspaceSink::new(&workspace).unwrap();
-    
+
     // workspace is empty, check it!
     assert_eq!(read_dir(&workspace).unwrap().count(), 0);
 
@@ -89,5 +89,5 @@ fn test_workspace_cleanup() {
 
     // clean workspace, and check there is no more file in it.
     clean_workspace(&workspace).unwrap();
-    assert_eq!(read_dir(&workspace).unwrap().count(), 0);    
+    assert_eq!(read_dir(&workspace).unwrap().count(), 0);
 }