zkas/compiler: Introduce .literal section in bincode, and add remaining code.

bin/zkas/src/main.rs

@@ -8,9 +8,7 @@ use clap::Parser as ClapParser;
 
 use darkfi::{
     cli_desc,
-    zkas::{
-        analyzer::Analyzer, compiler::Compiler, decoder::ZkBinary, lexer::Lexer, parser::Parser,
-    },
+    zkas::{Analyzer, Compiler, Lexer, Parser, ZkBinary},
 };
 
 #[derive(clap::Parser)]
@@ -51,14 +49,24 @@ fn main() {
             exit(1);
         }
     };
-    let source = source.replace("\t", "    ");
 
+    // Clean up tabs, and convert CRLF to LF.
+    let source = source.replace("\t", "    ").replace("\r\n", "\n");
+
+    // The lexer goes over the input file and separates its content into
+    // tokens that get fed into a parser.
     let lexer = Lexer::new(filename, source.chars());
     let tokens = lexer.lex();
 
+    // The parser goes over the tokens provided by the lexer and builds
+    // the initial AST, not caring much about the semantics, just enforcing
+    // syntax and general structure.
     let parser = Parser::new(filename, source.chars(), tokens);
     let (constants, witnesses, statements) = parser.parse();
 
+    // The analyzer goes through the initial AST provided by the parser and
+    // converts return and variable types to their correct forms, and also
+    // checks that the semantics of the ZK script are correct.
     let mut analyzer = Analyzer::new(filename, source.chars(), constants, witnesses, statements);
     analyzer.analyze_types();
 
@@ -80,6 +88,7 @@ fn main() {
         analyzer.constants,
         analyzer.witnesses,
         analyzer.statements,
+        analyzer.literals,
         !args.strip,
     );
 
@@ -98,12 +107,9 @@ fn main() {
         }
     };
 
-    match file.write_all(&bincode) {
-        Ok(_) => {}
-        Err(e) => {
-            eprintln!("Error: Failed to write bincode to \"{}\". {}", output, e);
-            exit(1);
-        }
+    if let Err(e) = file.write_all(&bincode) {
+        eprintln!("Error: Failed to write bincode to \"{}\". {}", output, e);
+        exit(1);
     };
 
     println!("Wrote output to {}", &output);

src/error.rs

@@ -289,7 +289,7 @@ pub enum Error {
     ConfigInvalid,
 
     #[error("Failed decoding bincode: {0}")]
-    ZkasDecoderError(&'static str),
+    ZkasDecoderError(String),
 
     #[cfg(feature = "regex")]
     #[error(transparent)]

src/zkas/compiler.rs

@@ -3,7 +3,7 @@ use std::str::Chars;
 use super::{
     ast::{Arg, Constant, Literal, Statement, StatementType, Witness},
     error::ErrorEmitter,
-    types::Type,
+    types::StackType,
 };
 use crate::util::serial::{serialize, VarInt};
 
@@ -69,6 +69,7 @@ impl Compiler {
 
         // In the .contract section, we write all our witness types, on the stack
         // they're in order of appearance.
+        bincode.extend_from_slice(b".contract");
         for i in &self.witnesses {
             tmp_stack.push(i.name.as_str());
             bincode.push(i.typ as u8);
@@ -90,7 +91,7 @@ impl Compiler {
                 match arg {
                     Arg::Var(arg) => {
                         if let Some(found) = Compiler::lookup_stack(&tmp_stack, &arg.name) {
-                            bincode.push(Type::Var as u8);
+                            bincode.push(StackType::Var as u8);
                             bincode.extend_from_slice(&serialize(&VarInt(found as u64)));
                             continue
                         }
@@ -103,7 +104,7 @@ impl Compiler {
                     }
                     Arg::Lit(lit) => {
                         if let Some(found) = Compiler::lookup_literal(&self.literals, &lit.name) {
-                            bincode.push(Type::Lit as u8);
+                            bincode.push(StackType::Lit as u8);
                             bincode.extend_from_slice(&serialize(&VarInt(found as u64)));
                             continue
                         }

src/zkas/decoder.rs

@@ -1,39 +1,52 @@
-use super::{compiler::MAGIC_BYTES, opcode::Opcode, types::Type};
+use super::{compiler::MAGIC_BYTES, types::StackType, LitType, Opcode, VarType};
 use crate::{
     util::serial::{deserialize_partial, VarInt},
-    Error::ZkasDecoderError,
+    Error::ZkasDecoderError as ZkasErr,
     Result,
 };
 
+/// A ZkBinary decoded from compiled zkas code.
+/// This is used by the zkvm.
 #[derive(Clone, Debug)]
 pub struct ZkBinary {
-    pub constants: Vec<(Type, String)>,
-    pub witnesses: Vec<Type>,
-    pub opcodes: Vec<(Opcode, Vec<usize>)>,
+    pub constants: Vec<(VarType, String)>,
+    pub literals: Vec<(LitType, String)>,
+    pub witnesses: Vec<VarType>,
+    pub opcodes: Vec<(Opcode, Vec<(StackType, usize)>)>,
+}
+
+// https://stackoverflow.com/questions/35901547/how-can-i-find-a-subsequence-in-a-u8-slice
+fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option<usize> {
+    haystack.windows(needle.len()).position(|window| window == needle)
 }
 
 impl ZkBinary {
     pub fn decode(bytes: &[u8]) -> Result<Self> {
         let magic_bytes = &bytes[0..4];
         if magic_bytes != MAGIC_BYTES {
-            return Err(ZkasDecoderError("Magic bytes are incorrect."))
+            return Err(ZkasErr("Magic bytes are incorrect.".to_string()))
         }
 
         let _binary_version = &bytes[4];
 
         let constants_offset = match find_subslice(bytes, b".constant") {
             Some(v) => v,
-            None => return Err(ZkasDecoderError("Could not find .constant section.")),
+            None => return Err(ZkasErr("Could not find .constant section".to_string())),
+        };
+
+        let literals_offset = match find_subslice(bytes, b".literal") {
+            Some(v) => v,
+            None => return Err(ZkasErr("Could not find .literal section".to_string())),
         };
 
         let contract_offset = match find_subslice(bytes, b".contract") {
             Some(v) => v,
-            None => return Err(ZkasDecoderError("Could not find .contract section")),
+            None => return Err(ZkasErr("Could not find .contract section".to_string())),
         };
 
         let circuit_offset = match find_subslice(bytes, b".circuit") {
             Some(v) => v,
-            None => return Err(ZkasDecoderError("Could not find .circuit section")),
+            None => return Err(ZkasErr("Could not find .circuit section".to_string())),
         };
 
         let debug_offset = match find_subslice(bytes, b".debug") {
@@ -41,36 +54,50 @@ impl ZkBinary {
             None => bytes.len(),
         };
 
-        if constants_offset > contract_offset {
-            return Err(ZkasDecoderError(".contract appeared before .constant"))
+        if constants_offset > literals_offset {
+            return Err(ZkasErr(".literal section appeared before .constant".to_string()))
+        }
+
+        if literals_offset > contract_offset {
+            return Err(ZkasErr(".contract section appeared before .literal".to_string()))
         }
 
         if contract_offset > circuit_offset {
-            return Err(ZkasDecoderError(".contract appeared before .circuit"))
+            return Err(ZkasErr(".circuit section appeared before .contract".to_string()))
         }
 
         if circuit_offset > debug_offset {
-            return Err(ZkasDecoderError(".circuit appeared before .debug or EOF"))
+            return Err(ZkasErr(".debug section appeared before .circuit or EOF".to_string()))
         }
 
-        let constants_section = &bytes[constants_offset + b".constant".len()..contract_offset];
+        let constants_section = &bytes[constants_offset + b".constant".len()..literals_offset];
+        let literals_section = &bytes[literals_offset + b".literal".len()..contract_offset];
         let contract_section = &bytes[contract_offset + b".contract".len()..circuit_offset];
         let circuit_section = &bytes[circuit_offset + b".circuit".len()..debug_offset];
 
         let constants = ZkBinary::parse_constants(constants_section)?;
+        let literals = ZkBinary::parse_literals(literals_section)?;
         let witnesses = ZkBinary::parse_contract(contract_section)?;
         let opcodes = ZkBinary::parse_circuit(circuit_section)?;
         // TODO: Debug info
 
-        Ok(Self { constants, witnesses, opcodes })
+        Ok(Self { constants, literals, witnesses, opcodes })
     }
 
-    fn parse_constants(bytes: &[u8]) -> Result<Vec<(Type, String)>> {
+    fn parse_constants(bytes: &[u8]) -> Result<Vec<(VarType, String)>> {
         let mut constants = vec![];
 
         let mut iter_offset = 0;
         while iter_offset < bytes.len() {
-            let c_type = Type::from_repr(bytes[iter_offset]);
+            let c_type = match VarType::from_repr(bytes[iter_offset]) {
+                Some(v) => v,
+                None => {
+                    return Err(ZkasErr(format!(
+                        "Could not decode constant VarType from {}",
+                        bytes[iter_offset],
+                    )))
+                }
+            };
             iter_offset += 1;
             let (name, offset) = deserialize_partial::<String>(&bytes[iter_offset..])?;
             iter_offset += offset;
@@ -81,12 +108,45 @@ impl ZkBinary {
         Ok(constants)
     }
 
-    fn parse_contract(bytes: &[u8]) -> Result<Vec<Type>> {
+    fn parse_literals(bytes: &[u8]) -> Result<Vec<(LitType, String)>> {
+        let mut literals = vec![];
+
+        let mut iter_offset = 0;
+        while iter_offset < bytes.len() {
+            let l_type = match LitType::from_repr(bytes[iter_offset]) {
+                Some(v) => v,
+                None => {
+                    return Err(ZkasErr(format!(
+                        "Could not decode literal LitType from {}",
+                        bytes[iter_offset],
+                    )))
+                }
+            };
+            iter_offset += 1;
+            let (name, offset) = deserialize_partial::<String>(&bytes[iter_offset..])?;
+            iter_offset += offset;
+
+            literals.push((l_type, name));
+        }
+
+        Ok(literals)
+    }
+
+    fn parse_contract(bytes: &[u8]) -> Result<Vec<VarType>> {
         let mut witnesses = vec![];
 
         let mut iter_offset = 0;
         while iter_offset < bytes.len() {
-            let w_type = Type::from_repr(bytes[iter_offset]);
+            let w_type = match VarType::from_repr(bytes[iter_offset]) {
+                Some(v) => v,
+                None => {
+                    return Err(ZkasErr(format!(
+                        "Could not decode witness VarType from {}",
+                        bytes[iter_offset],
+                    )))
+                }
+            };
+
             iter_offset += 1;
 
             witnesses.push(w_type);
@@ -95,12 +155,20 @@ impl ZkBinary {
         Ok(witnesses)
     }
 
-    fn parse_circuit(bytes: &[u8]) -> Result<Vec<(Opcode, Vec<usize>)>> {
+    fn parse_circuit(bytes: &[u8]) -> Result<Vec<(Opcode, Vec<(StackType, usize)>)>> {
         let mut opcodes = vec![];
 
         let mut iter_offset = 0;
         while iter_offset < bytes.len() {
-            let opcode = Opcode::from_repr(bytes[iter_offset]);
+            let opcode = match Opcode::from_repr(bytes[iter_offset]) {
+                Some(v) => v,
+                None => {
+                    return Err(ZkasErr(format!(
+                        "Could not decode Opcode from {}",
+                        bytes[iter_offset]
+                    )))
+                }
+            };
             iter_offset += 1;
 
             let (arg_num, offset) = deserialize_partial::<VarInt>(&bytes[iter_offset..])?;
@@ -108,9 +176,20 @@ impl ZkBinary {
 
             let mut args = vec![];
             for _ in 0..arg_num.0 {
+                let stack_type = bytes[iter_offset];
+                iter_offset += 1;
                 let (stack_index, offset) = deserialize_partial::<VarInt>(&bytes[iter_offset..])?;
                 iter_offset += offset;
-                args.push(stack_index.0 as usize); // FIXME
+                let stack_type = match StackType::from_repr(stack_type) {
+                    Some(v) => v,
+                    None => {
+                        return Err(ZkasErr(format!(
+                            "Could not decode StackType from {}",
+                            stack_type
+                        )))
+                    }
+                };
+                args.push((stack_type, stack_index.0 as usize)); // FIXME, why?
             }
 
             opcodes.push((opcode, args));
@@ -119,8 +198,3 @@ impl ZkBinary {
         Ok(opcodes)
     }
 }
-
-// https://stackoverflow.com/questions/35901547/how-can-i-find-a-subsequence-in-a-u8-slice
-fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option<usize> {
-    haystack.windows(needle.len()).position(|window| window == needle)
-}

src/zkas/mod.rs

@@ -1,18 +1,37 @@
-/// Semantic analyzer
-pub mod analyzer;
-/// AST
-pub mod ast;
-/// Compiler
-pub mod compiler;
-/// Binary decoder
-pub mod decoder;
+//! `src/zkas` is the library holding the zkas toolchain, consisting of a
+//! lexer, parser, static/semantic analyzers, a binary compiler, and a
+//! binary decoder.
+
 /// Error emitter
 mod error;
-/// Lexer module
-pub mod lexer;
+
 /// Language opcodes
 pub mod opcode;
-/// Parser module
-pub mod parser;
+pub use opcode::Opcode;
+
 /// Language types
 pub mod types;
+pub use types::{LitType, VarType};
+
+/// Language AST
+pub mod ast;
+
+/// Lexer module
+pub mod lexer;
+pub use lexer::Lexer;
+
+/// Parser module
+pub mod parser;
+pub use parser::Parser;
+
+/// Analyzer module
+pub mod analyzer;
+pub use analyzer::Analyzer;
+
+/// Compiler module
+pub mod compiler;
+pub use compiler::Compiler;
+
+/// Decoder module
+pub mod decoder;
+pub use decoder::ZkBinary;

src/zkas/opcode.rs

@@ -79,6 +79,28 @@ impl Opcode {
         }
     }
 
+    pub fn from_repr(b: u8) -> Option<Self> {
+        match b {
+            0x01 => Some(Self::EcAdd),
+            0x02 => Some(Self::EcMul),
+            0x03 => Some(Self::EcMulBase),
+            0x04 => Some(Self::EcMulShort),
+            0x08 => Some(Self::EcGetX),
+            0x09 => Some(Self::EcGetY),
+            0x10 => Some(Self::PoseidonHash),
+            0x20 => Some(Self::MerkleRoot),
+            0x30 => Some(Self::BaseAdd),
+            0x31 => Some(Self::BaseMul),
+            0x32 => Some(Self::BaseSub),
+            0x40 => Some(Self::WitnessBase),
+            0x50 => Some(Self::RangeCheck),
+            0x51 => Some(Self::LessThan),
+            0xf0 => Some(Self::ConstrainInstance),
+            0xff => Some(Self::DebugPrint),
+            _ => None,
+        }
+    }
+
     /// Return a tuple of vectors of types that are accepted by a specific opcode.
     /// `r.0` is the return type(s), and `r.1` is the argument type(s).
     pub fn arg_types(&self) -> (Vec<VarType>, Vec<VarType>) {

src/zkas/types.rs

@@ -1,3 +1,21 @@
+/// Stack types in bincode & vm
+#[derive(Clone, Debug)]
+#[repr(u8)]
+pub enum StackType {
+    Var = 0x00,
+    Lit = 0x01,
+}
+
+impl StackType {
+    pub fn from_repr(b: u8) -> Option<Self> {
+        match b {
+            0x00 => Some(Self::Var),
+            0x01 => Some(Self::Lit),
+            _ => None,
+        }
+    }
+}
+
 /// Varable types supported by the zkas VM
 #[derive(Copy, Clone, PartialEq, Debug)]
 #[repr(u8)]
@@ -39,6 +57,25 @@ pub enum VarType {
     Uint64 = 0x31,
 }
 
+impl VarType {
+    pub fn from_repr(b: u8) -> Option<Self> {
+        match b {
+            0x01 => Some(Self::EcPoint),
+            0x02 => Some(Self::EcFixedPoint),
+            0x03 => Some(Self::EcFixedPointShort),
+            0x04 => Some(Self::EcFixedPointBase),
+            0x10 => Some(Self::Base),
+            0x11 => Some(Self::BaseArray),
+            0x12 => Some(Self::Scalar),
+            0x13 => Some(Self::ScalarArray),
+            0x20 => Some(Self::MerklePath),
+            0x30 => Some(Self::Uint32),
+            0x31 => Some(Self::Uint64),
+            _ => None,
+        }
+    }
+}
+
 /// Literal types supported by the zkas VM
 #[derive(Copy, Clone, PartialEq, Debug)]
 #[repr(u8)]
@@ -51,6 +88,13 @@ pub enum LitType {
 }
 
 impl LitType {
+    pub fn from_repr(b: u8) -> Option<Self> {
+        match b {
+            0x01 => Some(Self::Uint64),
+            _ => None,
+        }
+    }
+
     pub fn to_vartype(&self) -> VarType {
         match self {
             Self::Dummy => VarType::Dummy,