hex and binary literals for integers.

Note that they can't be used on the command line since SnarkVM does the parsing there.
ProvableHQ · Oct 7, 2024 · 8b97172 · 8b97172
1 parent b391f51
commit 8b97172
Show file tree

Hide file tree

Showing 78 changed files with 991 additions and 164 deletions.
diff --git a/compiler/ast/src/expressions/literal.rs b/compiler/ast/src/expressions/literal.rs
@@ -104,3 +104,86 @@ impl Node for Literal {
         }
     }
 }
+
+struct DisplayDecimal<'a>(&'a Literal);
+
+impl Literal {
+    /// For displaying a literal as decimal, regardless of the radix in which it was parsed.
+    ///
+    /// In particular this is useful for outputting .aleo files.
+    pub fn display_decimal(&self) -> impl '_ + fmt::Display {
+        DisplayDecimal(self)
+    }
+}
+
+impl fmt::Display for DisplayDecimal<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match &self.0 {
+            Literal::Address(address, _, _) => write!(f, "{address}"),
+            Literal::Boolean(boolean, _, _) => write!(f, "{boolean}"),
+            Literal::Field(field, _, _) => write!(f, "{field}field"),
+            Literal::Group(group) => write!(f, "{group}group"),
+            Literal::Integer(type_, value, _, _) => {
+                if !value.starts_with("0x")
+                    && !value.starts_with("-0x")
+                    && !value.starts_with("0b")
+                    && !value.starts_with("-0b")
+                {
+                    // It's already decimal.
+                    return write!(f, "{value}{type_}");
+                }
+                let string = value.replace('_', "");
+                if value.starts_with("-") {
+                    let v = i128::from_str_by_radix(&string).expect("Failed to parse integer?");
+                    write!(f, "{v}{type_}")
+                } else {
+                    let v = u128::from_str_by_radix(&string).expect("Failed to parse integer?");
+                    write!(f, "{v}{type_}")
+                }
+            }
+            Literal::Scalar(scalar, _, _) => write!(f, "{scalar}scalar"),
+            Literal::String(string, _, _) => write!(f, "\"{string}\""),
+        }
+    }
+}
+
+/// This trait allows to parse integer literals of any type generically.
+///
+/// The literal may optionally start with a `-` and/or `0x` or `0b`.
+pub trait FromStrRadix: Sized {
+    fn from_str_by_radix(src: &str) -> Result<Self, std::num::ParseIntError>;
+}
+
+macro_rules! implement_from_str_radix {
+    ($($ty:ident)*) => {
+        $(
+            impl FromStrRadix for $ty {
+                fn from_str_by_radix(src: &str) -> Result<Self, std::num::ParseIntError> {
+                    if let Some(stripped) = src.strip_prefix("0x") {
+                        Self::from_str_radix(stripped, 16)
+                    } else if let Some(stripped) = src.strip_prefix("0b") {
+                        Self::from_str_radix(stripped, 2)
+                    } else if let Some(stripped) = src.strip_prefix("-0x") {
+                        // We have to remove the 0x prefix and put back in a - to use
+                        // std's parsing. Alternatively we could jump through
+                        // a few hoops to avoid allocating.
+                        let mut s = String::new();
+                        s.push('-');
+                        s.push_str(stripped);
+                        Self::from_str_radix(&s, 16)
+                    } else if let Some(stripped) = src.strip_prefix("-0b") {
+                        // Ditto.
+                        let mut s = String::new();
+                        s.push('-');
+                        s.push_str(stripped);
+                        Self::from_str_radix(&s, 2)
+                    } else {
+                        Self::from_str_radix(src, 10)
+                    }
+                }
+            }
+        )*
+    };
+}
+
+implement_from_str_radix! { u8 u16 u32 u64 u128 i8 i16 i32 i64 i128 }
diff --git a/compiler/ast/src/value/mod.rs b/compiler/ast/src/value/mod.rs
@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
 
-use crate::{GroupLiteral, Identifier, IntegerType, Literal, NodeID, Type};
+use crate::{FromStrRadix as _, GroupLiteral, Identifier, IntegerType, Literal, NodeID, Type};
 
 use leo_errors::{FlattenError, LeoError, Result, type_name};
 use leo_span::{Span, Symbol};
@@ -874,16 +874,16 @@ impl TryFrom<&Literal> for Value {
             Literal::Integer(integer_type, raw_string, span, _) => {
                 let string = raw_string.replace('_', "");
                 match integer_type {
-                    IntegerType::U8 => Self::U8(string.parse()?, *span),
-                    IntegerType::U16 => Self::U16(string.parse()?, *span),
-                    IntegerType::U32 => Self::U32(string.parse()?, *span),
-                    IntegerType::U64 => Self::U64(string.parse()?, *span),
-                    IntegerType::U128 => Self::U128(string.parse()?, *span),
-                    IntegerType::I8 => Self::I8(string.parse()?, *span),
-                    IntegerType::I16 => Self::I16(string.parse()?, *span),
-                    IntegerType::I32 => Self::I32(string.parse()?, *span),
-                    IntegerType::I64 => Self::I64(string.parse()?, *span),
-                    IntegerType::I128 => Self::I128(string.parse()?, *span),
+                    IntegerType::U8 => Self::U8(u8::from_str_by_radix(&string)?, *span),
+                    IntegerType::U16 => Self::U16(u16::from_str_by_radix(&string)?, *span),
+                    IntegerType::U32 => Self::U32(u32::from_str_by_radix(&string)?, *span),
+                    IntegerType::U64 => Self::U64(u64::from_str_by_radix(&string)?, *span),
+                    IntegerType::U128 => Self::U128(u128::from_str_by_radix(&string)?, *span),
+                    IntegerType::I8 => Self::I8(i8::from_str_by_radix(&string)?, *span),
+                    IntegerType::I16 => Self::I16(i16::from_str_by_radix(&string)?, *span),
+                    IntegerType::I32 => Self::I32(i32::from_str_by_radix(&string)?, *span),
+                    IntegerType::I64 => Self::I64(i64::from_str_by_radix(&string)?, *span),
+                    IntegerType::I128 => Self::I128(i128::from_str_by_radix(&string)?, *span),
                 }
             }
         })

diff --git a/compiler/parser/src/parser/expression.rs b/compiler/parser/src/parser/expression.rs
@@ -758,6 +758,15 @@ impl<N: Network> ParserContext<'_, N> {
                 let full_span = span + suffix_span;
                 let assert_no_whitespace = |x| assert_no_whitespace(span, suffix_span, &value, x);
                 match self.eat_any(INT_TYPES).then_some(&self.prev_token.token) {
+                    // Hex or binary literal on a noninteger is an error.
+                    Some(Token::Field) | Some(Token::Group) | Some(Token::Scalar)
+                        if value.starts_with("0x")
+                            || value.starts_with("0b")
+                            || value.starts_with("-0x")
+                            || value.starts_with("-0b") =>
+                    {
+                        return Err(ParserError::hexbin_literal_nonintegers(span).into());
+                    }
                     // Literal followed by `field`, e.g., `42field`.
                     Some(Token::Field) => {
                         assert_no_whitespace("field")?;

diff --git a/compiler/parser/src/tokenizer/lexer.rs b/compiler/parser/src/tokenizer/lexer.rs
@@ -160,25 +160,34 @@ impl Token {
     /// If there is input but no integer, this function returns the tuple consisting of
     /// length 0 and a dummy integer token that contains an empty string.
     /// However, this function is always called when the next character is a digit.
-    /// This function eats a sequence of one or more digits and underscores
-    /// (starting from a digit, as explained above, given when it is called),
-    /// which corresponds to a numeral in the ABNF grammar.
+    /// This function eats a sequence representing a decimal numeral, a hex
+    /// numeral (beginning with `0x`), or a binary numeral (beginning with `0b`), optionally
+    /// including underscores, which corresponds to a numeral in the ABNF grammar.
     fn eat_integer(input: &mut Peekable<impl Iterator<Item = char>>) -> Result<(usize, Token)> {
         if input.peek().is_none() {
             return Err(ParserError::lexer_empty_input().into());
         }
 
-        let mut int = String::new();
+        if !input.peek().unwrap().is_ascii_digit() {
+            return Ok((0, Token::Integer("".into())));
+        }
 
-        // Note that it is still impossible to have a number that starts with an `_` because eat_integer is only called when the first character is a digit.
-        while let Some(c) = input.next_if(|c| c.is_ascii_digit() || *c == '_') {
-            if c == '0' && matches!(input.peek(), Some('x')) {
-                int.push(c);
-                int.push(input.next().unwrap());
-                return Err(ParserError::lexer_hex_number_provided(int).into());
-            }
+        let mut int = String::new();
 
-            int.push(c);
+        let first = input.next().unwrap();
+        int.push(first);
+
+        if first == '0' && input.peek() == Some(&'x') {
+            int.push(input.next().unwrap());
+            // Don't use c.is_ascii_hexdigit because we want to disallow lowercase.
+            int.extend(
+                input.take_while(|&c| c.is_ascii_digit() || ['A', 'B', 'C', 'D', 'E', 'F'].contains(&c) || c == '_'),
+            );
+        } else if first == '0' && input.peek() == Some(&'b') {
+            int.push(input.next().unwrap());
+            int.extend(input.take_while(|&c| c.is_digit(2) || c == '_'));
+        } else {
+            int.extend(input.take_while(|&c| c.is_ascii_digit() || c == '_'));
         }
 
         Ok((int.len(), Token::Integer(int)))

diff --git a/compiler/passes/src/code_generation/visit_expressions.rs b/compiler/passes/src/code_generation/visit_expressions.rs
@@ -82,7 +82,9 @@ impl<'a> CodeGenerator<'a> {
     }
 
     fn visit_value(&mut self, input: &'a Literal) -> (String, String) {
-        (format!("{input}"), String::new())
+        // AVM can only parse decimal numbers.
+        let decimal_input = input.display_decimal();
+        (format!("{decimal_input}"), String::new())
     }
 
     fn visit_locator(&mut self, input: &'a LocatorExpression) -> (String, String) {

diff --git a/compiler/passes/src/type_checking/check_expressions.rs b/compiler/passes/src/type_checking/check_expressions.rs
@@ -23,7 +23,6 @@ use leo_span::{Span, Symbol, sym};
 use snarkvm::console::network::Network;
 
 use itertools::Itertools;
-use std::str::FromStr;
 
 fn return_incorrect_type(t1: Option<Type>, t2: Option<Type>, expected: &Option<Type>) -> Option<Type> {
     match (t1, t2) {
@@ -883,9 +882,9 @@ impl<'a, N: Network> ExpressionVisitor<'a> for TypeChecker<'a, N> {
     }
 
     fn visit_literal(&mut self, input: &'a Literal, expected: &Self::AdditionalInput) -> Self::Output {
-        fn parse_integer_literal<I: FromStr>(handler: &Handler, raw_string: &str, span: Span, type_string: &str) {
+        fn parse_integer_literal<I: FromStrRadix>(handler: &Handler, raw_string: &str, span: Span, type_string: &str) {
             let string = raw_string.replace('_', "");
-            if string.parse::<I>().is_err() {
+            if I::from_str_by_radix(&string).is_err() {
                 handler.emit_err(TypeCheckerError::invalid_int_value(string, type_string, span));
             }
         }

diff --git a/errors/src/errors/parser/parser_errors.rs b/errors/src/errors/parser/parser_errors.rs
@@ -176,6 +176,7 @@ create_messages!(
     }
 
     /// When a hex number is provided.
+    // TODO unused
     @backtraced
     lexer_hex_number_provided {
         args: (input: impl Display),
@@ -341,4 +342,11 @@ create_messages!(
         msg: format!("Cannot create an external record. Records can only be created in the program that they are defined in."),
         help: None,
     }
+
+    @formatted
+    hexbin_literal_nonintegers {
+        args: (),
+        msg: format!("Hex and binary literals may only be used for integer types."),
+        help: None,
+    }
 );
diff --git a/tests/expectations/compiler/finalize/get_incorrect_num_operands.out b/tests/expectations/compiler/finalize/get_incorrect_num_operands.out
@@ -2,4 +2,4 @@
 namespace: Compile
 expectation: Fail
 outputs:
-  - "Error [EPAR0370021]: The type of `tokens` has no associated function `get` that takes 2 argument(s).\n    --> compiler-test:18:9\n     |\n  18 |         tokens.get(true, true);\n     |         ^^^^^^^^^^^^^^^^^^^^^^\nError [EPAR0370021]: The type of `amounts` has no associated function `get` that takes 0 argument(s).\n    --> compiler-test:20:9\n     |\n  20 |         amounts.get();\n     |         ^^^^^^^^^^^^^"
+  - "Error [EPAR0370020]: The type of `tokens` has no associated function `get` that takes 2 argument(s).\n    --> compiler-test:18:9\n     |\n  18 |         tokens.get(true, true);\n     |         ^^^^^^^^^^^^^^^^^^^^^^\nError [EPAR0370020]: The type of `amounts` has no associated function `get` that takes 0 argument(s).\n    --> compiler-test:20:9\n     |\n  20 |         amounts.get();\n     |         ^^^^^^^^^^^^^"
diff --git a/tests/expectations/compiler/finalize/get_or_incorrect_num_operands.out b/tests/expectations/compiler/finalize/get_or_incorrect_num_operands.out
@@ -2,4 +2,4 @@
 namespace: Compile
 expectation: Fail
 outputs:
-  - "Error [EPAR0370021]: The type of `tokens` has no associated function `get_or_use` that takes 3 argument(s).\n    --> compiler-test:18:9\n     |\n  18 |         tokens.get_or_use(addr, amount, 1u128);\n     |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nError [EPAR0370021]: The type of `amounts` has no associated function `get_or_use` that takes 1 argument(s).\n    --> compiler-test:20:9\n     |\n  20 |         amounts.get_or_use(1u8);\n     |         ^^^^^^^^^^^^^^^^^^^^^^^\nError [EPAR0370021]: The type of `amounts` has no associated function `get_or_use` that takes 0 argument(s).\n    --> compiler-test:22:9\n     |\n  22 |         amounts.get_or_use();\n     |         ^^^^^^^^^^^^^^^^^^^^"
+  - "Error [EPAR0370020]: The type of `tokens` has no associated function `get_or_use` that takes 3 argument(s).\n    --> compiler-test:18:9\n     |\n  18 |         tokens.get_or_use(addr, amount, 1u128);\n     |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nError [EPAR0370020]: The type of `amounts` has no associated function `get_or_use` that takes 1 argument(s).\n    --> compiler-test:20:9\n     |\n  20 |         amounts.get_or_use(1u8);\n     |         ^^^^^^^^^^^^^^^^^^^^^^^\nError [EPAR0370020]: The type of `amounts` has no associated function `get_or_use` that takes 0 argument(s).\n    --> compiler-test:22:9\n     |\n  22 |         amounts.get_or_use();\n     |         ^^^^^^^^^^^^^^^^^^^^"
diff --git a/tests/expectations/compiler/finalize/set_incorrect_num_operands.out b/tests/expectations/compiler/finalize/set_incorrect_num_operands.out
@@ -2,4 +2,4 @@
 namespace: Compile
 expectation: Fail
 outputs:
-  - "Error [EPAR0370021]: The type of `tokens` has no associated function `set` that takes 3 argument(s).\n    --> compiler-test:18:9\n     |\n  18 |         tokens.set(addr, amount, 1u128);\n     |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nError [EPAR0370021]: The type of `amounts` has no associated function `set` that takes 1 argument(s).\n    --> compiler-test:20:9\n     |\n  20 |         amounts.set(1u8);\n     |         ^^^^^^^^^^^^^^^^\nError [EPAR0370021]: The type of `amounts` has no associated function `set` that takes 0 argument(s).\n    --> compiler-test:22:9\n     |\n  22 |         amounts.set();\n     |         ^^^^^^^^^^^^^"
+  - "Error [EPAR0370020]: The type of `tokens` has no associated function `set` that takes 3 argument(s).\n    --> compiler-test:18:9\n     |\n  18 |         tokens.set(addr, amount, 1u128);\n     |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nError [EPAR0370020]: The type of `amounts` has no associated function `set` that takes 1 argument(s).\n    --> compiler-test:20:9\n     |\n  20 |         amounts.set(1u8);\n     |         ^^^^^^^^^^^^^^^^\nError [EPAR0370020]: The type of `amounts` has no associated function `set` that takes 0 argument(s).\n    --> compiler-test:22:9\n     |\n  22 |         amounts.set();\n     |         ^^^^^^^^^^^^^"
diff --git a/tests/expectations/compiler/function/duplicate_definition_fail.out b/tests/expectations/compiler/function/duplicate_definition_fail.out
@@ -2,4 +2,4 @@
 namespace: Compile
 expectation: Fail
 outputs:
-  - "Error [EPAR0370032]: `console` statements are not yet supported.\n    --> compiler-test:5:9\n     |\n   5 |         console.log(\"{}\", 1u8);\n     |         ^^^^^^^\n     |\n     = Consider using `assert`, `assert_eq`, or `assert_neq` instead."
+  - "Error [EPAR0370031]: `console` statements are not yet supported.\n    --> compiler-test:5:9\n     |\n   5 |         console.log(\"{}\", 1u8);\n     |         ^^^^^^^\n     |\n     = Consider using `assert`, `assert_eq`, or `assert_neq` instead."
diff --git a/tests/expectations/compiler/function/duplicate_parameter_fail.out b/tests/expectations/compiler/function/duplicate_parameter_fail.out
@@ -2,4 +2,4 @@
 namespace: Compile
 expectation: Fail
 outputs:
-  - "Error [EPAR0370032]: `console` statements are not yet supported.\n    --> compiler-test:5:9\n     |\n   5 |         console.log(\"{}\", 1u8);\n     |         ^^^^^^^\n     |\n     = Consider using `assert`, `assert_eq`, or `assert_neq` instead."
+  - "Error [EPAR0370031]: `console` statements are not yet supported.\n    --> compiler-test:5:9\n     |\n   5 |         console.log(\"{}\", 1u8);\n     |         ^^^^^^^\n     |\n     = Consider using `assert`, `assert_eq`, or `assert_neq` instead."
diff --git a/tests/expectations/compiler/integers/i128/hex_and_bin.out b/tests/expectations/compiler/integers/i128/hex_and_bin.out
@@ -0,0 +1,18 @@
+---
+namespace: Compile
+expectation: Pass
+outputs:
+  - - compile:
+        - initial_symbol_table: b8fa9541b26d44c8348e7ded98e6bd1b235ee9c02e632192fb2da00234e84c5c
+          type_checked_symbol_table: 9cc9c032ecfa8ab95f117b6c2f6b5f77335cdc444604a28ddca7ea204c620277
+          unrolled_symbol_table: 9cc9c032ecfa8ab95f117b6c2f6b5f77335cdc444604a28ddca7ea204c620277
+          initial_ast: 1e34f3c30849e79eeba6b5ca17e438efb5bf24fa3ceb50ddc00133a8474be73c
+          unrolled_ast: 1e34f3c30849e79eeba6b5ca17e438efb5bf24fa3ceb50ddc00133a8474be73c
+          ssa_ast: 6926cc34fac90a4bed0c819030cf61f0f614e539e4b85b63c2abb3b8776cf532
+          flattened_ast: 108e01348ff9fd5dbbc00cc94208f98c5a5782a86ddf452ee59602c5bfe046f4
+          destructured_ast: d05df9936ea9ab1e577e8d4a14046ccd75af35b76dab552119728a2d240ed83e
+          inlined_ast: d05df9936ea9ab1e577e8d4a14046ccd75af35b76dab552119728a2d240ed83e
+          dce_ast: d05df9936ea9ab1e577e8d4a14046ccd75af35b76dab552119728a2d240ed83e
+          bytecode: 7b4ac23af746311a65be3314562eb9247cacd4b29ed1ea20d04b0dc8c06eef19
+          errors: ""
+          warnings: ""
diff --git a/tests/expectations/compiler/integers/i16/hex_and_bin.out b/tests/expectations/compiler/integers/i16/hex_and_bin.out
@@ -0,0 +1,18 @@
+---
+namespace: Compile
+expectation: Pass
+outputs:
+  - - compile:
+        - initial_symbol_table: 29029f1a16e25967a35a6ed07eb857a7a14c2aff01cbe5cc53065fb37de75e33
+          type_checked_symbol_table: b18c7816d649a45de2cb8867c4d7f791b48da710b8a68be9ad2e81a8acebbf09
+          unrolled_symbol_table: b18c7816d649a45de2cb8867c4d7f791b48da710b8a68be9ad2e81a8acebbf09
+          initial_ast: 40da0941a9022d4da4860586a31ab26d1062f58eb5c254a8729160cc34d62f30
+          unrolled_ast: 40da0941a9022d4da4860586a31ab26d1062f58eb5c254a8729160cc34d62f30
+          ssa_ast: 10914c235af1410bcc77ba6b7ef1378d414b655086650d6b6814b5adfa29cb17
+          flattened_ast: 042a582d89960ccf8683decbf3a0b3b44c66d53f8922ef878ae83317c16eb777
+          destructured_ast: f9434ee8cb2104ba7085d97a57f5e9ff6c25a30064fb1b7cfebcd6c1726b54d6
+          inlined_ast: f9434ee8cb2104ba7085d97a57f5e9ff6c25a30064fb1b7cfebcd6c1726b54d6
+          dce_ast: f9434ee8cb2104ba7085d97a57f5e9ff6c25a30064fb1b7cfebcd6c1726b54d6
+          bytecode: abbe9790219732acac590659c2aa27351a46df65ca5cc03c9def1889d5642d2d
+          errors: ""
+          warnings: ""
diff --git a/tests/expectations/compiler/integers/i32/hex_and_bin.out b/tests/expectations/compiler/integers/i32/hex_and_bin.out
@@ -0,0 +1,18 @@
+---
+namespace: Compile
+expectation: Pass
+outputs:
+  - - compile:
+        - initial_symbol_table: 64448615fb54198853ec0db1342703f538e30c529c17dc1de8ce21e73cb9aec0
+          type_checked_symbol_table: f91e02cc50da5608d6a4c2ddbb06a08d8887d7a271437ccc1f4f2dc7df4feb8b
+          unrolled_symbol_table: f91e02cc50da5608d6a4c2ddbb06a08d8887d7a271437ccc1f4f2dc7df4feb8b
+          initial_ast: d6e00a4cb2abea938e9eb0af35e058f5eb749a8d97dc56e434553511cfbf5bdc
+          unrolled_ast: d6e00a4cb2abea938e9eb0af35e058f5eb749a8d97dc56e434553511cfbf5bdc
+          ssa_ast: 9c01e57aa0ee03cf03b7da15f64c40a6fce773dac8afda6e8722f5d8d16d7730
+          flattened_ast: 5222a287ea1a208cb950a77630e32621375e0f9d1c78ee74828482c1a3d2d5fa
+          destructured_ast: 4dd127a21d0013d590115c8f373787c105d754b97ec0fc5e5014c7ddb32b6011
+          inlined_ast: 4dd127a21d0013d590115c8f373787c105d754b97ec0fc5e5014c7ddb32b6011
+          dce_ast: 4dd127a21d0013d590115c8f373787c105d754b97ec0fc5e5014c7ddb32b6011
+          bytecode: 60f3f7cf4996cce21e8854a05cd0018dbbc6f3836e82f110a7327a48d826604a
+          errors: ""
+          warnings: ""