fix: support hexadecimal numbers and words with underscores (#57)

Author: dsherret

src/ast.rs

@@ -68,10 +68,31 @@ impl<'a> From<Value<'a>> for serde_json::Value {
       }
       Value::BooleanLit(b) => serde_json::Value::Bool(b.value),
       Value::NullKeyword(_) => serde_json::Value::Null,
-      Value::NumberLit(num) => match serde_json::Number::from_str(num.value) {
-        Ok(number) => serde_json::Value::Number(number),
-        Err(_) => serde_json::Value::String(num.value.to_string()),
-      },
+      Value::NumberLit(num) => {
+        // Check if this is a hexadecimal literal (0x or 0X prefix)
+        let num_str = num.value.trim_start_matches('-');
+        if num_str.len() > 2 && (num_str.starts_with("0x") || num_str.starts_with("0X")) {
+          // Parse hexadecimal and convert to decimal
+          let hex_part = &num_str[2..];
+          match i64::from_str_radix(hex_part, 16) {
+            Ok(decimal_value) => {
+              let final_value = if num.value.starts_with('-') {
+                -decimal_value
+              } else {
+                decimal_value
+              };
+              serde_json::Value::Number(serde_json::Number::from(final_value))
+            }
+            Err(_) => serde_json::Value::String(num.value.to_string()),
+          }
+        } else {
+          // Standard decimal number
+          match serde_json::Number::from_str(num.value) {
+            Ok(number) => serde_json::Value::Number(number),
+            Err(_) => serde_json::Value::String(num.value.to_string()),
+          }
+        }
+      }
       Value::Object(obj) => {
         let mut map = serde_json::map::Map::new();
         for prop in obj.properties {

src/cst/mod.rs

@@ -1442,10 +1442,30 @@ impl CstNumberLit {
   pub fn to_serde_value(&self) -> Option<serde_json::Value> {
     use std::str::FromStr;
     let raw = self.0.borrow().value.clone();
-    match serde_json::Number::from_str(&raw) {
-      Ok(number) => Some(serde_json::Value::Number(number)),
-      // If the number is invalid, return it as a string (same behavior as AST conversion)
-      Err(_) => Some(serde_json::Value::String(raw)),
+
+    // check if this is a hexadecimal literal (0x or 0X prefix)
+    let num_str = raw.trim_start_matches('-');
+    if num_str.len() > 2 && (num_str.starts_with("0x") || num_str.starts_with("0X")) {
+      // parse hexadecimal and convert to decimal
+      let hex_part = &num_str[2..];
+      match i64::from_str_radix(hex_part, 16) {
+        Ok(decimal_value) => {
+          let final_value = if raw.starts_with('-') {
+            -decimal_value
+          } else {
+            decimal_value
+          };
+          Some(serde_json::Value::Number(serde_json::Number::from(final_value)))
+        }
+        Err(_) => Some(serde_json::Value::String(raw)),
+      }
+    } else {
+      // standard decimal number
+      match serde_json::Number::from_str(&raw) {
+        Ok(number) => Some(serde_json::Value::Number(number)),
+        // if the number is invalid, return it as a string (same behavior as AST conversion)
+        Err(_) => Some(serde_json::Value::String(raw)),
+      }
     }
   }
 }

src/parse_to_ast.rs

@@ -573,4 +573,33 @@ mod tests {
   fn error_correct_line_column_unicode_width() {
     assert_has_strict_error(r#"["🧑‍🦰", ["#, "Unterminated array on line 1 column 10");
   }
+
+  #[test]
+  fn it_should_parse_unquoted_keys_with_hex_and_trailing_comma() {
+    let text = r#"{
+      CP_CanFuncReqId: 0x7DF,  // 2015
+  }"#;
+    {
+      let parse_result = parse_to_ast(text, &Default::default(), &Default::default()).unwrap();
+
+      let value = parse_result.value.unwrap();
+      let obj = value.as_object().unwrap();
+      assert_eq!(obj.properties.len(), 1);
+      assert_eq!(obj.properties[0].name.as_str(), "CP_CanFuncReqId");
+
+      let number_value = obj.properties[0].value.as_number_lit().unwrap();
+      assert_eq!(number_value.value, "0x7DF");
+    }
+    #[cfg(feature = "serde")]
+    {
+      let value = crate::parse_to_serde_value(text, &Default::default()).unwrap().unwrap();
+      // hexadecimal numbers are converted to decimal in serde output
+      assert_eq!(
+        value,
+        serde_json::json!({
+          "CP_CanFuncReqId": 2015
+        })
+      );
+    }
+  }
 }

src/scanner.rs

@@ -160,6 +160,23 @@ impl<'a> Scanner<'a> {
 
     if self.is_zero() {
       self.move_next_char();
+
+      // check for hexadecimal literal (0x or 0X)
+      if matches!(self.current_char(), Some('x') | Some('X')) {
+        self.move_next_char();
+
+        // must have at least one hex digit
+        if !self.is_hex_digit() {
+          return Err(self.create_error_for_current_char(ParseErrorKind::ExpectedDigit));
+        }
+
+        while self.is_hex_digit() {
+          self.move_next_char();
+        }
+
+        let end_byte_index = self.byte_index;
+        return Ok(Token::Number(&self.file_text[start_byte_index..end_byte_index]));
+      }
     } else if self.is_one_nine() {
       self.move_next_char();
       while self.is_digit() {
@@ -288,10 +305,12 @@ impl<'a> Scanner<'a> {
     let start_byte_index = self.byte_index;
 
     while let Some(current_char) = self.current_char() {
-      if current_char.is_whitespace() || current_char == '\r' || current_char == '\n' || current_char == ':' {
+      // check for word terminators
+      if current_char.is_whitespace() || current_char == ':' {
         break;
       }
-      if !current_char.is_alphanumeric() && current_char != '-' {
+      // validate that the character is allowed in a word literal
+      if !current_char.is_alphanumeric() && current_char != '-' && current_char != '_' {
         return Err(self.create_error_for_current_token(ParseErrorKind::UnexpectedToken));
       }
 
@@ -382,6 +401,13 @@ impl<'a> Scanner<'a> {
     self.is_one_nine() || self.is_zero()
   }
 
+  fn is_hex_digit(&self) -> bool {
+    match self.current_char() {
+      Some(current_char) => current_char.is_ascii_hexdigit(),
+      _ => false,
+    }
+  }
+
   fn is_zero(&self) -> bool {
     self.current_char() == Some('0')
   }
@@ -496,6 +522,24 @@ mod tests {
     );
   }
 
+  #[test]
+  fn it_tokenizes_hexadecimal_numbers() {
+    assert_has_tokens(
+      "0x7DF, 0xFF, 0x123ABC, 0xabc, 0X1F",
+      vec![
+        Token::Number("0x7DF"),
+        Token::Comma,
+        Token::Number("0xFF"),
+        Token::Comma,
+        Token::Number("0x123ABC"),
+        Token::Comma,
+        Token::Number("0xabc"),
+        Token::Comma,
+        Token::Number("0X1F"),
+      ],
+    );
+  }
+
   #[test]
   fn it_errors_invalid_exponent() {
     assert_has_error(

src/serde.rs

@@ -84,4 +84,24 @@ mod tests {
 
     assert_eq!(result, Some(SerdeValue::Object(expected_value)));
   }
+
+  #[test]
+  fn it_should_parse_hexadecimal_numbers_to_decimal() {
+    let result = parse_to_serde_value(
+      r#"{
+        "hex1": 0x7DF,
+        "hex2": 0xFF,
+        "hex3": 0x10
+      }"#,
+      &Default::default(),
+    )
+    .unwrap();
+
+    let mut expected_value = serde_json::map::Map::new();
+    expected_value.insert("hex1".to_string(), SerdeValue::Number(serde_json::Number::from(2015)));
+    expected_value.insert("hex2".to_string(), SerdeValue::Number(serde_json::Number::from(255)));
+    expected_value.insert("hex3".to_string(), SerdeValue::Number(serde_json::Number::from(16)));
+
+    assert_eq!(result, Some(SerdeValue::Object(expected_value)));
+  }
 }