✨ feat: Add tokens to parse output for JNI

caoccao · Mar 18, 2024 · e22a430 · e22a430
1 parent 77406e2
commit e22a430
Show file tree

Hide file tree

Showing 3 changed files with 92 additions and 5 deletions.
diff --git a/rust/src/core.rs b/rust/src/core.rs
@@ -35,10 +35,7 @@ pub fn parse<'local>(code: String, options: options::ParseOptions) -> Result<out
     _ => parse_module(parse_params),
   };
   match result {
-    Ok(parsed_source) => Ok(outputs::ParseOutput {
-      module: parsed_source.is_module(),
-      script: parsed_source.is_script(),
-    }),
+    Ok(parsed_source) => Ok(outputs::ParseOutput::new(parsed_source, options.capture_tokens)),
     Err(e) => Err(e.to_string()),
   }
 }

diff --git a/rust/src/outputs.rs b/rust/src/outputs.rs
@@ -19,6 +19,9 @@ use jni::objects::{GlobalRef, JMethodID, JObject};
 use jni::sys::jvalue;
 use jni::JNIEnv;
 
+use deno_ast::swc::parser::token::TokenAndSpan;
+use deno_ast::ParsedSource;
+
 use std::ptr::null_mut;
 
 use crate::converter;
@@ -126,6 +129,22 @@ pub trait ToJniType {
 pub struct ParseOutput {
   pub module: bool,
   pub script: bool,
+  pub tokens: Option<Vec<TokenAndSpan>>,
+}
+
+impl ParseOutput {
+  pub fn new(parsed_source: ParsedSource, capture_tokens: bool) -> ParseOutput {
+    let tokens = if capture_tokens {
+      Some(parsed_source.tokens().to_vec())
+    } else {
+      None
+    };
+    ParseOutput {
+      module: parsed_source.is_module(),
+      script: parsed_source.is_script(),
+      tokens,
+    }
+  }
 }
 
 impl ToJniType for ParseOutput {

diff --git a/rust/tests/test_core.rs b/rust/tests/test_core.rs
@@ -15,7 +15,9 @@
 * limitations under the License.
 */
 
-use deno_ast::MediaType;
+use deno_ast::{
+  swc::{atoms::JsWord, common::Spanned, common::BytePos, parser::token::{IdentLike, Keyword, Token, Word}}, MediaType,
+};
 
 use swc4j::*;
 
@@ -45,6 +47,75 @@ fn test_parse_jsx_with_default_options() {
   assert!(!output.script);
 }
 
+#[test]
+fn test_parse_typescript_with_default_options() {
+  let code = "function add(a:number, b:number) { return a+b; }";
+  let options = options::ParseOptions {
+    media_type: MediaType::TypeScript,
+    ..Default::default()
+  };
+  let output = core::parse(code.to_owned(), options);
+  assert!(output.is_ok());
+  let output = output.unwrap();
+  assert!(output.module);
+  assert!(!output.script);
+  assert!(output.tokens.is_none());
+}
+
+#[test]
+fn test_parse_typescript_with_capture_tokens() {
+  let code = "function add(a:number, b:number) { return a+b; }";
+  let options = options::ParseOptions {
+    capture_tokens: true,
+    media_type: MediaType::TypeScript,
+    ..Default::default()
+  };
+  let output = core::parse(code.to_owned(), options);
+  assert!(output.is_ok());
+  let output = output.unwrap();
+  assert!(output.module);
+  assert!(!output.script);
+  assert!(output.tokens.is_some());
+  let tokens = output.tokens.unwrap();
+  /*
+   * TokenAndSpan { token: function, had_line_break: true, span: Span { lo: BytePos(1), hi: BytePos(9), ctxt: #0 } }
+   * TokenAndSpan { token: add, had_line_break: false, span: Span { lo: BytePos(10), hi: BytePos(13), ctxt: #0 } }
+   * TokenAndSpan { token: (, had_line_break: false, span: Span { lo: BytePos(13), hi: BytePos(14), ctxt: #0 } }
+   * TokenAndSpan { token: a, had_line_break: false, span: Span { lo: BytePos(14), hi: BytePos(15), ctxt: #0 } }
+   * TokenAndSpan { token: :, had_line_break: false, span: Span { lo: BytePos(15), hi: BytePos(16), ctxt: #0 } }
+   * TokenAndSpan { token: number, had_line_break: false, span: Span { lo: BytePos(16), hi: BytePos(22), ctxt: #0 } }
+   * TokenAndSpan { token: ,, had_line_break: false, span: Span { lo: BytePos(22), hi: BytePos(23), ctxt: #0 } }
+   * TokenAndSpan { token: b, had_line_break: false, span: Span { lo: BytePos(24), hi: BytePos(25), ctxt: #0 } }
+   * TokenAndSpan { token: :, had_line_break: false, span: Span { lo: BytePos(25), hi: BytePos(26), ctxt: #0 } }
+   * TokenAndSpan { token: number, had_line_break: false, span: Span { lo: BytePos(26), hi: BytePos(32), ctxt: #0 } }
+   * TokenAndSpan { token: ), had_line_break: false, span: Span { lo: BytePos(32), hi: BytePos(33), ctxt: #0 } }
+   * TokenAndSpan { token: {, had_line_break: false, span: Span { lo: BytePos(34), hi: BytePos(35), ctxt: #0 } }
+   * TokenAndSpan { token: return, had_line_break: false, span: Span { lo: BytePos(36), hi: BytePos(42), ctxt: #0 } }
+   * TokenAndSpan { token: a, had_line_break: false, span: Span { lo: BytePos(43), hi: BytePos(44), ctxt: #0 } }
+   * TokenAndSpan { token: +, had_line_break: false, span: Span { lo: BytePos(44), hi: BytePos(45), ctxt: #0 } }
+   * TokenAndSpan { token: b, had_line_break: false, span: Span { lo: BytePos(45), hi: BytePos(46), ctxt: #0 } }
+   * TokenAndSpan { token: ;, had_line_break: false, span: Span { lo: BytePos(46), hi: BytePos(47), ctxt: #0 } }
+   * TokenAndSpan { token: }, had_line_break: false, span: Span { lo: BytePos(48), hi: BytePos(49), ctxt: #0 } }
+   */
+  let t0 = &tokens[0];
+  let t1 = &tokens[1];
+  let t2 = &tokens[2];
+  let t3 = &tokens[3];
+  let t12 = &tokens[12];
+  assert_eq!(Token::Word(Word::Keyword(Keyword::Function)), t0.token);
+  assert_eq!(Token::Word(Word::Ident(IdentLike::Other(JsWord::new("add")))), t1.token);
+  assert_eq!(Token::LParen, t2.token);
+  assert_eq!(Token::Word(Word::Ident(IdentLike::Other(JsWord::new("a")))), t3.token);
+  assert_eq!(Token::Word(Word::Keyword(Keyword::Return)), t12.token);
+  assert!(t0.had_line_break);
+  assert!(!t1.had_line_break);
+  assert!(!t2.had_line_break);
+  assert!(!t3.had_line_break);
+  assert!(!t12.had_line_break);
+  assert_eq!(BytePos(1), t0.span_lo());
+  assert_eq!(BytePos(9), t0.span_hi());
+}
+
 #[test]
 fn test_parse_wrong_media_type() {
   let code = "function add(a:number, b:number) { return a+b; }";