cyrllic support init

metrico · Apr 14, 2023 · 9984762 · 9984762
1 parent b772705
commit 9984762
Show file tree

Hide file tree

Showing 3 changed files with 78 additions and 8 deletions.
diff --git a/patches/bnf+1.0.1.patch b/patches/bnf+1.0.1.patch
@@ -1,8 +1,27 @@
 diff --git a/node_modules/bnf/BnfRules.js b/node_modules/bnf/BnfRules.js
-index 1f949c1..aa002da 100755
+index 1f949c1..098e140 100755
 --- a/node_modules/bnf/BnfRules.js
 +++ b/node_modules/bnf/BnfRules.js
-@@ -135,9 +135,13 @@ exports.bnfRules = {
+@@ -19,6 +19,9 @@ exports.bnfRules = {
+
+     return false;
+   },
++  REALPHA(token){
++    return token.TryRe(/^\p{L}/u);
++  },
+   SYMBOL( token ){
+     if( token.CharIs( 33 )
+       || token.CharCodeRange( 35, 38 )
+@@ -35,7 +38,7 @@ exports.bnfRules = {
+   },
+   ANYCHAR( token ){
+     return token.Or( [
+-      token.Rule( "ALPHA" ),
++      token.Rule( "REALPHA" ),
+       token.Rule( "DIGIT" ),
+       token.Rule( "SYMBOL" ),
+       token.Rule( "ONEWSP" )
+@@ -135,9 +138,13 @@ exports.bnfRules = {
    ESCAQUOTE( token ){
      return token.TryString( Buffer.from( [ 92, 96 ] ) );
    },
@@ -16,23 +35,37 @@ index 1f949c1..aa002da 100755
        token.Rule( "ESCSQUOTE" ),
        token.Rule( "ANYCHAR" ),
        token.Rule( "QUOTE" ),
-@@ -162,6 +166,7 @@ exports.bnfRules = {
+@@ -162,6 +169,7 @@ exports.bnfRules = {
    //DO BE REMOVED IN THIS MAJOR VERSION! DO NOT USE!
    QEANYCHAR( token ){
      return token.Or( [
 +      token.Rule( "ESCSLASH" ),
        token.Rule( "ESCQUOTE" ),
        token.Rule( "ANYCHAR" ),
        token.Rule( "SQUOTE" ),
-@@ -186,6 +191,7 @@ exports.bnfRules = {
+@@ -186,6 +194,7 @@ exports.bnfRules = {
    //DO BE REMOVED IN THIS MAJOR VERSION! DO NOT USE!
    AQEANYCHAR( token ){
      return token.Or( [
 +      token.Rule( "ESCSLASH" ),
        token.Rule( "ESCAQUOTE" ),
        token.Rule( "ANYCHAR" ),
        token.Rule( "SQUOTE" ),
-@@ -285,7 +291,7 @@ exports.parserRules = {
+@@ -219,11 +228,12 @@ exports.bnfRules = {
+     */
+   },
+   AQLITERAL( token ){
+-    return token.And( [
++    const res = token.And( [
+       token.Rule( "AQUOTE" ),
+       token.Rule( "AQLITERALCHARS" ),
+       token.Rule( "AQUOTE" )
+     ]);
++    return res;
+   },
+   LITERAL( token ){
+     return token.Or( [
+@@ -285,7 +295,7 @@ exports.parserRules = {
      }
      else{
        //This can be optimized @LHF
@@ -41,7 +74,7 @@ index 1f949c1..aa002da 100755
          for( let t = 0; t < token._tokenTrees[i].length; t++ ){
            for( let line in token._tokenTrees[i][t].expected ){
              for( let char in token._tokenTrees[i][t].expected[line] ){
-@@ -295,7 +301,7 @@ exports.parserRules = {
+@@ -295,7 +305,7 @@ exports.parserRules = {
              }
            }
          }
@@ -50,11 +83,24 @@ index 1f949c1..aa002da 100755
        token._tokenTrees[0] = [];
 
        return false;
+diff --git a/node_modules/bnf/Script.js b/node_modules/bnf/Script.js
+index e5eb5b7..56bd641 100755
+--- a/node_modules/bnf/Script.js
++++ b/node_modules/bnf/Script.js
+@@ -15,7 +15,7 @@ exports.Script = class Script{
+   }
+
+   GetString( length, token ){
+-    let str = this.rawScript.substring( token.point, token.point + length );
++    let str = Buffer.from(this.rawScript).subarray( token.point, token.point + length ).toString('utf8');
+     token.Seek( length );
+     return str;
+   }
 diff --git a/node_modules/bnf/Token.js b/node_modules/bnf/Token.js
-index f592ae5..d8fc55a 100755
+index f592ae5..c76e103 100755
 --- a/node_modules/bnf/Token.js
 +++ b/node_modules/bnf/Token.js
-@@ -226,13 +226,13 @@ exports.Token = class Token{
+@@ -226,16 +226,27 @@ exports.Token = class Token{
 
    TryString( charBuffer ){
      let stringBuffer = Buffer.alloc( charBuffer.length );
@@ -70,3 +116,17 @@ index f592ae5..d8fc55a 100755
      return true;
    }
 
++  TryRe(pattern) {
++    const mtch = this.script.scriptBuffer.subarray(this.point).toString('utf-8').match(pattern);
++    if (mtch && mtch[0]) {
++      const b = Buffer.from(mtch[0]);
++      this.SetValue(mtch[0]);
++      this.point+=b.length;
++      return true;
++    }
++    return false;
++  }
++
+   SetChar( char ){
+     this.SetValue( this.GetChar() );
+     this.point++;
diff --git a/test/parser.test.js b/test/parser.test.js
@@ -19,6 +19,12 @@ it('should compile', () => {
   expect(res).toBeTruthy()
 })
 
+it('should compile log_stream_selector with ciryllic', () => {
+  const scr = '{et_dolorem=`тететёąĄ`}'
+  const script = bnf.ParseScript(scr)
+  expect(script).toBeTruthy()
+})
+
 it('should compile strings with escaped quotes', () => {
   const res = bnf.ParseScript('bytes_rate({run="kok\\"oko",u_ru_ru!="lolol",zozo=~"sssss"}  |~"atltlt" !~   "rmrmrm" [5m])')
   expect(res.rootToken.Children('log_stream_selector_rule').map(c => c.value)).toEqual(

diff --git a/test/plugins.test.js b/test/plugins.test.js
@@ -3,3 +3,7 @@ const { getPlg } = require('../plugins/engine')
 it('should glob', () => {
   expect(getPlg({ type: 'unwrap_registry' })).toBeTruthy()
 })
+
+it('should unicode chars', () => {
+  console.log('АąŚĄ'.match(/\p{L}/ug))
+})