From 89c04a0e144c5b70b9548d8f0cc7e80a82cd8eb6 Mon Sep 17 00:00:00 2001 From: Stephen Amar Date: Wed, 18 Dec 2024 13:50:23 -0800 Subject: [PATCH] Add std.native & move std.xz/gzip to this (#241) - Add support std.native - Move our non-standard std.xz/std.gzip to this. - Add gzip support to scala native. --- sjsonnet/src-native/sjsonnet/Platform.scala | 22 +++++-- sjsonnet/src/sjsonnet/ReadWriter.scala | 4 ++ sjsonnet/src/sjsonnet/Std.scala | 66 +++++++++++-------- sjsonnet/test/src-jvm-native/ErrorTests.scala | 5 +- sjsonnet/test/src-jvm/sjsonnet/Example.java | 4 +- .../test/src-jvm/sjsonnet/StdGzipTests.scala | 10 +++ .../test/src-jvm/sjsonnet/StdXzTests.scala | 11 +++- .../src-native/sjsonnet/StdGzipTests.scala | 16 +++++ 8 files changed, 101 insertions(+), 37 deletions(-) create mode 100644 sjsonnet/test/src-native/sjsonnet/StdGzipTests.scala diff --git a/sjsonnet/src-native/sjsonnet/Platform.scala b/sjsonnet/src-native/sjsonnet/Platform.scala index 0a88ce0f..dd88e7c8 100644 --- a/sjsonnet/src-native/sjsonnet/Platform.scala +++ b/sjsonnet/src-native/sjsonnet/Platform.scala @@ -1,12 +1,26 @@ package sjsonnet -import java.io.File + +import java.io.{ByteArrayOutputStream, File} +import java.util.Base64 +import java.util.zip.GZIPOutputStream + object Platform { - def gzipBytes(s: Array[Byte]): String = { - throw new Exception("GZip not implemented in Scala Native") + def gzipBytes(b: Array[Byte]): String = { + val outputStream: ByteArrayOutputStream = new ByteArrayOutputStream(b.length) + val gzip: GZIPOutputStream = new GZIPOutputStream(outputStream) + try { + gzip.write(b) + } finally { + gzip.close() + outputStream.close() + } + Base64.getEncoder.encodeToString(outputStream.toByteArray) } + def gzipString(s: String): String = { - throw new Exception("GZip not implemented in Scala Native") + gzipBytes(s.getBytes()) } + def xzBytes(s: Array[Byte], compressionLevel: Option[Int]): String = { throw new Exception("XZ not implemented in Scala Native") } diff --git a/sjsonnet/src/sjsonnet/ReadWriter.scala b/sjsonnet/src/sjsonnet/ReadWriter.scala index 82e38eeb..46dec93a 100644 --- a/sjsonnet/src/sjsonnet/ReadWriter.scala +++ b/sjsonnet/src/sjsonnet/ReadWriter.scala @@ -44,4 +44,8 @@ object ReadWriter{ def apply(t: Val) = t.asFunc def write(pos: Position, t: Val.Func) = t } + implicit object BuiltinRead extends ReadWriter[Val.Builtin] { + def apply(t: Val) = t.asInstanceOf[Val.Builtin] + def write(pos: Position, t: Val.Builtin) = t + } } diff --git a/sjsonnet/src/sjsonnet/Std.scala b/sjsonnet/src/sjsonnet/Std.scala index 5d5a76f6..ff217e1b 100644 --- a/sjsonnet/src/sjsonnet/Std.scala +++ b/sjsonnet/src/sjsonnet/Std.scala @@ -16,11 +16,39 @@ import scala.util.matching.Regex * in Scala code. Uses `builtin` and other helpers to handle the common wrapper * logic automatically */ -class Std { +class Std(private val additionalNativeFunctions: Map[String, Val.Builtin] = Map.empty) { private val dummyPos: Position = new Position(null, 0) private val emptyLazyArray = new Array[Lazy](0) private val leadingWhiteSpacePattern = Pattern.compile("^[ \t\n\f\r\u0085\u00A0']+") private val trailingWhiteSpacePattern = Pattern.compile("[ \t\n\f\r\u0085\u00A0']+$") + private val oldNativeFunctions = Map( + builtin("gzip", "v"){ (_, _, v: Val) => + v match{ + case Val.Str(_, value) => Platform.gzipString(value) + case arr: Val.Arr => Platform.gzipBytes(arr.iterator.map(_.cast[Val.Num].value.toByte).toArray) + case x => Error.fail("Cannot gzip encode " + x.prettyName) + } + }, + + builtinWithDefaults("xz", "v" -> null, "compressionLevel" -> Val.Null(dummyPos)){ (args, pos, ev) => + val compressionLevel: Option[Int] = args(1) match { + case Val.Null(_) => + // Use default compression level if the user didn't set one + None + case Val.Num(_, n) => + Some(n.toInt) + case x => + Error.fail("Cannot xz encode with compression level " + x.prettyName) + } + args(0) match { + case Val.Str(_, value) => Platform.xzString(value, compressionLevel) + case arr: Val.Arr => Platform.xzBytes(arr.iterator.map(_.cast[Val.Num].value.toByte).toArray, compressionLevel) + case x => Error.fail("Cannot xz encode " + x.prettyName) + } + }, + ) + require(oldNativeFunctions.forall(k => !additionalNativeFunctions.contains(k._1)), "Conflicting native functions") + private val nativeFunctions = oldNativeFunctions ++ additionalNativeFunctions private object AssertEqual extends Val.Builtin2("assertEqual", "a", "b") { def evalRhs(v1: Val, v2: Val, ev: EvalScope, pos: Position): Val = { @@ -1285,31 +1313,6 @@ class Std { new Val.Arr(pos, Base64.getDecoder().decode(s).map(i => Val.Num(pos, i))) }, - builtin("gzip", "v"){ (pos, ev, v: Val) => - v match{ - case Val.Str(_, value) => Platform.gzipString(value) - case arr: Val.Arr => Platform.gzipBytes(arr.iterator.map(_.cast[Val.Num].value.toByte).toArray) - case x => Error.fail("Cannot gzip encode " + x.prettyName) - } - }, - - builtinWithDefaults("xz", "v" -> null, "compressionLevel" -> Val.Null(dummyPos)){ (args, pos, ev) => - val compressionLevel: Option[Int] = args(1) match { - case Val.Null(_) => - // Use default compression level if the user didn't set one - None - case Val.Num(_, n) => - Some(n.toInt) - case x => - Error.fail("Cannot xz encode with compression level " + x.prettyName) - } - args(0) match { - case Val.Str(_, value) => Platform.xzString(value, compressionLevel) - case arr: Val.Arr => Platform.xzBytes(arr.iterator.map(_.cast[Val.Num].value.toByte).toArray, compressionLevel) - case x => Error.fail("Cannot xz encode " + x.prettyName) - } - }, - builtin(EncodeUTF8), builtin(DecodeUTF8), @@ -1511,8 +1514,15 @@ class Std { builtin(MaxArray), builtin("primitiveEquals", "x", "y") { (_, ev, x: Val, y: Val) => x.isInstanceOf[y.type] && ev.compare(x, y) == 0 - } - ) + }, + builtin("native", "name") { (pos, ev, name: String) => + if (nativeFunctions.contains(name)) { + nativeFunctions(name) + } else { + Error.fail("Native function " + name + " not found", pos)(ev) + } + }, + ) ++ oldNativeFunctions private def toSetArrOrString(args: Array[Val], idx: Int, pos: Position, ev: EvalScope) = { args(idx) match { diff --git a/sjsonnet/test/src-jvm-native/ErrorTests.scala b/sjsonnet/test/src-jvm-native/ErrorTests.scala index 5fede204..2f521750 100644 --- a/sjsonnet/test/src-jvm-native/ErrorTests.scala +++ b/sjsonnet/test/src-jvm-native/ErrorTests.scala @@ -259,9 +259,8 @@ object ErrorTests extends TestSuite{ ) } test("native_not_found") - check( - """sjsonnet.Error: Field does not exist: native - | at [Select native].(sjsonnet/test/resources/test_suite/error.native_not_found.jsonnet:17:4) - | at [Apply1].(sjsonnet/test/resources/test_suite/error.native_not_found.jsonnet:17:11) + """sjsonnet.Error: Native function non_existent_native not found + | at [std.native].(sjsonnet/test/resources/test_suite/error.native_not_found.jsonnet:17:11) |""".stripMargin ) test("obj_assert") - { diff --git a/sjsonnet/test/src-jvm/sjsonnet/Example.java b/sjsonnet/test/src-jvm/sjsonnet/Example.java index 44499f68..a171c3c8 100644 --- a/sjsonnet/test/src-jvm/sjsonnet/Example.java +++ b/sjsonnet/test/src-jvm/sjsonnet/Example.java @@ -1,5 +1,7 @@ package sjsonnet; +import scala.collection.immutable.Map$; + public class Example { public void example(){ sjsonnet.SjsonnetMain.main0( @@ -11,7 +13,7 @@ public void example(){ os.package$.MODULE$.pwd(), scala.None$.empty(), scala.None$.empty(), - new sjsonnet.Std().Std() + new sjsonnet.Std(Map$.MODULE$.empty()).Std() ); } } diff --git a/sjsonnet/test/src-jvm/sjsonnet/StdGzipTests.scala b/sjsonnet/test/src-jvm/sjsonnet/StdGzipTests.scala index 5f759682..c08d890a 100644 --- a/sjsonnet/test/src-jvm/sjsonnet/StdGzipTests.scala +++ b/sjsonnet/test/src-jvm/sjsonnet/StdGzipTests.scala @@ -16,6 +16,16 @@ object StdGzipTests extends TestSuite { case s if s >= 16 => "H4sIAAAAAAAA/8vIBACsKpPYAgAAAA==" case _ => "H4sIAAAAAAAAAMvIBACsKpPYAgAAAA==" }) + eval("""std.native('gzip')([1, 2])""") ==> ujson.Str(Runtime.version().feature() match { + // https://bugs.openjdk.org/browse/JDK-8244706 + case s if s >= 16 => "H4sIAAAAAAAA/2NkAgCSQsy2AgAAAA==" + case _ => "H4sIAAAAAAAAAGNkAgCSQsy2AgAAAA==" + }) + eval("""std.native('gzip')("hi")""") ==> ujson.Str(Runtime.version().feature() match { + // https://bugs.openjdk.org/browse/JDK-8244706 + case s if s >= 16 => "H4sIAAAAAAAA/8vIBACsKpPYAgAAAA==" + case _ => "H4sIAAAAAAAAAMvIBACsKpPYAgAAAA==" + }) } } } diff --git a/sjsonnet/test/src-jvm/sjsonnet/StdXzTests.scala b/sjsonnet/test/src-jvm/sjsonnet/StdXzTests.scala index 4c4a3a06..75fa7e1e 100644 --- a/sjsonnet/test/src-jvm/sjsonnet/StdXzTests.scala +++ b/sjsonnet/test/src-jvm/sjsonnet/StdXzTests.scala @@ -10,11 +10,20 @@ object StdXzTests extends TestSuite { eval("""std.xz("hi")""") ==> ujson.Str("/Td6WFoAAATm1rRGAgAhARYAAAB0L+WjAQABaGkAAAD+qTgRvMqlSAABGgLcLqV+H7bzfQEAAAAABFla") eval("""std.xz([1, 2], compressionLevel = 0)""") ==> ujson.Str("/Td6WFoAAATm1rRGAgAhAQwAAACPmEGcAQABAQIAAADRC9qlUgJ94gABGgLcLqV+H7bzfQEAAAAABFla") eval("""std.xz("hi", compressionLevel = 1)""") ==> ujson.Str("/Td6WFoAAATm1rRGAgAhARAAAACocI6GAQABaGkAAAD+qTgRvMqlSAABGgLcLqV+H7bzfQEAAAAABFla") - val ex = intercept[Exception] { + var ex = intercept[Exception] { // Compression level 10 is invalid eval("""std.xz("hi", 10)""") } assert(ex.getMessage.contains("Unsupported preset: 10")) + eval("""std.native('xz')([1, 2])""") ==> ujson.Str("/Td6WFoAAATm1rRGAgAhARYAAAB0L+WjAQABAQIAAADRC9qlUgJ94gABGgLcLqV+H7bzfQEAAAAABFla") + eval("""std.native('xz')("hi")""") ==> ujson.Str("/Td6WFoAAATm1rRGAgAhARYAAAB0L+WjAQABaGkAAAD+qTgRvMqlSAABGgLcLqV+H7bzfQEAAAAABFla") + eval("""std.native('xz')([1, 2], compressionLevel = 0)""") ==> ujson.Str("/Td6WFoAAATm1rRGAgAhAQwAAACPmEGcAQABAQIAAADRC9qlUgJ94gABGgLcLqV+H7bzfQEAAAAABFla") + eval("""std.native('xz')("hi", compressionLevel = 1)""") ==> ujson.Str("/Td6WFoAAATm1rRGAgAhARAAAACocI6GAQABaGkAAAD+qTgRvMqlSAABGgLcLqV+H7bzfQEAAAAABFla") + ex = intercept[Exception] { + // Compression level 10 is invalid + eval("""std.native('xz')("hi", 10)""") + } + assert(ex.getMessage.contains("Unsupported preset: 10")) } } } diff --git a/sjsonnet/test/src-native/sjsonnet/StdGzipTests.scala b/sjsonnet/test/src-native/sjsonnet/StdGzipTests.scala new file mode 100644 index 00000000..bebdef6b --- /dev/null +++ b/sjsonnet/test/src-native/sjsonnet/StdGzipTests.scala @@ -0,0 +1,16 @@ +package sjsonnet + +import sjsonnet.TestUtils.eval +import utest._ + +object StdGzipTests extends TestSuite { + val tests = Tests { + test("gzip"){ + eval("""std.gzip([1, 2])""") ==> ujson.Str("H4sIAAAAAAAAAGNkAgCSQsy2AgAAAA==") + eval("""std.gzip("hi")""") ==> ujson.Str("H4sIAAAAAAAAAMvIBACsKpPYAgAAAA==") + eval("""std.native('gzip')([1, 2])""") ==> ujson.Str("H4sIAAAAAAAAAGNkAgCSQsy2AgAAAA==") + eval("""std.native('gzip')("hi")""") ==> ujson.Str("H4sIAAAAAAAAAMvIBACsKpPYAgAAAA==") + } + } +} +