From e044f86981c2f0c8d7b88d6b353537c85205a2be Mon Sep 17 00:00:00 2001 From: Jens Wille Date: Tue, 12 Jul 2022 14:44:30 +0200 Subject: [PATCH] Add `flatten()` Fix function. (#122) --- README.md | 8 + .../org/metafacture/metafix/FixMethod.java | 8 + .../metafacture/metafix/api/FixFunction.java | 7 + .../metafix/MetafixMethodTest.java | 150 ++++++++++++++++++ .../method/fromJson/toJson/flatten/todo.txt | 1 - 5 files changed, 173 insertions(+), 1 deletion(-) delete mode 100644 metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flatten/todo.txt diff --git a/README.md b/README.md index 170c5a52..151568ad 100644 --- a/README.md +++ b/README.md @@ -457,6 +457,14 @@ Only keeps field values that match the regular expression pattern. filter("", "") ``` +##### `flatten` + +Flattens a nested array field. + +```perl +flatten("") +``` + ##### `index` Returns the index position of a substring in a field and replaces the field value with this number. diff --git a/metafix/src/main/java/org/metafacture/metafix/FixMethod.java b/metafix/src/main/java/org/metafacture/metafix/FixMethod.java index 5c96bd79..4932dfba 100644 --- a/metafix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/metafix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -389,6 +389,14 @@ public void apply(final Metafix metafix, final Record record, final List ); } }, + flatten { + @Override + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + record.transform(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(newArray(flatten(a.stream())))) + ); + } + }, index { @Override public void apply(final Metafix metafix, final Record record, final List params, final Map options) { diff --git a/metafix/src/main/java/org/metafacture/metafix/api/FixFunction.java b/metafix/src/main/java/org/metafacture/metafix/api/FixFunction.java index 865958e8..95736293 100644 --- a/metafix/src/main/java/org/metafacture/metafix/api/FixFunction.java +++ b/metafix/src/main/java/org/metafacture/metafix/api/FixFunction.java @@ -55,4 +55,11 @@ default Stream unique(final Stream stream) { return stream.filter(set::add); } + default Stream flatten(final Stream stream) { + return stream.flatMap(v -> v.extractType((m, c) -> m + .ifArray(a -> c.accept(flatten(a.stream()))) + .orElse(w -> c.accept(Stream.of(w))) + )); + } + } diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java index ab77e3f4..49bdbe5b 100644 --- a/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java @@ -1109,6 +1109,156 @@ public void shouldFilterArrayObjectValues() { ); } + @Test + public void shouldFlattenFlatArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "flatten('flat')" + ), + i -> { + i.startRecord("1"); + i.literal("flat", "1"); + i.literal("flat", "2"); + i.literal("flat", "3"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("flat", "1"); + o.get().literal("flat", "2"); + o.get().literal("flat", "3"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldFlattenFlatHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "flatten('flat')" + ), + i -> { + i.startRecord("1"); + i.startEntity("flat"); + i.literal("a", "1"); + i.literal("a", "2"); + i.literal("a", "3"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("flat"); + o.get().literal("a", "1"); + o.get().literal("a", "2"); + o.get().literal("a", "3"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldFlattenFlatString() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "flatten('flat')" + ), + i -> { + i.startRecord("1"); + i.literal("flat", "1"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("flat", "1"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldFlattenNestedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "flatten('deep[]')" + ), + i -> { + i.startRecord("1"); + i.startEntity("deep[]"); + i.literal("1", "1"); + i.startEntity("2[]"); + i.literal("1", "2"); + i.literal("2", "3"); + i.endEntity(); + i.startEntity("3[]"); + i.startEntity("1[]"); + i.literal("1", "4"); + i.literal("2", "5"); + i.endEntity(); + i.literal("2", "6"); + i.endEntity(); + i.literal("4", "7"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("deep[]"); + o.get().literal("1", "1"); + o.get().literal("2", "2"); + o.get().literal("3", "3"); + o.get().literal("4", "4"); + o.get().literal("5", "5"); + o.get().literal("6", "6"); + o.get().literal("7", "7"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldFlattenNestedArrayWithHashes() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "flatten('deep[]')" + ), + i -> { + i.startRecord("1"); + i.startEntity("deep[]"); + i.literal("1", "1"); + i.startEntity("2"); + i.literal("a", "2"); + i.literal("a", "3"); + i.endEntity(); + i.startEntity("3[]"); + i.startEntity("1"); + i.literal("a", "4"); + i.literal("a", "5"); + i.endEntity(); + i.literal("2", "6"); + i.endEntity(); + i.literal("4", "7"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("deep[]"); + o.get().literal("1", "1"); + o.get().startEntity("2"); + o.get().literal("a", "2"); + o.get().literal("a", "3"); + o.get().endEntity(); + o.get().startEntity("3"); + o.get().literal("a", "4"); + o.get().literal("a", "5"); + o.get().endEntity(); + o.get().literal("4", "6"); + o.get().literal("5", "7"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + @Test public void shouldGetFirstIndexOfSubstring() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( diff --git a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flatten/todo.txt b/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flatten/todo.txt deleted file mode 100644 index 1b5e6dca..00000000 --- a/metafix/src/test/resources/org/metafacture/metafix/integration/method/fromJson/toJson/flatten/todo.txt +++ /dev/null @@ -1 +0,0 @@ -See issue #122