diff --git a/build.gradle b/build.gradle index 0b9e46ac..855ecc08 100644 --- a/build.gradle +++ b/build.gradle @@ -23,6 +23,7 @@ subprojects { ext { versions = [ 'ace': '1.3.3', + 'equalsverifier': '3.8.2', 'jetty': '9.4.14.v20181114', 'jquery': '3.3.1-1', 'junit_jupiter': '5.4.2', diff --git a/metafix/build.gradle b/metafix/build.gradle index 07cc980a..ad03509d 100644 --- a/metafix/build.gradle +++ b/metafix/build.gradle @@ -23,6 +23,7 @@ dependencies { implementation "org.metafacture:metafacture-flowcontrol:${versions.metafacture}" implementation "org.metafacture:metamorph:${versions.metafacture}" + testImplementation "nl.jqno.equalsverifier:equalsverifier:${versions.equalsverifier}" testImplementation "org.mockito:mockito-core:${versions.mockito}" testImplementation "org.mockito:mockito-junit-jupiter:${versions.mockito}" } diff --git a/metafix/src/main/java/org/metafacture/metafix/FixMethod.java b/metafix/src/main/java/org/metafacture/metafix/FixMethod.java index b5b51a6b..3ad98280 100644 --- a/metafix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/metafix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -20,16 +20,30 @@ import org.metafacture.metamorph.maps.FileMap; import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.UnaryOperator; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.stream.Stream; enum FixMethod { // SCRIPT-LEVEL METHODS: + nothing { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + // do nothing + } + }, put_filemap { public void apply(final Metafix metafix, final Record record, final List params, final Map options) { final String fileName = params.get(0); @@ -99,7 +113,7 @@ public void apply(final Metafix metafix, final Record record, final List record.getList(field, a -> record.put(field, Value.newHash(h -> { for (int i = 1; i < a.size(); i = i + 2) { - h.put(a.get(i - 1).toString(), a.get(i)); + h.put(a.get(i - 1).asString(), a.get(i)); } }))); } @@ -116,7 +130,7 @@ public void apply(final Metafix metafix, final Record record, final List record.getList(field, a -> a.forEach(v -> { final Pattern p = Pattern.compile(params.get(1)); - final Matcher m = p.matcher(v.toString()); + final Matcher m = p.matcher(v.asString()); if (m.matches()) { record.remove(field); @@ -153,13 +167,21 @@ public void apply(final Metafix metafix, final Record record, final List record.replace(params.get(0), params.subList(1, params.size()).stream() .filter(f -> literalString(f) || record.find(f) != null) .map(f -> literalString(f) ? new Value(f.substring(1)) : record.findList(f, null).asArray().get(0)) - .map(Value::toString).collect(Collectors.joining(joinChar != null ? joinChar : " "))); + .map(Value::asString).collect(Collectors.joining(joinChar != null ? joinChar : " "))); } private boolean literalString(final String s) { return s.startsWith("~"); } }, + random { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String field = params.get(0); + final int max = getInteger(params, 1); + + record.replace(field, String.valueOf(RANDOM.nextInt(max))); + } + }, reject { public void apply(final Metafix metafix, final Record record, final List params, final Map options) { record.setReject(true); @@ -170,6 +192,36 @@ public void apply(final Metafix metafix, final Record record, final List params.forEach(record::removeNested); } }, + rename { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String search = params.get(1); + final String replace = params.get(2); + + final UnaryOperator operator = s -> s.replaceAll(search, replace); + + record.transformField(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(renameArray(a, operator))) + .ifHash(h -> c.accept(renameHash(h, operator))) + .orElseThrow() + ); + } + + private Value renameArray(final Value.Array array, final UnaryOperator operator) { + return Value.newArray(a -> array.forEach(v -> a.add(renameValue(v, operator)))); + } + + private Value renameHash(final Value.Hash hash, final UnaryOperator operator) { + return Value.newHash(h -> hash.forEach((f, v) -> h.put(operator.apply(f), renameValue(v, operator)))); + } + + private Value renameValue(final Value value, final UnaryOperator operator) { + return value.extractType((m, c) -> m + .ifArray(a -> c.accept(renameArray(a, operator))) + .ifHash(h -> c.accept(renameHash(h, operator))) + .orElse(c) + ); + } + }, retain { public void apply(final Metafix metafix, final Record record, final List params, final Map options) { record.retainFields(params); @@ -183,16 +235,13 @@ public void apply(final Metafix metafix, final Record record, final List record.addAll(field.replace(DOT_APPEND, EMPTY), toAdd); } else { - record.put(field, Value.newArray(a -> toAdd.forEach(s -> a.add(new Value(s))))); + record.put(field, newArray(toAdd.stream().map(Value::new))); } } }, set_field { public void apply(final Metafix metafix, final Record record, final List params, final Map options) { - final String field = params.get(0); - - record.remove(field); - record.replace(field, params.get(1)); + record.replace(params.get(0), params.get(1)); } }, set_hash { @@ -220,16 +269,56 @@ public void apply(final Metafix metafix, final Record record, final List // TODO SPEC: switch to morph-style named params in general? + append { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String value = params.get(1); + record.transformFields(params, s -> s + value); + } + }, capitalize { public void apply(final Metafix metafix, final Record record, final List params, final Map options) { record.transformFields(params, s -> s.substring(0, 1).toUpperCase() + s.substring(1)); } }, + count { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + record.transformField(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(new Value(a.size()))) + .ifHash(h -> c.accept(new Value(h.size()))) + ); + } + }, downcase { public void apply(final Metafix metafix, final Record record, final List params, final Map options) { record.transformFields(params, String::toLowerCase); } }, + filter { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final Pattern search = Pattern.compile(params.get(1)); + final boolean invert = getBoolean(options, "invert"); + + final Predicate predicate = s -> search.matcher(s.asString()).find(); + + record.transformField(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(newArray(a.stream().filter(invert ? predicate.negate() : predicate)))) + ); + } + }, + index { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String search = params.get(1); + record.transformFields(params, s -> String.valueOf(s.indexOf(search))); // TODO: multiple + } + }, + join_field { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String joinChar = params.size() > 1 ? params.get(1) : ""; + record.transformField(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(new Value(a.stream().map(Value::asString).collect(Collectors.joining(joinChar))))) + ); + } + }, lookup { public void apply(final Metafix metafix, final Record record, final List params, final Map options) { final Map map; @@ -251,9 +340,73 @@ public void apply(final Metafix metafix, final Record record, final List record.transformFields(params, k -> map.getOrDefault(k, defaultValue)); } }, + prepend { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String value = params.get(1); + record.transformFields(params, s -> value + s); + } + }, + replace_all { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String search = params.get(1); + final String replace = params.get(2); + + record.transformFields(params, s -> s.replaceAll(search, replace)); + } + }, + reverse { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + record.transformField(params.get(0), (m, c) -> m + .ifArray(a -> { + final List list = a.stream().collect(Collectors.toList()); + Collections.reverse(list); + c.accept(new Value(list)); + }) + .ifString(s -> c.accept(new Value(new StringBuilder(s).reverse().toString()))) + ); + } + }, + sort_field { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final boolean numeric = getBoolean(options, "numeric"); + final boolean reverse = getBoolean(options, "reverse"); + final boolean uniq = getBoolean(options, "uniq"); + + final Function function = Value::asString; + final Comparator comparator = numeric ? + Comparator.comparing(function.andThen(Integer::parseInt)) : Comparator.comparing(function); + + record.transformField(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(new Value((uniq ? unique(a.stream()) : a.stream()) + .sorted(reverse ? comparator.reversed() : comparator).collect(Collectors.toList())))) + ); + } + }, + split_field { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String splitChar = params.size() > 1 ? params.get(1) : "\\s+"; + final Pattern splitPattern = Pattern.compile(splitChar); + + final Function splitFunction = s -> + newArray(Arrays.stream(splitPattern.split(s)).map(Value::new)); + + record.transformField(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(newArray(a.stream().map(Value::asString).map(splitFunction)))) + .ifHash(h -> c.accept(Value.newHash(n -> h.forEach((f, w) -> n.put(f, splitFunction.apply(w.asString())))))) + .ifString(s -> c.accept(splitFunction.apply(s))) + ); + } + }, substring { public void apply(final Metafix metafix, final Record record, final List params, final Map options) { - record.transformFields(params, s -> s.substring(Integer.parseInt(params.get(1)), Integer.parseInt(params.get(2)) - 1)); + record.transformFields(params, s -> s.substring(getInteger(params, 1), getInteger(params, 2) - 1)); + } + }, + sum { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + record.transformField(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(new Value(a.stream().map(Value::asString).mapToInt(Integer::parseInt).sum()))) + ); } }, trim { @@ -261,6 +414,13 @@ public void apply(final Metafix metafix, final Record record, final List record.transformFields(params, String::trim); } }, + uniq { + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + record.transformField(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(newArray(unique(a.stream())))) + ); + } + }, upcase { public void apply(final Metafix metafix, final Record record, final List params, final Map options) { record.transformFields(params, String::toUpperCase); @@ -275,6 +435,25 @@ public void apply(final Metafix metafix, final Record record, final List private static final String FILEMAP_SEPARATOR_OPTION = "sep_char"; private static final String FILEMAP_DEFAULT_SEPARATOR = ","; + private static final Random RANDOM = new Random(); + + private static boolean getBoolean(final Map options, final String key) { + return Boolean.parseBoolean(options.get(key)); + } + + private static int getInteger(final List params, final int index) { + return Integer.parseInt(params.get(index)); + } + + private static Value newArray(final Stream stream) { + return Value.newArray(a -> stream.forEach(a::add)); + } + + private static Stream unique(final Stream stream) { + final Set set = new HashSet<>(); + return stream.filter(set::add); + } + abstract void apply(Metafix metafix, Record record, List params, Map options); } diff --git a/metafix/src/main/java/org/metafacture/metafix/Metafix.java b/metafix/src/main/java/org/metafacture/metafix/Metafix.java index aeb1dbb5..07c3283f 100644 --- a/metafix/src/main/java/org/metafacture/metafix/Metafix.java +++ b/metafix/src/main/java/org/metafacture/metafix/Metafix.java @@ -132,7 +132,7 @@ public void startRecord(final String identifier) { public void endRecord() { entityCountStack.removeLast(); if (!entityCountStack.isEmpty()) { - throw new IllegalStateException(ENTITIES_NOT_BALANCED); + throw new MetafactureException(new IllegalStateException(ENTITIES_NOT_BALANCED)); } flattener.endRecord(); LOG.debug("End record, walking fix: {}", currentRecord); @@ -153,9 +153,11 @@ private void emit(final String field, final Value value) { } for (int i = 0; i < array.size(); ++i) { + final Value currentValue = array.get(i); final String fieldName = isMulti ? String.valueOf(i + 1) : field; - array.get(i).matchType() + currentValue.matchType() + .ifArray(a -> emit(fieldName, currentValue)) .ifHash(h -> { outputStreamReceiver.startEntity(fieldName); h.forEach(this::emit); diff --git a/metafix/src/main/java/org/metafacture/metafix/Value.java b/metafix/src/main/java/org/metafacture/metafix/Value.java index 17dfef5a..653a2980 100644 --- a/metafix/src/main/java/org/metafacture/metafix/Value.java +++ b/metafix/src/main/java/org/metafacture/metafix/Value.java @@ -17,6 +17,7 @@ package org.metafacture.metafix; import org.metafacture.commons.tries.SimpleRegexTrie; +import org.metafacture.framework.MetafactureException; import java.util.ArrayList; import java.util.Arrays; @@ -26,6 +27,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; @@ -105,6 +107,10 @@ public Value(final String string) { this.string = string; } + public Value(final int integer) { + this(String.valueOf(integer)); + } + public static Value newArray() { return newArray(null); } @@ -205,12 +211,37 @@ public TypeMatcher matchType() { return new TypeMatcher(this); } - private T extractType(final BiConsumer> consumer) { + public T extractType(final BiConsumer> consumer) { final AtomicReference result = new AtomicReference<>(); consumer.accept(matchType(), result::set); return result.get(); } + @Override + public final boolean equals(final Object object) { + if (object == this) { + return true; + } + + if (!(object instanceof Value)) { + return false; + } + + final Value other = (Value) object; + return Objects.equals(type, other.type) && + Objects.equals(array, other.array) && + Objects.equals(hash, other.hash) && + Objects.equals(string, other.string); + } + + @Override + public final int hashCode() { + return Objects.hashCode(type) + + Objects.hashCode(array) + + Objects.hashCode(hash) + + Objects.hashCode(string); + } + @Override public String toString() { return isNull() ? null : extractType((m, c) -> m @@ -268,7 +299,7 @@ public void orElse(final Consumer consumer) { public void orElseThrow() { orElse(v -> { final String types = expected.stream().map(Type::name).collect(Collectors.joining(" or ")); - throw new IllegalStateException("expected " + types + ", got " + value.type); + throw new MetafactureException(new IllegalStateException("expected " + types + ", got " + value.type)); }); } @@ -281,7 +312,7 @@ private TypeMatcher match(final Type type, final Consumer consumer, final return this; } else { - throw new IllegalStateException("already expecting " + type); + throw new MetafactureException(new IllegalStateException("already expecting " + type)); } } @@ -427,7 +458,7 @@ else if (isNumber(field)) { } } - list.removeIf(v -> v == null); + list.removeIf(v -> Value.isNull(v)); } private void transformFields(final int index, final String[] fields, final UnaryOperator operator) { @@ -784,6 +815,18 @@ private void appendValue(final String[] newName, final Value v) { } } + public void transformField(final String field, final BiConsumer> consumer) { + final Value oldValue = find(field); + + if (oldValue != null) { + final Value newValue = oldValue.extractType(consumer); + + if (newValue != null) { + insert(InsertMode.REPLACE, split(field), newValue); + } + } + } + public void transformFields(final List params, final UnaryOperator operator) { transformFields(split(params.get(0)), operator); } @@ -806,7 +849,9 @@ private void transformFields(final String[] fields, final UnaryOperator map.remove(f); if (operator != null) { - value.asList(a -> a.forEach(v -> append(f, operator.apply(v.toString())))); + value.matchType() + .ifString(s -> append(f, operator.apply(s))) + .orElseThrow(); } } else { diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java index bcebe58b..d901f7f6 100644 --- a/metafix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java @@ -39,6 +39,8 @@ public class MetafixLookupTest { private static final String CSV_MAP = "src/test/resources/org/metafacture/metafix/maps/test.csv"; private static final String TSV_MAP = "src/test/resources/org/metafacture/metafix/maps/test.tsv"; + private static final String LOOKUP = "lookup('title.*',"; + @Mock private StreamReceiver streamReceiver; @@ -48,14 +50,14 @@ public MetafixLookupTest() { @Test public void inline() { assertMap( - "lookup('title', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)" + LOOKUP + " Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)" ); } @Test public void inlineMultilineIndent() { assertMap( - "lookup('title',", + LOOKUP, " Aloha: Alohaeha,", " Moin: 'Moin zäme',", " __default: Tach)" @@ -65,7 +67,7 @@ public void inlineMultilineIndent() { @Test public void inlineDotNotationNested() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "lookup('data.title', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)" + "lookup('data.title.*', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)" ), i -> { i.startRecord("1"); @@ -92,14 +94,14 @@ public void inlineDotNotationNested() { @Test public void csv() { assertMap( - "lookup('title', '" + CSV_MAP + "')" + LOOKUP + " '" + CSV_MAP + "')" ); } @Test public void tsv() { assertMap( - "lookup('title', '" + TSV_MAP + "', sep_char:'\t')" + LOOKUP + " '" + TSV_MAP + "', sep_char:'\t')" ); } @@ -107,7 +109,7 @@ public void tsv() { public void shouldLookupInSeparateInternalMap() { assertMap( "put_map('testMap', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)", - "lookup('title', 'testMap')" + LOOKUP + " 'testMap')" ); } @@ -115,7 +117,7 @@ public void shouldLookupInSeparateInternalMap() { public void shouldLookupInSeparateExternalFileMap() { assertMap( "put_filemap('" + CSV_MAP + "')", - "lookup('title', '" + CSV_MAP + "')" + LOOKUP + " '" + CSV_MAP + "')" ); } @@ -123,7 +125,7 @@ public void shouldLookupInSeparateExternalFileMap() { public void shouldLookupInSeparateExternalFileMapWithName() { assertMap( "put_filemap('" + CSV_MAP + "', 'testMap')", - "lookup('title', 'testMap')" + LOOKUP + " 'testMap')" ); } @@ -131,7 +133,7 @@ public void shouldLookupInSeparateExternalFileMapWithName() { public void shouldLookupInSeparateExternalFileMapWithOptions() { assertMap( "put_filemap('" + TSV_MAP + "', sep_char: '\t')", - "lookup('title', '" + TSV_MAP + "')" + LOOKUP + " '" + TSV_MAP + "')" ); } @@ -139,7 +141,7 @@ public void shouldLookupInSeparateExternalFileMapWithOptions() { public void shouldLookupInSeparateExternalFileMapWithNameAndOptions() { assertMap( "put_filemap('" + TSV_MAP + "', 'testMap', sep_char: '\t')", - "lookup('title', 'testMap')" + LOOKUP + " 'testMap')" ); } @@ -148,7 +150,7 @@ public void shouldDefineMultipleSeparateMaps() { assertMap( "put_map('testMap', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)", "put_map('testMap2', __default: Hi)", - "lookup('title', 'testMap')" + LOOKUP + " 'testMap')" ); } @@ -157,7 +159,7 @@ public void shouldOverwriteExistingSeparateMap() { assertMap( "put_map('testMap', __default: Hi)", "put_filemap('" + CSV_MAP + "', 'testMap')", - "lookup('title', 'testMap')" + LOOKUP + " 'testMap')" ); } @@ -165,7 +167,7 @@ public void shouldOverwriteExistingSeparateMap() { public void shouldIgnoreOptionsOnLookupInSeparateInternalMap() { assertMap( "put_map('testMap', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)", - "lookup('title', 'testMap', __default: Hi)" + LOOKUP + " 'testMap', __default: Hi)" ); } @@ -173,14 +175,14 @@ public void shouldIgnoreOptionsOnLookupInSeparateInternalMap() { public void shouldIgnoreOptionsOnLookupInSeparateExternalFileMap() { assertMap( "put_filemap('" + CSV_MAP + "')", - "lookup('title', '" + CSV_MAP + "', sep_char: '\t')" + LOOKUP + " '" + CSV_MAP + "', sep_char: '\t')" ); } @Test public void shouldNotLookupInExternalFileMapWithWrongOptions() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "lookup('title', '" + CSV_MAP + "', sep_char: '\t')" + LOOKUP + " '" + CSV_MAP + "', sep_char: '\t')" ), i -> { i.startRecord("1"); @@ -191,6 +193,8 @@ public void shouldNotLookupInExternalFileMapWithWrongOptions() { }, o -> { o.get().startRecord("1"); + o.get().startEntity("title"); + o.get().endEntity(); o.get().endRecord(); } ); @@ -199,8 +203,8 @@ public void shouldNotLookupInExternalFileMapWithWrongOptions() { @Test public void shouldIgnoreOptionsOnSubsequentLookupInExternalFileMap() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "lookup('title', '" + CSV_MAP + "')", - "lookup('title', '" + CSV_MAP + "', sep_char: '\t')" + LOOKUP + " '" + CSV_MAP + "')", + LOOKUP + " '" + CSV_MAP + "', sep_char: '\t')" ), i -> { i.startRecord("1"); @@ -223,9 +227,9 @@ public void shouldIgnoreOptionsOnSubsequentLookupInExternalFileMap() { @Test public void shouldFailLookupInUnknownNamedMap() { - Assertions.assertThrows(MorphExecutionException.class, () -> + final Throwable exception = Assertions.assertThrows(MorphExecutionException.class, () -> MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "lookup('title', 'testMap')" + LOOKUP + " 'testMap')" ), i -> { i.startRecord("1"); @@ -236,9 +240,10 @@ public void shouldFailLookupInUnknownNamedMap() { }, o -> { } - ), - "File not found: testMap" + ) ); + + Assertions.assertEquals("File not found: testMap", exception.getMessage()); } private void assertMap(final String... fixDef) { diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java index dda5c6c9..d3b8fed2 100644 --- a/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java @@ -16,11 +16,10 @@ package org.metafacture.metafix; -import org.metafacture.framework.MetafactureException; import org.metafacture.framework.StreamReceiver; import com.google.common.collect.ImmutableMap; -import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -44,212 +43,196 @@ public MetafixMethodTest() { } @Test - public void upcase() { + public void shouldUpcaseString() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "upcase('title')"), + "upcase('title')" + ), i -> { i.startRecord("1"); - i.endRecord(); - - i.startRecord("2"); i.literal("title", "marc"); - i.literal("title", "json"); - i.endRecord(); - - i.startRecord("3"); i.endRecord(); - }, o -> { + }, + o -> { o.get().startRecord("1"); + o.get().literal("title", "MARC"); o.get().endRecord(); - - o.get().startRecord("2"); - o.get().startEntity("title"); - o.get().literal("1", "MARC"); - o.get().literal("2", "JSON"); - o.get().endEntity(); - o.get().endRecord(); - - o.get().startRecord("3"); - o.get().endRecord(); - }); + } + ); } @Test - public void upcaseDotNotationNested() { + public void shouldUpcaseDotNotationNested() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "upcase('data.title')"), + "upcase('data.title')" + ), i -> { i.startRecord("1"); i.startEntity("data"); i.literal("title", "marc"); - i.literal("title", "json"); i.endEntity(); i.endRecord(); - }, (o, f) -> { + }, + o -> { o.get().startRecord("1"); o.get().startEntity("data"); - o.get().startEntity("title"); - o.get().literal("1", "MARC"); - o.get().literal("2", "JSON"); - f.apply(2).endEntity(); + o.get().literal("title", "MARC"); + o.get().endEntity(); o.get().endRecord(); - }); + } + ); } @Test - public void downcase() { + public void shouldDowncaseString() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "downcase('title')"), + "downcase('title')" + ), i -> { i.startRecord("1"); + i.literal("title", "MARC"); i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "marc"); + o.get().endRecord(); + } + ); + } - i.startRecord("2"); + @Test + public void shouldDowncaseStringsInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "downcase('title.*')" + ), + i -> { + i.startRecord("1"); i.literal("title", "MARC"); i.literal("title", "Json"); i.endRecord(); - - i.startRecord("3"); - i.endRecord(); - }, o -> { + }, + o -> { o.get().startRecord("1"); - o.get().endRecord(); - - o.get().startRecord("2"); o.get().startEntity("title"); o.get().literal("1", "marc"); o.get().literal("2", "json"); o.get().endEntity(); o.get().endRecord(); - - o.get().startRecord("3"); - o.get().endRecord(); - }); + } + ); } @Test - public void capitalize() { + public void shouldCapitalizeString() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "capitalize('title')"), + "capitalize('title')" + ), i -> { i.startRecord("1"); + i.literal("title", "marc"); i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "Marc"); + o.get().endRecord(); + } + ); + } - i.startRecord("2"); + @Test + public void shouldCapitalizeStringsInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "capitalize('title.*')" + ), + i -> { + i.startRecord("1"); i.literal("title", "marc"); i.literal("title", "json"); i.endRecord(); - - i.startRecord("3"); - i.endRecord(); - }, o -> { + }, + o -> { o.get().startRecord("1"); - o.get().endRecord(); - - o.get().startRecord("2"); o.get().startEntity("title"); o.get().literal("1", "Marc"); o.get().literal("2", "Json"); o.get().endEntity(); o.get().endRecord(); + } + ); + } - o.get().startRecord("3"); - o.get().endRecord(); - }); + @Test + public void shouldNotCapitalizeArray() { + MetafixTestHelpers.assertThrows(IllegalStateException.class, "expected String, got Array", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "capitalize('title')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "marc"); + i.literal("title", "json"); + i.endRecord(); + }, + o -> { + } + ) + ); } @Test - public void substring() { + public void shouldGetSubstringOfString() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "substring('title', '0', '2')"), + "substring('title', '0', '2')" + ), i -> { i.startRecord("1"); - i.endRecord(); - - i.startRecord("2"); i.literal("title", "marc"); - i.literal("title", "json"); - i.endRecord(); - - i.startRecord("3"); i.endRecord(); - }, o -> { + }, + o -> { o.get().startRecord("1"); + o.get().literal("title", "m"); o.get().endRecord(); - - o.get().startRecord("2"); - o.get().startEntity("title"); - o.get().literal("1", "m"); - o.get().literal("2", "j"); - o.get().endEntity(); - o.get().endRecord(); - - o.get().startRecord("3"); - o.get().endRecord(); - }); + } + ); } @Test - public void substringWithVar() { + public void shouldGetSubstringWithVar() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "substring('title', '0', '$[end]')"), - ImmutableMap.of("end", "3"), + "substring('title', '0', '$[end]')" + ), + ImmutableMap.of("end", "3"), i -> { i.startRecord("1"); - i.endRecord(); - - i.startRecord("2"); i.literal("title", "marc"); - i.literal("title", "json"); - i.endRecord(); - - i.startRecord("3"); i.endRecord(); - }, o -> { + }, + o -> { o.get().startRecord("1"); + o.get().literal("title", "ma"); o.get().endRecord(); - - o.get().startRecord("2"); - o.get().startEntity("title"); - o.get().literal("1", "ma"); - o.get().literal("2", "js"); - o.get().endEntity(); - o.get().endRecord(); - - o.get().startRecord("3"); - o.get().endRecord(); - }); + } + ); } @Test - public void trim() { + public void shouldTrimString() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "trim('title')"), + "trim('title')" + ), i -> { i.startRecord("1"); - i.endRecord(); - - i.startRecord("2"); i.literal("title", " marc "); - i.literal("title", " json "); - i.endRecord(); - - i.startRecord("3"); i.endRecord(); - }, o -> { + }, + o -> { o.get().startRecord("1"); + o.get().literal("title", "marc"); o.get().endRecord(); - - o.get().startRecord("2"); - o.get().startEntity("title"); - o.get().literal("1", "marc"); - o.get().literal("2", "json"); - o.get().endEntity(); - o.get().endRecord(); - - o.get().startRecord("3"); - o.get().endRecord(); - }); + } + ); } @Test @@ -351,7 +334,7 @@ public void parseTextMixedGroups() { @Test public void parseTextEscapedGroups() { - Assertions.assertThrows(MetafactureException.class, () -> + MetafixTestHelpers.assertThrows(IllegalArgumentException.class, "No group with name ", () -> MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "parse_text(data, '(?.)(.)\\\\(?.\\\\)')" ), @@ -362,14 +345,13 @@ public void parseTextEscapedGroups() { }, o -> { } - ), - "No group with name " + ) ); } @Test public void parseTextQuotedGroups() { - Assertions.assertThrows(MetafactureException.class, () -> + MetafixTestHelpers.assertThrows(IllegalArgumentException.class, "No group with name ", () -> MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "parse_text(data, '(?.)(.)\\\\Q(?.)\\\\E')" ), @@ -380,8 +362,7 @@ public void parseTextQuotedGroups() { }, o -> { } - ), - "No group with name " + ) ); } @@ -471,4 +452,873 @@ public void characterClass() { o.get().endRecord(); }); } + + @Test + public void shouldDoNothing() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "nothing()" + ), + i -> { + i.startRecord("1"); + i.literal("title", "marc"); + i.literal("title", "json"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("title"); + o.get().literal("1", "marc"); + o.get().literal("2", "json"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldAppendValue() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "append(title, ' ?!')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "metafix"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "metafix ?!"); + o.get().endRecord(); + } + ); + } + + @Test + // TODO: Fix order (`animols.name` should stay before `animols.type`) + public void shouldAppendValueInHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "append(animols.name, ' boss')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animols"); + i.literal("name", "bird"); + i.literal("type", "TEST"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animols"); + o.get().literal("type", "TEST"); + o.get().literal("name", "bird boss"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldAppendValueInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "append('animals[].1', ' is cool')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "dog is cool"); + o.get().literal("2", "cat"); + o.get().literal("3", "zebra"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldAppendValueInEntireArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "append('animals[].*', ' is cool')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "dog is cool"); + o.get().literal("2", "cat is cool"); + o.get().literal("3", "zebra is cool"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldNotAppendValueToArray() { + MetafixTestHelpers.assertThrows(IllegalStateException.class, "expected String, got Array", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "append('animals[]', ' is cool')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldNotAppendValueToHash() { + MetafixTestHelpers.assertThrows(IllegalStateException.class, "expected String, got Hash", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "append('animals', ' is cool')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldCountNumberOfValuesInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "count(numbers)" + ), + i -> { + i.startRecord("1"); + i.literal("numbers", "41"); + i.literal("numbers", "42"); + i.literal("numbers", "6"); + i.literal("numbers", "6"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("numbers", "4"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldCountNumberOfValuesInHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "count(person)" + ), + i -> { + i.startRecord("1"); + i.startEntity("person"); + i.literal("name", "François"); + i.literal("age", "12"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("person", "2"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldFilterArrayValues() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "filter(animals, '[Cc]at')" + ), + i -> { + i.startRecord("1"); + i.literal("animals", "Lion"); + i.literal("animals", "Cat"); + i.literal("animals", "Tiger"); + i.literal("animals", "Bobcat"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals"); + o.get().literal("1", "Cat"); + o.get().literal("2", "Bobcat"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldFilterArrayValuesInverted() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "filter(animals, '[Cc]at', invert: 'true')" + ), + i -> { + i.startRecord("1"); + i.literal("animals", "Lion"); + i.literal("animals", "Cat"); + i.literal("animals", "Tiger"); + i.literal("animals", "Bobcat"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals"); + o.get().literal("1", "Lion"); + o.get().literal("2", "Tiger"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldFilterArrayObjectValues() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "filter('animals[]', '[Cc]at')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "Lion"); + i.literal("2", "Cat"); + i.literal("3", "Tiger"); + i.literal("4", "Bobcat"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "Cat"); + o.get().literal("2", "Bobcat"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldGetFirstIndexOfSubstring() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "index(animal, 'n')" + ), + i -> { + i.startRecord("1"); + i.literal("animal", "bunny"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("animal", "2"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldGetIndexOfSubstring() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "index(title, 't')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "metafix"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "2"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldJoinArrayField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "join_field(numbers, '/')" + ), + i -> { + i.startRecord("1"); + i.literal("numbers", "6"); + i.literal("numbers", "42"); + i.literal("numbers", "41"); + i.literal("numbers", "6"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("numbers", "6/42/41/6"); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldJoinArrayObjectField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "join_field('animals[]', ',')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("animals[]", "dog,cat,zebra"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldPrependValue() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "prepend(title, 'I love ')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "metafix"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "I love metafix"); + o.get().endRecord(); + } + ); + } + + @Test + // TODO: Fix order (`animols.name` should stay before `animols.type`) + public void shouldPrependValueInHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "prepend(animols.name, 'nice ')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animols"); + i.literal("name", "bird"); + i.literal("type", "TEST"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animols"); + o.get().literal("type", "TEST"); + o.get().literal("name", "nice bird"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldPrependValueInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "prepend('animals[].1', 'cool ')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "cool dog"); + o.get().literal("2", "cat"); + o.get().literal("3", "zebra"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldPrependValueInEntireArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "prepend('animals[].*', 'cool ')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "cool dog"); + o.get().literal("2", "cool cat"); + o.get().literal("3", "cool zebra"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldNotPrependValueToArray() { + MetafixTestHelpers.assertThrows(IllegalStateException.class, "expected String, got Array", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "prepend('animals[]', 'cool ')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldReplaceAllRegexes() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "replace_all(title, '[aei]', 'X')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "metafix"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "mXtXfXx"); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldReplaceAllRegexesInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "replace_all('animals[].*', a, QR)" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "dog"); + o.get().literal("2", "cQRt"); + o.get().literal("3", "zebrQR"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReverseString() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "reverse(title)" + ), + i -> { + i.startRecord("1"); + i.literal("title", "metafix"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "xifatem"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldReverseArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "reverse(title)" + ), + i -> { + i.startRecord("1"); + i.literal("title", "marc"); + i.literal("title", "json"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("title"); + o.get().literal("1", "json"); + o.get().literal("2", "marc"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldSortField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "sort_field(numbers)" + ), + i -> { + i.startRecord("1"); + i.literal("numbers", "6"); + i.literal("numbers", "42"); + i.literal("numbers", "41"); + i.literal("numbers", "6"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("numbers"); + o.get().literal("1", "41"); + o.get().literal("2", "42"); + o.get().literal("3", "6"); + o.get().literal("4", "6"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldSortFieldNumerically() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "sort_field(numbers, numeric: 'true')" + ), + i -> { + i.startRecord("1"); + i.literal("numbers", "6"); + i.literal("numbers", "42"); + i.literal("numbers", "41"); + i.literal("numbers", "6"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("numbers"); + o.get().literal("1", "6"); + o.get().literal("2", "6"); + o.get().literal("3", "41"); + o.get().literal("4", "42"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldFailToSortNumericallyWithInvalidNumber() { + MetafixTestHelpers.assertThrows(NumberFormatException.class, "For input string: \"x\"", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "sort_field(numbers, numeric: 'true')" + ), + i -> { + i.startRecord("1"); + i.literal("numbers", "6"); + i.literal("numbers", "42"); + i.literal("numbers", "x"); + i.literal("numbers", "6"); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldSortFieldInReverse() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "sort_field(numbers, reverse: 'true')" + ), + i -> { + i.startRecord("1"); + i.literal("numbers", "6"); + i.literal("numbers", "42"); + i.literal("numbers", "41"); + i.literal("numbers", "6"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("numbers"); + o.get().literal("1", "6"); + o.get().literal("2", "6"); + o.get().literal("3", "42"); + o.get().literal("4", "41"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldSortFieldNumericallyInReverse() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "sort_field(numbers, numeric: 'true', reverse: 'true')" + ), + i -> { + i.startRecord("1"); + i.literal("numbers", "6"); + i.literal("numbers", "42"); + i.literal("numbers", "41"); + i.literal("numbers", "6"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("numbers"); + o.get().literal("1", "42"); + o.get().literal("2", "41"); + o.get().literal("3", "6"); + o.get().literal("4", "6"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldSortFieldAndRemoveDuplicates() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "sort_field(numbers, uniq: 'true')" + ), + i -> { + i.startRecord("1"); + i.literal("numbers", "6"); + i.literal("numbers", "42"); + i.literal("numbers", "41"); + i.literal("numbers", "6"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("numbers"); + o.get().literal("1", "41"); + o.get().literal("2", "42"); + o.get().literal("3", "6"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldSplitStringField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "split_field(date, '-')" + ), + i -> { + i.startRecord("1"); + i.literal("date", "1918-17-16"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("date"); + o.get().literal("1", "1918"); + o.get().literal("2", "17"); + o.get().literal("3", "16"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldSplitArrayField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "split_field(date, '-')" + ), + i -> { + i.startRecord("1"); + i.literal("date", "1918-17-16"); + i.literal("date", "2021-22-23"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("date"); + o.get().startEntity("1"); + o.get().literal("1", "1918"); + o.get().literal("2", "17"); + o.get().literal("3", "16"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("1", "2021"); + o.get().literal("2", "22"); + o.get().literal("3", "23"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldSplitHashField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "split_field(date, '-')" + ), + i -> { + i.startRecord("1"); + i.startEntity("date"); + i.literal("a", "1918-17-16"); + i.literal("b", "2021-22-23"); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("date"); + o.get().startEntity("a"); + o.get().literal("1", "1918"); + o.get().literal("2", "17"); + o.get().literal("3", "16"); + o.get().endEntity(); + o.get().startEntity("b"); + o.get().literal("1", "2021"); + o.get().literal("2", "22"); + o.get().literal("3", "23"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + @Disabled("See https://github.com/metafacture/metafacture-fix/issues/100") + public void shouldSplitNestedField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "split_field('others[].*.tools', '--')" + ), + i -> { + i.startRecord("1"); + i.startEntity("others[]"); + i.startEntity("1"); + i.literal("tools", "hammer--saw--bow"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("others[]"); + o.get().startEntity("1"); + o.get().startEntity("tools"); + o.get().literal("1", "hammer"); + o.get().literal("2", "saw"); + o.get().literal("3", "bow"); + f.apply(3).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldSumNumbers() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "sum(numbers)" + ), + i -> { + i.startRecord("1"); + i.literal("numbers", "41"); + i.literal("numbers", "42"); + i.literal("numbers", "6"); + i.literal("numbers", "6"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("numbers", "95"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldRemoveDuplicates() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "uniq(numbers)" + ), + i -> { + i.startRecord("1"); + i.literal("numbers", "41"); + i.literal("numbers", "42"); + i.literal("numbers", "6"); + i.literal("numbers", "6"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("numbers"); + o.get().literal("1", "41"); + o.get().literal("2", "42"); + o.get().literal("3", "6"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + } diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java index 14c3624b..83a125e8 100644 --- a/metafix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java @@ -16,13 +16,12 @@ package org.metafacture.metafix; -import org.metafacture.framework.MetafactureException; import org.metafacture.framework.StreamReceiver; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -895,26 +894,31 @@ public void addFieldToObjectByIndexInIndexedArray() { @Test public void addFieldToFirstObjectMissing() { - assertThrowsOnEmptyRecord("add_field('animals[].$first.kind','nice')"); + assertThrowsOnEmptyRecord("$first"); } @Test public void addFieldToLastObjectMissing() { - assertThrowsOnEmptyRecord("add_field('animals[].$last.kind','nice')"); + assertThrowsOnEmptyRecord("$last"); } @Test public void addFieldToObjectByIndexMissing() { - assertThrowsOnEmptyRecord("add_field('animals[].2.kind','nice')"); + assertThrowsOnEmptyRecord("2"); } - private void assertThrowsOnEmptyRecord(final String fix) { - Assertions.assertThrows(MetafactureException.class, () -> { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(fix), + private void assertThrowsOnEmptyRecord(final String index) { + MetafixTestHelpers.assertThrows(IllegalArgumentException.class, "Using ref, but can't find: " + index + " in: {}", () -> { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "add_field('animals[]." + index + ".kind','nice')" + ), i -> { i.startRecord("1"); i.endRecord(); - }, o -> { }); + }, + o -> { + } + ); }); } @@ -1595,42 +1599,43 @@ public void accessArrayByIndex() { } @Test - public void accessArrayImplicit() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "upcase('name')"), - i -> { - i.startRecord("1"); - i.literal("name", "max"); - i.literal("name", "mo"); - i.endRecord(); - }, (o, f) -> { - o.get().startRecord("1"); - o.get().startEntity("name"); - o.get().literal("1", "MAX"); - o.get().literal("2", "MO"); - o.get().endEntity(); - o.get().endRecord(); - }); + public void shouldNotAccessArrayImplicitly() { + MetafixTestHelpers.assertThrows(IllegalStateException.class, "expected String, got Array", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "upcase('name')" + ), + i -> { + i.startRecord("1"); + i.literal("name", "max"); + i.literal("name", "mo"); + i.endRecord(); + }, + o -> { + } + ) + ); } @Test - // TODO: WDCD? explicit * for array fields? - public void accessArrayByWildcard() { + public void shouldAccessArrayByWildcard() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "upcase('name.*')"), + "upcase('name.*')" + ), i -> { i.startRecord("1"); i.literal("name", "max"); i.literal("name", "mo"); i.endRecord(); - }, (o, f) -> { + }, + o -> { o.get().startRecord("1"); o.get().startEntity("name"); o.get().literal("1", "MAX"); o.get().literal("2", "MO"); o.get().endEntity(); o.get().endRecord(); - }); + } + ); } @Test @@ -1742,4 +1747,270 @@ public void accessArrayOfObjectsByDoListBind() { }); } + @Test + public void shouldAddRandomNumber() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "random(test, '100')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "marc"); + i.literal("title", "json"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("title"); + o.get().literal("1", "marc"); + o.get().literal("2", "json"); + o.get().endEntity(); + o.get().literal(ArgumentMatchers.eq("test"), ArgumentMatchers.argThat(i -> Integer.parseInt(i) < 100)); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldReplaceExistingValueWithRandomNumber() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "random(others, '100')" + ), + i -> { + i.startRecord("1"); + i.literal("others", "human"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal(ArgumentMatchers.eq("others"), ArgumentMatchers.argThat(i -> Integer.parseInt(i) < 100)); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldAddRandomNumberToMarkedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "random('animals[].$append', '100')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "cat"); + i.literal("2", "dog"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "cat"); + o.get().literal("2", "dog"); + o.get().literal(ArgumentMatchers.eq("3"), ArgumentMatchers.argThat(i -> Integer.parseInt(i) < 100)); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldAddObjectWithRandomNumberToMarkedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('bnimals[]')", + "random('bnimals[].$append.number', '100')" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("bnimals[]"); + o.get().startEntity("1"); + o.get().literal(ArgumentMatchers.eq("number"), ArgumentMatchers.argThat(i -> Integer.parseInt(i) < 100)); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldAddRandomNumberToUnmarkedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "random('animals.$append', '100')" + ), + i -> { + i.startRecord("1"); + i.literal("animals", "cat"); + i.literal("animals", "dog"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals"); + o.get().literal("1", "cat"); + o.get().literal("2", "dog"); + o.get().literal(ArgumentMatchers.eq("3"), ArgumentMatchers.argThat(i -> Integer.parseInt(i) < 100)); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + @Disabled("See https://github.com/metafacture/metafacture-fix/issues/100") + public void shouldAddRandomNumberToUnmarkedArrayObject() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "random('animals.$append', '100')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals"); + i.literal("1", "cat"); + i.literal("2", "dog"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals"); + o.get().literal("1", "cat"); + o.get().literal("2", "dog"); + o.get().literal(ArgumentMatchers.eq("3"), ArgumentMatchers.argThat(i -> Integer.parseInt(i) < 100)); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldRenameFieldsInHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "rename(your, '[ae]', X)" + ), + i -> { + i.startRecord("1"); + i.startEntity("your"); + i.literal("name", "nicolas"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("your"); + o.get().literal("nXmX", "nicolas"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldRecursivelyRenameFieldsInHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "rename(others, ani, QR)" + ), + i -> { + i.startRecord("1"); + i.startEntity("others"); + i.literal("animal", "human"); + i.literal("canister", "metall"); + i.startEntity("area"); + i.literal("ani", "test"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("others"); + o.get().literal("QRmal", "human"); + o.get().literal("cQRster", "metall"); + o.get().startEntity("area"); + o.get().literal("QR", "test"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldRecursivelyRenameFieldsInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "rename('animals[]', ani, XY)" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.startEntity("1"); + i.literal("animal", "dog"); + i.endEntity(); + i.startEntity("2"); + i.literal("animal", "cat"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().startEntity("1"); + o.get().literal("XYmal", "dog"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("XYmal", "cat"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + @Disabled("java.lang.ArrayIndexOutOfBoundsException: 0; see https://github.com/metafacture/metafacture-fix/issues/100") + public void shouldRenameAllFieldsInHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "rename('.', ani, XY)" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals"); + i.literal("animal", "dog"); + i.literal("animal", "cat"); + i.endEntity(); + i.startEntity("others"); + i.literal("animal", "human"); + i.literal("canister", "metall"); + i.startEntity("area"); + i.literal("ani", "test"); + i.endEntity(); + i.endEntity(); + i.startEntity("fictional"); + i.literal("animal", "unicorn"); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("XYmals"); + o.get().startEntity("XYmal"); + o.get().literal("1", "dog"); + o.get().literal("2", "cat"); + f.apply(2).endEntity(); + o.get().startEntity("others"); + o.get().literal("XYmal", "human"); + o.get().literal("cXYster", "metall"); + o.get().startEntity("area"); + o.get().literal("XY", "test"); + f.apply(2).endEntity(); + o.get().startEntity("fictional"); + o.get().literal("XYmal", "unicorn"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + } diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixScriptTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixScriptTest.java index 2396f050..1a71db9a 100644 --- a/metafix/src/test/java/org/metafacture/metafix/MetafixScriptTest.java +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixScriptTest.java @@ -127,6 +127,11 @@ public void shouldPutExternalFileMapWithNameAndOptions() { assertMap("put_filemap('" + TSV_MAP + "', '" + MAP_NAME + "', sep_char: '\t')", MAP_NAME); } + @Test + public void shouldDoNothing() { + assertFix("nothing()", f -> { }); + } + private void assertVar(final String fixDef, final Map vars, final Map result) { assertFix(fixDef, vars, f -> result.forEach((k, v) -> Assertions.assertEquals(v, f.getVars().get(k)))); } diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java b/metafix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java index cde64818..89b51ca4 100644 --- a/metafix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java @@ -16,9 +16,11 @@ package org.metafacture.metafix; +import org.metafacture.framework.MetafactureException; import org.metafacture.framework.StreamReceiver; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.function.Executable; import org.mockito.InOrder; import org.mockito.Mockito; import org.mockito.exceptions.base.MockitoAssertionError; @@ -43,6 +45,12 @@ public final class MetafixTestHelpers { private MetafixTestHelpers() { } + public static void assertThrows(final Class expectedClass, final String expectedMessage, final Executable executable) { + final Throwable exception = Assertions.assertThrows(MetafactureException.class, executable).getCause(); + Assertions.assertSame(expectedClass, exception.getClass()); + Assertions.assertEquals(expectedMessage, exception.getMessage()); + } + public static void assertFix(final StreamReceiver receiver, final List fixDef, final Consumer in, final Consumer> out) { assertFix(receiver, fixDef, in, (s, f) -> out.accept(s), Metafix.NO_VARS); diff --git a/metafix/src/test/java/org/metafacture/metafix/ValueTest.java b/metafix/src/test/java/org/metafacture/metafix/ValueTest.java new file mode 100644 index 00000000..68120145 --- /dev/null +++ b/metafix/src/test/java/org/metafacture/metafix/ValueTest.java @@ -0,0 +1,35 @@ +/* + * Copyright 2022 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metafix; + +import nl.jqno.equalsverifier.EqualsVerifier; +import org.junit.jupiter.api.Test; + +public class ValueTest { + + public ValueTest() { + } + + @Test + public void shouldSatisfyEqualsContract() { + EqualsVerifier.forClass(Value.class) + .withPrefabValues(Value.class, Value.newArray(), Value.newHash()) + .withPrefabValues(Value.Hash.class, Value.newHash().asHash(), Value.newHash(h -> h.put("k", new Value("v"))).asHash()) + .verify(); + } + +}