diff --git a/build.gradle b/build.gradle index bd4aea8b..855ecc08 100644 --- a/build.gradle +++ b/build.gradle @@ -32,7 +32,7 @@ subprojects { 'mockito': '2.27.0', 'requirejs': '2.3.6', 'slf4j': '1.7.21', - 'xtext': '2.17.0', + 'xtext': '2.26.0.M2', 'guava': '29.0-jre' ] } diff --git a/metafix/src/main/java/org/metafacture/metafix/FixMethod.java b/metafix/src/main/java/org/metafacture/metafix/FixMethod.java index b54a8590..d91ca193 100644 --- a/metafix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/metafix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -32,7 +32,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.stream.Stream; public enum FixMethod implements FixFunction { @@ -122,7 +121,7 @@ public void apply(final Metafix metafix, final Record record, final List record.getList(field, a -> record.put(field, Value.newHash(h -> { for (int i = 1; i < a.size(); i = i + 2) { - h.put(a.get(i - 1).toString(), a.get(i)); + h.put(a.get(i - 1).asString(), a.get(i)); } }))); } @@ -141,7 +140,7 @@ public void apply(final Metafix metafix, final Record record, final List record.getList(field, a -> a.forEach(v -> { final Pattern p = Pattern.compile(params.get(1)); - final Matcher m = p.matcher(v.toString()); + final Matcher m = p.matcher(v.asString()); if (m.matches()) { record.remove(field); @@ -179,7 +178,7 @@ public void apply(final Metafix metafix, final Record record, final List record.replace(params.get(0), params.subList(1, params.size()).stream() .filter(f -> literalString(f) || record.find(f) != null) .map(f -> literalString(f) ? new Value(f.substring(1)) : record.findList(f, null).asArray().get(0)) - .map(Value::toString).collect(Collectors.joining(joinChar != null ? joinChar : " "))); + .map(Value::asString).collect(Collectors.joining(joinChar != null ? joinChar : " "))); } private boolean literalString(final String s) { @@ -192,7 +191,7 @@ public void apply(final Metafix metafix, final Record record, final List final String field = params.get(0); final int max = getInteger(params, 1); - record.append(field, String.valueOf(RANDOM.nextInt(max))); + record.replace(field, String.valueOf(RANDOM.nextInt(max))); } }, reject { @@ -210,14 +209,32 @@ public void apply(final Metafix metafix, final Record record, final List rename { @Override public void apply(final Metafix metafix, final Record record, final List params, final Map options) { - record.transformField(params.get(0), v -> { - final String search = params.get(1); - final String replace = params.get(2); + final String search = params.get(1); + final String replace = params.get(2); - // TODO: recurse into arrays/values - return v.isHash() ? Value.newHash(h -> - v.asHash().forEach((f, w) -> h.put(f.replaceAll(search, replace), w))) : null; - }); + final UnaryOperator operator = s -> s.replaceAll(search, replace); + + record.transformField(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(renameArray(a, operator))) + .ifHash(h -> c.accept(renameHash(h, operator))) + .orElseThrow() + ); + } + + private Value renameArray(final Value.Array array, final UnaryOperator operator) { + return Value.newArray(a -> array.forEach(v -> a.add(renameValue(v, operator)))); + } + + private Value renameHash(final Value.Hash hash, final UnaryOperator operator) { + return Value.newHash(h -> hash.forEach((f, v) -> h.put(operator.apply(f), renameValue(v, operator)))); + } + + private Value renameValue(final Value value, final UnaryOperator operator) { + return value.extractType((m, c) -> m + .ifArray(a -> c.accept(renameArray(a, operator))) + .ifHash(h -> c.accept(renameHash(h, operator))) + .orElse(c) + ); } }, retain { @@ -242,10 +259,7 @@ public void apply(final Metafix metafix, final Record record, final List set_field { @Override public void apply(final Metafix metafix, final Record record, final List params, final Map options) { - final String field = params.get(0); - - record.remove(field); - record.replace(field, params.get(1)); + record.replace(params.get(0), params.get(1)); } }, set_hash { @@ -291,8 +305,10 @@ public void apply(final Metafix metafix, final Record record, final List count { @Override public void apply(final Metafix metafix, final Record record, final List params, final Map options) { - record.transformField(params.get(0), v -> - v.isArray() ? new Value(v.asArray().size()) : v.isHash() ? new Value(v.asHash().size()) : null); + record.transformField(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(new Value(a.size()))) + .ifHash(h -> c.accept(new Value(h.size()))) + ); } }, downcase { @@ -304,13 +320,14 @@ public void apply(final Metafix metafix, final Record record, final List filter { @Override public void apply(final Metafix metafix, final Record record, final List params, final Map options) { - record.transformField(params.get(0), v -> { - final Pattern search = Pattern.compile(params.get(1)); - final boolean invert = getBoolean(options, "invert"); + final Pattern search = Pattern.compile(params.get(1)); + final boolean invert = getBoolean(options, "invert"); - final Predicate predicate = s -> search.matcher(s.asString()).find(); - return v.isArray() ? newArray(v.asArray().stream().filter(invert ? predicate.negate() : predicate)) : null; - }); + final Predicate predicate = s -> search.matcher(s.asString()).find(); + + record.transformField(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(newArray(a.stream().filter(invert ? predicate.negate() : predicate)))) + ); } }, index { @@ -323,10 +340,10 @@ public void apply(final Metafix metafix, final Record record, final List join_field { @Override public void apply(final Metafix metafix, final Record record, final List params, final Map options) { - record.transformField(params.get(0), v -> { - final String joinChar = params.size() > 1 ? params.get(1) : ""; - return v.isArray() ? new Value(v.asArray().stream().map(Value::toString).collect(Collectors.joining(joinChar))) : null; - }); + final String joinChar = params.size() > 1 ? params.get(1) : ""; + record.transformField(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(new Value(a.stream().map(Value::asString).collect(Collectors.joining(joinChar))))) + ); } }, lookup { @@ -370,57 +387,47 @@ public void apply(final Metafix metafix, final Record record, final List reverse { @Override public void apply(final Metafix metafix, final Record record, final List params, final Map options) { - record.transformField(params.get(0), v -> { - final Value result; - - if (v.isString()) { - result = new Value(new StringBuilder(v.asString()).reverse().toString()); - } - else if (v.isArray()) { - final List list = v.asArray().stream().collect(Collectors.toList()); - Collections.reverse(list); - result = new Value(list); - } - else { - result = null; - } - - return result; - }); + record.transformField(params.get(0), (m, c) -> m + .ifArray(a -> { + final List list = a.stream().collect(Collectors.toList()); + Collections.reverse(list); + c.accept(new Value(list)); + }) + .ifString(s -> c.accept(new Value(new StringBuilder(s).reverse().toString()))) + ); } }, sort_field { @Override public void apply(final Metafix metafix, final Record record, final List params, final Map options) { - record.transformField(params.get(0), v -> { - final boolean numeric = getBoolean(options, "numeric"); - final boolean reverse = getBoolean(options, "reverse"); - final boolean uniq = getBoolean(options, "uniq"); + final boolean numeric = getBoolean(options, "numeric"); + final boolean reverse = getBoolean(options, "reverse"); + final boolean uniq = getBoolean(options, "uniq"); - final Stream stream = v.asArray().stream(); - final Function function = Value::asString; - final Comparator comparator = numeric ? - Comparator.comparing(function.andThen(Integer::parseInt)) : Comparator.comparing(function); + final Function function = Value::asString; + final Comparator comparator = numeric ? + Comparator.comparing(function.andThen(Integer::parseInt)) : Comparator.comparing(function); - return v.isArray() ? new Value((uniq ? unique(stream) : stream) - .sorted(reverse ? comparator.reversed() : comparator).collect(Collectors.toList())) : null; - }); + record.transformField(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(new Value((uniq ? unique(a.stream()) : a.stream()) + .sorted(reverse ? comparator.reversed() : comparator).collect(Collectors.toList())))) + ); } }, split_field { @Override public void apply(final Metafix metafix, final Record record, final List params, final Map options) { - record.transformField(params.get(0), v -> { - final String splitChar = params.size() > 1 ? params.get(1) : "\\s+"; - final Pattern splitPattern = Pattern.compile(splitChar); + final String splitChar = params.size() > 1 ? params.get(1) : "\\s+"; + final Pattern splitPattern = Pattern.compile(splitChar); - final UnaryOperator splitOperator = s -> - newArray(Arrays.stream(splitPattern.split(s.asString())).map(Value::new)); + final Function splitFunction = s -> + newArray(Arrays.stream(splitPattern.split(s)).map(Value::new)); - return v.isString() ? splitOperator.apply(v) : v.isArray() ? - newArray(v.asArray().stream().map(splitOperator)) : v.isHash() ? - Value.newHash(h -> v.asHash().forEach((f, s) -> h.put(f, splitOperator.apply(s)))) : null; - }); + record.transformField(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(newArray(a.stream().map(Value::asString).map(splitFunction)))) + .ifHash(h -> c.accept(Value.newHash(n -> h.forEach((f, w) -> n.put(f, splitFunction.apply(w.asString())))))) + .ifString(s -> c.accept(splitFunction.apply(s))) + ); } }, substring { @@ -432,8 +439,9 @@ public void apply(final Metafix metafix, final Record record, final List sum { @Override public void apply(final Metafix metafix, final Record record, final List params, final Map options) { - record.transformField(params.get(0), v -> - v.isArray() ? new Value(v.asArray().stream().map(Value::asString).mapToInt(Integer::parseInt).sum()) : null); + record.transformField(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(new Value(a.stream().map(Value::asString).mapToInt(Integer::parseInt).sum()))) + ); } }, trim { @@ -445,7 +453,9 @@ public void apply(final Metafix metafix, final Record record, final List uniq { @Override public void apply(final Metafix metafix, final Record record, final List params, final Map options) { - record.transformField(params.get(0), v -> v.isArray() ? newArray(unique(v.asArray().stream())) : null); + record.transformField(params.get(0), (m, c) -> m + .ifArray(a -> c.accept(newArray(unique(a.stream())))) + ); } }, upcase { diff --git a/metafix/src/main/java/org/metafacture/metafix/Metafix.java b/metafix/src/main/java/org/metafacture/metafix/Metafix.java index bdd432d3..07c3283f 100644 --- a/metafix/src/main/java/org/metafacture/metafix/Metafix.java +++ b/metafix/src/main/java/org/metafacture/metafix/Metafix.java @@ -77,7 +77,7 @@ public class Metafix implements StreamPipe, Maps { // checkstyle private int entityCount; private StreamReceiver outputStreamReceiver; private String recordIdentifier; - private List entities = new ArrayList<>(); + private List entities = new ArrayList<>(); public Metafix() { flattener.setReceiver(new DefaultStreamReceiver() { @@ -132,7 +132,7 @@ public void startRecord(final String identifier) { public void endRecord() { entityCountStack.removeLast(); if (!entityCountStack.isEmpty()) { - throw new IllegalStateException(ENTITIES_NOT_BALANCED); + throw new MetafactureException(new IllegalStateException(ENTITIES_NOT_BALANCED)); } flattener.endRecord(); LOG.debug("End record, walking fix: {}", currentRecord); @@ -172,20 +172,33 @@ private void emit(final String field, final Value value) { }); } + private void addValue(final String name, final Value value) { + final int index = entityCountStack.peek() - 1; + if (index < 0 || entities.size() <= index) { + currentRecord.add(name, value); + } + else { + entities.get(index).matchType() + .ifArray(a -> a.add(value)) + .ifHash(h -> h.add(name, value)) + .orElseThrow(); + } + } + @Override public void startEntity(final String name) { if (name == null) { throw new IllegalArgumentException("Entity name must not be null."); } - ++entityCount; - final Integer currentEntityIndex = entityCountStack.peek() - 1; - final Value.Hash previousEntity = currentEntityIndex < 0 || - entities.size() <= currentEntityIndex ? null : entities.get(currentEntityIndex); - entityCountStack.push(Integer.valueOf(entityCount)); + + final Value value = name.endsWith(ARRAY_MARKER) ? Value.newArray() : Value.newHash(); + // TODO: Remove array marker? => name.substring(0, name.length() - ARRAY_MARKER.length()); + + addValue(name, value); + entities.add(value); + + entityCountStack.push(Integer.valueOf(++entityCount)); flattener.startEntity(name); - final Value value = Value.newHash(); - (previousEntity != null ? previousEntity : currentRecord).add(name, value); - entities.add(value.asHash()); } @Override @@ -197,10 +210,7 @@ public void endEntity() { @Override public void literal(final String name, final String value) { LOG.debug("Putting '{}': '{}'", name, value); - final Integer currentEntityIndex = entityCountStack.peek() - 1; - final Value.Hash currentEntity = currentEntityIndex < 0 || - entities.size() <= currentEntityIndex ? null : entities.get(currentEntityIndex); - (currentEntity != null ? currentEntity : currentRecord).add(name, new Value(value)); + addValue(name, new Value(value)); // TODO: keep flattener as option? // flattener.literal(name, value); } diff --git a/metafix/src/main/java/org/metafacture/metafix/Value.java b/metafix/src/main/java/org/metafacture/metafix/Value.java index 8edbed6e..653a2980 100644 --- a/metafix/src/main/java/org/metafacture/metafix/Value.java +++ b/metafix/src/main/java/org/metafacture/metafix/Value.java @@ -17,6 +17,7 @@ package org.metafacture.metafix; import org.metafacture.commons.tries.SimpleRegexTrie; +import org.metafacture.framework.MetafactureException; import java.util.ArrayList; import java.util.Arrays; @@ -210,7 +211,7 @@ public TypeMatcher matchType() { return new TypeMatcher(this); } - private T extractType(final BiConsumer> consumer) { + public T extractType(final BiConsumer> consumer) { final AtomicReference result = new AtomicReference<>(); consumer.accept(matchType(), result::set); return result.get(); @@ -298,7 +299,7 @@ public void orElse(final Consumer consumer) { public void orElseThrow() { orElse(v -> { final String types = expected.stream().map(Type::name).collect(Collectors.joining(" or ")); - throw new IllegalStateException("expected " + types + ", got " + value.type); + throw new MetafactureException(new IllegalStateException("expected " + types + ", got " + value.type)); }); } @@ -311,7 +312,7 @@ private TypeMatcher match(final Type type, final Consumer consumer, final return this; } else { - throw new IllegalStateException("already expecting " + type); + throw new MetafactureException(new IllegalStateException("already expecting " + type)); } } @@ -335,21 +336,6 @@ void apply(final Hash hash, final String field, final Value value) { void apply(final Hash hash, final String field, final Value value) { hash.add(field, value); } - }, - /* For an indexed representation of arrays as hashes with 1, 2, 3 etc. keys. - * i.e. ["a", "b", "c"] as { "1":"a", "2":"b", "3": "c" } - * This is what is produced by JsonDecoder and Metafix itself for arrays. - * TODO? maybe this would be a good general internal representation, resulting - * in every value being either a hash or a string, no more separate array type.*/ - INDEXED { - @Override - void apply(final Hash hash, final String field, final Value value) { - hash.add(nextIndex(hash), field.equals(ReservedField.$append.name()) ? value : newHash(h -> h.put(field, value))); - } - - private String nextIndex(final Hash hash) { - return "" + (hash.size() + 1) /* TODO? check if keys are actually all ints? */; - } }; abstract void apply(Hash hash, String field, Value value); @@ -472,7 +458,7 @@ else if (isNumber(field)) { } } - list.removeIf(v -> v == null); + list.removeIf(v -> Value.isNull(v)); } private void transformFields(final int index, final String[] fields, final UnaryOperator operator) { @@ -727,7 +713,7 @@ private Value insert(final InsertMode mode, final String[] fields, final Value n // TODO: move impl into enum elements, here call only value.insert value.matchType() .ifArray(a -> a.insert(mode, tail, newValue)) - .ifHash(h -> h.insert(insertMode(mode, field, tail), tail, newValue)) + .ifHash(h -> h.insert(mode, tail, newValue)) .orElseThrow(); } } @@ -738,7 +724,7 @@ private Value insert(final InsertMode mode, final String[] fields, final Value n private Value processRef(final InsertMode mode, final Value newValue, final String field, final String[] tail) { final Value referencedValue = getReferencedValue(field); if (referencedValue != null) { - return referencedValue.asHash().insert(insertMode(mode, field, tail), tail, newValue); + return referencedValue.asHash().insert(mode, tail, newValue); } else { throw new IllegalArgumentException("Using ref, but can't find: " + field + " in: " + this); @@ -768,16 +754,6 @@ private Value getReferencedValue(final String field) { return referencedValue; } - private InsertMode insertMode(final InsertMode mode, final String field, final String[] tail) { - // if the field is marked as array, this hash should be smth. like { 1=a, 2=b } - final boolean isIndexedArray = field.endsWith(Metafix.ARRAY_MARKER); - final boolean nextIsRef = tail.length > 0 && ( - tail[0].startsWith(ReservedField.$first.name()) || - tail[0].startsWith(ReservedField.$last.name()) || - isNumber(tail[0])); - return isIndexedArray && !nextIsRef ? InsertMode.INDEXED : mode; - } - /** * Removes the given field/value pair from this hash. * @@ -839,11 +815,11 @@ private void appendValue(final String[] newName, final Value v) { } } - public void transformField(final String field, final UnaryOperator operator) { + public void transformField(final String field, final BiConsumer> consumer) { final Value oldValue = find(field); if (oldValue != null) { - final Value newValue = operator.apply(oldValue); + final Value newValue = oldValue.extractType(consumer); if (newValue != null) { insert(InsertMode.REPLACE, split(field), newValue); @@ -873,7 +849,9 @@ private void transformFields(final String[] fields, final UnaryOperator map.remove(f); if (operator != null) { - value.asList(a -> a.forEach(v -> append(f, operator.apply(v.toString())))); + value.matchType() + .ifString(s -> append(f, operator.apply(s))) + .orElseThrow(); } } else { diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixBindTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixBindTest.java index d8bc3e3f..9329a068 100644 --- a/metafix/src/test/java/org/metafacture/metafix/MetafixBindTest.java +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixBindTest.java @@ -21,10 +21,7 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.extension.RegisterExtension; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; import org.mockito.junit.jupiter.MockitoExtension; import java.util.Arrays; @@ -37,9 +34,6 @@ @ExtendWith(MockitoExtension.class) public class MetafixBindTest { - @RegisterExtension - private MockitoRule mockitoRule = MockitoJUnit.rule(); - @Mock private StreamReceiver streamReceiver; diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixIfTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixIfTest.java index d57a963a..d550b4a0 100644 --- a/metafix/src/test/java/org/metafacture/metafix/MetafixIfTest.java +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixIfTest.java @@ -21,10 +21,7 @@ import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.extension.RegisterExtension; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; import org.mockito.junit.jupiter.MockitoExtension; import java.util.Arrays; @@ -37,9 +34,6 @@ @ExtendWith(MockitoExtension.class) public class MetafixIfTest { - @RegisterExtension - private MockitoRule mockitoRule = MockitoJUnit.rule(); - @Mock private StreamReceiver streamReceiver; diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java index 6d554aab..d901f7f6 100644 --- a/metafix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java @@ -22,10 +22,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.extension.RegisterExtension; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; import org.mockito.junit.jupiter.MockitoExtension; import java.util.Arrays; @@ -42,8 +39,7 @@ public class MetafixLookupTest { private static final String CSV_MAP = "src/test/resources/org/metafacture/metafix/maps/test.csv"; private static final String TSV_MAP = "src/test/resources/org/metafacture/metafix/maps/test.tsv"; - @RegisterExtension - private MockitoRule mockitoRule = MockitoJUnit.rule(); + private static final String LOOKUP = "lookup('title.*',"; @Mock private StreamReceiver streamReceiver; @@ -54,14 +50,14 @@ public MetafixLookupTest() { @Test public void inline() { assertMap( - "lookup('title', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)" + LOOKUP + " Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)" ); } @Test public void inlineMultilineIndent() { assertMap( - "lookup('title',", + LOOKUP, " Aloha: Alohaeha,", " Moin: 'Moin zäme',", " __default: Tach)" @@ -71,7 +67,7 @@ public void inlineMultilineIndent() { @Test public void inlineDotNotationNested() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "lookup('data.title', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)" + "lookup('data.title.*', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)" ), i -> { i.startRecord("1"); @@ -98,14 +94,14 @@ public void inlineDotNotationNested() { @Test public void csv() { assertMap( - "lookup('title', '" + CSV_MAP + "')" + LOOKUP + " '" + CSV_MAP + "')" ); } @Test public void tsv() { assertMap( - "lookup('title', '" + TSV_MAP + "', sep_char:'\t')" + LOOKUP + " '" + TSV_MAP + "', sep_char:'\t')" ); } @@ -113,7 +109,7 @@ public void tsv() { public void shouldLookupInSeparateInternalMap() { assertMap( "put_map('testMap', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)", - "lookup('title', 'testMap')" + LOOKUP + " 'testMap')" ); } @@ -121,7 +117,7 @@ public void shouldLookupInSeparateInternalMap() { public void shouldLookupInSeparateExternalFileMap() { assertMap( "put_filemap('" + CSV_MAP + "')", - "lookup('title', '" + CSV_MAP + "')" + LOOKUP + " '" + CSV_MAP + "')" ); } @@ -129,7 +125,7 @@ public void shouldLookupInSeparateExternalFileMap() { public void shouldLookupInSeparateExternalFileMapWithName() { assertMap( "put_filemap('" + CSV_MAP + "', 'testMap')", - "lookup('title', 'testMap')" + LOOKUP + " 'testMap')" ); } @@ -137,7 +133,7 @@ public void shouldLookupInSeparateExternalFileMapWithName() { public void shouldLookupInSeparateExternalFileMapWithOptions() { assertMap( "put_filemap('" + TSV_MAP + "', sep_char: '\t')", - "lookup('title', '" + TSV_MAP + "')" + LOOKUP + " '" + TSV_MAP + "')" ); } @@ -145,7 +141,7 @@ public void shouldLookupInSeparateExternalFileMapWithOptions() { public void shouldLookupInSeparateExternalFileMapWithNameAndOptions() { assertMap( "put_filemap('" + TSV_MAP + "', 'testMap', sep_char: '\t')", - "lookup('title', 'testMap')" + LOOKUP + " 'testMap')" ); } @@ -154,7 +150,7 @@ public void shouldDefineMultipleSeparateMaps() { assertMap( "put_map('testMap', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)", "put_map('testMap2', __default: Hi)", - "lookup('title', 'testMap')" + LOOKUP + " 'testMap')" ); } @@ -163,7 +159,7 @@ public void shouldOverwriteExistingSeparateMap() { assertMap( "put_map('testMap', __default: Hi)", "put_filemap('" + CSV_MAP + "', 'testMap')", - "lookup('title', 'testMap')" + LOOKUP + " 'testMap')" ); } @@ -171,7 +167,7 @@ public void shouldOverwriteExistingSeparateMap() { public void shouldIgnoreOptionsOnLookupInSeparateInternalMap() { assertMap( "put_map('testMap', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)", - "lookup('title', 'testMap', __default: Hi)" + LOOKUP + " 'testMap', __default: Hi)" ); } @@ -179,14 +175,14 @@ public void shouldIgnoreOptionsOnLookupInSeparateInternalMap() { public void shouldIgnoreOptionsOnLookupInSeparateExternalFileMap() { assertMap( "put_filemap('" + CSV_MAP + "')", - "lookup('title', '" + CSV_MAP + "', sep_char: '\t')" + LOOKUP + " '" + CSV_MAP + "', sep_char: '\t')" ); } @Test public void shouldNotLookupInExternalFileMapWithWrongOptions() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "lookup('title', '" + CSV_MAP + "', sep_char: '\t')" + LOOKUP + " '" + CSV_MAP + "', sep_char: '\t')" ), i -> { i.startRecord("1"); @@ -197,6 +193,8 @@ public void shouldNotLookupInExternalFileMapWithWrongOptions() { }, o -> { o.get().startRecord("1"); + o.get().startEntity("title"); + o.get().endEntity(); o.get().endRecord(); } ); @@ -205,8 +203,8 @@ public void shouldNotLookupInExternalFileMapWithWrongOptions() { @Test public void shouldIgnoreOptionsOnSubsequentLookupInExternalFileMap() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "lookup('title', '" + CSV_MAP + "')", - "lookup('title', '" + CSV_MAP + "', sep_char: '\t')" + LOOKUP + " '" + CSV_MAP + "')", + LOOKUP + " '" + CSV_MAP + "', sep_char: '\t')" ), i -> { i.startRecord("1"); @@ -231,7 +229,7 @@ public void shouldIgnoreOptionsOnSubsequentLookupInExternalFileMap() { public void shouldFailLookupInUnknownNamedMap() { final Throwable exception = Assertions.assertThrows(MorphExecutionException.class, () -> MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "lookup('title', 'testMap')" + LOOKUP + " 'testMap')" ), i -> { i.startRecord("1"); diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java index d58793ac..ce1bf92c 100644 --- a/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java @@ -16,18 +16,13 @@ package org.metafacture.metafix; -import org.metafacture.framework.MetafactureException; import org.metafacture.framework.StreamReceiver; import com.google.common.collect.ImmutableMap; -import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.jupiter.api.function.Executable; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; import org.mockito.junit.jupiter.MockitoExtension; import java.util.Arrays; @@ -41,9 +36,6 @@ @ExtendWith(MockitoExtension.class) public class MetafixMethodTest { - @RegisterExtension - private MockitoRule mockitoRule = MockitoJUnit.rule(); - @Mock private StreamReceiver streamReceiver; @@ -51,212 +43,196 @@ public MetafixMethodTest() { } @Test - public void upcase() { + public void shouldUpcaseString() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "upcase('title')"), + "upcase('title')" + ), i -> { i.startRecord("1"); - i.endRecord(); - - i.startRecord("2"); i.literal("title", "marc"); - i.literal("title", "json"); i.endRecord(); - - i.startRecord("3"); - i.endRecord(); - }, o -> { + }, + o -> { o.get().startRecord("1"); + o.get().literal("title", "MARC"); o.get().endRecord(); - - o.get().startRecord("2"); - o.get().startEntity("title"); - o.get().literal("1", "MARC"); - o.get().literal("2", "JSON"); - o.get().endEntity(); - o.get().endRecord(); - - o.get().startRecord("3"); - o.get().endRecord(); - }); + } + ); } @Test - public void upcaseDotNotationNested() { + public void shouldUpcaseDotNotationNested() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "upcase('data.title')"), + "upcase('data.title')" + ), i -> { i.startRecord("1"); i.startEntity("data"); i.literal("title", "marc"); - i.literal("title", "json"); i.endEntity(); i.endRecord(); - }, (o, f) -> { + }, + o -> { o.get().startRecord("1"); o.get().startEntity("data"); - o.get().startEntity("title"); - o.get().literal("1", "MARC"); - o.get().literal("2", "JSON"); - f.apply(2).endEntity(); + o.get().literal("title", "MARC"); + o.get().endEntity(); o.get().endRecord(); - }); + } + ); } @Test - public void downcase() { + public void shouldDowncaseString() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "downcase('title')"), + "downcase('title')" + ), i -> { i.startRecord("1"); + i.literal("title", "MARC"); i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "marc"); + o.get().endRecord(); + } + ); + } - i.startRecord("2"); + @Test + public void shouldDowncaseStringsInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "downcase('title.*')" + ), + i -> { + i.startRecord("1"); i.literal("title", "MARC"); i.literal("title", "Json"); i.endRecord(); - - i.startRecord("3"); - i.endRecord(); - }, o -> { + }, + o -> { o.get().startRecord("1"); - o.get().endRecord(); - - o.get().startRecord("2"); o.get().startEntity("title"); o.get().literal("1", "marc"); o.get().literal("2", "json"); o.get().endEntity(); o.get().endRecord(); - - o.get().startRecord("3"); - o.get().endRecord(); - }); + } + ); } @Test - public void capitalize() { + public void shouldCapitalizeString() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "capitalize('title')"), + "capitalize('title')" + ), i -> { i.startRecord("1"); + i.literal("title", "marc"); i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("title", "Marc"); + o.get().endRecord(); + } + ); + } - i.startRecord("2"); + @Test + public void shouldCapitalizeStringsInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "capitalize('title.*')" + ), + i -> { + i.startRecord("1"); i.literal("title", "marc"); i.literal("title", "json"); i.endRecord(); - - i.startRecord("3"); - i.endRecord(); - }, o -> { + }, + o -> { o.get().startRecord("1"); - o.get().endRecord(); - - o.get().startRecord("2"); o.get().startEntity("title"); o.get().literal("1", "Marc"); o.get().literal("2", "Json"); o.get().endEntity(); o.get().endRecord(); + } + ); + } - o.get().startRecord("3"); - o.get().endRecord(); - }); + @Test + public void shouldNotCapitalizeArray() { + MetafixTestHelpers.assertThrows(IllegalStateException.class, "expected String, got Array", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "capitalize('title')" + ), + i -> { + i.startRecord("1"); + i.literal("title", "marc"); + i.literal("title", "json"); + i.endRecord(); + }, + o -> { + } + ) + ); } @Test - public void substring() { + public void shouldGetSubstringOfString() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "substring('title', '0', '2')"), + "substring('title', '0', '2')" + ), i -> { i.startRecord("1"); - i.endRecord(); - - i.startRecord("2"); i.literal("title", "marc"); - i.literal("title", "json"); i.endRecord(); - - i.startRecord("3"); - i.endRecord(); - }, o -> { + }, + o -> { o.get().startRecord("1"); + o.get().literal("title", "m"); o.get().endRecord(); - - o.get().startRecord("2"); - o.get().startEntity("title"); - o.get().literal("1", "m"); - o.get().literal("2", "j"); - o.get().endEntity(); - o.get().endRecord(); - - o.get().startRecord("3"); - o.get().endRecord(); - }); + } + ); } @Test - public void substringWithVar() { + public void shouldGetSubstringWithVar() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "substring('title', '0', '$[end]')"), - ImmutableMap.of("end", "3"), + "substring('title', '0', '$[end]')" + ), + ImmutableMap.of("end", "3"), i -> { i.startRecord("1"); - i.endRecord(); - - i.startRecord("2"); i.literal("title", "marc"); - i.literal("title", "json"); - i.endRecord(); - - i.startRecord("3"); i.endRecord(); - }, o -> { + }, + o -> { o.get().startRecord("1"); + o.get().literal("title", "ma"); o.get().endRecord(); - - o.get().startRecord("2"); - o.get().startEntity("title"); - o.get().literal("1", "ma"); - o.get().literal("2", "js"); - o.get().endEntity(); - o.get().endRecord(); - - o.get().startRecord("3"); - o.get().endRecord(); - }); + } + ); } @Test - public void trim() { + public void shouldTrimString() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "trim('title')"), + "trim('title')" + ), i -> { i.startRecord("1"); - i.endRecord(); - - i.startRecord("2"); i.literal("title", " marc "); - i.literal("title", " json "); i.endRecord(); - - i.startRecord("3"); - i.endRecord(); - }, o -> { + }, + o -> { o.get().startRecord("1"); + o.get().literal("title", "marc"); o.get().endRecord(); - - o.get().startRecord("2"); - o.get().startEntity("title"); - o.get().literal("1", "marc"); - o.get().literal("2", "json"); - o.get().endEntity(); - o.get().endRecord(); - - o.get().startRecord("3"); - o.get().endRecord(); - }); + } + ); } @Test @@ -358,7 +334,7 @@ public void parseTextMixedGroups() { @Test public void parseTextEscapedGroups() { - assertThrows(IllegalArgumentException.class, "No group with name ", () -> + MetafixTestHelpers.assertThrows(IllegalArgumentException.class, "No group with name ", () -> MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "parse_text(data, '(?.)(.)\\\\(?.\\\\)')" ), @@ -375,7 +351,7 @@ public void parseTextEscapedGroups() { @Test public void parseTextQuotedGroups() { - assertThrows(IllegalArgumentException.class, "No group with name ", () -> + MetafixTestHelpers.assertThrows(IllegalArgumentException.class, "No group with name ", () -> MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "parse_text(data, '(?.)(.)\\\\Q(?.)\\\\E')" ), @@ -517,6 +493,128 @@ public void shouldAppendValue() { ); } + @Test + // TODO: Fix order (`animols.name` should stay before `animols.type`) + public void shouldAppendValueInHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "append(animols.name, ' boss')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animols"); + i.literal("name", "bird"); + i.literal("type", "TEST"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animols"); + o.get().literal("type", "TEST"); + o.get().literal("name", "bird boss"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldAppendValueInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "append('animals[].1', ' is cool')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "dog is cool"); + o.get().literal("2", "cat"); + o.get().literal("3", "zebra"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldAppendValueInEntireArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "append('animals[].*', ' is cool')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "dog is cool"); + o.get().literal("2", "cat is cool"); + o.get().literal("3", "zebra is cool"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldNotAppendValueToArray() { + MetafixTestHelpers.assertThrows(IllegalStateException.class, "expected String, got Array", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "append('animals[]', ' is cool')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldNotAppendValueToHash() { + MetafixTestHelpers.assertThrows(IllegalStateException.class, "expected String, got Hash", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "append('animals', ' is cool')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + @Test public void shouldCountNumberOfValuesInArray() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( @@ -607,6 +705,51 @@ public void shouldFilterArrayValuesInverted() { ); } + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldFilterArrayObjectValues() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "filter('animals[]', '[Cc]at')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "Lion"); + i.literal("2", "Cat"); + i.literal("3", "Tiger"); + i.literal("4", "Bobcat"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "Cat"); + o.get().literal("2", "Bobcat"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldGetFirstIndexOfSubstring() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "index(animal, 'n')" + ), + i -> { + i.startRecord("1"); + i.literal("animal", "bunny"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("animal", "2"); + o.get().endRecord(); + } + ); + } + @Test public void shouldGetIndexOfSubstring() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( @@ -646,6 +789,29 @@ public void shouldJoinArrayField() { ); } + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldJoinArrayObjectField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "join_field('animals[]', ',')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("animals[]", "dog,cat,zebra"); + o.get().endRecord(); + } + ); + } + @Test public void shouldPrependValue() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( @@ -664,6 +830,106 @@ public void shouldPrependValue() { ); } + @Test + // TODO: Fix order (`animols.name` should stay before `animols.type`) + public void shouldPrependValueInHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "prepend(animols.name, 'nice ')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animols"); + i.literal("name", "bird"); + i.literal("type", "TEST"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animols"); + o.get().literal("type", "TEST"); + o.get().literal("name", "nice bird"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldPrependValueInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "prepend('animals[].1', 'cool ')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "cool dog"); + o.get().literal("2", "cat"); + o.get().literal("3", "zebra"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldPrependValueInEntireArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "prepend('animals[].*', 'cool ')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "cool dog"); + o.get().literal("2", "cool cat"); + o.get().literal("3", "cool zebra"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldNotPrependValueToArray() { + MetafixTestHelpers.assertThrows(IllegalStateException.class, "expected String, got Array", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "prepend('animals[]', 'cool ')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + @Test public void shouldReplaceAllRegexes() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( @@ -682,6 +948,33 @@ public void shouldReplaceAllRegexes() { ); } + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldReplaceAllRegexesInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "replace_all('animals[].*', a, QR)" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "dog"); + i.literal("2", "cat"); + i.literal("3", "zebra"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "dog"); + o.get().literal("2", "cQRt"); + o.get().literal("3", "zebrQR"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + @Test public void shouldReverseString() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( @@ -776,7 +1069,7 @@ public void shouldSortFieldNumerically() { @Test public void shouldFailToSortNumericallyWithInvalidNumber() { - assertThrows(NumberFormatException.class, "For input string: \"x\"", () -> + MetafixTestHelpers.assertThrows(NumberFormatException.class, "For input string: \"x\"", () -> MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "sort_field(numbers, numeric: 'true')" ), @@ -953,6 +1246,35 @@ public void shouldSplitHashField() { ); } + @Test + @Disabled("See https://github.com/metafacture/metafacture-fix/issues/100") + public void shouldSplitNestedField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "split_field('others[].*.tools', '--')" + ), + i -> { + i.startRecord("1"); + i.startEntity("others[]"); + i.startEntity("1"); + i.literal("tools", "hammer--saw--bow"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("others[]"); + o.get().startEntity("1"); + o.get().startEntity("tools"); + o.get().literal("1", "hammer"); + o.get().literal("2", "saw"); + o.get().literal("3", "bow"); + f.apply(3).endEntity(); + o.get().endRecord(); + } + ); + } + @Test public void shouldSumNumbers() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( @@ -1020,10 +1342,4 @@ public void shouldApplyCustomJavaFunction() { ); } - private void assertThrows(final Class expectedClass, final String expectedMessage, final Executable executable) { - final Throwable exception = Assertions.assertThrows(MetafactureException.class, executable).getCause(); - Assertions.assertSame(expectedClass, exception.getClass()); - Assertions.assertEquals(expectedMessage, exception.getMessage()); - } - } diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java index d1ce65c4..83a125e8 100644 --- a/metafix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java @@ -16,18 +16,13 @@ package org.metafacture.metafix; -import org.metafacture.framework.MetafactureException; import org.metafacture.framework.StreamReceiver; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.extension.RegisterExtension; import org.mockito.ArgumentMatchers; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; import org.mockito.junit.jupiter.MockitoExtension; import java.util.Arrays; @@ -41,9 +36,6 @@ @ExtendWith(MockitoExtension.class) public class MetafixRecordTest { - @RegisterExtension - private MockitoRule mockitoRule = MockitoJUnit.rule(); - @Mock private StreamReceiver streamReceiver; @@ -484,6 +476,7 @@ public void complexAppendWithArrayOfObjects() { @Test public void appendWithWildcard() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('stringimals[]')", "copy_field('?nimal', 'stringimals[].$append')" ), i -> { @@ -549,6 +542,7 @@ public void simpleCopyWithWildcard() { @Test public void appendWithMultipleWildcards() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('stringimals[]')", "copy_field('?ni??l', 'stringimals[].$append')" ), i -> { @@ -579,6 +573,7 @@ public void appendWithMultipleWildcards() { @Test public void appendWithAsteriksWildcard() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('stringimals[]')", "copy_field('*al', 'stringimals[].$append')" ), i -> { @@ -608,6 +603,7 @@ public void appendWithAsteriksWildcard() { @Test public void appendWithBracketWildcard() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('stringimals[]')", "copy_field('[ac]nimal', 'stringimals[].$append')" ), i -> { @@ -637,6 +633,7 @@ public void appendWithBracketWildcard() { // See https://github.com/metafacture/metafacture-fix/issues/89 public void appendWithAsteriksWildcardAtTheEnd() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('stringimals[]')", "copy_field('ani*', 'stringimals[].$append')" ), i -> { @@ -897,26 +894,31 @@ public void addFieldToObjectByIndexInIndexedArray() { @Test public void addFieldToFirstObjectMissing() { - assertThrowsOnEmptyRecord("add_field('animals[].$first.kind','nice')"); + assertThrowsOnEmptyRecord("$first"); } @Test public void addFieldToLastObjectMissing() { - assertThrowsOnEmptyRecord("add_field('animals[].$last.kind','nice')"); + assertThrowsOnEmptyRecord("$last"); } @Test public void addFieldToObjectByIndexMissing() { - assertThrowsOnEmptyRecord("add_field('animals[].2.kind','nice')"); + assertThrowsOnEmptyRecord("2"); } - private void assertThrowsOnEmptyRecord(final String fix) { - Assertions.assertThrows(MetafactureException.class, () -> { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(fix), + private void assertThrowsOnEmptyRecord(final String index) { + MetafixTestHelpers.assertThrows(IllegalArgumentException.class, "Using ref, but can't find: " + index + " in: {}", () -> { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "add_field('animals[]." + index + ".kind','nice')" + ), i -> { i.startRecord("1"); i.endRecord(); - }, o -> { }); + }, + o -> { + } + ); }); } @@ -1597,42 +1599,43 @@ public void accessArrayByIndex() { } @Test - public void accessArrayImplicit() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "upcase('name')"), - i -> { - i.startRecord("1"); - i.literal("name", "max"); - i.literal("name", "mo"); - i.endRecord(); - }, (o, f) -> { - o.get().startRecord("1"); - o.get().startEntity("name"); - o.get().literal("1", "MAX"); - o.get().literal("2", "MO"); - o.get().endEntity(); - o.get().endRecord(); - }); + public void shouldNotAccessArrayImplicitly() { + MetafixTestHelpers.assertThrows(IllegalStateException.class, "expected String, got Array", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "upcase('name')" + ), + i -> { + i.startRecord("1"); + i.literal("name", "max"); + i.literal("name", "mo"); + i.endRecord(); + }, + o -> { + } + ) + ); } @Test - // TODO: WDCD? explicit * for array fields? - public void accessArrayByWildcard() { + public void shouldAccessArrayByWildcard() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "upcase('name.*')"), + "upcase('name.*')" + ), i -> { i.startRecord("1"); i.literal("name", "max"); i.literal("name", "mo"); i.endRecord(); - }, (o, f) -> { + }, + o -> { o.get().startRecord("1"); o.get().startEntity("name"); o.get().literal("1", "MAX"); o.get().literal("2", "MO"); o.get().endEntity(); o.get().endRecord(); - }); + } + ); } @Test @@ -1767,6 +1770,120 @@ public void shouldAddRandomNumber() { ); } + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldReplaceExistingValueWithRandomNumber() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "random(others, '100')" + ), + i -> { + i.startRecord("1"); + i.literal("others", "human"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal(ArgumentMatchers.eq("others"), ArgumentMatchers.argThat(i -> Integer.parseInt(i) < 100)); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldAddRandomNumberToMarkedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "random('animals[].$append', '100')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.literal("1", "cat"); + i.literal("2", "dog"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().literal("1", "cat"); + o.get().literal("2", "dog"); + o.get().literal(ArgumentMatchers.eq("3"), ArgumentMatchers.argThat(i -> Integer.parseInt(i) < 100)); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldAddObjectWithRandomNumberToMarkedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('bnimals[]')", + "random('bnimals[].$append.number', '100')" + ), + i -> { + i.startRecord("1"); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("bnimals[]"); + o.get().startEntity("1"); + o.get().literal(ArgumentMatchers.eq("number"), ArgumentMatchers.argThat(i -> Integer.parseInt(i) < 100)); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldAddRandomNumberToUnmarkedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "random('animals.$append', '100')" + ), + i -> { + i.startRecord("1"); + i.literal("animals", "cat"); + i.literal("animals", "dog"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals"); + o.get().literal("1", "cat"); + o.get().literal("2", "dog"); + o.get().literal(ArgumentMatchers.eq("3"), ArgumentMatchers.argThat(i -> Integer.parseInt(i) < 100)); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + @Disabled("See https://github.com/metafacture/metafacture-fix/issues/100") + public void shouldAddRandomNumberToUnmarkedArrayObject() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "random('animals.$append', '100')" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals"); + i.literal("1", "cat"); + i.literal("2", "dog"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("animals"); + o.get().literal("1", "cat"); + o.get().literal("2", "dog"); + o.get().literal(ArgumentMatchers.eq("3"), ArgumentMatchers.argThat(i -> Integer.parseInt(i) < 100)); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + @Test public void shouldRenameFieldsInHash() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( @@ -1789,4 +1906,111 @@ public void shouldRenameFieldsInHash() { ); } + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldRecursivelyRenameFieldsInHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "rename(others, ani, QR)" + ), + i -> { + i.startRecord("1"); + i.startEntity("others"); + i.literal("animal", "human"); + i.literal("canister", "metall"); + i.startEntity("area"); + i.literal("ani", "test"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("others"); + o.get().literal("QRmal", "human"); + o.get().literal("cQRster", "metall"); + o.get().startEntity("area"); + o.get().literal("QR", "test"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + // See https://github.com/metafacture/metafacture-fix/issues/100 + public void shouldRecursivelyRenameFieldsInArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "rename('animals[]', ani, XY)" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals[]"); + i.startEntity("1"); + i.literal("animal", "dog"); + i.endEntity(); + i.startEntity("2"); + i.literal("animal", "cat"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("animals[]"); + o.get().startEntity("1"); + o.get().literal("XYmal", "dog"); + o.get().endEntity(); + o.get().startEntity("2"); + o.get().literal("XYmal", "cat"); + f.apply(2).endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + @Disabled("java.lang.ArrayIndexOutOfBoundsException: 0; see https://github.com/metafacture/metafacture-fix/issues/100") + public void shouldRenameAllFieldsInHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "rename('.', ani, XY)" + ), + i -> { + i.startRecord("1"); + i.startEntity("animals"); + i.literal("animal", "dog"); + i.literal("animal", "cat"); + i.endEntity(); + i.startEntity("others"); + i.literal("animal", "human"); + i.literal("canister", "metall"); + i.startEntity("area"); + i.literal("ani", "test"); + i.endEntity(); + i.endEntity(); + i.startEntity("fictional"); + i.literal("animal", "unicorn"); + i.endEntity(); + i.endRecord(); + }, + (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("XYmals"); + o.get().startEntity("XYmal"); + o.get().literal("1", "dog"); + o.get().literal("2", "cat"); + f.apply(2).endEntity(); + o.get().startEntity("others"); + o.get().literal("XYmal", "human"); + o.get().literal("cXYster", "metall"); + o.get().startEntity("area"); + o.get().literal("XY", "test"); + f.apply(2).endEntity(); + o.get().startEntity("fictional"); + o.get().literal("XYmal", "unicorn"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + } diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixSelectorTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixSelectorTest.java index 934412ad..0e2a5884 100644 --- a/metafix/src/test/java/org/metafacture/metafix/MetafixSelectorTest.java +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixSelectorTest.java @@ -21,10 +21,7 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.extension.RegisterExtension; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; import org.mockito.junit.jupiter.MockitoExtension; import java.util.Arrays; @@ -39,9 +36,6 @@ @Disabled("TODO: support Fix-style selectors https://github.com/LibreCat/Catmandu/wiki/Selectors") public final class MetafixSelectorTest { - @RegisterExtension - private MockitoRule mockitoRule = MockitoJUnit.rule(); - @Mock private StreamReceiver streamReceiver; diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java b/metafix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java index cde64818..89b51ca4 100644 --- a/metafix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java @@ -16,9 +16,11 @@ package org.metafacture.metafix; +import org.metafacture.framework.MetafactureException; import org.metafacture.framework.StreamReceiver; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.function.Executable; import org.mockito.InOrder; import org.mockito.Mockito; import org.mockito.exceptions.base.MockitoAssertionError; @@ -43,6 +45,12 @@ public final class MetafixTestHelpers { private MetafixTestHelpers() { } + public static void assertThrows(final Class expectedClass, final String expectedMessage, final Executable executable) { + final Throwable exception = Assertions.assertThrows(MetafactureException.class, executable).getCause(); + Assertions.assertSame(expectedClass, exception.getClass()); + Assertions.assertEquals(expectedMessage, exception.getMessage()); + } + public static void assertFix(final StreamReceiver receiver, final List fixDef, final Consumer in, final Consumer> out) { assertFix(receiver, fixDef, in, (s, f) -> out.accept(s), Metafix.NO_VARS);