From 28aecbab148fe62533b09bdae44f3c21d5492242 Mon Sep 17 00:00:00 2001 From: Jens Wille Date: Tue, 16 Nov 2021 11:51:39 +0100 Subject: [PATCH] Move static record/hash methods to Value/Hash class. (#64) --- .../org/metafacture/metafix/FixMethod.java | 216 ++++-------------- .../org/metafacture/metafix/FixPredicate.java | 4 +- .../java/org/metafacture/metafix/Metafix.java | 57 +---- .../metafix/RecordTransformer.java | 5 +- .../java/org/metafacture/metafix/Value.java | 200 ++++++++++++++++ .../metafacture/metafix/HashValueTest.java | 2 +- 6 files changed, 252 insertions(+), 232 deletions(-) diff --git a/metafix/src/main/java/org/metafacture/metafix/FixMethod.java b/metafix/src/main/java/org/metafacture/metafix/FixMethod.java index 9fd636ea..c4415c6b 100644 --- a/metafix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/metafix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -21,7 +21,6 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -32,8 +31,10 @@ enum FixMethod { set_field { public void apply(final Record record, final List params, final Map options) { - record.remove(params.get(0)); - insert(InsertMode.REPLACE, record, split(params.get(0)), params.get(1)); + final String field = params.get(0); + + record.remove(field); + record.replace(field, params.get(1)); } }, set_array { @@ -41,7 +42,7 @@ public void apply(final Record record, final List params, final Map toAdd = params.subList(1, params.size()); if (field.endsWith(DOT_APPEND)) { - Metafix.addAll(record, field.replace(DOT_APPEND, EMPTY), toAdd); + record.addAll(field.replace(DOT_APPEND, EMPTY), toAdd); } else { record.put(field, Value.newArray(a -> toAdd.forEach(s -> a.add(new Value(s))))); @@ -65,13 +66,15 @@ public void apply(final Record record, final List params, final Map params, final Map options) { - final String fieldName = params.get(0); - Metafix.asList(record.get(fieldName), a -> a.forEach(recordEntry -> { + final String field = params.get(0); + + record.getList(field, a -> a.forEach(recordEntry -> { if (recordEntry.isHash()) { - record.remove(fieldName); - recordEntry.asHash().forEach((subFieldName, value) -> { - Metafix.add(record, fieldName, new Value(subFieldName)); - Metafix.add(record, fieldName, value); + record.remove(field); + + recordEntry.asHash().forEach((subField, value) -> { + record.add(field, new Value(subField)); + record.add(field, value); }); } })); @@ -79,49 +82,55 @@ public void apply(final Record record, final List params, final Map params, final Map options) { - Metafix.asList(record.get(params.get(0)), values -> record.put(params.get(0), Value.newHash(h -> { - for (int i = 1; i < values.size(); i = i + 2) { - h.put(values.get(i - 1).toString(), values.get(i)); + final String field = params.get(0); + + record.getList(field, a -> record.put(field, Value.newHash(h -> { + for (int i = 1; i < a.size(); i = i + 2) { + h.put(a.get(i - 1).toString(), a.get(i)); } }))); } }, add_field { public void apply(final Record record, final List params, final Map options) { - insert(InsertMode.APPEND, record, split(params.get(0)), params.get(1)); + record.append(params.get(0), params.get(1)); } }, move_field { public void apply(final Record record, final List params, final Map options) { - copy(record, params); - remove(record, split(params.get(0))); + record.copy(params); + record.removeNested(params.get(0)); } }, copy_field { public void apply(final Record record, final List params, final Map options) { - copy(record, params); + record.copy(params); } }, remove_field { public void apply(final Record record, final List params, final Map options) { - params.forEach(p -> remove(record, split(p))); + params.forEach(record::removeNested); } }, format { public void apply(final Record record, final List params, final Map options) { - Metafix.asList(record.get(params.get(0)), oldValues -> { + final String field = params.get(0); + + record.getList(field, oldValues -> { final String newValue = String.format(params.get(1), oldValues.stream().toArray()); - record.replace(params.get(0), new Value(Arrays.asList(new Value(newValue)))); + record.replace(field, new Value(Arrays.asList(new Value(newValue)))); }); } }, parse_text { public void apply(final Record record, final List params, final Map options) { - Metafix.asList(record.get(params.get(0)), a -> a.forEach(v -> { + final String field = params.get(0); + + record.getList(field, a -> a.forEach(v -> { final Pattern p = Pattern.compile(params.get(1)); final Matcher m = p.matcher(v.toString()); if (m.matches()) { - record.remove(params.get(0)); + record.remove(field); /** * {@code Pattern.namedGroups()} not available as API, @@ -139,11 +148,11 @@ public void apply(final Record record, final List params, final Map params, final Map params, final Map options) { final String joinChar = options.get("join_char"); - insert(InsertMode.REPLACE, record, split(params.get(0)), params.subList(1, params.size()).stream() - .filter(f -> literalString(f) || find(record, split(f)) != null) - .map(f -> literalString(f) ? new Value(f.substring(1)) : Metafix.asList(find(record, split(f)), null).asArray().get(0)) + record.replace(params.get(0), params.subList(1, params.size()).stream() + .filter(f -> literalString(f) || record.find(f) != null) + .map(f -> literalString(f) ? new Value(f.substring(1)) : record.findList(f, null).asArray().get(0)) .map(Value::toString).collect(Collectors.joining(joinChar != null ? joinChar : " "))); } @@ -183,33 +192,32 @@ public void apply(final Record record, final List params, final Map params, final Map options) { - applyToFields(record, params, - s -> s.substring(Integer.parseInt(params.get(1)), Integer.parseInt(params.get(2)) - 1)); + record.transformFields(params, s -> s.substring(Integer.parseInt(params.get(1)), Integer.parseInt(params.get(2)) - 1)); } }, trim { public void apply(final Record record, final List params, final Map options) { - applyToFields(record, params, s -> s.trim()); + record.transformFields(params, String::trim); } }, upcase { public void apply(final Record record, final List params, final Map options) { - applyToFields(record, params, s -> s.toUpperCase()); + record.transformFields(params, String::toUpperCase); } }, downcase { public void apply(final Record record, final List params, final Map options) { - applyToFields(record, params, s -> s.toLowerCase()); + record.transformFields(params, String::toLowerCase); } }, capitalize { public void apply(final Record record, final List params, final Map options) { - applyToFields(record, params, s -> s.substring(0, 1).toUpperCase() + s.substring(1)); + record.transformFields(params, s -> s.substring(0, 1).toUpperCase() + s.substring(1)); } }, lookup { public void apply(final Record record, final List params, final Map options) { - applyToFields(record, params, s -> { + record.transformFields(params, s -> { final Map map = buildMap(options, params.size() <= 1 ? null : params.get(1)); return map.getOrDefault(s, map.get("__default")); // TODO Catmandu uses 'default' }); @@ -234,145 +242,9 @@ private Map fileMap(final String location, final String separato private static final Pattern NAMED_GROUP_PATTERN = Pattern.compile("\\(\\?<(.+?)>"); - private static final String NESTED = "Nested non-map / non-list: "; private static final String EMPTY = ""; - private static final String APPEND = "$append"; - private static final String DOT_APPEND = "." + APPEND; - private static final String LAST = "$last"; - - private static void applyToFields(final Record record, final List params, final Function fun) { - final String field = params.get(0); - final Value found = find(record, split(field)); - if (found != null) { - remove(record, split(field)); - - if (fun != null) { - Metafix.asList(found, a -> a.forEach(old -> insert(InsertMode.APPEND, record, split(field), fun.apply(old.toString())))); - } - } - } - - private static Value insert(final InsertMode mode, final Value.Hash hash, final String[] fields, final String value) { - final String currentField = fields[0]; - - if (fields.length == 1) { - mode.apply(hash, currentField, value); - } - else { - final String[] remainingFields = Arrays.copyOfRange(fields, 1, fields.length); - final Value nested = insertNested(mode, hash, value, currentField, remainingFields); - hash.put(currentField, nested); - } - - return new Value(hash); - } - - private static Value insertNested(final InsertMode mode, final Value.Hash hash, final String value, final String currentField, final String[] remainingFields) { - if (!hash.containsField(currentField)) { - hash.put(currentField, Value.newHash()); - } - final Value nested = hash.get(currentField); - final Value result; - if (nested.isHash()) { - result = insert(mode, nested.asHash(), remainingFields, value); - } - else if (nested.isArray()) { - processList(mode, value, remainingFields, nested.asArray()); - result = hash.get(currentField); - } - else { - throw new IllegalStateException(NESTED + nested); - } - return result; - } - - private static void processList(final InsertMode mode, final String value, final String[] remainingFields, final Value.Array nestedList) { - final Value nestedMap; - switch (remainingFields[0]) { - case APPEND: - nestedList.add(Value.newHash(h -> insert(mode, h, Arrays.copyOfRange(remainingFields, 1, remainingFields.length), value))); - break; - case LAST: - final Value last = nestedList.get(nestedList.size() - 1); - if (last.isHash()) { - insert(mode, last.asHash(), Arrays.copyOfRange(remainingFields, 1, remainingFields.length), value); - } - break; - default: - nestedList.add(Value.newHash(h -> insert(mode, h, remainingFields, value))); - break; - } - } - - static Value find(final Value.Hash hash, final String[] fields) { - final String currentField = fields[0]; - if (!hash.containsField(currentField) || fields.length == 1) { - return hash.get(currentField); - } - final String[] remainingFields = Arrays.copyOfRange(fields, 1, fields.length); - return findNested(hash, currentField, remainingFields); - } - - private static Value findNested(final Value.Hash hash, final String currentField, final String[] remainingFields) { - final Value nested = hash.get(currentField); - - // TODO: array of maps, like in insertNested - - if (nested.isHash()) { - return find(nested.asHash(), remainingFields); - } - - throw new IllegalStateException(NESTED + nested); - } - - private static Value remove(final Value.Hash hash, final String[] fields) { - final String currentField = fields[0]; - if (fields.length == 1) { - hash.remove(currentField); - } - if (!hash.containsField(currentField)) { - return new Value(hash); - } - final String[] remainingFields = Arrays.copyOfRange(fields, 1, fields.length); - return removeNested(hash, currentField, remainingFields); - } - - private static Value removeNested(final Value.Hash hash, final String currentField, final String[] remainingFields) { - final Value nested = hash.get(currentField); - if (nested.isHash()) { - return remove(nested.asHash(), remainingFields); - } - throw new IllegalStateException(NESTED + nested); - } - - private static void copy(final Record record, final List params) { - final String oldName = params.get(0); - final String newName = params.get(1); - final Value value = find(record, split(oldName)); - Metafix.asList(value, vs -> vs.forEach(v -> insert(InsertMode.APPEND, record, split(newName), v.toString()))); - } - - static String[] split(final String s) { - return s.split("\\."); - } - - private enum InsertMode { - REPLACE { - @Override - void apply(final Value.Hash hash, final String field, final String value) { - hash.put(field, new Value(value)); - } - }, - APPEND { - @Override - void apply(final Value.Hash hash, final String field, final String value) { - final Value oldValue = hash.get(field); - final Value newValue = new Value(value); - hash.put(field, oldValue == null ? newValue : Metafix.merged(oldValue, newValue)); - } - }; - abstract void apply(Value.Hash hash, String field, String value); - } + private static final String DOT_APPEND = "." + Value.Hash.APPEND_FIELD; abstract void apply(Record record, List params, Map options); + } diff --git a/metafix/src/main/java/org/metafacture/metafix/FixPredicate.java b/metafix/src/main/java/org/metafacture/metafix/FixPredicate.java index 7d2795c6..866eafc2 100644 --- a/metafix/src/main/java/org/metafacture/metafix/FixPredicate.java +++ b/metafix/src/main/java/org/metafacture/metafix/FixPredicate.java @@ -66,8 +66,8 @@ protected boolean test(final Record record, final String fieldName, final Predic }; boolean testStream(final Record record, final String fieldName, final Predicate> p) { - final Value value = FixMethod.find(record, FixMethod.split(fieldName)); - return value != null && p.test(Metafix.asList(value, null).asArray().stream()); + final Value value = record.find(fieldName); + return value != null && p.test(value.asList(null).asArray().stream()); } public boolean test(final Record record, final FixPredicate p, final List params) { diff --git a/metafix/src/main/java/org/metafacture/metafix/Metafix.java b/metafix/src/main/java/org/metafacture/metafix/Metafix.java index 1163e5f5..1e03be87 100644 --- a/metafix/src/main/java/org/metafacture/metafix/Metafix.java +++ b/metafix/src/main/java/org/metafacture/metafix/Metafix.java @@ -39,7 +39,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.function.Consumer; /** * Transforms a data stream sent via the {@link StreamReceiver} interface. Use @@ -147,7 +146,7 @@ public void endRecord() { } private void emit(final String field, final Value value) { - asList(value, array -> { + Value.asList(value, array -> { final boolean isMulti = array.size() > 1 || value.isArray(); if (isMulti) { outputStreamReceiver.startEntity(field + "[]"); @@ -195,7 +194,7 @@ private Value.Hash currentEntity(final String name, final Value.Hash previousEnt else { final Value value = Value.newHash(); currentEntity = value.asHash(); - add(previousEntity != null ? previousEntity : currentRecord, name, value); + (previousEntity != null ? previousEntity : currentRecord).add(name, value); } return currentEntity; } @@ -212,7 +211,7 @@ public void literal(final String name, final String value) { final Integer currentEntityIndex = entityCountStack.peek() - 1; final Value.Hash currentEntity = currentEntityIndex < 0 || entities.size() <= currentEntityIndex ? null : entities.get(currentEntityIndex); - add(currentEntity != null ? currentEntity : currentRecord, name, new Value(value)); + (currentEntity != null ? currentEntity : currentRecord).add(name, new Value(value)); // TODO: keep flattener as option? // flattener.literal(name, value); } @@ -251,54 +250,4 @@ public Record getCurrentRecord() { return currentRecord; } - static void addAll(final Value.Hash hash, final String fieldName, final List values) { - values.forEach(value -> add(hash, fieldName, new Value(value))); - } - - static void addAll(final Value.Hash hash, final Value.Hash values) { - values.forEach((fieldName, value) -> add(hash, fieldName, value)); - } - - static void add(final Value.Hash hash, final String name, final Value newValue) { - final Value oldValue = hash.get(name); - hash.put(name, oldValue == null ? newValue : merged(oldValue, newValue)); - } - - static Value merged(final Value value1, final Value value2) { - if (value1.isHash() && value2.isHash()) { - final Value.Hash hash = value1.asHash(); - value2.asHash().forEach(hash::put); - return value1; - } - else { - return asList(value1, a1 -> asList(value2, a2 -> a2.forEach(a1::add))); - } - } - - static Value asList(final Value value, final Consumer consumer) { - final Value result; - - if (Value.isNull(value)) { - result = null; - } - else if (value.isArray()) { - if (consumer != null) { - consumer.accept(value.asArray()); - } - - result = value; - } - else { - result = Value.newArray(a -> { - a.add(value); - - if (consumer != null) { - consumer.accept(a); - } - }); - } - - return result; - } - } diff --git a/metafix/src/main/java/org/metafacture/metafix/RecordTransformer.java b/metafix/src/main/java/org/metafacture/metafix/RecordTransformer.java index 8f0182c7..2ced5c0a 100644 --- a/metafix/src/main/java/org/metafacture/metafix/RecordTransformer.java +++ b/metafix/src/main/java/org/metafacture/metafix/RecordTransformer.java @@ -93,9 +93,8 @@ private void processBind(final Do theDo, final EList params) { if (theDo.getName().equals("list")) { // TODO impl multiple binds via FixBind enum final Map options = options(theDo.getOptions()); final Record fullRecord = record.shallowClone(); - final Value values = FixMethod.find(record, FixMethod.split(options.get("path"))); - Metafix.asList(values, a -> a.forEach(value -> { + record.findList(options.get("path"), a -> a.forEach(value -> { // for each value, bind the current record/scope/context to the given var name: record = new Record(); record.put(options.get("var"), value); @@ -104,7 +103,7 @@ record = new Record(); record.remove(options.get("var")); // and remember the things we added while bound (this probably needs some tweaking): - Metafix.addAll(fullRecord, record); + fullRecord.addAll(record); })); record = fullRecord; diff --git a/metafix/src/main/java/org/metafacture/metafix/Value.java b/metafix/src/main/java/org/metafacture/metafix/Value.java index 511da671..dbf9469a 100644 --- a/metafix/src/main/java/org/metafacture/metafix/Value.java +++ b/metafix/src/main/java/org/metafacture/metafix/Value.java @@ -17,12 +17,14 @@ package org.metafacture.metafix; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.function.BiConsumer; import java.util.function.Consumer; +import java.util.function.UnaryOperator; import java.util.stream.Stream; /** @@ -173,6 +175,40 @@ public String asString() { } } + public static Value asList(final Value value, final Consumer consumer) { + return isNull(value) ? null : value.asList(consumer); + } + + public Value asList(final Consumer consumer) { + if (isArray()) { + if (consumer != null) { + consumer.accept(asArray()); + } + + return this; + } + else { + return Value.newArray(a -> { + a.add(this); + + if (consumer != null) { + consumer.accept(a); + } + }); + } + } + + public Value merge(final Value value) { + if (isHash() && value.isHash()) { + final Value.Hash asHash = asHash(); + value.asHash().forEach(asHash::put); + return this; + } + else { + return asList(a1 -> value.asList(a2 -> a2.forEach(a1::add))); + } + } + @Override public String toString() { final String result; @@ -263,6 +299,11 @@ public String asString() { */ public static class Hash extends AbstractValueType { + /*package-private*/ static final String APPEND_FIELD = "$append"; + private static final String LAST_FIELD = "$last"; + + private static final String FIELD_PATH_SEPARATOR = "\\."; + private final Map map = new LinkedHashMap<>(); /** @@ -324,6 +365,14 @@ public void replace(final String field, final Value value) { } } + public Value replace(final String fieldPath, final String newValue) { + return insert(InsertMode.REPLACE, fieldPath, newValue); + } + + public Value append(final String fieldPath, final String newValue) { + return insert(InsertMode.APPEND, fieldPath, newValue); + } + /** * Retrieves the field value from this hash. * @@ -334,6 +383,102 @@ public Value get(final String field) { return map.get(field); } + public Value find(final String fieldPath) { + return find(split(fieldPath)); + } + + private Value find(final String[] fields) { + final String field = fields[0]; + + return fields.length == 1 || !containsField(field) ? get(field) : + findNested(field, Arrays.copyOfRange(fields, 1, fields.length)); + } + + private Value findNested(final String field, final String[] remainingFields) { + final Value value = get(field); + + // TODO: array of maps, like in insert nested + + if (value.isHash()) { + return value.asHash().find(remainingFields); + } + + throw new IllegalStateException("expected hash, got " + value.type); + } + + public Value findList(final String fieldPath, final Consumer consumer) { + return Value.asList(find(fieldPath), consumer); + } + + public Value getList(final String field, final Consumer consumer) { + return Value.asList(get(field), consumer); + } + + private String[] split(final String fieldPath) { + return fieldPath.split(FIELD_PATH_SEPARATOR); + } + + public void addAll(final String field, final List values) { + values.forEach(value -> add(field, new Value(value))); + } + + public void addAll(final Value.Hash hash) { + hash.forEach(this::add); + } + + public void add(final String field, final Value newValue) { + final Value oldValue = get(field); + put(field, oldValue == null ? newValue : oldValue.merge(newValue)); + } + + public Value insert(final InsertMode mode, final String fieldPath, final String newValue) { + return insert(mode, split(fieldPath), newValue); + } + + private Value insert(final InsertMode mode, final String[] fields, final String newValue) { + final String field = fields[0]; + + if (fields.length == 1) { + mode.apply(this, field, newValue); + } + else { + if (!containsField(field)) { + put(field, Value.newHash()); + } + + final String[] remainingFields = Arrays.copyOfRange(fields, 1, fields.length); + final String[] nestedFields = Arrays.copyOfRange(remainingFields, 1, remainingFields.length); + final Value value = get(field); + + if (value.isHash()) { + value.asHash().insert(mode, remainingFields, newValue); + } + else if (value.isArray()) { + final Value.Array array = value.asArray(); + + switch (remainingFields[0]) { + case APPEND_FIELD: + array.add(Value.newHash(h -> h.insert(mode, nestedFields, newValue))); + break; + case LAST_FIELD: + final Value last = array.get(array.size() - 1); + if (last.isHash()) { + last.asHash().insert(mode, nestedFields, newValue); + } + break; + default: + array.add(Value.newHash(h -> h.insert(mode, remainingFields, newValue))); + break; + } + } + else { + throw new IllegalStateException("expected array or hash, got " + value.type); + } + } + + return new Value(this); + } + /** * Removes the given field/value pair from this hash. * @@ -343,6 +488,40 @@ public void remove(final String field) { map.remove(field); } + public void removeNested(final String fieldPath) { + removeNested(split(fieldPath)); + } + + private void removeNested(final String[] fields) { + final String field = fields[0]; + + if (fields.length == 1) { + remove(field); + } + else if (containsField(field)) { + get(field).asHash().removeNested(Arrays.copyOfRange(fields, 1, fields.length)); + } + } + + public void copy(final List params) { + final String oldName = params.get(0); + final String newName = params.get(1); + findList(oldName, a -> a.forEach(v -> append(newName, v.toString()))); + } + + public void transformFields(final List params, final UnaryOperator operator) { + final String field = params.get(0); + final Value value = find(field); + + if (value != null) { + removeNested(field); + + if (operator != null) { + value.asList(a -> a.forEach(v -> append(field, operator.apply(v.toString())))); + } + } + } + /** * Retains only the given field/value pairs in this hash. * @@ -379,6 +558,27 @@ public String asString() { return map.toString(); } + private enum InsertMode { + + REPLACE { + @Override + void apply(final Value.Hash hash, final String field, final String value) { + hash.put(field, new Value(value)); + } + }, + APPEND { + @Override + void apply(final Value.Hash hash, final String field, final String value) { + final Value oldValue = hash.get(field); + final Value newValue = new Value(value); + hash.put(field, oldValue == null ? newValue : oldValue.merge(newValue)); + } + }; + + abstract void apply(Value.Hash hash, String field, String value); + + } + } } diff --git a/metafix/src/test/java/org/metafacture/metafix/HashValueTest.java b/metafix/src/test/java/org/metafacture/metafix/HashValueTest.java index 9d5ffdfb..a26e6030 100644 --- a/metafix/src/test/java/org/metafacture/metafix/HashValueTest.java +++ b/metafix/src/test/java/org/metafacture/metafix/HashValueTest.java @@ -129,7 +129,7 @@ public void shouldReplaceExistingField() { public void shouldNotReplaceExistingFieldWithNullValue() { final Value.Hash hash = newHash(); hash.put(FIELD, VALUE); - hash.replace(FIELD, null); + hash.replace(FIELD, (Value) null); Assertions.assertEquals(VALUE, hash.get(FIELD)); }