Skip to content
This repository has been archived by the owner on Jan 27, 2025. It is now read-only.

Commit

Permalink
Merge pull request #103 from metafacture/100-addFixFunctions
Browse files Browse the repository at this point in the history 
Add Fix functions from Catmandu Cheat Sheet.
  • Loading branch information
@blackwinter
blackwinter authored Jan 26, 2022
2 parents f1fcf99 + 1eebaac commit 1159022
Show file tree
Hide file tree
Showing 11 changed files with 1,598 additions and 196 deletions.
1 change: 1 addition & 0 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ subprojects {
ext {
versions = [
'ace': '1.3.3',
'equalsverifier': '3.8.2',
'jetty': '9.4.14.v20181114',
'jquery': '3.3.1-1',
'junit_jupiter': '5.4.2',
Expand Down
1 change: 1 addition & 0 deletions metafix/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ dependencies {
implementation "org.metafacture:metafacture-flowcontrol:${versions.metafacture}"
implementation "org.metafacture:metamorph:${versions.metafacture}"

testImplementation "nl.jqno.equalsverifier:equalsverifier:${versions.equalsverifier}"
testImplementation "org.mockito:mockito-core:${versions.mockito}"
testImplementation "org.mockito:mockito-junit-jupiter:${versions.mockito}"
}
Expand Down
197 changes: 188 additions & 9 deletions metafix/src/main/java/org/metafacture/metafix/FixMethod.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,30 @@
import org.metafacture.metamorph.maps.FileMap;

import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.UnaryOperator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;

enum FixMethod {

// SCRIPT-LEVEL METHODS:

nothing {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
// do nothing
}
},
put_filemap {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
final String fileName = params.get(0);
Expand Down Expand Up @@ -99,7 +113,7 @@ public void apply(final Metafix metafix, final Record record, final List<String>

record.getList(field, a -> record.put(field, Value.newHash(h -> {
for (int i = 1; i < a.size(); i = i + 2) {
h.put(a.get(i - 1).toString(), a.get(i));
h.put(a.get(i - 1).asString(), a.get(i));
}
})));
}
Expand All @@ -116,7 +130,7 @@ public void apply(final Metafix metafix, final Record record, final List<String>

record.getList(field, a -> a.forEach(v -> {
final Pattern p = Pattern.compile(params.get(1));
final Matcher m = p.matcher(v.toString());
final Matcher m = p.matcher(v.asString());
if (m.matches()) {
record.remove(field);

Expand Down Expand Up @@ -153,13 +167,21 @@ public void apply(final Metafix metafix, final Record record, final List<String>
record.replace(params.get(0), params.subList(1, params.size()).stream()
.filter(f -> literalString(f) || record.find(f) != null)
.map(f -> literalString(f) ? new Value(f.substring(1)) : record.findList(f, null).asArray().get(0))
.map(Value::toString).collect(Collectors.joining(joinChar != null ? joinChar : " ")));
.map(Value::asString).collect(Collectors.joining(joinChar != null ? joinChar : " ")));
}

private boolean literalString(final String s) {
return s.startsWith("~");
}
},
random {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
final String field = params.get(0);
final int max = getInteger(params, 1);

record.replace(field, String.valueOf(RANDOM.nextInt(max)));
}
},
reject {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
record.setReject(true);
Expand All @@ -170,6 +192,36 @@ public void apply(final Metafix metafix, final Record record, final List<String>
params.forEach(record::removeNested);
}
},
rename {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
final String search = params.get(1);
final String replace = params.get(2);

final UnaryOperator<String> operator = s -> s.replaceAll(search, replace);

record.transformField(params.get(0), (m, c) -> m
.ifArray(a -> c.accept(renameArray(a, operator)))
.ifHash(h -> c.accept(renameHash(h, operator)))
.orElseThrow()
);
}

private Value renameArray(final Value.Array array, final UnaryOperator<String> operator) {
return Value.newArray(a -> array.forEach(v -> a.add(renameValue(v, operator))));
}

private Value renameHash(final Value.Hash hash, final UnaryOperator<String> operator) {
return Value.newHash(h -> hash.forEach((f, v) -> h.put(operator.apply(f), renameValue(v, operator))));
}

private Value renameValue(final Value value, final UnaryOperator<String> operator) {
return value.extractType((m, c) -> m
.ifArray(a -> c.accept(renameArray(a, operator)))
.ifHash(h -> c.accept(renameHash(h, operator)))
.orElse(c)
);
}
},
retain {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
record.retainFields(params);
Expand All @@ -183,16 +235,13 @@ public void apply(final Metafix metafix, final Record record, final List<String>
record.addAll(field.replace(DOT_APPEND, EMPTY), toAdd);
}
else {
record.put(field, Value.newArray(a -> toAdd.forEach(s -> a.add(new Value(s)))));
record.put(field, newArray(toAdd.stream().map(Value::new)));
}
}
},
set_field {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
final String field = params.get(0);

record.remove(field);
record.replace(field, params.get(1));
record.replace(params.get(0), params.get(1));
}
},
set_hash {
Expand Down Expand Up @@ -220,16 +269,56 @@ public void apply(final Metafix metafix, final Record record, final List<String>

// TODO SPEC: switch to morph-style named params in general?

append {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
final String value = params.get(1);
record.transformFields(params, s -> s + value);
}
},
capitalize {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
record.transformFields(params, s -> s.substring(0, 1).toUpperCase() + s.substring(1));
}
},
count {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
record.transformField(params.get(0), (m, c) -> m
.ifArray(a -> c.accept(new Value(a.size())))
.ifHash(h -> c.accept(new Value(h.size())))
);
}
},
downcase {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
record.transformFields(params, String::toLowerCase);
}
},
filter {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
final Pattern search = Pattern.compile(params.get(1));
final boolean invert = getBoolean(options, "invert");

final Predicate<Value> predicate = s -> search.matcher(s.asString()).find();

record.transformField(params.get(0), (m, c) -> m
.ifArray(a -> c.accept(newArray(a.stream().filter(invert ? predicate.negate() : predicate))))
);
}
},
index {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
final String search = params.get(1);
record.transformFields(params, s -> String.valueOf(s.indexOf(search))); // TODO: multiple
}
},
join_field {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
final String joinChar = params.size() > 1 ? params.get(1) : "";
record.transformField(params.get(0), (m, c) -> m
.ifArray(a -> c.accept(new Value(a.stream().map(Value::asString).collect(Collectors.joining(joinChar)))))
);
}
},
lookup {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
final Map<String, String> map;
Expand All @@ -251,16 +340,87 @@ public void apply(final Metafix metafix, final Record record, final List<String>
record.transformFields(params, k -> map.getOrDefault(k, defaultValue));
}
},
prepend {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
final String value = params.get(1);
record.transformFields(params, s -> value + s);
}
},
replace_all {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
final String search = params.get(1);
final String replace = params.get(2);

record.transformFields(params, s -> s.replaceAll(search, replace));
}
},
reverse {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
record.transformField(params.get(0), (m, c) -> m
.ifArray(a -> {
final List<Value> list = a.stream().collect(Collectors.toList());
Collections.reverse(list);
c.accept(new Value(list));
})
.ifString(s -> c.accept(new Value(new StringBuilder(s).reverse().toString())))
);
}
},
sort_field {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
final boolean numeric = getBoolean(options, "numeric");
final boolean reverse = getBoolean(options, "reverse");
final boolean uniq = getBoolean(options, "uniq");

final Function<Value, String> function = Value::asString;
final Comparator<Value> comparator = numeric ?
Comparator.comparing(function.andThen(Integer::parseInt)) : Comparator.comparing(function);

record.transformField(params.get(0), (m, c) -> m
.ifArray(a -> c.accept(new Value((uniq ? unique(a.stream()) : a.stream())
.sorted(reverse ? comparator.reversed() : comparator).collect(Collectors.toList()))))
);
}
},
split_field {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
final String splitChar = params.size() > 1 ? params.get(1) : "\\s+";
final Pattern splitPattern = Pattern.compile(splitChar);

final Function<String, Value> splitFunction = s ->
newArray(Arrays.stream(splitPattern.split(s)).map(Value::new));

record.transformField(params.get(0), (m, c) -> m
.ifArray(a -> c.accept(newArray(a.stream().map(Value::asString).map(splitFunction))))
.ifHash(h -> c.accept(Value.newHash(n -> h.forEach((f, w) -> n.put(f, splitFunction.apply(w.asString()))))))
.ifString(s -> c.accept(splitFunction.apply(s)))
);
}
},
substring {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
record.transformFields(params, s -> s.substring(Integer.parseInt(params.get(1)), Integer.parseInt(params.get(2)) - 1));
record.transformFields(params, s -> s.substring(getInteger(params, 1), getInteger(params, 2) - 1));
}
},
sum {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
record.transformField(params.get(0), (m, c) -> m
.ifArray(a -> c.accept(new Value(a.stream().map(Value::asString).mapToInt(Integer::parseInt).sum())))
);
}
},
trim {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
record.transformFields(params, String::trim);
}
},
uniq {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
record.transformField(params.get(0), (m, c) -> m
.ifArray(a -> c.accept(newArray(unique(a.stream()))))
);
}
},
upcase {
public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
record.transformFields(params, String::toUpperCase);
Expand All @@ -275,6 +435,25 @@ public void apply(final Metafix metafix, final Record record, final List<String>
private static final String FILEMAP_SEPARATOR_OPTION = "sep_char";
private static final String FILEMAP_DEFAULT_SEPARATOR = ",";

private static final Random RANDOM = new Random();

private static boolean getBoolean(final Map<String, String> options, final String key) {
return Boolean.parseBoolean(options.get(key));
}

private static int getInteger(final List<String> params, final int index) {
return Integer.parseInt(params.get(index));
}

private static Value newArray(final Stream<Value> stream) {
return Value.newArray(a -> stream.forEach(a::add));
}

private static Stream<Value> unique(final Stream<Value> stream) {
final Set<Value> set = new HashSet<>();
return stream.filter(set::add);
}

abstract void apply(Metafix metafix, Record record, List<String> params, Map<String, String> options);

}
6 changes: 4 additions & 2 deletions metafix/src/main/java/org/metafacture/metafix/Metafix.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ public void startRecord(final String identifier) {
public void endRecord() {
entityCountStack.removeLast();
if (!entityCountStack.isEmpty()) {
throw new IllegalStateException(ENTITIES_NOT_BALANCED);
throw new MetafactureException(new IllegalStateException(ENTITIES_NOT_BALANCED));
}
flattener.endRecord();
LOG.debug("End record, walking fix: {}", currentRecord);
Expand All @@ -153,9 +153,11 @@ private void emit(final String field, final Value value) {
}

for (int i = 0; i < array.size(); ++i) {
final Value currentValue = array.get(i);
final String fieldName = isMulti ? String.valueOf(i + 1) : field;

array.get(i).matchType()
currentValue.matchType()
.ifArray(a -> emit(fieldName, currentValue))
.ifHash(h -> {
outputStreamReceiver.startEntity(fieldName);
h.forEach(this::emit);
Expand Down
Loading

0 comments on commit 1159022

Please sign in to comment.