From f328a1360bbb4e576133db9dd1741a47591b234a Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Fri, 2 Jul 2021 15:39:25 +0200 Subject: [PATCH 01/55] Re-approach record-based Metafix implementation (#35) Starting from a record map, not the event-based morph classes Following Catmandu Fix cheat sheet at https://github.com/LibreCat/Catmandu/wiki/Fixes-Cheat-Sheet --- .../org/metafacture/fix/web/FixServlet.java | 4 +- .../metafacture/fix/FixStandaloneSetup.java | 52 + .../org/metafacture/metamorph/FixBuilder.java | 525 ---------- .../metafacture/metamorph/FixFunction.java | 93 -- .../org/metafacture/metamorph/FixMethod.java | 143 +++ .../metafacture/metamorph/FixPredicate.java | 78 ++ .../org/metafacture/metamorph/Metafix.java | 414 +------- .../metafacture/metamorph/MetafixFilter.java | 125 --- .../metamorph/RecordTransformer.java | 147 +++ .../metafacture/metamorph/MetafixApiTest.java | 214 ---- .../metamorph/MetafixBindTest.java | 176 ++++ .../metafacture/metamorph/MetafixDslTest.java | 955 ------------------ ...xRecordTest.java => MetafixFieldTest.java} | 157 ++- .../metamorph/MetafixFilterTest.java | 92 -- .../metafacture/metamorph/MetafixIfTest.java | 337 ++++-- .../metamorph/MetafixLookupTest.java | 142 +++ .../metamorph/MetafixMethodTest.java | 316 ++++++ .../metamorph/MetafixSelectorTest.java | 88 ++ .../metamorph/MetafixTestHelpers.java | 19 +- 19 files changed, 1492 insertions(+), 2585 deletions(-) delete mode 100644 org.metafacture.fix/src/main/java/org/metafacture/metamorph/FixBuilder.java delete mode 100644 org.metafacture.fix/src/main/java/org/metafacture/metamorph/FixFunction.java create mode 100644 org.metafacture.fix/src/main/java/org/metafacture/metamorph/FixMethod.java create mode 100644 org.metafacture.fix/src/main/java/org/metafacture/metamorph/FixPredicate.java delete mode 100644 org.metafacture.fix/src/main/java/org/metafacture/metamorph/MetafixFilter.java create mode 100644 org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java delete mode 100644 org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixApiTest.java create mode 100644 org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixBindTest.java delete mode 100644 org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixDslTest.java rename org.metafacture.fix/src/test/java/org/metafacture/metamorph/{MetafixRecordTest.java => MetafixFieldTest.java} (52%) delete mode 100644 org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixFilterTest.java create mode 100644 org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixLookupTest.java create mode 100644 org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixMethodTest.java create mode 100644 org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixSelectorTest.java diff --git a/org.metafacture.fix.web/src/main/java/org/metafacture/fix/web/FixServlet.java b/org.metafacture.fix.web/src/main/java/org/metafacture/fix/web/FixServlet.java index 2b675426..95fc5443 100644 --- a/org.metafacture.fix.web/src/main/java/org/metafacture/fix/web/FixServlet.java +++ b/org.metafacture.fix.web/src/main/java/org/metafacture/fix/web/FixServlet.java @@ -1,6 +1,6 @@ package org.metafacture.fix.web; -import org.metafacture.metamorph.Metafix; +import org.metafacture.fix.FixStandaloneSetup; import org.metafacture.runner.Flux; import org.antlr.runtime.RecognitionException; @@ -106,7 +106,7 @@ private boolean process(final HttpServletRequest request, final HttpServletRespo } private String absPathToTempFile(final String content, final String suffix) throws IOException { - return Metafix.absPathToTempFile(new StringReader(content), suffix); + return FixStandaloneSetup.absPathToTempFile(new StringReader(content), suffix); } } diff --git a/org.metafacture.fix/src/main/java/org/metafacture/fix/FixStandaloneSetup.java b/org.metafacture.fix/src/main/java/org/metafacture/fix/FixStandaloneSetup.java index b4f86a0c..a6bbec11 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/fix/FixStandaloneSetup.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/fix/FixStandaloneSetup.java @@ -1,5 +1,23 @@ package org.metafacture.fix; +import org.metafacture.fix.fix.Fix; + +import org.eclipse.emf.common.util.URI; +import org.eclipse.emf.ecore.resource.Resource; +import org.eclipse.xtext.resource.XtextResource; +import org.eclipse.xtext.resource.XtextResourceSet; +import org.eclipse.xtext.util.CancelIndicator; +import org.eclipse.xtext.validation.CheckMode; +import org.eclipse.xtext.validation.IResourceValidator; +import org.eclipse.xtext.validation.Issue; +import com.google.common.io.CharStreams; +import com.google.inject.Injector; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Reader; + /** * Initialization support for running Xtext languages without Equinox extension registry. */ @@ -12,4 +30,38 @@ public static void doSetup() { new FixStandaloneSetup().createInjectorAndDoEMFRegistration(); } + public static Fix parseFix(final Reader fixDef) { + // TODO: do this only once per application + final Injector injector = new FixStandaloneSetup().createInjectorAndDoEMFRegistration(); + FixStandaloneSetup.doSetup(); + + try { + final URI uri = URI.createFileURI(absPathToTempFile(fixDef, ".fix")); + final Resource resource = injector.getInstance(XtextResourceSet.class).getResource(uri, true); + final IResourceValidator validator = ((XtextResource) resource).getResourceServiceProvider().getResourceValidator(); + + for (final Issue issue : validator.validate(resource, CheckMode.ALL, CancelIndicator.NullImpl)) { + System.err.println(issue.getMessage()); + } + + return (Fix) resource.getContents().get(0); + } + catch (final IOException e) { + e.printStackTrace(); + return null; + } + } + + public static String absPathToTempFile(final Reader fixDef, final String suffix) throws IOException { + // TODO: avoid temp file creation + final File file = File.createTempFile("metafix", suffix); + file.deleteOnExit(); + + try (FileWriter out = new FileWriter(file)) { + CharStreams.copy(fixDef, out); + } + + return file.getAbsolutePath(); + } + } diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/FixBuilder.java b/org.metafacture.fix/src/main/java/org/metafacture/metamorph/FixBuilder.java deleted file mode 100644 index 3e118919..00000000 --- a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/FixBuilder.java +++ /dev/null @@ -1,525 +0,0 @@ -/* - * Copyright 2013, 2020 Deutsche Nationalbibliothek and others - * - * Licensed under the Apache License, Version 2.0 the "License"; - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.metafacture.metamorph; - -import org.metafacture.commons.StringUtil; -import org.metafacture.fix.fix.Do; -import org.metafacture.fix.fix.Expression; -import org.metafacture.fix.fix.Fix; -import org.metafacture.fix.fix.If; -import org.metafacture.fix.fix.MethodCall; -import org.metafacture.fix.fix.Options; -import org.metafacture.metamorph.api.Collect; -import org.metafacture.metamorph.api.ConditionAware; -import org.metafacture.metamorph.api.FlushListener; -import org.metafacture.metamorph.api.Function; -import org.metafacture.metamorph.api.InterceptorFactory; -import org.metafacture.metamorph.api.NamedValuePipe; -import org.metafacture.metamorph.functions.Constant; -import org.metafacture.metamorph.functions.NotEquals; -import org.metafacture.metamorph.functions.Replace; - -import com.google.common.collect.Multimap; -import org.eclipse.emf.common.util.EList; -import org.eclipse.xtext.xbase.lib.Pair; - -import java.util.Deque; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.regex.Pattern; - -/** - * Builds a {@link Metafix} from a Fix DSL description - * - * @author Markus Michael Geipel (MorphBuilder) - * @author Christoph Böhme (MorphBuilder) - * @author Fabian Steeg (FixBuilder) - * - */ -public class FixBuilder { // checkstyle-disable-line ClassDataAbstractionCoupling|ClassFanOutComplexity - - static final String ARRAY_MARKER = "[]"; - private static final String FLUSH_WITH = "flushWith"; - private static final String RECORD = "record"; - - private final Deque stack = new LinkedList<>(); - private final InterceptorFactory interceptorFactory; - private final Metafix metafix; - - private CollectFactory collectFactory; - private FunctionFactory functionFactory; - - public FixBuilder(final Metafix metafix, final InterceptorFactory interceptorFactory) { - this.metafix = metafix; - this.interceptorFactory = interceptorFactory; - - stack.push(new StackFrame(metafix)); - - collectFactory = new CollectFactory(); - functionFactory = new FunctionFactory(); - // morph: not-equals, replace, fix: not_equals, replace_all - functionFactory.registerClass("not_equals", NotEquals.class); - functionFactory.registerClass("replace_all", Replace.class); - } - - Metafix getMetafix() { - return metafix; - } - - public void walk(final Fix fix) { - processSubexpressions(fix.getElements(), null); - } - - private void processBind(final Expression expression, final EList params) { - final String firstParam = resolvedAttribute(params, 1); - final Do theDo = (Do) expression; - Collect collect = null; - - // Special bind cases, no generic no-args collectors - switch (expression.getName()) { - case "entity": - collect = createEntity(firstParam); - break; - case "array": - collect = createEntity(firstParam + ARRAY_MARKER); - break; - case "map": - final NamedValuePipe enterDataMap = enterDataMap(params, false); - processSubexpressions(theDo.getElements(), firstParam); - exitData(); - if (enterDataMap instanceof Entity) { - exitCollectorAndFlushWith(firstParam); - } - return; - default: - break; - } - - // try generic no-args collectors, registered in collectFactory - if (collect == null) { - if (!collectFactory.containsKey(expression.getName())) { - throw new IllegalArgumentException("Collector " + expression.getName() + - " not found"); - } - final Map attributes = resolvedAttributeMap(params, theDo.getOptions()); - // flushWith should not be passed to the headPipe object via a - // setter (see newInstance): - attributes.remove(FLUSH_WITH); - collect = collectFactory.newInstance(expression.getName(), attributes); - } - if (collect != null) { - stack.push(new StackFrame(collect)); - processSubexpressions(theDo.getElements(), firstParam); - // must be set after recursive calls to flush descendants before parent - final String flushWith = resolvedAttribute(resolvedAttributeMap(params, theDo.getOptions()), FLUSH_WITH); - exitCollectorAndFlushWith(flushWith); - } - } - - protected final String resolveVars(final String string) { - return string == null ? null : StringUtil.format(string, Metafix.VAR_START, Metafix.VAR_END, false, metafix.getVars()); - } - - protected final Map resolvedAttributeMap(final List params, final Options options) { - final Map attributes = new HashMap(); - if (options == null) { - return attributes; - } - final EList keys = options.getKeys(); - final EList values = options.getValues(); - for (int i = 0; i < keys.size() && i < values.size(); i = i + 1) { - attributes.put(resolveVars(keys.get(i)), resolveVars(values.get(i))); - } - return attributes; - } - - private Collect createEntity(final String name) { - final Entity entity = new Entity(() -> metafix.getStreamReceiver()); - entity.setName(name); - return entity; - } - - protected void exitCollectorAndFlushWith(final String flushWith) { - final StackFrame currentCollect = stack.pop(); - final NamedValuePipe tailPipe = currentCollect.getPipe(); - - final NamedValuePipe interceptor = interceptorFactory.createNamedValueInterceptor(); - final NamedValuePipe delegate; - if (interceptor == null || tailPipe instanceof Entity) { - // The result of entity collectors cannot be intercepted - // because they only use the receive/emit interface for - // signalling while the actual data is transferred using - // a custom mechanism. In order for this to work the Entity - // class checks whether source and receiver are an - // instances of Entity. If an interceptor is inserted between - // entity elements this mechanism will break. - delegate = tailPipe; - } - else { - delegate = interceptor; - delegate.addNamedValueSource(tailPipe); - } - - final StackFrame parent = stack.peek(); - - if (parent.isInEntity()) { - ((Entity) parent.getPipe()).setNameSource(delegate); - } - else if (parent.isInCondition()) { - ((ConditionAware) parent.getPipe()).setConditionSource(delegate); - } - else { - parent.getPipe().addNamedValueSource(delegate); - } - final Collect collector = (Collect) tailPipe; - if (null != flushWith) { - collector.setWaitForFlush(true); - registerFlush(flushWith, collector); - } - } - - private void registerFlush(final String flushWith, final FlushListener flushListener) { - final String[] keysSplit = Pattern.compile("|", Pattern.LITERAL).split(flushWith); - for (final String key : keysSplit) { - final FlushListener interceptor = interceptorFactory.createFlushInterceptor(flushListener); - final FlushListener delegate; - if (interceptor == null) { - delegate = flushListener; - } - else { - delegate = interceptor; - } - if (key.equals(RECORD)) { - metafix.registerRecordEndFlush(delegate); - } - else { - metafix.registerNamedValueReceiver(key, new Flush(delegate)); - } - } - } - - private void processSubexpressions(final List expressions, final String superSource) { - for (final Expression sub : expressions) { - final EList p = sub.getParams(); - String source = resolvedAttribute(p, 1); - if (source == null && superSource != null) { - source = superSource; - } - if (sub instanceof Do) { - processBind(sub, p); - } - else if (sub instanceof If) { - processConditional(sub, p); - } - else { - processFunction(sub, p, source); - } - } - } - - private void processConditional(final Expression expression, final EList p) { - final boolean isTopLevelIf = stack.peek().pipe instanceof Metafix; - // If we're at the top level, we wrap the If into a collector: - if (isTopLevelIf) { - final Collect collect = collectFactory.newInstance("choose"); - stack.push(new StackFrame(collect)); - } - enterIf(); - final If theIf = (If) expression; - enterDataMap(p, false); - final Map attributes = resolvedAttributeMap(p, null); - attributes.put("string", resolvedAttribute(p, 2)); // for contains & equals morph functions - // TODO: support morph functions: regexp -> match, morph quantors as prefixes: none_, all_, any_ - runMetamorphFunction(expression.getName(), attributes); - exitData(); - // The Metamorph IF acts like a guard, executing not nested statements, but following statements: - exitIf(); - processSubexpressions(theIf.getElements(), resolvedAttribute(p, 1)); - // As a draft for a record mode, we might do something like this instead: - if (metafix.isRecordMode() && testConditional(theIf.getName(), p)) { - processSubexpressions(theIf.getElements(), resolvedAttribute(p, 1)); - } - // If we're at the top level, we close the wrapping collector: - if (isTopLevelIf) { - exitCollectorAndFlushWith(RECORD); - } - } - - private boolean testConditional(final String conditional, final EList p) { - System.out.printf(": %s p: %s\n", conditional, p); - boolean result = false; - final String field = resolvedAttribute(p, 1); - final String value = resolvedAttribute(p, 2); - final Multimap map = metafix.getCurrentRecord(); - System.out.printf("<%s>: field: %s value: %s in: %s\n", conditional, field, value, map); - switch (conditional) { - case "any_match": - result = map.containsKey(field) && map.get(field).stream().anyMatch(v -> v.matches(value)); - break; - case "all_match": - result = map.containsKey(field) && map.get(field).stream().allMatch(v -> v.matches(value)); - break; - default: - } - return result; - } - - private void processFunction(final Expression expression, final List params, final String source) { - final FixFunction functionToRun = findFixFunction(expression); - if (functionToRun != null) { - System.out.printf("Running Fix function %s, params %s, source %s\n", expression.getName(), params, source); - functionToRun.apply(this, expression, params, source); - } - else { - final Options options = ((MethodCall) expression).getOptions(); - final Map attributes = resolvedAttributeMap(params, options); - runMetamorphFunction(expression.getName(), attributes); - } - } - - private FixFunction findFixFunction(final Expression expression) { - for (final FixFunction exp : FixFunction.values()) { - if (exp.name().equalsIgnoreCase(expression.getName())) { - return exp; - } - } - return null; - } - - private void runMetamorphFunction(final String name, final Map attributes) { - if (functionFactory.containsKey(name)) { - final String flushWith = attributes.remove(FLUSH_WITH); - final Function function = functionFactory.newInstance(name, attributes); - if (null != flushWith) { - registerFlush(flushWith, function); - } - function.setMaps(metafix); - - final StackFrame head = stack.peek(); - final NamedValuePipe interceptor = interceptorFactory.createNamedValueInterceptor(); - final NamedValuePipe delegate; - if (interceptor == null) { - delegate = function; - } - else { - delegate = interceptor; - function.addNamedValueSource(delegate); - } - delegate.addNamedValueSource(head.getPipe()); - head.setPipe(function); - } - else { - throw new IllegalArgumentException(name + " not found"); - } - } - - void exitData() { - final NamedValuePipe dataPipe = stack.pop().getPipe(); - - final NamedValuePipe interceptor = interceptorFactory.createNamedValueInterceptor(); - final NamedValuePipe delegate; - if (interceptor == null) { - delegate = dataPipe; - } - else { - delegate = interceptor; - delegate.addNamedValueSource(dataPipe); - } - - final StackFrame parent = stack.peek(); - - if (parent.isInEntity()) { - ((Entity) parent.getPipe()).setNameSource(delegate); - } - else if (parent.isInCondition()) { - ((ConditionAware) parent.getPipe()).setConditionSource(delegate); - } - else { - parent.getPipe().addNamedValueSource(delegate); - } - } - - NamedValuePipe enterDataMap(final List params, final boolean standalone) { - Entity entity = null; - final Data data = new Data(); - String dataName = resolvedAttribute(params, 2); - final String resolvedAttribute = resolvedAttribute(params, 2); - if (resolvedAttribute != null && resolvedAttribute.contains(".")) { - final String[] keyElements = resolvedAttribute.split("\\."); - final Pair firstAndLast = createEntities(keyElements); - firstAndLast.getValue().addNamedValueSource(data); - entity = firstAndLast.getKey(); - stack.push(new StackFrame(entity)); - dataName = keyElements[keyElements.length - 1]; - } - data.setName(dataName); - final String source = resolvedAttribute(params, 1); - metafix.registerNamedValueReceiver(source, getDelegate(data)); - final StackFrame frame = new StackFrame(data); - if (!standalone) { - frame.setInEntity(true); - } - stack.push(frame); - return entity != null ? entity : data; - } - - void enterDataAdd(final List params) { - final String resolvedAttribute = resolvedAttribute(params, 1); - if (resolvedAttribute.contains(".")) { - addNestedField(params, resolvedAttribute); - } - else { - final Data newDate = new Data(); - newDate.setName(resolvedAttribute); - final Constant constant = new Constant(); - constant.setValue(resolvedAttribute(params, 2)); - newDate.addNamedValueSource(constant); - metafix.registerNamedValueReceiver("_id", constant); - stack.push(new StackFrame(newDate)); - } - } - - void enterDataFunction(final String fieldName, final NamedValuePipe function, final boolean standalone) { - if (standalone) { - final Data data = new Data(); - data.setName("@" + fieldName); - metafix.registerNamedValueReceiver(fieldName, getDelegate(data, function)); - stack.push(new StackFrame(data)); - return; - } - final StackFrame head = stack.peek(); - final NamedValuePipe interceptor = interceptorFactory.createNamedValueInterceptor(); - final NamedValuePipe delegate; - if (interceptor == null) { - delegate = function; - } - else { - delegate = interceptor; - function.addNamedValueSource(delegate); - } - delegate.addNamedValueSource(head.getPipe()); - head.setPipe(function); - } - - void enterIf() { - assert stack.peek().getPipe() instanceof ConditionAware : - "statement `if` is not allowed in the current element"; - - stack.peek().setInCondition(true); - } - - void exitIf() { - stack.peek().setInCondition(false); - } - - private void addNestedField(final List params, final String resolvedAttribute) { - final String[] keyElements = resolvedAttribute.split("\\."); - final Pair firstAndLast = createEntities(keyElements); - final Constant constant = new Constant(); - constant.setValue(resolvedAttribute(params, 2)); - final Data newData = new Data(); - newData.setName(keyElements[keyElements.length - 1]); - newData.addNamedValueSource(constant); - firstAndLast.getValue().addNamedValueSource(newData); - metafix.registerNamedValueReceiver("_id", constant); - stack.push(new StackFrame(firstAndLast.getKey())); - } - - private Pair createEntities(final String[] keyElements) { - Entity firstEntity = null; - Entity lastEntity = null; - for (int i = 0; i < keyElements.length - 1; i = i + 1) { - final Entity currentEntity = new Entity(() -> metafix.getStreamReceiver()); - currentEntity.setName(keyElements[i]); - if (firstEntity == null) { - firstEntity = currentEntity; - } - if (lastEntity != null) { - lastEntity.addNamedValueSource(currentEntity); - } - lastEntity = currentEntity; - } - return new Pair<>(firstEntity, lastEntity); - } - - private NamedValuePipe getDelegate(final NamedValuePipe dataForDelegate) { - return getDelegate(dataForDelegate, interceptorFactory.createNamedValueInterceptor()); - } - - private NamedValuePipe getDelegate(final NamedValuePipe dataForDelegate, final NamedValuePipe interceptor) { - final NamedValuePipe delegate; - - if (interceptor == null) { - delegate = dataForDelegate; - } - else { - delegate = interceptor; - dataForDelegate.addNamedValueSource(delegate); - } - - return delegate; - } - - private String resolvedAttribute(final List params, final int i) { - return params.size() < i ? null : resolveVars(params.get(i - 1)); - } - - private String resolvedAttribute(final Map attributes, final String string) { - return resolveVars(attributes.get(string)); - } - - private static class StackFrame { - - private NamedValuePipe pipe; - - private boolean inEntity; - - private boolean inCondition; - - private StackFrame(final NamedValuePipe pipe) { - this.pipe = pipe; - } - - public void setInCondition(final boolean inCondition) { - this.inCondition = inCondition; - } - - public boolean isInCondition() { - return inCondition; - } - - public NamedValuePipe getPipe() { - return pipe; - } - - public void setPipe(final NamedValuePipe pipe) { - this.pipe = pipe; - } - - public void setInEntity(final boolean inEntity) { - this.inEntity = inEntity; - } - - public boolean isInEntity() { - return inEntity; - } - - } - -} diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/FixFunction.java b/org.metafacture.fix/src/main/java/org/metafacture/metamorph/FixFunction.java deleted file mode 100644 index 1f298241..00000000 --- a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/FixFunction.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2020 Fabian Steeg, hbz - * - * Licensed under the Apache License, Version 2.0 the "License"; - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.metafacture.metamorph; - -import org.metafacture.fix.fix.Expression; -import org.metafacture.fix.fix.MethodCall; -import org.metafacture.fix.fix.Options; -import org.metafacture.metamorph.api.NamedValuePipe; -import org.metafacture.metamorph.functions.Lookup; -import org.metafacture.metamorph.maps.FileMap; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -enum FixFunction { - // fix-style functions, not (like) in morph - MAP { - public void apply(final FixBuilder builder, final Expression expression, final List params, - final String source) { - final NamedValuePipe enterDataMap = builder.enterDataMap(params, false); - builder.exitData(); - if (enterDataMap instanceof Entity) { - builder.exitCollectorAndFlushWith(null); - } - } - }, - ADD_FIELD { - public void apply(final FixBuilder builder, final Expression expression, final List params, - final String source) { - builder.enterDataAdd(params); - builder.exitData(); - } - }, - LOOKUP { - public void apply(final FixBuilder builder, final Expression expression, final List params, - final String source) { - final Lookup lookup = new Lookup(); - lookup.setMaps(builder.getMetafix()); - final Map map = buildMap(expression); - final String name = map.hashCode() + ""; - lookup.setMap(name); - builder.getMetafix().putMap(name, map); - builder.enterDataFunction(source, lookup, false); - } - - private Map buildMap(final Expression expression) { - final Map options = options((MethodCall) expression); - final String file = options.get("in"); - final String sep = "separator"; - final boolean useFileMap = file != null && - (options.size() == 1 || options.size() == 2 && options.containsKey(sep)); - final Map map = useFileMap ? fileMap(file, options.get(sep)) : options; - return map; - } - - private Map options(final MethodCall method) { - final Options options = method.getOptions(); - final Map map = new HashMap<>(); - if (options != null) { - for (int i = 0; i < options.getKeys().size(); i += 1) { - map.put(options.getKeys().get(i), options.getValues().get(i)); - } - } - return map; - } - - private Map fileMap(final String secondParam, final String separator) { - final FileMap fileMap = new FileMap(); - if (separator != null) { - fileMap.setSeparator(separator); - } - fileMap.setFile(secondParam); - return fileMap; - } - - }; - public abstract void apply(FixBuilder builder, Expression expression, List params, String source); -} diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metamorph/FixMethod.java new file mode 100644 index 00000000..6bac4bc8 --- /dev/null +++ b/org.metafacture.fix/src/main/java/org/metafacture/metamorph/FixMethod.java @@ -0,0 +1,143 @@ +/* + * Copyright 2021 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metamorph; + +import org.metafacture.metamorph.maps.FileMap; + +import com.google.common.collect.Multimap; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +enum FixMethod { + + // RECORD-LEVEL METHODS: + + set_field { + public void apply(final Multimap record, final List params, + final Map options) { + record.replaceValues(params.get(0), Arrays.asList(params.get(1))); + } + }, + add_field { + public void apply(final Multimap record, final List params, + final Map options) { + record.put(params.get(0), params.get(1)); + } + }, + move_field { + public void apply(final Multimap record, final List params, + final Map options) { + final String oldFieldName = params.get(0); + final String newFieldName = params.get(1); + record.putAll(newFieldName, record.get(oldFieldName)); + record.removeAll(oldFieldName); + } + }, + copy_field { + public void apply(final Multimap record, final List params, + final Map options) { + final String oldName = params.get(0); + final String newName = params.get(1); + record.putAll(newName, record.get(oldName)); + } + }, + remove_field { + public void apply(final Multimap record, final List params, + final Map options) { + record.removeAll(params.get(0)); + } + }, + + // FIELD-LEVEL METHODS: + + substring { + @SuppressWarnings("checkstyle:MagicNumber") // TODO: switch to morph-style named params in general? + public void apply(final Multimap record, final List params, + final Map options) { + applyToFields(record, params, + s -> s.substring(Integer.parseInt(params.get(1)), Integer.parseInt(params.get(2)) - 1)); + } + }, + trim { + public void apply(final Multimap record, final List params, + final Map options) { + applyToFields(record, params, s -> s.trim()); + } + }, + upcase { + public void apply(final Multimap record, final List params, + final Map options) { + applyToFields(record, params, s -> s.toUpperCase()); + } + }, + downcase { + public void apply(final Multimap record, final List params, + final Map options) { + applyToFields(record, params, s -> s.toLowerCase()); + } + }, + capitalize { + public void apply(final Multimap record, final List params, + final Map options) { + applyToFields(record, params, s -> s.substring(0, 1).toUpperCase() + s.substring(1)); + } + }, + lookup { + public void apply(final Multimap record, final List params, + final Map options) { + applyToFields(record, params, s -> { + final Map map = buildMap(options, params.size() <= 1 ? null : params.get(1)); + return map.getOrDefault(s, map.get("__default")); // TODO Catmandu uses 'default' + }); + } + + private Map buildMap(final Map options, final String fileLocation) { + final String sep = "sep_char"; + final Map map = fileLocation != null ? fileMap(fileLocation, options.get(sep)) : options; + return map; + } + + private Map fileMap(final String location, final String separator) { + final FileMap fileMap = new FileMap(); + fileMap.setSeparator(","); // CSV as default + if (separator != null) { // override with option + fileMap.setSeparator(separator); + } + fileMap.setFile(location); + return fileMap; + } + }; + + private static void applyToFields(final Multimap record, final List params, + final Function fun) { + final String key = params.get(0); + if (record.containsKey(key)) { + final Collection olds = new ArrayList(record.get(key)); + olds.forEach(old -> { + record.remove(key, old); + record.put(key, fun.apply(old)); + }); + } + } + + abstract void apply(Multimap record, List params, Map options); +} diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/FixPredicate.java b/org.metafacture.fix/src/main/java/org/metafacture/metamorph/FixPredicate.java new file mode 100644 index 00000000..ed57ee26 --- /dev/null +++ b/org.metafacture.fix/src/main/java/org/metafacture/metamorph/FixPredicate.java @@ -0,0 +1,78 @@ +/* + * Copyright 2021 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metamorph; + +import com.google.common.collect.Multimap; + +import java.util.List; +import java.util.function.Predicate; +import java.util.stream.Stream; + +enum FixPredicate { + + contain { + public Predicate of(final String value) { + return v -> v.contains(value); + } + }, + equal { + public Predicate of(final String value) { + return v -> v.equals(value); + } + }, + match { + public Predicate of(final String value) { + return v -> v.matches(value); + } + }; + + abstract Predicate of(String value); + + enum Quantifier { + + all { + @Override + public boolean test(final Multimap record, final FixPredicate p, + final List params) { + return test(record, params.get(0), s -> s.allMatch(p.of(params.get(1)))); + } + + }, + any { + @Override + public boolean test(final Multimap record, final FixPredicate p, + final List params) { + return test(record, params.get(0), s -> s.anyMatch(p.of(params.get(1)))); + } + }, + none { + @Override + public boolean test(final Multimap record, final FixPredicate p, + final List params) { + final String fieldName = params.get(0); + final String valueToTest = params.get(1); + return !record.containsKey(fieldName) || record.get(fieldName).stream().noneMatch(p.of(valueToTest)); + } + }; + + boolean test(final Multimap record, final String fieldName, final Predicate> f) { + return record.containsKey(fieldName) && f.test(record.get(fieldName).stream()); + } + + abstract boolean test(Multimap record, FixPredicate p, List params); + } +} diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/Metafix.java b/org.metafacture.fix/src/main/java/org/metafacture/metamorph/Metafix.java index 7c3e68fd..f76814d8 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/Metafix.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metamorph/Metafix.java @@ -14,128 +14,56 @@ * limitations under the License. */ +//TODO: move all classes here to fix package + package org.metafacture.metamorph; -import org.metafacture.commons.tries.SimpleRegexTrie; import org.metafacture.fix.FixStandaloneSetup; import org.metafacture.fix.fix.Expression; import org.metafacture.fix.fix.Fix; -import org.metafacture.flowcontrol.StreamBuffer; -import org.metafacture.framework.StandardEventNames; import org.metafacture.framework.StreamPipe; import org.metafacture.framework.StreamReceiver; import org.metafacture.framework.helpers.DefaultStreamReceiver; import org.metafacture.mangling.StreamFlattener; -import org.metafacture.metamorph.api.FlushListener; -import org.metafacture.metamorph.api.InterceptorFactory; -import org.metafacture.metamorph.api.Maps; -import org.metafacture.metamorph.api.MorphErrorHandler; -import org.metafacture.metamorph.api.NamedValuePipe; -import org.metafacture.metamorph.api.NamedValueReceiver; -import org.metafacture.metamorph.api.NamedValueSource; -import org.metafacture.metamorph.api.SourceLocation; -import com.google.common.collect.HashMultimap; +import com.google.common.collect.LinkedListMultimap; import com.google.common.collect.Multimap; -import com.google.common.io.CharStreams; -import com.google.inject.Injector; -import org.eclipse.emf.common.util.URI; -import org.eclipse.emf.ecore.resource.Resource; -import org.eclipse.xtext.resource.XtextResource; -import org.eclipse.xtext.resource.XtextResourceSet; -import org.eclipse.xtext.util.CancelIndicator; -import org.eclipse.xtext.validation.CheckMode; -import org.eclipse.xtext.validation.IResourceValidator; -import org.eclipse.xtext.validation.Issue; -import java.io.Closeable; -import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; import java.io.Reader; import java.io.StringReader; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.Deque; -import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; /** * Transforms a data stream sent via the {@link StreamReceiver} interface. Use - * {@link FixBuilder} to create an instance based on a Fix DSL description. + * {@link RecordTransformer} to transform based on a Fix DSL description. * * @author Markus Michael Geipel (Metamorph) * @author Christoph Böhme (Metamorph) * @author Fabian Steeg (Metafix) */ -// TODO: implement ConditionAware to support top-level `if` like in MetafixRecordTest -public class Metafix implements StreamPipe, NamedValuePipe, Maps { // checkstyle-disable-line ClassDataAbstractionCoupling|ClassFanOutComplexity +public class Metafix implements StreamPipe { - public static final String ELSE_KEYWORD = "_else"; - public static final char FEEDBACK_CHAR = '@'; - public static final char ESCAPE_CHAR = '\\'; - public static final String METADATA = "__meta"; public static final String VAR_START = "$["; public static final String VAR_END = "]"; - + static final Map NO_VARS = Collections.emptyMap(); private static final String ENTITIES_NOT_BALANCED = "Entity starts and ends are not balanced"; - private static final Object ELSE_NESTED_KEYWORD = "_elseNested"; - private static final Object ELSE_FLATTENED_KEYWORD = "_elseFlattened"; - - private static final InterceptorFactory NULL_INTERCEPTOR_FACTORY = new NullInterceptorFactory(); - private static final Map NO_VARS = Collections.emptyMap(); - - private Multimap currentRecord = HashMultimap.create(); - - // warning: auxiliary class WildcardRegistry in WildcardDataRegistry.java should not be accessed from outside its own source file - //private final Registry dataRegistry = new WildcardRegistry<>(); - private final Registry dataRegistry = new Registry() { - private final SimpleRegexTrie trie = new SimpleRegexTrie<>(); - @Override - public void register(final String path, final NamedValueReceiver value) { - trie.put(path, value); - } - - @Override - public List get(final String path) { - return trie.get(path); - } - }; - - private final List resources = new ArrayList<>(); - private final List elseSources = new ArrayList<>(); - private final Map> maps = new HashMap<>(); + // TODO: Use SimpleRegexTrie / WildcardTrie for wildcard, alternation and character class support + private Multimap currentRecord = LinkedListMultimap.create(); + private Fix fix; + private final List expressions = new ArrayList<>(); + private Map vars = NO_VARS; private final StreamFlattener flattener = new StreamFlattener(); - private final Deque entityCountStack = new LinkedList<>(); - private int currentEntityCount; private int entityCount; - private StreamReceiver outputStreamReceiver; - private MorphErrorHandler errorHandler = new DefaultErrorHandler(); - private int recordCount; - private final List recordEndListener = new ArrayList<>(); - - private final List expressions = new ArrayList<>(); - private Map vars = NO_VARS; - - private boolean elseNested; - private boolean elseNestedEntityStarted; - private String currentLiteralName; - - private final StreamBuffer buffer = new StreamBuffer(); - private boolean needsReplay; - private Fix fix; - private InterceptorFactory interceptorFactory; - private boolean recordMode; public Metafix() { init(); @@ -146,15 +74,7 @@ public Metafix(final String fixDef) throws FileNotFoundException { } public Metafix(final String fixDef, final Map vars) throws FileNotFoundException { - this(fixDef, vars, NULL_INTERCEPTOR_FACTORY); - } - - public Metafix(final String fixDef, final InterceptorFactory interceptorFactory) throws FileNotFoundException { - this(fixDef, NO_VARS, interceptorFactory); - } - - public Metafix(final String fixDef, final Map vars, final InterceptorFactory interceptorFactory) throws FileNotFoundException { - this(fixDef.endsWith(".fix") ? new FileReader(fixDef) : new StringReader(fixDef), vars, interceptorFactory); + this(fixDef.endsWith(".fix") ? new FileReader(fixDef) : new StringReader(fixDef), vars); } public Metafix(final Reader morphDef) { @@ -162,34 +82,10 @@ public Metafix(final Reader morphDef) { } public Metafix(final Reader fixDef, final Map vars) { - this(fixDef, vars, NULL_INTERCEPTOR_FACTORY); - } - - public Metafix(final Reader fixDef, final InterceptorFactory interceptorFactory) { - this(fixDef, NO_VARS, interceptorFactory); - } - - public Metafix(final Reader fixDef, final Map vars, final InterceptorFactory interceptorFactory) { - buildPipeline(fixDef, vars, interceptorFactory); + buildPipeline(fixDef, vars); init(); } - public Metafix(final InputStream fixDef) { - this(fixDef, NO_VARS); - } - - public Metafix(final InputStream fixDef, final Map vars) { - this(fixDef, vars, NULL_INTERCEPTOR_FACTORY); - } - - public Metafix(final InputStream fixDef, final InterceptorFactory interceptorFactory) { - this(fixDef, NO_VARS, interceptorFactory); - } - - public Metafix(final InputStream fixDef, final Map vars, final InterceptorFactory interceptorFactory) { - this(new InputStreamReader(fixDef), vars, interceptorFactory); - } - public List getExpressions() { return expressions; } @@ -198,131 +94,45 @@ private void init() { flattener.setReceiver(new DefaultStreamReceiver() { @Override public void literal(final String name, final String value) { - dispatch(name, value, getElseSources(), false); + // TODO: set up logging + System.out.printf("Putting '%s':'%s'\n", name, value); + currentRecord.put(name, value); } }); } - private void buildPipeline(final Reader fixDef, final Map theVars, final InterceptorFactory interceptorF) { - final Fix f = parseFix(fixDef); + private void buildPipeline(final Reader fixDef, final Map theVars) { + final Fix f = FixStandaloneSetup.parseFix(fixDef); this.fix = f; this.vars = theVars; - this.interceptorFactory = interceptorF; - // TODO: FixInterpreter no longer needed? - // new FixInterpreter().run(this, fix); - new FixBuilder(this, interceptorFactory).walk(fix); - } - - private Fix parseFix(final Reader fixDef) { - // TODO: do this only once per application - final Injector injector = new FixStandaloneSetup().createInjectorAndDoEMFRegistration(); - FixStandaloneSetup.doSetup(); - - try { - final URI uri = URI.createFileURI(absPathToTempFile(fixDef, ".fix")); - final Resource resource = injector.getInstance(XtextResourceSet.class).getResource(uri, true); - final IResourceValidator validator = ((XtextResource) resource).getResourceServiceProvider().getResourceValidator(); - - for (final Issue issue : validator.validate(resource, CheckMode.ALL, CancelIndicator.NullImpl)) { - System.err.println(issue.getMessage()); - } - - return (Fix) resource.getContents().get(0); - } - catch (final IOException e) { - e.printStackTrace(); - return null; - } - } - - public static String absPathToTempFile(final Reader fixDef, final String suffix) throws IOException { - // TODO: avoid temp file creation - final File file = File.createTempFile("metafix", suffix); - file.deleteOnExit(); - - try (FileWriter out = new FileWriter(file)) { - CharStreams.copy(fixDef, out); - } - - return file.getAbsolutePath(); - } - - protected List getElseSources() { - return elseSources; - } - - protected void setEntityMarker(final String entityMarker) { - flattener.setEntityMarker(entityMarker); - } - - protected void setErrorHandler(final MorphErrorHandler errorHandler) { - this.errorHandler = errorHandler; - } - - protected void registerNamedValueReceiver(final String source, final NamedValueReceiver data) { - if (ELSE_NESTED_KEYWORD.equals(source)) { - elseNested = true; - } - if (ELSE_KEYWORD.equals(source) || ELSE_FLATTENED_KEYWORD.equals(source) || elseNested) { - if (elseSources.isEmpty()) { - elseSources.add(data); - } - else { - System.out.println("Only one of '_else', '_elseFlattened' and '_elseNested' is allowed. Ignoring the superflous ones."); - } - } - else { - dataRegistry.register(source, data); - } } @Override public void startRecord(final String identifier) { - buffer.clear(); - buffer.startRecord(identifier); - + currentRecord = LinkedListMultimap.create(); System.out.printf("Start record: %s\n", currentRecord); flattener.startRecord(identifier); entityCountStack.clear(); - entityCount = 0; - currentEntityCount = 0; entityCountStack.add(Integer.valueOf(entityCount)); - - ++recordCount; - recordCount %= Integer.MAX_VALUE; - - if (!recordMode) { - outputStreamReceiver.startRecord(identifier); - } - else if (!needsReplay) { - needsReplay = true; - currentRecord = HashMultimap.create(); - outputStreamReceiver.startRecord(identifier); - } - dispatch(StandardEventNames.ID, identifier, null, false); + outputStreamReceiver.startRecord(identifier); } @Override public void endRecord() { - for (final FlushListener listener : recordEndListener) { - listener.flush(recordCount, currentEntityCount); - } - entityCountStack.removeLast(); - if (!entityCountStack.isEmpty()) { throw new IllegalStateException(ENTITIES_NOT_BALANCED); } - flattener.endRecord(); - buffer.endRecord(); - if (recordMode && needsReplay) { - System.out.printf("End record, replay with: %s\n", currentRecord); - new FixBuilder(this, interceptorFactory).walk(fix); - buffer.replay(); - needsReplay = false; - } + System.out.printf("End record, walking fix: %s\n", currentRecord); + final RecordTransformer transformer = new RecordTransformer(currentRecord, vars, fix); + currentRecord = transformer.transform(); + System.out.println("Sending results to " + outputStreamReceiver); + currentRecord.entries().forEach(e -> { + outputStreamReceiver.literal(e.getKey(), e.getValue()); + // TODO: send actual entities for `nested.fields` + }); outputStreamReceiver.endRecord(); } @@ -331,96 +141,30 @@ public void startEntity(final String name) { if (name == null) { throw new IllegalArgumentException("Entity name must not be null."); } - ++entityCount; - currentEntityCount = entityCount; entityCountStack.push(Integer.valueOf(entityCount)); - flattener.startEntity(name); - buffer.startEntity(name); } @Override public void endEntity() { - dispatch(flattener.getCurrentPath(), "", getElseSources(), true); - currentEntityCount = entityCountStack.pop().intValue(); + entityCountStack.pop().intValue(); flattener.endEntity(); - buffer.endEntity(); } @Override public void literal(final String name, final String value) { - currentLiteralName = name; flattener.literal(name, value); - buffer.literal(name, value); } @Override public void resetStream() { - // TODO: Implement proper reset handling outputStreamReceiver.resetStream(); - buffer.clear(); } @Override public void closeStream() { - for (final Closeable closeable : resources) { - try { - closeable.close(); - } - catch (final IOException e) { - errorHandler.error(e); - } - } - outputStreamReceiver.closeStream(); - buffer.clear(); - } - - private void dispatch(final String path, final String value, final List fallback, final boolean endEntity) { - final List matchingData = findMatchingData(path, fallback); - - if (matchingData != null) { - send(path, value, matchingData); - } - else if (fallback != null) { - if (endEntity) { - if (elseNestedEntityStarted) { - outputStreamReceiver.endEntity(); - elseNestedEntityStarted = false; - } - } - else { - final String entityName = elseNested ? flattener.getCurrentEntityName() : null; - - if (entityName != null) { - if (findMatchingData(entityName, fallback) == null) { - if (!elseNestedEntityStarted) { - outputStreamReceiver.startEntity(entityName); - elseNestedEntityStarted = true; - } - - send(escapeFeedbackChar(currentLiteralName), value, fallback); - } - } - else { - send(escapeFeedbackChar(path), value, fallback); - } - } - } - } - - private List findMatchingData(final String path, final List fallback) { - final List matchingData = dataRegistry.get(path); - return matchingData != null && !matchingData.isEmpty() ? matchingData : null; - } - - private void send(final String key, final String value, final List dataList) { - System.out.printf("Sending '%s':'%s' to %s\n", key, value, dataList); - currentRecord.put(key, value); - for (final NamedValueReceiver data : dataList) { - data.receive(key, value, null, recordCount, currentEntityCount); - } } /** @@ -431,9 +175,7 @@ public R setReceiver(final R streamReceiver) { if (streamReceiver == null) { throw new IllegalArgumentException("'streamReceiver' must not be null"); } - outputStreamReceiver = streamReceiver; - buffer.setReceiver(this); return streamReceiver; } @@ -441,96 +183,6 @@ public StreamReceiver getStreamReceiver() { return outputStreamReceiver; } - @Override - public void receive(final String name, final String value, final NamedValueSource source, final int recordCountParam, final int entityCountParam) { - if (name == null) { - throw new IllegalArgumentException( - "encountered literal with name='null'. This indicates a bug in a function or a collector."); - } - - if (startsWithFeedbackChar(name)) { - dispatch(name, value, null, false); - return; - } - - final String unescapedName; - - if (name.length() > 1 && name.charAt(0) == ESCAPE_CHAR && (name.charAt(1) == FEEDBACK_CHAR || name.charAt(1) == ESCAPE_CHAR)) { - unescapedName = name.substring(1); - } - else { - unescapedName = name; - } - outputStreamReceiver.literal(unescapedName, value); - } - - private boolean startsWithFeedbackChar(final String name) { - return name.length() != 0 && name.charAt(0) == FEEDBACK_CHAR; - } - - private String escapeFeedbackChar(final String name) { - return name == null ? null : (startsWithFeedbackChar(name) ? ESCAPE_CHAR : "") + name; - } - - @Override - public Map getMap(final String mapName) { - return maps.getOrDefault(mapName, Collections.emptyMap()); - } - - @Override - public String getValue(final String mapName, final String key) { - final Map map = getMap(mapName); - return map.containsKey(key) ? map.get(key) : map.get(Maps.DEFAULT_MAP_KEY); - } - - @Override - public Map putMap(final String mapName, final Map map) { - if (map instanceof Closeable) { - final Closeable closable = (Closeable) map; - resources.add(closable); - } - - return maps.put(mapName, map); - } - - @Override - public String putValue(final String mapName, final String key, final String value) { - return maps.computeIfAbsent(mapName, k -> new HashMap<>()).put(key, value); - } - - @Override - public Collection getMapNames() { - return Collections.unmodifiableSet(maps.keySet()); - } - - public void registerRecordEndFlush(final FlushListener flushListener) { - recordEndListener.add(flushListener); - } - - @Override - public void addNamedValueSource(final NamedValueSource namedValueSource) { - namedValueSource.setNamedValueReceiver(this); - } - - @Override - public void setNamedValueReceiver(final NamedValueReceiver receiver) { - throw new UnsupportedOperationException("The Metafix object cannot act as a NamedValueSender"); - } - - @Override - public void setSourceLocation(final SourceLocation sourceLocation) { - // Nothing to do - // Metafix does not have a source location (we could - // in theory use the location of the module in a flux - // script) - } - - @Override - public SourceLocation getSourceLocation() { - // Metafix does not have a source location - return null; - } - public Map getVars() { return vars; } @@ -539,12 +191,4 @@ public Multimap getCurrentRecord() { return currentRecord; } - public void setRecordMode(final boolean recordMode) { - this.recordMode = recordMode; - } - - public boolean isRecordMode() { - return recordMode; - } - } diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/MetafixFilter.java b/org.metafacture.fix/src/main/java/org/metafacture/metamorph/MetafixFilter.java deleted file mode 100644 index 667507b5..00000000 --- a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/MetafixFilter.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright 2013, 2021 Deutsche Nationalbibliothek and others - * - * Licensed under the Apache License, Version 2.0 the "License"; - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.metafacture.metamorph; - -import org.metafacture.flowcontrol.StreamBuffer; -import org.metafacture.framework.FluxCommand; -import org.metafacture.framework.StreamReceiver; -import org.metafacture.framework.annotations.Description; -import org.metafacture.framework.annotations.In; -import org.metafacture.framework.annotations.Out; -import org.metafacture.framework.helpers.DefaultStreamPipe; -import org.metafacture.javaintegration.SingleValue; -import org.metafacture.metamorph.api.InterceptorFactory; - -import java.io.FileNotFoundException; -import java.util.Map; - -/** - * Filters a stream based on a fix definition. A record is accepted if the - * morph returns at least one non empty value. - * - * @author Markus Michael Geipel - * @author Fabian Steeg, hbz (adapted for Metafix) - * - */ -@Description("Filters a stream based on a fix definition. A record is accepted if the fix returns at least one non empty value.") -@In(StreamReceiver.class) -@Out(StreamReceiver.class) -@FluxCommand("fix-filter") -public final class MetafixFilter extends DefaultStreamPipe { - - private final StreamBuffer buffer = new StreamBuffer(); - private final SingleValue singleValue = new SingleValue(); - private final Metafix metafix; - - public MetafixFilter(final String morphDef) throws FileNotFoundException { - metafix = new Metafix(morphDef); - metafix.setReceiver(singleValue); - } - - public MetafixFilter(final Metafix metamorph) { - this.metafix = metamorph; - metamorph.setReceiver(singleValue); - } - - public MetafixFilter(final String morphDef, final Map vars) throws FileNotFoundException { - metafix = new Metafix(morphDef, vars); - metafix.setReceiver(singleValue); - } - - public MetafixFilter(final String morphDef, final InterceptorFactory interceptorFactory) throws FileNotFoundException { - metafix = new Metafix(morphDef, interceptorFactory); - metafix.setReceiver(singleValue); - } - - @Override - protected void onSetReceiver() { - buffer.setReceiver(getReceiver()); - } - - private void dispatch() { - final String key = singleValue.getValue(); - if (!key.isEmpty()) { - buffer.replay(); - } - buffer.clear(); - } - - @Override - public void startRecord(final String identifier) { - buffer.startRecord(identifier); - metafix.startRecord(identifier); - } - - @Override - public void endRecord() { - buffer.endRecord(); - metafix.endRecord(); - dispatch(); - } - - @Override - public void startEntity(final String name) { - buffer.startEntity(name); - metafix.startEntity(name); - } - - @Override - public void endEntity() { - buffer.endEntity(); - metafix.endEntity(); - } - - @Override - public void literal(final String name, final String value) { - buffer.literal(name, value); - metafix.literal(name, value); - } - - @Override - protected void onResetStream() { - buffer.clear(); - metafix.resetStream(); - } - - @Override - protected void onCloseStream() { - buffer.clear(); - metafix.closeStream(); - } -} diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java b/org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java new file mode 100644 index 00000000..ef47dc13 --- /dev/null +++ b/org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java @@ -0,0 +1,147 @@ +/* + * Copyright 2021 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metamorph; + +import org.metafacture.commons.StringUtil; +import org.metafacture.fix.fix.Do; +import org.metafacture.fix.fix.Expression; +import org.metafacture.fix.fix.Fix; +import org.metafacture.fix.fix.If; +import org.metafacture.fix.fix.MethodCall; +import org.metafacture.fix.fix.Options; +import org.metafacture.metamorph.FixPredicate.Quantifier; + +import org.eclipse.emf.common.util.EList; +import com.google.common.collect.LinkedListMultimap; +import com.google.common.collect.Multimap; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Transform a record using a Fix + * + * @author Fabian Steeg + * + */ +class RecordTransformer { + + private Fix fix; + private Multimap record; + private Map vars; + + RecordTransformer(final Multimap record, final Map vars, final Fix fix) { + this.record = LinkedListMultimap.create(record); + this.vars = vars; + this.fix = fix; + } + + Multimap transform() { + processSubexpressions(fix.getElements()); + return record; + } + + Multimap getRecord() { + return record; + } + + private String resolveVars(final String string) { + return string == null ? null : StringUtil.format(string, Metafix.VAR_START, Metafix.VAR_END, false, vars); + } + + private void processSubexpressions(final List expressions) { + for (final Expression sub : expressions) { + final EList params = sub.getParams(); + if (sub instanceof Do) { + processBind(sub, params); + } + else if (sub instanceof If) { + processConditional(sub, params); + } + else { + processFunction(sub, params); + } + } + } + + private void processBind(final Expression expression, final EList params) { + // TODO: implement Fix Binds + // final String firstParam = resolvedAttribute(params, 1); + // final Do theDo = (Do) expression; + // TODO, possibly: use morph collectors here + // final CollectFactory collectFactory = new CollectFactory(); + // final Map attributes = resolvedAttributeMap(params, theDo.getOptions()); + // final Collect collect = collectFactory.newInstance(expression.getName(), attributes); + } + + private void processConditional(final Expression expression, final EList parameters) { + final If theIf = (If) expression; + if (testConditional(theIf.getName(), parameters)) { + processSubexpressions(theIf.getElements()); + } + } + + private boolean testConditional(final String conditional, final EList params) { + System.out.printf(": %s parameters: %s\n", conditional, params); + boolean result = false; + if (!conditional.contains("_")) { + throw new IllegalArgumentException("Missing quantifier prefix (all_, any_, none_) for " + conditional); + } + final String[] quantifierAndPredicate = conditional.split("_"); + try { + final Quantifier quantifier = FixPredicate.Quantifier.valueOf(quantifierAndPredicate[0]); + final FixPredicate predicate = FixPredicate.valueOf(quantifierAndPredicate[1]); + result = quantifier.test(record, predicate, params); + } + catch (final IllegalArgumentException e) { + e.printStackTrace(); + } + // TODO, possibly: use morph functions here (& in processFunction): + // final FunctionFactory functionFactory = new FunctionFactory(); + // functionFactory.registerClass("not_equals", NotEquals.class); + // functionFactory.registerClass("replace_all", Replace.class); + // final Function function = functionFactory.newInstance(conditional, + // resolvedAttributeMap(parameters, theIf.getOptions())); + return result; + } + + private void processFunction(final Expression expression, final List params) { + try { + final FixMethod method = FixMethod.valueOf(expression.getName()); + final List resolvedParams = params.stream().map(this::resolveVars).collect(Collectors.toList()); + final Map options = options((MethodCall) expression); + method.apply(record, resolvedParams, options); + } + catch (final IllegalArgumentException e) { + e.printStackTrace(); + } + } + + private Map options(final MethodCall method) { + final Options options = method.getOptions(); + final Map map = new HashMap<>(); + if (options != null) { + for (int i = 0; i < options.getKeys().size(); i += 1) { + map.put(options.getKeys().get(i), options.getValues().get(i)); + } + } + return map; + } + +} diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixApiTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixApiTest.java deleted file mode 100644 index 2efed161..00000000 --- a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixApiTest.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright 2013, 2019 Deutsche Nationalbibliothek and others - * - * Licensed under the Apache License, Version 2.0 the "License"; - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.metafacture.metamorph; - -import org.metafacture.framework.helpers.DefaultStreamReceiver; -import org.metafacture.metamorph.api.Maps; -import org.metafacture.metamorph.api.NamedValueReceiver; - -import org.junit.Assert; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.mockito.ArgumentMatchers; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.util.HashMap; -import java.util.Map; - -/** - * Tests the basic functionality of Metafix via API. - * - * @author Markus Michael Geipel (MetamorphTest) - * @author Christoph Böhme (rewrite MetamorphTest) - * @author Fabian Steeg (MetafixApiTest) - */ -@ExtendWith(MockitoExtension.class) -public class MetafixApiTest { - - private static final String OUT_NAME = "outName"; - - private static final String TEST_ENTITY = "testEntity"; - private static final String TEST_LITERAL = "testLiteral"; - private static final String TEST_MAP = "testMap"; - private static final String TEST_VALUE = "testValue"; - - @RegisterExtension - private MockitoRule mockitoRule = MockitoJUnit.rule(); - - @Mock - private NamedValueReceiver namedValueReceiver; - - private Metafix metafix; - - public MetafixApiTest() { - } - - @BeforeEach - public void createSystemUnderTest() { - metafix = new Metafix(); - metafix.setReceiver(new DefaultStreamReceiver()); - } - - @Test - public void shouldMapMatchingPath() { - setupSimpleMappingMorph(); - - metafix.startRecord(""); - metafix.literal(TEST_ENTITY + "." + TEST_LITERAL, TEST_VALUE); - - receive(); - } - - @Test - public void shouldNotMapNonMatchingPath() { - setupSimpleMappingMorph(); - - metafix.startRecord(""); - metafix.literal("nonMatching.path", TEST_VALUE); - - neverReceive(); - } - - @Test - public void shouldMapMatchingLiteralInMatchingEntity() { - setupSimpleMappingMorph(); - - metafix.startRecord(""); - metafix.startEntity(TEST_ENTITY); - metafix.literal(TEST_LITERAL, TEST_VALUE); - - receive(); - } - - @Test - public void shouldNotMapNonMatchingLiteralInMatchingEntity() { - setupSimpleMappingMorph(); - - metafix.startRecord(""); - metafix.startEntity(TEST_ENTITY); - metafix.literal("nonMatching", TEST_VALUE); - - neverReceive(); - } - - @Test - public void shouldNotMapMatchingLiteralInNonMatchingEntity() { - setupSimpleMappingMorph(); - - metafix.startRecord(""); - metafix.startEntity("nonMatching"); - metafix.literal(TEST_LITERAL, TEST_VALUE); - - neverReceive(); - } - - @Test - public void shouldNotMapLiteralWithoutMatchingEntity() { - setupSimpleMappingMorph(); - - metafix.startRecord(""); - metafix.literal(TEST_LITERAL, TEST_VALUE); - - neverReceive(); - } - - @Test - public void shouldReturnValueFromNestedMap() { - final Map map = new HashMap<>(); - map.put(OUT_NAME, TEST_VALUE); - - metafix.putMap(TEST_MAP, map); - - Assert.assertNotNull(metafix.getMap(TEST_MAP)); - Assert.assertEquals(TEST_VALUE, metafix.getValue(TEST_MAP, OUT_NAME)); - } - - @Test - public void shouldReturnDefaultValueIfMapIsKnownButNameIsUnknown() { - final Map map = new HashMap<>(); - map.put(Maps.DEFAULT_MAP_KEY, "defaultValue"); - - metafix.putMap(TEST_MAP, map); - - Assert.assertEquals("defaultValue", metafix.getValue(TEST_MAP, "nameNotInMap")); - } - - @Test - public void shouldFeedbackLiteralsStartingWithAtIntoMetamorph() { - final Data dataIn; - dataIn = new Data(); - dataIn.setName("@feedback"); - metafix.addNamedValueSource(dataIn); - metafix.registerNamedValueReceiver(TEST_LITERAL, dataIn); - - final Data dataOut = new Data(); - dataOut.setName(OUT_NAME); - dataOut.setNamedValueReceiver(namedValueReceiver); - metafix.registerNamedValueReceiver("@feedback", dataOut); - - metafix.startRecord(""); - metafix.literal(TEST_LITERAL, TEST_VALUE); - - receive(); - } - - @Test - public void shouldThrowIllegalStateExceptionIfEntityIsNotClosed() { - Assertions.assertThrows(IllegalStateException.class, () -> { - metafix.startRecord(""); - metafix.startEntity(TEST_ENTITY); - metafix.startEntity(TEST_ENTITY); - metafix.endEntity(); - metafix.endRecord(); // Exception expected - }); - } - - /** - * Creates the Metafix object structure that corresponds to the Metafix DSL - * statement {@code map(testEntity.testLiteral, outName)}. - */ - private void setupSimpleMappingMorph() { - final Data data = new Data(); - data.setName(OUT_NAME); - data.setNamedValueReceiver(namedValueReceiver); - metafix.registerNamedValueReceiver(TEST_ENTITY + '.' + TEST_LITERAL, data); - } - - private void receive() { - Mockito.verify(namedValueReceiver).receive( - ArgumentMatchers.eq(OUT_NAME), - ArgumentMatchers.eq(TEST_VALUE), - ArgumentMatchers.isA(Data.class), - ArgumentMatchers.anyInt(), - ArgumentMatchers.anyInt() - ); - - Mockito.verifyNoMoreInteractions(namedValueReceiver); - } - - private void neverReceive() { - Mockito.verifyZeroInteractions(namedValueReceiver); - } - -} diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixBindTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixBindTest.java new file mode 100644 index 00000000..8c9aa6be --- /dev/null +++ b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixBindTest.java @@ -0,0 +1,176 @@ +/* + * Copyright 2021 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metamorph; + +import org.metafacture.framework.StreamReceiver; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Arrays; + +/** + * Test Metafix binds. + * + * @author Fabian Steeg + */ +@ExtendWith(MockitoExtension.class) +@SuppressWarnings("checkstyle:MultipleStringLiterals") +@Disabled // implement Fix-style binds (with or without collectors) +public class MetafixBindTest { + + @RegisterExtension + private MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Mock + private StreamReceiver streamReceiver; + + public MetafixBindTest() { + } + + @Test + public void ifInCollector() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "do entity('author')", + " map('name')", + " if all_contain('name', 'University')", // + " add_field('type', 'Organization')", // + " end", + "end"), // + i -> { + i.startRecord("1"); + i.literal("name", "A University"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().startEntity("author"); + o.get().literal("type", "Organization"); + o.get().literal("name", "A University"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void ifInCollectorMultiRecords() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "do entity('author')", + " map('name')", + " if all_contain('name', 'University')", // + " add_field('type', 'Organization')", // + " end", + "end"), // + i -> { + i.startRecord("1"); + i.literal("name", "Max"); + i.endRecord(); + // + i.startRecord("2"); + i.literal("name", "A University"); + i.endRecord(); + // + i.startRecord("3"); + i.literal("name", "Mary"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().startEntity("author"); + o.get().literal("name", "Max"); + o.get().endEntity(); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().startEntity("author"); + o.get().literal("type", "Organization"); + o.get().literal("name", "A University"); + o.get().endEntity(); + o.get().endRecord(); + // + o.get().startRecord("3"); + o.get().startEntity("author"); + o.get().literal("name", "Mary"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void ifInCollectorChoose() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "do choose(flushWith: 'record')", + " if all_contain('name', 'University')", // + " add_field('type', 'Organization')", // + " end", // + "end"), // + i -> { + i.startRecord("1"); + i.literal("name", "Max University"); + i.endRecord(); + // + i.startRecord("2"); + i.literal("name", "Max Musterman"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().endRecord(); + }); + } + + @Test + public void ifInCollectorCombine() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "do combine(name: 'fullName', value: '${first} ${last}')", // + " if all_contain('author.type', 'Person')", // + " map('author.first', 'first')", // + " map('author.last', 'last')", // + " end", // + "end"), // + i -> { + i.startRecord("1"); + i.startEntity("author"); + i.literal("type", "Organization"); + i.endEntity(); + i.endRecord(); + // + i.startRecord("2"); + i.startEntity("author"); + i.literal("first", "Max"); + i.literal("last", "Musterman"); + i.literal("type", "DifferentiatedPerson"); + i.endEntity(); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().literal("fullName", "Max Musterman"); + o.get().endRecord(); + }); + } + +} diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixDslTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixDslTest.java deleted file mode 100644 index 35a03553..00000000 --- a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixDslTest.java +++ /dev/null @@ -1,955 +0,0 @@ -/* - * Copyright 2013, 2019 Deutsche Nationalbibliothek and others - * - * Licensed under the Apache License, Version 2.0 the "License"; - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.metafacture.metamorph; - -import org.metafacture.framework.StreamReceiver; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.mockito.ArgumentMatchers; -import org.mockito.InOrder; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; -import org.mockito.junit.jupiter.MockitoExtension; -import com.google.common.collect.ImmutableMap; - -import java.io.FileNotFoundException; -import java.net.URISyntaxException; -import java.util.Collections; -import java.util.Map; - -/** - * Tests the basic functionality of Metafix via DSL. - * - * @author Christoph Böhme (MetamorphTest) - * @author Fabian Steeg (MetafixDslTest) - */ -@ExtendWith(MockitoExtension.class) -public class MetafixDslTest { - - private static final String LITERAL_A = "lit-A"; - private static final String LITERAL_ALOHA = "Aloha"; - private static final String LITERAL_B = "lit-B"; - private static final String LITERAL_HAWAII = "Hawaii"; - private static final String LITERAL_MOIN = "Moin"; - private static final String LITERAL_MOIN_MOIN = "Moin Moin"; - private static final String LITERAL_LANGEOOG = "Langeoog"; - private static final String LITERAL_BALTRUM = "Baltrum"; - - @RegisterExtension - private MockitoRule mockitoRule = MockitoJUnit.rule(); - - @Mock - private StreamReceiver streamReceiver; - - public MetafixDslTest() { - } - - @Test - public void shouldMapLiteral() { - final Metafix metafix = fix( - "map(a,b)" - ); - - metafix.startRecord("1"); - metafix.literal("a", LITERAL_ALOHA); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal("b", LITERAL_ALOHA); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldMapAndFilterNeqLiteral() { - final Metafix metafix = fix( - "do map(a,b)", // checkstyle-disable-line MultipleStringLiterals - " not_equals(string: '')", - "end" // checkstyle-disable-line MultipleStringLiterals - ); - - metafix.startRecord("1"); - metafix.literal("a", ""); - metafix.literal("a", LITERAL_ALOHA); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal("b", LITERAL_ALOHA); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldMapAndFilterEqLiteral() { - final Metafix metafix = fix( - "do map(a,b)", // checkstyle-disable-line MultipleStringLiterals - " equals(string: '')", - "end" // checkstyle-disable-line MultipleStringLiterals - ); - - metafix.startRecord("1"); - metafix.literal("a", ""); - metafix.literal("a", LITERAL_ALOHA); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal("b", ""); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldMapAndFilterRegexLiteral() { - final Metafix metafix = fix( - "do map(a,b)", // checkstyle-disable-line MultipleStringLiterals - " regexp(match: '.+')", - "end" // checkstyle-disable-line MultipleStringLiterals - ); - - metafix.startRecord("1"); - metafix.literal("a", ""); - metafix.literal("a", LITERAL_ALOHA); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal("b", LITERAL_ALOHA); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldMapAndChangeLiteralSingle() { - final Metafix metafix = fix( - "do map(a,b)", // checkstyle-disable-line MultipleStringLiterals - " replace_all(pattern: 'a-val', with: 'b-val')", - "end" // checkstyle-disable-line MultipleStringLiterals - ); - - metafix.startRecord("1"); - metafix.literal("a", "a-val"); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal("b", "b-val"); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldMapAndChangeLiteralMulti() { - final Metafix metafix = fix( - "do map(a,b)", - " replace_all(pattern: 'a-val', with: 'b-val')", - " compose(prefix: 'pre-')", - " compose(postfix: '-post')", - "end" // checkstyle-disable-line MultipleStringLiterals - ); - - metafix.startRecord("1"); - metafix.literal("a", "a-val"); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal("b", "pre-b-val-post"); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldMapNested() { - final Metafix metafix = fix( - "map(a,b.c)" - ); - - metafix.startRecord("1"); - metafix.literal("a", LITERAL_ALOHA); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).startEntity("b"); - ordered.verify(streamReceiver).literal("c", LITERAL_ALOHA); - ordered.verify(streamReceiver).endEntity(); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldMapNestedMulti() { - final Metafix metafix = fix( - "map(a,b)", - "map(a.c, b.c)", - "map(a.d, b.d)" - ); - - metafix.startRecord("1"); - metafix.literal("a.c", LITERAL_A); - metafix.literal("a.d", LITERAL_B); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).startEntity("b"); - ordered.verify(streamReceiver).literal("c", LITERAL_A); - ordered.verify(streamReceiver).literal("d", LITERAL_B); - ordered.verify(streamReceiver).endEntity(); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldAddLiteral() { - final Metafix metafix = fix( - "add_field(a,'A')" - ); - - metafix.startRecord("1"); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal("a", "A"); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldAddEntity() { - final Metafix metafix = fix( - "add_field(a.b,'AB')" - ); - - metafix.startRecord("1"); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).startEntity("a"); - ordered.verify(streamReceiver).literal("b", "AB"); - ordered.verify(streamReceiver).endEntity(); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldAddEntityNested() { - final Metafix metafix = fix( - "add_field(a.b.c.d.e,'ABCDE')" - ); - - metafix.startRecord("1"); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).startEntity("a"); - ordered.verify(streamReceiver).startEntity("b"); - ordered.verify(streamReceiver).startEntity("c"); - ordered.verify(streamReceiver).startEntity("d"); - ordered.verify(streamReceiver).literal("e", "ABCDE"); - ordered.verify(streamReceiver, Mockito.times(4)).endEntity(); // checkstyle-disable-line MagicNumber - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldReplaceInLiteral() { - final Metafix metafix = fix( - "do map(a)", // checkstyle-disable-line MultipleStringLiterals - " replace_all(pattern: 'a', with: 'b')", - "end" - ); - - metafix.startRecord("1"); - metafix.literal("a", LITERAL_ALOHA); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal("a", LITERAL_ALOHA.replaceAll("a", "b")); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldReplaceWithEntities() { - final Metafix metafix = fix( - "do map(a.b, a.b)", - " replace_all(pattern: 'a', with: 'b')", - "end" - ); - - metafix.startRecord("1"); - metafix.startEntity("a"); - metafix.literal("b", LITERAL_ALOHA); - metafix.endEntity(); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).startEntity("a"); - ordered.verify(streamReceiver).literal("b", LITERAL_ALOHA.replaceAll("a", "b")); - ordered.verify(streamReceiver).endEntity(); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldTrim() { - final Metafix metafix = fix( - "do map(a,b)", - " trim()", - "end" - ); - - metafix.startRecord("1"); - metafix.literal("a", "\t" + LITERAL_ALOHA + " "); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal("b", LITERAL_ALOHA); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void appendLiteral() { - final Metafix metafix = fix( - "do map(a)", - " compose(postfix: 'eha')", - "end" - ); - - metafix.startRecord("1"); - metafix.literal("a", LITERAL_ALOHA); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal("a", LITERAL_ALOHA + "eha"); // checkstyle-disable-line MultipleStringLiterals - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void prependLiteral() { - final Metafix metafix = fix( - "do map(a)", - " compose(prefix: 'eha')", - "end" - ); - - metafix.startRecord("1"); - metafix.literal("a", LITERAL_ALOHA); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal("a", "eha" + LITERAL_ALOHA); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void prependLiteralWithVarVal() { - final Metafix metafix = fix( - ImmutableMap.of("pre", "eha"), - "do map(a)", - " compose(prefix: '$[pre]')", - "end" - ); - metafix.startRecord("1"); - metafix.literal("a", LITERAL_ALOHA); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal("a", "eha" + LITERAL_ALOHA); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void prependLiteralWithVarKey() { - final Metafix metafix = fix( - ImmutableMap.of("composeOperation", "prefix"), - "do map(a)", - " compose('$[composeOperation]': 'eha')", - "end" - ); - metafix.startRecord("1"); - metafix.literal("a", LITERAL_ALOHA); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal("a", "eha" + LITERAL_ALOHA); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldHandleUnmatchedLiteralsInElseSource() { - final Metafix metafix = fix( - "map(Sylt,Hawaii)", - "map(_else)" - ); - - metafix.startRecord("1"); - metafix.literal(LITERAL_LANGEOOG, LITERAL_MOIN); - metafix.literal("Sylt", LITERAL_ALOHA); - metafix.literal(LITERAL_BALTRUM, LITERAL_MOIN_MOIN); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal(LITERAL_LANGEOOG, LITERAL_MOIN); - ordered.verify(streamReceiver).literal(LITERAL_HAWAII, LITERAL_ALOHA); - ordered.verify(streamReceiver).literal(LITERAL_BALTRUM, LITERAL_MOIN_MOIN); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldAllowTreatingEntityEndEventsAsLiterals() { - final Metafix metafix = fix( - "map(e1)", - "map(e1.e2)", - "map(e1.e2.d)" - ); - - metafix.startRecord("entity end info"); - metafix.startEntity("e1"); - metafix.startEntity("e2"); - metafix.literal("d", "a"); - metafix.endEntity(); - metafix.endEntity(); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("entity end info"); - ordered.verify(streamReceiver).literal("e1.e2.d", "a"); - ordered.verify(streamReceiver).literal("e1.e2", ""); - ordered.verify(streamReceiver).literal("e1", ""); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldLookupInline() { - verifyLookup(fix( - "do map(a)", // checkstyle-disable-line MultipleStringLiterals - " lookup(Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)", // checkstyle-disable-line MultipleStringLiterals - "end" - )); - } - - @Test - public void shouldLookupInTsvFile() throws URISyntaxException { - verifyLookup(fix( - "do map(a)", - " lookup(in: 'src/test/java/org/metafacture/metamorph/maps/test.tsv')", - "end" - )); - } - - @Test - public void shouldLookupInCsvFile() throws URISyntaxException { - verifyLookup(fix( - "do map(a)", - " lookup(in: 'src/test/java/org/metafacture/metamorph/maps/test.csv', separator: ',')", - "end" - )); - } - - private void verifyLookup(final Metafix metafix) { - metafix.startRecord("1"); - metafix.literal("a", LITERAL_ALOHA); - metafix.literal("a", LITERAL_MOIN); - metafix.literal("a", LITERAL_HAWAII); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal("a", "Alohaeha"); // checkstyle-disable-line MultipleStringLiterals - ordered.verify(streamReceiver).literal("a", "Moin zäme"); // checkstyle-disable-line MultipleStringLiterals - ordered.verify(streamReceiver).literal("a", "Tach"); // checkstyle-disable-line MultipleStringLiterals - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldMapAndLookupInline() { - verifyMapAndLookup(fix( - "do map(a,b)", - " lookup(Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)", // checkstyle-disable-line MultipleStringLiterals - "end" - )); - } - - @Test - public void shouldMapAndLookupInTsvFile() { - verifyMapAndLookup(fix( - "do map(a,b)", - " lookup(in: 'src/test/java/org/metafacture/metamorph/maps/test.tsv')", // checkstyle-disable-line MultipleStringLiterals - "end" - )); - } - - @Test - public void shouldMapAndLookupInCsvFile() { - verifyMapAndLookup(fix( - "do map(a,b)", - " lookup(in: 'src/test/java/org/metafacture/metamorph/maps/test.csv', separator: ',')", // checkstyle-disable-line MultipleStringLiterals - "end" - )); - } - - private void verifyMapAndLookup(final Metafix metafix) { - metafix.startRecord("1"); - metafix.literal("a", LITERAL_ALOHA); - metafix.literal("a", LITERAL_MOIN); - metafix.literal("a", LITERAL_HAWAII); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal("b", "Alohaeha"); - ordered.verify(streamReceiver).literal("b", "Moin zäme"); - ordered.verify(streamReceiver).literal("b", "Tach"); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldMapToExplicitEntityAndLookupInline() { - final Metafix metafix = fix( - "do entity('c')", - " do map(a,b)", - " lookup(Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)", - " end", // checkstyle-disable-line MultipleStringLiterals - "end" - ); - - metafix.startRecord("1"); - metafix.literal("a", LITERAL_ALOHA); - metafix.literal("a", LITERAL_MOIN); - metafix.literal("a", LITERAL_HAWAII); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).startEntity("c"); - ordered.verify(streamReceiver).literal("b", "Alohaeha"); - ordered.verify(streamReceiver).literal("b", "Moin zäme"); - ordered.verify(streamReceiver).literal("b", "Tach"); - ordered.verify(streamReceiver).endEntity(); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldMapToImplicitEntityAndLookupInline() { - final Metafix metafix = fix( - "do map(a,c.b)", - " lookup(Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)", - "end" - ); - - metafix.startRecord("1"); - metafix.literal("a", LITERAL_ALOHA); - metafix.literal("a", LITERAL_MOIN); - metafix.literal("a", LITERAL_HAWAII); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).startEntity("c"); - ordered.verify(streamReceiver).literal("b", "Alohaeha"); - ordered.verify(streamReceiver).literal("b", "Moin zäme"); - ordered.verify(streamReceiver).literal("b", "Tach"); - ordered.verify(streamReceiver).endEntity(); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldMapAndChangeAndLookupInline() { - final Metafix metafix = fix( - "do map(a,b)", - " replace_all(pattern: 'lit-A', with: 'Aloha')", - " replace_all(pattern: 'lit-B', with: 'Moin')", - " lookup(Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)", - "end" - ); - - metafix.startRecord("1"); - metafix.literal("a", LITERAL_A); - metafix.literal("a", LITERAL_B); - metafix.literal("a", LITERAL_HAWAII); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal("b", "Alohaeha"); - ordered.verify(streamReceiver).literal("b", "Moin zäme"); - ordered.verify(streamReceiver).literal("b", "Tach"); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldCombineLiterals() { - final Metafix metafix = fix( - "do combine(name: d, value: '${place}, ${greet}')", - " map(a, greet)", - " map(b, place)", - "end", - "map(c, e)" - ); - - metafix.startRecord("1"); - metafix.literal("a", LITERAL_ALOHA); - metafix.literal("b", LITERAL_HAWAII); - metafix.literal("c", LITERAL_MOIN); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal("d", LITERAL_HAWAII + ", " + LITERAL_ALOHA); - ordered.verify(streamReceiver).literal("e", LITERAL_MOIN); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldDoEntityAndMap() { - final Metafix metafix = fix( - "do entity(b)", - " map(a, c)", - "end" - ); - - metafix.startRecord("1"); - metafix.literal("a", LITERAL_A); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).startEntity("b"); - ordered.verify(streamReceiver).literal("c", LITERAL_A); - ordered.verify(streamReceiver).endEntity(); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldMapAndChangeNestedExplicit() { - final Metafix metafix = fix( - "do entity(b)", - " do map(a, c)", - " replace_all(pattern: 'A', with: 'B')", - " end", - "end" - ); - - metafix.startRecord("1"); - metafix.literal("a", LITERAL_A); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).startEntity("b"); - ordered.verify(streamReceiver).literal("c", LITERAL_B); - ordered.verify(streamReceiver).endEntity(); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldMapAndChangeNestedImplicit() { - final Metafix metafix = fix( - "do map(a, b.c)", - " replace_all(pattern: 'A', with: 'B')", - "end", - "map(x, y)" - ); - - metafix.startRecord("1"); - metafix.literal("a", LITERAL_A); - metafix.literal("x", LITERAL_A); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).startEntity("b"); - ordered.verify(streamReceiver).literal("c", LITERAL_B); - ordered.verify(streamReceiver).endEntity(); - ordered.verify(streamReceiver).literal("y", LITERAL_A); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldCombineToEntity() { - final Metafix metafix = fix( - "do entity(d)", - " do combine(name: a, value: '${place}, ${greet}')", - " map(a, greet)", - " map(b, place)", - " end", - "end", - "map(c, e)" - ); - - metafix.startRecord("1"); - metafix.literal("a", LITERAL_ALOHA); - metafix.literal("b", LITERAL_HAWAII); - metafix.literal("c", LITERAL_MOIN); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).startEntity("d"); - ordered.verify(streamReceiver).literal("a", LITERAL_HAWAII + ", " + LITERAL_ALOHA); - ordered.verify(streamReceiver).endEntity(); - ordered.verify(streamReceiver).literal("e", LITERAL_MOIN); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - public void shouldChooseFirstMapFlushWith() { - final Metafix metafix = fix( - "do choose(flushWith: 'd|record')", - " map(a, c)", - " map(b, c)", - "end", - "map(d,e)" // checkstyle-disable-line MultipleStringLiterals - ); - - metafix.startRecord("1"); - metafix.literal("b", LITERAL_B); - metafix.literal("a", LITERAL_A); - metafix.literal("d", LITERAL_B); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal("c", LITERAL_A); - ordered.verify(streamReceiver).literal("e", LITERAL_B); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - @Disabled("Fix choose flushing") - public void shouldChooseFirstMap() { - final Metafix metafix = fix( - "do choose()", - " map(a, c)", - " map(b, c)", - "end", - "map(d,e)" - ); - - metafix.startRecord("1"); - metafix.literal("b", LITERAL_B); - metafix.literal("a", LITERAL_A); - metafix.literal("d", LITERAL_B); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal("c", LITERAL_A); - ordered.verify(streamReceiver).literal("e", LITERAL_B); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - @Disabled("Fix choose flushing") - public void shouldChooseFirstDoMap() { - final Metafix metafix = fix( - "do choose()", - " do map(a, c)", - " not_equals('')", - " end", - " do map(b, c)", - " not_equals('')", - " end", - "end", - "map(d,e)" - ); - - metafix.startRecord("1"); - metafix.literal("b", LITERAL_B); - metafix.literal("a", LITERAL_A); - metafix.literal("d", LITERAL_B); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal("c", LITERAL_A); - ordered.verify(streamReceiver).literal("e", LITERAL_B); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - @Test - @Disabled("Fix syntax") - public void shouldUseCustomEntityMarker() { - final Metafix metafix = fix( - "map(entity~literal,data)" - ); - - metafix.startRecord("1"); - metafix.startEntity("entity"); - metafix.literal("literal", LITERAL_ALOHA); - metafix.endEntity(); - metafix.endRecord(); - - Mockito.verify(streamReceiver).literal("data", LITERAL_ALOHA); - } - - @Test - public void shouldMatchCharacterWithQuestionMarkWildcard() { - final Metafix metafix = fix( - "map('lit-?')" - ); - - metafix.startRecord("1"); - metafix.literal("lit", LITERAL_MOIN); - metafix.literal(LITERAL_A, LITERAL_ALOHA); - metafix.literal(LITERAL_B, "Aloha 'oe"); - metafix.endRecord(); - - Mockito.verify(streamReceiver).literal(LITERAL_A, LITERAL_ALOHA); - Mockito.verify(streamReceiver).literal(LITERAL_B, "Aloha 'oe"); - Mockito.verify(streamReceiver, Mockito.times(2)).literal(ArgumentMatchers.any(), ArgumentMatchers.any()); - } - - @Test - public void shouldMatchCharactersInCharacterClass() { - final Metafix metafix = fix( - "map('lit-[AB]')" - ); - - metafix.startRecord("1"); - metafix.literal(LITERAL_A, LITERAL_HAWAII); - metafix.literal(LITERAL_B, "Oahu"); - metafix.literal("lit-C", "Fehmarn"); - metafix.endRecord(); - - Mockito.verify(streamReceiver).literal(LITERAL_A, LITERAL_HAWAII); - Mockito.verify(streamReceiver).literal(LITERAL_B, "Oahu"); - Mockito.verify(streamReceiver, Mockito.times(2)).literal(ArgumentMatchers.any(), ArgumentMatchers.any()); - } - - @Test - public void shouldNotFeedbackJsonLdKeywords() { - final Metafix metafix = fix( - "map(_else)" - ); - - metafix.startRecord("1"); - metafix.literal("@id", LITERAL_HAWAII); - metafix.endRecord(); - - Mockito.verify(streamReceiver).literal("@id", LITERAL_HAWAII); - } - - @Test - public void shouldNotFeedbackJsonLdKeywordsNested() { - final Metafix metafix = fix( - "do entity('USA')", - " map('USA.Sylt', 'Hawaii')", - "end", - "map(_elseNested)" - ); - - metafix.startRecord("1"); - metafix.literal("Shikotan", "Aekap"); - metafix.startEntity("Germany"); - metafix.literal(LITERAL_LANGEOOG, LITERAL_MOIN); - metafix.literal(LITERAL_BALTRUM, LITERAL_MOIN_MOIN); - metafix.endEntity(); - metafix.startEntity("USA"); - metafix.literal("Sylt", LITERAL_ALOHA); - metafix.endEntity(); - metafix.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal("Shikotan", "Aekap"); - ordered.verify(streamReceiver).startEntity("Germany"); - ordered.verify(streamReceiver).literal(LITERAL_LANGEOOG, "Moin"); - ordered.verify(streamReceiver).literal(LITERAL_BALTRUM, LITERAL_MOIN_MOIN); - ordered.verify(streamReceiver).endEntity(); - ordered.verify(streamReceiver).startEntity("USA"); - ordered.verify(streamReceiver).literal("Hawaii", "Aloha"); - ordered.verify(streamReceiver).endEntity(); - ordered.verify(streamReceiver).endRecord(); - } - - @Test - @Disabled("Fix syntax") - public void shouldReplaceVariables() { - final Metafix metafix = fix( - "vars(in: Honolulu, out: Hawaii)", - "map($[in],$[out])" - ); - - metafix.startRecord("1"); - metafix.literal("Honolulu", LITERAL_ALOHA); - metafix.endRecord(); - - Mockito.verify(streamReceiver).literal(LITERAL_HAWAII, LITERAL_ALOHA); - } - - private Metafix fix(final String... fix) { - return fix(Collections.emptyMap(), fix); - } - - private Metafix fix(final Map vars, final String... fix) { - final String fixString = String.join("\n", fix); - System.out.println("\nFix string: " + fixString); - - Metafix metafix = null; - try { - metafix = new Metafix(fixString, vars); - metafix.setReceiver(streamReceiver); - } - catch (final FileNotFoundException e) { - e.printStackTrace(); - } - return metafix; - } - -} diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixRecordTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixFieldTest.java similarity index 52% rename from org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixRecordTest.java rename to org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixFieldTest.java index 6382ce41..9f9a72cc 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixRecordTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixFieldTest.java @@ -18,7 +18,6 @@ import org.metafacture.framework.StreamReceiver; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.RegisterExtension; @@ -30,16 +29,14 @@ import java.util.Arrays; /** - * Tests the non-field-streaming record functionality of Metafix via DSL. - * - * See https://github.com/metafacture/metafacture-fix/issues/35 + * Tests Metafix field / record level methods. Following the cheat sheet + * examples at https://github.com/LibreCat/Catmandu/wiki/Fixes-Cheat-Sheet * * @author Fabian Steeg */ @ExtendWith(MockitoExtension.class) @SuppressWarnings("checkstyle:MultipleStringLiterals") -@Disabled // Temporarily disabled while working on adapting `if` from metamorph -public class MetafixRecordTest { +public class MetafixFieldTest { @RegisterExtension private MockitoRule mockitoRule = MockitoJUnit.rule(); @@ -47,181 +44,163 @@ public class MetafixRecordTest { @Mock private StreamReceiver streamReceiver; - public MetafixRecordTest() { + public MetafixFieldTest() { } @Test - public void ifAnyMatch() { + public void set() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if any_match('name', '.*University.*')", // - " add_field('type', 'Organization')", // - "end"), // + "set_field('my.name','patrick')", + "set_field('your.name','nicolas')"), // i -> { i.startRecord("1"); - i.literal("name", "Max Musterman"); i.endRecord(); // i.startRecord("2"); - i.literal("name", "Some University"); - i.literal("name", "Filibandrina"); - i.endRecord(); - }, o -> { - o.get().startRecord("1"); - o.get().endRecord(); - // - o.get().startRecord("2"); - o.get().literal("type", "Organization"); - o.get().endRecord(); - }); - } - - @Test - public void ifAllMatch() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if all_match('name', '.*University.*')", // - " add_field('type', 'Organization')", // - "end"), // - i -> { - i.startRecord("1"); - i.literal("name", "Max Musterman"); - i.literal("name", "A University"); + i.startEntity("my"); + i.literal("name", "max"); + i.endEntity(); + i.startEntity("your"); + i.literal("name", "mo"); + i.endEntity(); i.endRecord(); // - i.startRecord("2"); - i.literal("name", "Some University"); - i.literal("name", "University Filibandrina"); + i.startRecord("3"); i.endRecord(); }, o -> { o.get().startRecord("1"); + o.get().literal("my.name", "patrick"); + o.get().literal("your.name", "nicolas"); o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("type", "Organization"); + o.get().literal("my.name", "patrick"); + o.get().literal("your.name", "nicolas"); + o.get().endRecord(); + // + o.get().startRecord("3"); + o.get().literal("my.name", "patrick"); + o.get().literal("your.name", "nicolas"); o.get().endRecord(); }); } @Test - public void ifAnyMatchNested() { + public void add() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if any_match('author.name.label', '.*University.*')", // - " add_field('author.type', 'Organization')", // - "end"), // + "add_field('my.name','nicolas')"), // i -> { i.startRecord("1"); - i.startEntity("author"); - i.startEntity("name"); - i.literal("label", "Max Musterman"); - i.endEntity(); - i.endEntity(); i.endRecord(); // i.startRecord("2"); - i.startEntity("author"); - i.startEntity("name"); - i.literal("label", "Some University"); - i.endEntity(); + i.startEntity("my"); + i.literal("name", "max"); i.endEntity(); i.endRecord(); + // + i.startRecord("3"); + i.endRecord(); }, o -> { o.get().startRecord("1"); + o.get().literal("my.name", "nicolas"); o.get().endRecord(); // o.get().startRecord("2"); - o.get().startEntity("author"); - o.get().literal("type", "Organization"); - o.get().endEntity(); + o.get().literal("my.name", "max"); + o.get().literal("my.name", "nicolas"); + o.get().endRecord(); + // + o.get().startRecord("3"); + o.get().literal("my.name", "nicolas"); o.get().endRecord(); }); } @Test - @Disabled // TODO: fix events for this record order - public void ifAnyMatchFirst() { + public void move() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if any_match('name', '.*University.*')", // - " add_field('type', 'Organization')", // - "end"), // + "move_field('my.name','your.name')", + "move_field('missing','whatever')"), // i -> { i.startRecord("1"); - i.literal("name", "Some University"); i.endRecord(); // i.startRecord("2"); - i.literal("name", "Max Musterman"); + i.startEntity("my"); + i.literal("name", "max"); + i.endEntity(); + i.endRecord(); + // + i.startRecord("3"); i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("type", "Organization"); o.get().endRecord(); // o.get().startRecord("2"); + o.get().literal("your.name", "max"); + o.get().endRecord(); + // + o.get().startRecord("3"); o.get().endRecord(); }); } @Test - @Disabled // TODO: support else block - public void ifAnyMatchElse() { + public void copy() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if any_match('name', '.*University.*')", // - " add_field('type', 'Organization')", // - "else", // - " add_field('type', 'Person')", // - "end"), // + "copy_field('your.name','your.name2')"), // i -> { i.startRecord("1"); - i.literal("name", "Max Musterman"); i.endRecord(); // i.startRecord("2"); - i.literal("name", "Some University"); + i.startEntity("your"); + i.literal("name", "max"); + i.endEntity(); + i.endRecord(); + // + i.startRecord("3"); i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("type", "Person"); o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("type", "Organization"); + o.get().literal("your.name", "max"); + o.get().literal("your.name2", "max"); + o.get().endRecord(); + // + o.get().startRecord("3"); o.get().endRecord(); }); } @Test - @Disabled // TODO: support elsif block - public void ifAnyMatchElsif() { + public void remove() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if any_match('name', '.*University.*')", // - " add_field('type', 'Organization')", // - "elsif any_match('name', '[^ ]* [^ ]*')", // - " add_field('type', 'Person')", // - "else", // - " add_field('type', 'Unknown')", // - "end"), // + "remove_field('your.name')"), // i -> { i.startRecord("1"); - i.literal("name", "Max Musterman"); i.endRecord(); // i.startRecord("2"); - i.literal("name", "Some University"); + i.startEntity("your"); + i.literal("name", "max"); + i.endEntity(); i.endRecord(); // i.startRecord("3"); - i.literal("name", "Filibandrina"); i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("type", "Person"); o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("type", "Organization"); o.get().endRecord(); // o.get().startRecord("3"); - o.get().literal("type", "Unknown"); o.get().endRecord(); }); } diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixFilterTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixFilterTest.java deleted file mode 100644 index bb2c9291..00000000 --- a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixFilterTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2013, 2021 Deutsche Nationalbibliothek and others - * - * Licensed under the Apache License, Version 2.0 the "License"; - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.metafacture.metamorph; - -import org.metafacture.framework.StreamReceiver; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.mockito.InOrder; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.io.FileNotFoundException; -import java.util.Collections; -import java.util.Map; - -/** - * Tests for class {@link MetafixFilter}. - * - * @author Fabian Steeg - * - */ -@ExtendWith(MockitoExtension.class) -public final class MetafixFilterTest { - - private static final String VAL = "val"; - - @RegisterExtension - private MockitoRule mockitoRule = MockitoJUnit.rule(); - - @Mock - private StreamReceiver streamReceiver; - - public MetafixFilterTest() { } - - @Test - public void shouldFilterRecords() { - final MetafixFilter filter = filter("map(a)"); - - filter.startRecord("1"); - filter.literal("a", VAL); - filter.endRecord(); - - filter.startRecord("2"); - filter.literal("b", VAL); - filter.endRecord(); - - final InOrder ordered = Mockito.inOrder(streamReceiver); - ordered.verify(streamReceiver).startRecord("1"); - ordered.verify(streamReceiver).literal("a", VAL); - ordered.verify(streamReceiver).endRecord(); - ordered.verifyNoMoreInteractions(); - } - - private MetafixFilter filter(final String... filter) { - return filter(Collections.emptyMap(), filter); - } - - private MetafixFilter filter(final Map vars, final String... fix) { - final String fixString = String.join("\n", fix); - System.out.println("\nFix filter string: " + fixString); - - MetafixFilter filter = null; - try { - filter = new MetafixFilter(fixString, vars); - filter.setReceiver(streamReceiver); - } - catch (final FileNotFoundException e) { - e.printStackTrace(); - } - return filter; - } - -} diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixIfTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixIfTest.java index 98868eee..de7b21f2 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixIfTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixIfTest.java @@ -30,9 +30,7 @@ import java.util.Arrays; /** - * Tests the `if` Metamorph mechanism adapted for Metafix via DSL. - * - * See https://github.com/metafacture/metafacture-fix/issues/10 + * Test Metafix `if` conditionals. * * @author Fabian Steeg */ @@ -50,80 +48,147 @@ public MetafixIfTest() { } @Test - public void ifTopLevel() { + public void ifAny() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "map('name')", - "if contains('name', 'University')", // + "if any_contain('name', 'University')", // " add_field('type', 'Organization')", // "end"), // i -> { i.startRecord("1"); + i.literal("name", "Mary"); i.literal("name", "A University"); i.endRecord(); + // + i.startRecord("2"); + i.literal("name", "Mary"); + i.literal("name", "Max"); + i.endRecord(); + // + i.startRecord("3"); + i.endRecord(); }, o -> { o.get().startRecord("1"); + o.get().literal("name", "Mary"); o.get().literal("name", "A University"); o.get().literal("type", "Organization"); o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().literal("name", "Mary"); + o.get().literal("name", "Max"); + o.get().endRecord(); + // + o.get().startRecord("3"); + o.get().endRecord(); }); } @Test - public void ifTopLevelWithEqualsFunction() { + public void ifAll() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if equals('name', 'University')", // + "if all_contain('name', 'University')", // " add_field('type', 'Organization')", // - "end", // - "map('name')"), + "end"), // i -> { i.startRecord("1"); - i.literal("name", "University"); + i.literal("name", "Mary"); + i.literal("name", "A University"); + i.endRecord(); + // + i.startRecord("2"); + i.literal("name", "Great University"); + i.literal("name", "A University"); + i.endRecord(); + // + i.startRecord("3"); i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("name", "University"); + o.get().literal("name", "Mary"); + o.get().literal("name", "A University"); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().literal("name", "Great University"); + o.get().literal("name", "A University"); o.get().literal("type", "Organization"); o.get().endRecord(); + // + o.get().startRecord("3"); + o.get().endRecord(); }); } @Test - public void ifInCollector() { + public void ifNone() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "do entity('author')", - " map('name')", - " if contains('name', 'University')", // - " add_field('type', 'Organization')", // - " end", + "if none_contain('name', 'University')", // + " add_field('type', 'Person')", // "end"), // i -> { i.startRecord("1"); + i.literal("name", "Mary"); i.literal("name", "A University"); i.endRecord(); + // + i.startRecord("2"); + i.literal("name", "Max"); + i.literal("name", "Mary"); + i.endRecord(); + // + i.startRecord("3"); + i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().startEntity("author"); - o.get().literal("type", "Organization"); + o.get().literal("name", "Mary"); o.get().literal("name", "A University"); - o.get().endEntity(); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().literal("name", "Max"); + o.get().literal("name", "Mary"); + o.get().literal("type", "Person"); + o.get().endRecord(); + // + o.get().startRecord("3"); + o.get().literal("type", "Person"); o.get().endRecord(); }); } @Test - public void ifTopLevelMultiRecords() { + public void ifEqual() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "map('name')", - "if contains('name', 'University')", // + "if all_equal('name', 'University')", // " add_field('type', 'Organization')", // + "end"), + i -> { + i.startRecord("1"); + i.literal("name", "University"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("name", "University"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + }); + } + + @Test + public void ifContainMoveField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "if all_contain('name', 'University')", // + " move_field('name', 'orgName')", // "end"), // i -> { i.startRecord("1"); i.literal("name", "Max"); i.endRecord(); + // i.startRecord("2"); i.literal("name", "A University"); i.endRecord(); + // i.startRecord("3"); i.literal("name", "Mary"); i.endRecord(); @@ -131,10 +196,11 @@ public void ifTopLevelMultiRecords() { o.get().startRecord("1"); o.get().literal("name", "Max"); o.get().endRecord(); + // o.get().startRecord("2"); - o.get().literal("name", "A University"); - o.get().literal("type", "Organization"); + o.get().literal("orgName", "A University"); o.get().endRecord(); + // o.get().startRecord("3"); o.get().literal("name", "Mary"); o.get().endRecord(); @@ -142,176 +208,251 @@ public void ifTopLevelMultiRecords() { } @Test - public void ifTopLevelMultiRecordsMapField() { + public void moveAndAddIfContain() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if contains('name', 'University')", // - " map('name', 'orgName')", // + "move_field('name', 'author.name')", + "if all_contain('author.name', 'University')", // + " add_field('author.type', 'Organization')", // "end"), // i -> { i.startRecord("1"); - i.literal("name", "Max"); + i.literal("name", "A University"); i.endRecord(); - i.startRecord("2"); + }, o -> { + o.get().startRecord("1"); + o.get().literal("author.name", "A University"); + o.get().literal("author.type", "Organization"); + o.get().endRecord(); + }); + } + + @Test + public void ifContainMultipleAddField() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "if all_contain('name', 'University')", // + " add_field('type', 'Organization')", // + " add_field('comment', 'type was guessed')", // + "end"), // + i -> { + i.startRecord("1"); i.literal("name", "A University"); i.endRecord(); - i.startRecord("3"); - i.literal("name", "Mary"); + }, o -> { + o.get().startRecord("1"); + o.get().literal("name", "A University"); + o.get().literal("type", "Organization"); + o.get().literal("comment", "type was guessed"); + o.get().endRecord(); + }); + } + + @Test + public void ifAnyMatch() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "if any_match('name', '.*University.*')", // + " add_field('type', 'Organization')", // + "end"), // + i -> { + i.startRecord("1"); + i.literal("name", "Max"); + i.endRecord(); + // + i.startRecord("2"); + i.literal("name", "Some University"); + i.literal("name", "Filibandrina"); i.endRecord(); }, o -> { o.get().startRecord("1"); + o.get().literal("name", "Max"); o.get().endRecord(); + // o.get().startRecord("2"); - o.get().literal("orgName", "A University"); - o.get().endRecord(); - o.get().startRecord("3"); + o.get().literal("name", "Some University"); + o.get().literal("name", "Filibandrina"); + o.get().literal("type", "Organization"); o.get().endRecord(); }); } @Test - @Disabled // TODO: in collector, `map` not firing when `if` fails - public void ifInCollectorMultiRecords() { + public void ifAllMatch() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "do entity('author')", - " map('name')", - " if contains('name', 'University')", // + "if all_match('name', '.*University.*')", // " add_field('type', 'Organization')", // - " end", "end"), // i -> { i.startRecord("1"); i.literal("name", "Max"); - i.endRecord(); - i.startRecord("2"); i.literal("name", "A University"); i.endRecord(); - i.startRecord("3"); - i.literal("name", "Mary"); + // + i.startRecord("2"); + i.literal("name", "Some University"); + i.literal("name", "University Filibandrina"); i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().startEntity("author"); o.get().literal("name", "Max"); - o.get().endEntity(); + o.get().literal("name", "A University"); o.get().endRecord(); + // o.get().startRecord("2"); - o.get().startEntity("author"); + o.get().literal("name", "Some University"); + o.get().literal("name", "University Filibandrina"); o.get().literal("type", "Organization"); - o.get().literal("name", "A University"); - o.get().endEntity(); - o.get().endRecord(); - o.get().startRecord("3"); - o.get().startEntity("author"); - o.get().literal("name", "Mary"); - o.get().endEntity(); o.get().endRecord(); }); } @Test - // Without `entity`, but nested.entity.syntax: two entities, same name - public void mapAndTestNestedTwoEntities() { + public void ifAnyMatchNested() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "map('name', 'author.name')", - "if contains('name', 'University')", // - " add_field('author.type', 'Organization')", // + "if any_match('author.name.label', '.*University.*')", // + " add_field('author.type', 'Organization')", // "end"), // i -> { i.startRecord("1"); - i.literal("name", "A University"); + i.startEntity("author"); + i.startEntity("name"); + i.literal("label", "Max"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + // + i.startRecord("2"); + i.startEntity("author"); + i.startEntity("name"); + i.literal("label", "Some University"); + i.endEntity(); + i.endEntity(); i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().startEntity("author"); - o.get().literal("type", "Organization"); - o.get().endEntity(); - o.get().startEntity("author"); - o.get().literal("name", "A University"); - o.get().endEntity(); + o.get().literal("author.name.label", "Max"); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().literal("author.name.label", "Some University"); + o.get().literal("author.type", "Organization"); o.get().endRecord(); }); } @Test - // Top-level `if` constructs internally wrap a choose collector: - public void ifInCollectorChoose() { + public void ifAnyMatchFirstRecord() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "do choose(flushWith: 'record')", - " if contains('name', 'University')", // + "if any_match('name', '.*University.*')", // " add_field('type', 'Organization')", // - " end", // "end"), // i -> { i.startRecord("1"); - i.literal("name", "Max University"); + i.literal("name", "Some University"); i.endRecord(); // i.startRecord("2"); - i.literal("name", "Max Musterman"); + i.literal("name", "Max"); i.endRecord(); }, o -> { o.get().startRecord("1"); + o.get().literal("name", "Some University"); o.get().literal("type", "Organization"); o.get().endRecord(); // o.get().startRecord("2"); + o.get().literal("name", "Max"); o.get().endRecord(); }); } @Test - public void ifInCollectorCombine() { + public void ifAnyMatchLastRecord() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "do combine(name: 'fullName', value: '${first} ${last}')", // - " if contains('author.type', 'Person')", // - " map('author.first', 'first')", // - " map('author.last', 'last')", // - " end", // + "if any_match('name', '.*University.*')", // + " add_field('type', 'Organization')", // "end"), // i -> { i.startRecord("1"); - i.startEntity("author"); - i.literal("type", "Organization"); - i.endEntity(); + i.literal("name", "Max"); i.endRecord(); // i.startRecord("2"); - i.startEntity("author"); - i.literal("first", "Max"); - i.literal("last", "Musterman"); - i.literal("type", "DifferentiatedPerson"); - i.endEntity(); + i.literal("name", "Some University"); i.endRecord(); }, o -> { o.get().startRecord("1"); + o.get().literal("name", "Max"); o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("fullName", "Max Musterman"); + o.get().literal("name", "Some University"); + o.get().literal("type", "Organization"); o.get().endRecord(); }); } @Test - @Disabled // TODO: second `add_field` not firing - public void ifTopLevelMultipleAddField() { + @Disabled // TODO: support else block + public void ifAnyMatchElse() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "map('name')", - "if contains('name', 'University')", // + "if any_match('name', '.*University.*')", // " add_field('type', 'Organization')", // - " add_field('comment', 'type was guessed')", // + "else", // + " add_field('type', 'Person')", // "end"), // i -> { i.startRecord("1"); - i.literal("name", "A University"); + i.literal("name", "Max"); + i.endRecord(); + // + i.startRecord("2"); + i.literal("name", "Some University"); i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("name", "A University"); + o.get().literal("type", "Person"); + o.get().endRecord(); + // + o.get().startRecord("2"); o.get().literal("type", "Organization"); - o.get().literal("comment", "type was guessed"); o.get().endRecord(); }); } + @Test + @Disabled // TODO: support elsif block + public void ifAnyMatchElsif() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "if any_match('name', '.*University.*')", // + " add_field('type', 'Organization')", // + "elsif any_match('name', '[^ ]* [^ ]*')", // + " add_field('type', 'Person')", // + "else", // + " add_field('type', 'Unknown')", // + "end"), // + i -> { + i.startRecord("1"); + i.literal("name", "Max"); + i.endRecord(); + // + i.startRecord("2"); + i.literal("name", "Some University"); + i.endRecord(); + // + i.startRecord("3"); + i.literal("name", "Filibandrina"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("type", "Person"); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().literal("type", "Organization"); + o.get().endRecord(); + // + o.get().startRecord("3"); + o.get().literal("type", "Unknown"); + o.get().endRecord(); + }); + } } diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixLookupTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixLookupTest.java new file mode 100644 index 00000000..9e4c8582 --- /dev/null +++ b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixLookupTest.java @@ -0,0 +1,142 @@ +/* + * Copyright 2021 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metamorph; + +import org.metafacture.framework.StreamReceiver; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Arrays; + +/** + * Tests Metafix lookup. Following the cheat sheet examples at + * https://github.com/LibreCat/Catmandu/wiki/Fixes-Cheat-Sheet + * + * @author Fabian Steeg + */ +@ExtendWith(MockitoExtension.class) +@SuppressWarnings("checkstyle:MultipleStringLiterals") +public class MetafixLookupTest { + + @RegisterExtension + private MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Mock + private StreamReceiver streamReceiver; + + public MetafixLookupTest() { + } + + @Test + public void inline() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "lookup('title', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)"), // + i -> { + i.startRecord("1"); + i.endRecord(); + // + i.startRecord("2"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Hey"); + i.endRecord(); + // + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().literal("title", "Alohaeha"); + o.get().literal("title", "Moin zäme"); + o.get().literal("title", "Tach"); + o.get().endRecord(); + // + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + public void csv() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "lookup('title', 'src/test/java/org/metafacture/metamorph/maps/test.csv')"), // + i -> { + i.startRecord("1"); + i.endRecord(); + // + i.startRecord("2"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Hey"); + i.endRecord(); + // + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().literal("title", "Alohaeha"); + o.get().literal("title", "Moin zäme"); + o.get().literal("title", "Tach"); + o.get().endRecord(); + // + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + public void tsv() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "lookup('title', 'src/test/java/org/metafacture/metamorph/maps/test.tsv', sep_char:'\t')"), // + i -> { + i.startRecord("1"); + i.endRecord(); + // + i.startRecord("2"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Hey"); + i.endRecord(); + // + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().literal("title", "Alohaeha"); + o.get().literal("title", "Moin zäme"); + o.get().literal("title", "Tach"); + o.get().endRecord(); + // + o.get().startRecord("3"); + o.get().endRecord(); + }); + } +} diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixMethodTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixMethodTest.java new file mode 100644 index 00000000..ec9c9a18 --- /dev/null +++ b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixMethodTest.java @@ -0,0 +1,316 @@ +/* + * Copyright 2021 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metamorph; + +import org.metafacture.framework.StreamReceiver; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.mockito.junit.jupiter.MockitoExtension; +import com.google.common.collect.ImmutableMap; + +import java.util.Arrays; + +/** + * Tests Metafix field level methods. Following the cheat sheet examples at + * https://github.com/LibreCat/Catmandu/wiki/Fixes-Cheat-Sheet + * + * @author Fabian Steeg + */ +@ExtendWith(MockitoExtension.class) +@SuppressWarnings("checkstyle:MultipleStringLiterals") +public class MetafixMethodTest { + + @RegisterExtension + private MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Mock + private StreamReceiver streamReceiver; + + public MetafixMethodTest() { + } + + @Test + public void upcase() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "upcase('title')"), // + i -> { + i.startRecord("1"); + i.endRecord(); + // + i.startRecord("2"); + i.literal("title", "marc"); + i.literal("title", "json"); + i.endRecord(); + // + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().literal("title", "MARC"); + o.get().literal("title", "JSON"); + o.get().endRecord(); + // + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + public void downcase() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "downcase('title')"), // + i -> { + i.startRecord("1"); + i.endRecord(); + // + i.startRecord("2"); + i.literal("title", "MARC"); + i.literal("title", "Json"); + i.endRecord(); + // + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().literal("title", "marc"); + o.get().literal("title", "json"); + o.get().endRecord(); + // + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + public void capitalize() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "capitalize('title')"), // + i -> { + i.startRecord("1"); + i.endRecord(); + // + i.startRecord("2"); + i.literal("title", "marc"); + i.literal("title", "json"); + i.endRecord(); + // + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().literal("title", "Marc"); + o.get().literal("title", "Json"); + o.get().endRecord(); + // + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + public void substring() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "substring('title', '0', '2')"), // + i -> { + i.startRecord("1"); + i.endRecord(); + // + i.startRecord("2"); + i.literal("title", "marc"); + i.literal("title", "json"); + i.endRecord(); + // + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().literal("title", "m"); + o.get().literal("title", "j"); + o.get().endRecord(); + // + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + public void substringWithVar() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "substring('title', '0', '$[end]')"), // + ImmutableMap.of("end", "3"), + i -> { + i.startRecord("1"); + i.endRecord(); + // + i.startRecord("2"); + i.literal("title", "marc"); + i.literal("title", "json"); + i.endRecord(); + // + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().literal("title", "ma"); + o.get().literal("title", "js"); + o.get().endRecord(); + // + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + public void trim() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "trim('title')"), // + i -> { + i.startRecord("1"); + i.endRecord(); + // + i.startRecord("2"); + i.literal("title", " marc "); + i.literal("title", " json "); + i.endRecord(); + // + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().literal("title", "marc"); + o.get().literal("title", "json"); + o.get().endRecord(); + // + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + @Disabled // Use SimpleRegexTrie/WildcardTrie + public void alternation() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "trim('title-1|title-2')"), // + i -> { + i.startRecord("1"); + i.endRecord(); + // + i.startRecord("2"); + i.literal("title-1", " marc "); + i.literal("title-2", " json "); + i.endRecord(); + // + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().literal("title-2", "marc"); + o.get().literal("title-1", "json"); + o.get().endRecord(); + // + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + @Disabled // Use SimpleRegexTrie/WildcardTrie + public void wildcard() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "trim('title-?')"), // + i -> { + i.startRecord("1"); + i.endRecord(); + // + i.startRecord("2"); + i.literal("title-1", " marc "); + i.literal("title-2", " json "); + i.endRecord(); + // + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().literal("title-2", "marc"); + o.get().literal("title-1", "json"); + o.get().endRecord(); + // + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + @Disabled // Use SimpleRegexTrie + public void characterClass() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "trim('title-[12]')"), // + i -> { + i.startRecord("1"); + i.endRecord(); + // + i.startRecord("2"); + i.literal("title-1", " marc "); + i.literal("title-2", " json "); + i.endRecord(); + // + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().literal("title-2", "marc"); + o.get().literal("title-1", "json"); + o.get().endRecord(); + // + o.get().startRecord("3"); + o.get().endRecord(); + }); + } +} diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixSelectorTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixSelectorTest.java new file mode 100644 index 00000000..95021061 --- /dev/null +++ b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixSelectorTest.java @@ -0,0 +1,88 @@ +/* + * Copyright 2021 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.metamorph; + +import org.metafacture.framework.StreamReceiver; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Arrays; + +/** + * Test Metafix selectors. + * + * @author Fabian Steeg + * + */ +@ExtendWith(MockitoExtension.class) +@Disabled // TODO: support Fix-style selectors https://github.com/LibreCat/Catmandu/wiki/Selectors +public final class MetafixSelectorTest { + + @RegisterExtension + private MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Mock + private StreamReceiver streamReceiver; + + public MetafixSelectorTest() { } + + @Test + public void reject() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "reject exists(error)"), // + i -> { + i.startRecord("1"); + i.literal("error", "details"); + i.endRecord(); + // + i.startRecord("2"); + i.endRecord(); + }, o -> { + o.get().startRecord("2"); + o.get().literal("name", "Mary"); + o.get().endRecord(); + }); + } + + @Test + public void rejectWithExplicitConditional() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "if exists(error)", // + " reject()", // + "end"), // + i -> { + i.startRecord("1"); + i.literal("error", "details"); + i.endRecord(); + // + i.startRecord("2"); + i.endRecord(); + }, o -> { + o.get().startRecord("2"); + o.get().literal("name", "Mary"); + o.get().endRecord(); + }); + } + +} diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixTestHelpers.java b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixTestHelpers.java index 08d01a60..91e8060c 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixTestHelpers.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixTestHelpers.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 hbz NRW + * Copyright 2020, 2021 hbz NRW * * Licensed under the Apache License, Version 2.0 the "License"; * you may not use this file except in compliance with the License. @@ -23,8 +23,8 @@ import org.mockito.exceptions.base.MockitoAssertionError; import java.io.FileNotFoundException; -import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.IntFunction; @@ -42,13 +42,18 @@ private MetafixTestHelpers() { } public static void assertFix(final StreamReceiver receiver, final List fixDef, final Consumer in, final Consumer> out) { - assertFix(receiver, fixDef, in, (s, f) -> out.accept(s)); + assertFix(receiver, fixDef, in, (s, f) -> out.accept(s), Metafix.NO_VARS); + } + + public static void assertFix(final StreamReceiver receiver, final List fixDef, + final Map vars, final Consumer in, final Consumer> out) { + assertFix(receiver, fixDef, in, (s, f) -> out.accept(s), vars); } public static void assertFix(final StreamReceiver receiver, final List fixLines, final Consumer in, - final BiConsumer, IntFunction> out) { + final BiConsumer, IntFunction> out, final Map vars) { final String fixString = String.join("\n", fixLines); - final Metafix metafix = fix(receiver, fixString); + final Metafix metafix = fix(receiver, fixString, vars); final InOrder ordered = Mockito.inOrder(receiver); try { in.accept(metafix); @@ -62,11 +67,11 @@ public static void assertFix(final StreamReceiver receiver, final List f } } - static Metafix fix(final StreamReceiver receiver, final String fix) { + static Metafix fix(final StreamReceiver receiver, final String fix, final Map vars) { System.out.println("\nFix string: " + fix); Metafix metafix = null; try { - metafix = new Metafix(fix, Collections.emptyMap()); + metafix = new Metafix(fix, vars); metafix.setReceiver(receiver); } catch (final FileNotFoundException e) { From 3045dbb91d8dfc2e77f46e824a9ef9c0e8445c41 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Thu, 19 Aug 2021 11:41:41 +0200 Subject: [PATCH 02/55] Implement `elsif` and `else` blocks (#35) See https://github.com/LibreCat/Catmandu/wiki/Conditionals --- .../metamorph/RecordTransformer.java | 18 ++++++++++++++---- .../metafacture/metamorph/MetafixIfTest.java | 10 ++++++---- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java b/org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java index ef47dc13..e926d654 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java @@ -18,6 +18,8 @@ import org.metafacture.commons.StringUtil; import org.metafacture.fix.fix.Do; +import org.metafacture.fix.fix.ElsIf; +import org.metafacture.fix.fix.Else; import org.metafacture.fix.fix.Expression; import org.metafacture.fix.fix.Fix; import org.metafacture.fix.fix.If; @@ -90,10 +92,18 @@ private void processBind(final Expression expression, final EList params // final Collect collect = collectFactory.newInstance(expression.getName(), attributes); } - private void processConditional(final Expression expression, final EList parameters) { - final If theIf = (If) expression; - if (testConditional(theIf.getName(), parameters)) { - processSubexpressions(theIf.getElements()); + private void processConditional(final Expression exp, final EList parameters) { + final If ifExp = (If) exp; + final ElsIf elsIfExp = ifExp.getElseIf(); + final Else elseExp = ifExp.getElse(); + if (testConditional(ifExp.getName(), parameters)) { + processSubexpressions(ifExp.getElements()); + } + else if (elsIfExp != null && testConditional(elsIfExp.getName(), elsIfExp.getParams())) { + processSubexpressions(elsIfExp.getElements()); + } + else if (elseExp != null) { + processSubexpressions(elseExp.getElements()); } } diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixIfTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixIfTest.java index de7b21f2..20d626bf 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixIfTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixIfTest.java @@ -18,7 +18,6 @@ import org.metafacture.framework.StreamReceiver; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.RegisterExtension; @@ -391,7 +390,6 @@ public void ifAnyMatchLastRecord() { } @Test - @Disabled // TODO: support else block public void ifAnyMatchElse() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// "if any_match('name', '.*University.*')", // @@ -409,17 +407,18 @@ public void ifAnyMatchElse() { i.endRecord(); }, o -> { o.get().startRecord("1"); + o.get().literal("name", "Max"); o.get().literal("type", "Person"); o.get().endRecord(); // o.get().startRecord("2"); + o.get().literal("name", "Some University"); o.get().literal("type", "Organization"); o.get().endRecord(); }); } @Test - @Disabled // TODO: support elsif block public void ifAnyMatchElsif() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// "if any_match('name', '.*University.*')", // @@ -431,7 +430,7 @@ public void ifAnyMatchElsif() { "end"), // i -> { i.startRecord("1"); - i.literal("name", "Max"); + i.literal("name", "Max Power"); i.endRecord(); // i.startRecord("2"); @@ -443,14 +442,17 @@ public void ifAnyMatchElsif() { i.endRecord(); }, o -> { o.get().startRecord("1"); + o.get().literal("name", "Max Power"); o.get().literal("type", "Person"); o.get().endRecord(); // o.get().startRecord("2"); + o.get().literal("name", "Some University"); o.get().literal("type", "Organization"); o.get().endRecord(); // o.get().startRecord("3"); + o.get().literal("name", "Filibandrina"); o.get().literal("type", "Unknown"); o.get().endRecord(); }); From 698125939149559b223f53a920fcd0ec5ef5b961 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Thu, 19 Aug 2021 12:27:40 +0200 Subject: [PATCH 03/55] Implement `unless` conditional (#35) See https://github.com/LibreCat/Catmandu/wiki/Conditionals --- .../main/java/org/metafacture/fix/Fix.xtext | 8 ++++++ .../metamorph/RecordTransformer.java | 15 ++++++++--- .../metafacture/metamorph/MetafixIfTest.java | 26 +++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/fix/Fix.xtext b/org.metafacture.fix/src/main/java/org/metafacture/fix/Fix.xtext index 0da72b5d..42c12067 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/fix/Fix.xtext +++ b/org.metafacture.fix/src/main/java/org/metafacture/fix/Fix.xtext @@ -9,11 +9,19 @@ Fix: Expression: Do | + Unless | If | MethodCall | {Expression} SL_COMMENT ; +Unless: + 'unless' name = ValidID '(' ( params += (QualifiedName|STRING) ( ',' params += (QualifiedName|STRING) )* )? ')' BEGIN + elements += Expression* + END + 'end' +; + If: 'if' name = ValidID '(' ( params += (QualifiedName|STRING) ( ',' params += (QualifiedName|STRING) )* )? ')' BEGIN elements += Expression* diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java b/org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java index e926d654..0aeb07c6 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java @@ -25,6 +25,7 @@ import org.metafacture.fix.fix.If; import org.metafacture.fix.fix.MethodCall; import org.metafacture.fix.fix.Options; +import org.metafacture.fix.fix.Unless; import org.metafacture.metamorph.FixPredicate.Quantifier; import org.eclipse.emf.common.util.EList; @@ -74,7 +75,10 @@ private void processSubexpressions(final List expressions) { processBind(sub, params); } else if (sub instanceof If) { - processConditional(sub, params); + processIf((If) sub, params); + } + else if (sub instanceof Unless) { + processUnless((Unless) sub, params); } else { processFunction(sub, params); @@ -92,8 +96,7 @@ private void processBind(final Expression expression, final EList params // final Collect collect = collectFactory.newInstance(expression.getName(), attributes); } - private void processConditional(final Expression exp, final EList parameters) { - final If ifExp = (If) exp; + private void processIf(final If ifExp, final EList parameters) { final ElsIf elsIfExp = ifExp.getElseIf(); final Else elseExp = ifExp.getElse(); if (testConditional(ifExp.getName(), parameters)) { @@ -107,6 +110,12 @@ else if (elseExp != null) { } } + private void processUnless(final Unless unless, final EList parameters) { + if (!testConditional(unless.getName(), parameters)) { + processSubexpressions(unless.getElements()); + } + } + private boolean testConditional(final String conditional, final EList params) { System.out.printf(": %s parameters: %s\n", conditional, params); boolean result = false; diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixIfTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixIfTest.java index 20d626bf..559b09a9 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixIfTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixIfTest.java @@ -389,6 +389,32 @@ public void ifAnyMatchLastRecord() { }); } + @Test + public void unlessAnyMatch() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "unless any_match('name', '.*University.*')", // + " add_field('type', 'Person')", // + "end"), // + i -> { + i.startRecord("1"); + i.literal("name", "Max"); + i.endRecord(); + // + i.startRecord("2"); + i.literal("name", "Some University"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("name", "Max"); + o.get().literal("type", "Person"); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().literal("name", "Some University"); + o.get().endRecord(); + }); + } + @Test public void ifAnyMatchElse() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// From e7b872a80850e1db0bec4831cfd0e57914a5607d Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Thu, 19 Aug 2021 14:34:41 +0200 Subject: [PATCH 04/55] Basic `list` bind implementation (#35) See https://github.com/LibreCat/Catmandu/wiki/Binds --- .../metamorph/RecordTransformer.java | 37 +++++++++++++------ .../metamorph/MetafixBindTest.java | 27 +++++++++++++- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java b/org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java index 0aeb07c6..636f572e 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java @@ -29,6 +29,7 @@ import org.metafacture.metamorph.FixPredicate.Quantifier; import org.eclipse.emf.common.util.EList; +import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.LinkedListMultimap; import com.google.common.collect.Multimap; @@ -72,7 +73,7 @@ private void processSubexpressions(final List expressions) { for (final Expression sub : expressions) { final EList params = sub.getParams(); if (sub instanceof Do) { - processBind(sub, params); + processBind((Do) sub, params); } else if (sub instanceof If) { processIf((If) sub, params); @@ -86,14 +87,27 @@ else if (sub instanceof Unless) { } } - private void processBind(final Expression expression, final EList params) { - // TODO: implement Fix Binds - // final String firstParam = resolvedAttribute(params, 1); - // final Do theDo = (Do) expression; - // TODO, possibly: use morph collectors here - // final CollectFactory collectFactory = new CollectFactory(); - // final Map attributes = resolvedAttributeMap(params, theDo.getOptions()); - // final Collect collect = collectFactory.newInstance(expression.getName(), attributes); + private void processBind(final Do theDo, final EList params) { + if (theDo.getName().equals("list")) { // TODO impl multiple binds via FixBind enum + final Map options = options(theDo.getOptions()); + final Multimap fullRecord = LinkedListMultimap.create(record); + record.get(options.get("path")).forEach(val -> { + // for each val, bind the current record/scope/context to the given var name: + record = LinkedListMultimap.create(ImmutableMultimap.of(options.get("var"), val)); + processSubexpressions(theDo.getElements()); + record.removeAll(options.get("var")); + // and remember the things we added while bound (this probably needs some tweaking): + fullRecord.putAll(record); + }); + record = fullRecord; + } + else { + System.out.println("Unprocessed bind: " + theDo); + // TODO, possibly: use morph collectors here + // final CollectFactory collectFactory = new CollectFactory(); + // final Map attributes = resolvedAttributeMap(params, theDo.getOptions()); + // final Collect collect = collectFactory.newInstance(expression.getName(), attributes); + } } private void processIf(final If ifExp, final EList parameters) { @@ -144,7 +158,7 @@ private void processFunction(final Expression expression, final List par try { final FixMethod method = FixMethod.valueOf(expression.getName()); final List resolvedParams = params.stream().map(this::resolveVars).collect(Collectors.toList()); - final Map options = options((MethodCall) expression); + final Map options = options(((MethodCall) expression).getOptions()); method.apply(record, resolvedParams, options); } catch (final IllegalArgumentException e) { @@ -152,8 +166,7 @@ private void processFunction(final Expression expression, final List par } } - private Map options(final MethodCall method) { - final Options options = method.getOptions(); + private Map options(final Options options) { final Map map = new HashMap<>(); if (options != null) { for (int i = 0; i < options.getKeys().size(); i += 1) { diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixBindTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixBindTest.java index 8c9aa6be..89a5c500 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixBindTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixBindTest.java @@ -36,7 +36,6 @@ */ @ExtendWith(MockitoExtension.class) @SuppressWarnings("checkstyle:MultipleStringLiterals") -@Disabled // implement Fix-style binds (with or without collectors) public class MetafixBindTest { @RegisterExtension @@ -49,6 +48,29 @@ public MetafixBindTest() { } @Test + public void doList() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "do list('path': 'name', 'var': 'n')", + " upcase('n')", + " trim('n')", + " copy_field('n', 'author')", + "end", + "remove_field('name')"), // + i -> { + i.startRecord("1"); + i.literal("name", " A University"); + i.literal("name", "Max "); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("author", "A UNIVERSITY"); + o.get().literal("author", "MAX"); + o.get().endRecord(); + }); + } + + @Test + @Disabled // implement Fix-style binds with collectors? public void ifInCollector() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// "do entity('author')", @@ -72,6 +94,7 @@ public void ifInCollector() { } @Test + @Disabled // implement Fix-style binds with collectors? public void ifInCollectorMultiRecords() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// "do entity('author')", @@ -115,6 +138,7 @@ public void ifInCollectorMultiRecords() { } @Test + @Disabled // implement Fix-style binds with collectors? public void ifInCollectorChoose() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// "do choose(flushWith: 'record')", @@ -141,6 +165,7 @@ public void ifInCollectorChoose() { } @Test + @Disabled // implement Fix-style binds with collectors? public void ifInCollectorCombine() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// "do combine(name: 'fullName', value: '${first} ${last}')", // From ebcf650a221440623d56d75eb5fba70d134d09b2 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Thu, 19 Aug 2021 15:35:17 +0200 Subject: [PATCH 05/55] Update flux-commands.properties (#35) MetafixFilter was removed in f328a13 --- org.metafacture.fix/src/main/resources/flux-commands.properties | 1 - 1 file changed, 1 deletion(-) diff --git a/org.metafacture.fix/src/main/resources/flux-commands.properties b/org.metafacture.fix/src/main/resources/flux-commands.properties index f765ad3c..eeba828f 100644 --- a/org.metafacture.fix/src/main/resources/flux-commands.properties +++ b/org.metafacture.fix/src/main/resources/flux-commands.properties @@ -13,4 +13,3 @@ # limitations under the License. # fix org.metafacture.metamorph.Metafix -fix-filter org.metafacture.metamorph.MetafixFilter From a786f73a4a654578902c6f4ee1cad9456dc32bfa Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Thu, 19 Aug 2021 16:48:38 +0200 Subject: [PATCH 06/55] Rename non-generated `fix` package to `metafix` (#35) --- .../{fix => metafix}/ide/FixIdeModule.java | 4 +- .../{fix => metafix}/ide/FixIdeSetup.java | 6 +- .../contentassist/antlr/FixTokenSource.java | 5 +- org.metafacture.fix.web/build.gradle | 2 +- .../org/metafacture/fix/web/FixServlet.java | 112 ------------------ .../metafacture/fix/web/ServerLauncher.java | 73 ------------ .../{fix => metafix}/web/FixWebModule.java | 7 +- .../{fix => metafix}/web/FixWebSetup.java | 8 +- ...e Fix (fix) Language Infrastructure.launch | 4 +- org.metafacture.fix/build.gradle | 8 +- .../metafacture/{fix => metafix}/Fix.xtext | 4 +- .../{fix => metafix}/FixRuntimeModule.java | 4 +- .../{fix => metafix}/FixStandaloneSetup.java | 4 +- .../{fix => metafix}/GenerateFix.mwe2 | 4 +- .../interpreter/FixInterpreter.java | 18 +-- .../jvmmodel/FixJvmModelInferrer.java | 4 +- .../parser/antlr/FixTokenSource.java | 4 +- .../scoping/FixScopeProvider.java | 2 +- .../validation/FixValidator.java | 2 +- .../validation/XtextValidator.java | 2 +- .../org/metafacture/metamorph/Metafix.java | 6 +- .../metamorph/RecordTransformer.java | 18 +-- .../tests/FixParsingTest.java | 4 +- .../tests/InterpreterTest.java | 6 +- 24 files changed, 61 insertions(+), 250 deletions(-) rename org.metafacture.fix.ide/src/main/java/org/metafacture/{fix => metafix}/ide/FixIdeModule.java (80%) rename org.metafacture.fix.ide/src/main/java/org/metafacture/{fix => metafix}/ide/FixIdeSetup.java (76%) rename org.metafacture.fix.ide/src/main/java/org/metafacture/{fix => metafix}/ide/contentassist/antlr/FixTokenSource.java (84%) delete mode 100644 org.metafacture.fix.web/src/main/java/org/metafacture/fix/web/FixServlet.java delete mode 100644 org.metafacture.fix.web/src/main/java/org/metafacture/fix/web/ServerLauncher.java rename org.metafacture.fix.web/src/main/java/org/metafacture/{fix => metafix}/web/FixWebModule.java (71%) rename org.metafacture.fix.web/src/main/java/org/metafacture/{fix => metafix}/web/FixWebSetup.java (71%) rename org.metafacture.fix/src/main/java/org/metafacture/{fix => metafix}/Fix.xtext (94%) rename org.metafacture.fix/src/main/java/org/metafacture/{fix => metafix}/FixRuntimeModule.java (77%) rename org.metafacture.fix/src/main/java/org/metafacture/{fix => metafix}/FixStandaloneSetup.java (96%) rename org.metafacture.fix/src/main/java/org/metafacture/{fix => metafix}/GenerateFix.mwe2 (93%) rename org.metafacture.fix/src/main/java/org/metafacture/{fix => metafix}/interpreter/FixInterpreter.java (87%) rename org.metafacture.fix/src/main/java/org/metafacture/{fix => metafix}/jvmmodel/FixJvmModelInferrer.java (95%) rename org.metafacture.fix/src/main/java/org/metafacture/{fix => metafix}/parser/antlr/FixTokenSource.java (84%) rename org.metafacture.fix/src/main/java/org/metafacture/{fix => metafix}/scoping/FixScopeProvider.java (87%) rename org.metafacture.fix/src/main/java/org/metafacture/{fix => metafix}/validation/FixValidator.java (84%) rename org.metafacture.fix/src/main/java/org/metafacture/{fix => metafix}/validation/XtextValidator.java (97%) rename org.metafacture.fix/src/test/java/org/metafacture/{fix => metafix}/tests/FixParsingTest.java (97%) rename org.metafacture.fix/src/test/java/org/metafacture/{fix => metafix}/tests/InterpreterTest.java (92%) diff --git a/org.metafacture.fix.ide/src/main/java/org/metafacture/fix/ide/FixIdeModule.java b/org.metafacture.fix.ide/src/main/java/org/metafacture/metafix/ide/FixIdeModule.java similarity index 80% rename from org.metafacture.fix.ide/src/main/java/org/metafacture/fix/ide/FixIdeModule.java rename to org.metafacture.fix.ide/src/main/java/org/metafacture/metafix/ide/FixIdeModule.java index 226c18c4..4b9f4785 100644 --- a/org.metafacture.fix.ide/src/main/java/org/metafacture/fix/ide/FixIdeModule.java +++ b/org.metafacture.fix.ide/src/main/java/org/metafacture/metafix/ide/FixIdeModule.java @@ -1,11 +1,9 @@ -package org.metafacture.fix.ide; +package org.metafacture.metafix.ide; /** * Use this class to register ide components. */ public class FixIdeModule extends AbstractFixIdeModule { - public FixIdeModule() { } - } diff --git a/org.metafacture.fix.ide/src/main/java/org/metafacture/fix/ide/FixIdeSetup.java b/org.metafacture.fix.ide/src/main/java/org/metafacture/metafix/ide/FixIdeSetup.java similarity index 76% rename from org.metafacture.fix.ide/src/main/java/org/metafacture/fix/ide/FixIdeSetup.java rename to org.metafacture.fix.ide/src/main/java/org/metafacture/metafix/ide/FixIdeSetup.java index 2d446884..2dd6f9fc 100644 --- a/org.metafacture.fix.ide/src/main/java/org/metafacture/fix/ide/FixIdeSetup.java +++ b/org.metafacture.fix.ide/src/main/java/org/metafacture/metafix/ide/FixIdeSetup.java @@ -1,7 +1,7 @@ -package org.metafacture.fix.ide; +package org.metafacture.metafix.ide; -import org.metafacture.fix.FixRuntimeModule; -import org.metafacture.fix.FixStandaloneSetup; +import org.metafacture.metafix.FixRuntimeModule; +import org.metafacture.metafix.FixStandaloneSetup; import com.google.inject.Guice; import com.google.inject.Injector; diff --git a/org.metafacture.fix.ide/src/main/java/org/metafacture/fix/ide/contentassist/antlr/FixTokenSource.java b/org.metafacture.fix.ide/src/main/java/org/metafacture/metafix/ide/contentassist/antlr/FixTokenSource.java similarity index 84% rename from org.metafacture.fix.ide/src/main/java/org/metafacture/fix/ide/contentassist/antlr/FixTokenSource.java rename to org.metafacture.fix.ide/src/main/java/org/metafacture/metafix/ide/contentassist/antlr/FixTokenSource.java index 579e345f..138a7f9d 100644 --- a/org.metafacture.fix.ide/src/main/java/org/metafacture/fix/ide/contentassist/antlr/FixTokenSource.java +++ b/org.metafacture.fix.ide/src/main/java/org/metafacture/metafix/ide/contentassist/antlr/FixTokenSource.java @@ -1,6 +1,6 @@ -package org.metafacture.fix.ide.contentassist.antlr; +package org.metafacture.metafix.ide.contentassist.antlr; -import org.metafacture.fix.ide.contentassist.antlr.internal.InternalFixParser; +import org.metafacture.metafix.ide.contentassist.antlr.internal.InternalFixParser; import org.antlr.runtime.Token; import org.antlr.runtime.TokenSource; @@ -31,5 +31,4 @@ protected int getEndTokenType() { protected boolean shouldEmitPendingEndTokens() { return false; } - } diff --git a/org.metafacture.fix.web/build.gradle b/org.metafacture.fix.web/build.gradle index 72a1cbfa..0c4bdcc8 100644 --- a/org.metafacture.fix.web/build.gradle +++ b/org.metafacture.fix.web/build.gradle @@ -27,7 +27,7 @@ dependencies { task jettyRun(type: JavaExec) { dependsOn(sourceSets.main.runtimeClasspath) classpath = sourceSets.main.runtimeClasspath.filter { it.exists() } - main = 'org.metafacture.fix.web.ServerLauncher' + main = 'org.metafacture.metafix.web.ServerLauncher' standardInput = System.in group = 'run' description = 'Starts an example Jetty server with your language' diff --git a/org.metafacture.fix.web/src/main/java/org/metafacture/fix/web/FixServlet.java b/org.metafacture.fix.web/src/main/java/org/metafacture/fix/web/FixServlet.java deleted file mode 100644 index 95fc5443..00000000 --- a/org.metafacture.fix.web/src/main/java/org/metafacture/fix/web/FixServlet.java +++ /dev/null @@ -1,112 +0,0 @@ -package org.metafacture.fix.web; - -import org.metafacture.fix.FixStandaloneSetup; -import org.metafacture.runner.Flux; - -import org.antlr.runtime.RecognitionException; -import org.eclipse.xtext.util.DisposableRegistry; -import org.eclipse.xtext.web.servlet.XtextServlet; -import org.eclipse.xtext.xbase.lib.InputOutput; - -import java.io.IOException; -import java.io.StringReader; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Map; -import javax.servlet.ServletException; -import javax.servlet.annotation.WebServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -/** - * Deploy this class into a servlet container to enable DSL-specific services. - */ -@WebServlet(name = "XtextServices", urlPatterns = "/xtext-service/*") -public class FixServlet extends XtextServlet { - - private static final String COMMAND_FIX = "fix"; - - private static final String PARAM_DATA = "data"; - private static final String PARAM_FIX = "fix"; - private static final String PARAM_FLUX = "flux"; - - private DisposableRegistry disposableRegistry; - - public FixServlet() { - } - - @Override - public void init() throws ServletException { - disposableRegistry = new FixWebSetup().createInjectorAndDoEMFRegistration().getInstance(DisposableRegistry.class); - } - - @Override - public void destroy() { - if (disposableRegistry != null) { - disposableRegistry.dispose(); - disposableRegistry = null; - } - - super.destroy(); - } - - @Override - public void doPost(final HttpServletRequest request, final HttpServletResponse response) throws IOException, ServletException { - InputOutput.println("POST Request: " + request); - - if (!request.getPathInfo().endsWith("/run") || !process(request, response)) { - super.doPost(request, response); - } - } - - @Override - public void doGet(final HttpServletRequest request, final HttpServletResponse response) throws IOException, ServletException { - InputOutput.println("GET Request: " + request); - - if (!process(request, response)) { - super.doGet(request, response); - } - } - - private boolean process(final HttpServletRequest request, final HttpServletResponse response) throws IOException { - final Map params = request.getParameterMap(); - - if (!params.containsKey(PARAM_DATA) || !params.containsKey(PARAM_FLUX) || !params.containsKey(PARAM_FIX)) { - return false; - } - - final StringBuilder builder = new StringBuilder(); - - final String inData = request.getParameter(PARAM_DATA); - builder.append(inData == null || inData.isEmpty() ? "" : - "\"" + absPathToTempFile(inData, ".txt") + "\"|open-file|"); - - final String fixFile = absPathToTempFile(request.getParameter(PARAM_FIX), ".fix"); - final String outFile = absPathToTempFile("", ".txt"); - - builder.append(request.getParameter(PARAM_FLUX).replaceAll("\\s?\\|\\s?", "|").replace( - "|" + COMMAND_FIX + "|", - "|org.metafacture.metamorph.Metafix(fixFile=\"" + fixFile + "\")|")); - builder.append("|write(\""); - builder.append(outFile); - builder.append("\");"); - - final String fullFlux = builder.toString(); - InputOutput.println("full flux: " + fullFlux); - - try { - Flux.main(new String[] {absPathToTempFile(fullFlux, ".flux")}); - } - catch (final RecognitionException e) { - throw new RuntimeException(e); - } - - Files.copy(Paths.get(outFile), response.getOutputStream()); - return true; - } - - private String absPathToTempFile(final String content, final String suffix) throws IOException { - return FixStandaloneSetup.absPathToTempFile(new StringReader(content), suffix); - } - -} diff --git a/org.metafacture.fix.web/src/main/java/org/metafacture/fix/web/ServerLauncher.java b/org.metafacture.fix.web/src/main/java/org/metafacture/fix/web/ServerLauncher.java deleted file mode 100644 index 1526dafe..00000000 --- a/org.metafacture.fix.web/src/main/java/org/metafacture/fix/web/ServerLauncher.java +++ /dev/null @@ -1,73 +0,0 @@ -package org.metafacture.fix.web; - -import org.eclipse.jetty.annotations.AnnotationConfiguration; -import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.util.log.Slf4jLog; -import org.eclipse.jetty.webapp.Configuration; -import org.eclipse.jetty.webapp.MetaInfConfiguration; -import org.eclipse.jetty.webapp.WebAppContext; -import org.eclipse.jetty.webapp.WebInfConfiguration; -import org.eclipse.jetty.webapp.WebXmlConfiguration; -import org.eclipse.xtext.xbase.lib.Exceptions; - -import java.net.InetSocketAddress; - -/** - * This program starts an HTTP server for testing the web integration of your DSL. - * Just execute it and point a web browser to http://localhost:8080/ - */ -public class ServerLauncher { // checkstyle-disable-line ClassDataAbstractionCoupling - - private ServerLauncher() { - throw new IllegalAccessError("Utility class"); - } - - public static void main(final String[] args) { - final WebAppContext context = new WebAppContext(); - context.setResourceBase("src/main/webapp"); - context.setWelcomeFiles(new String[] {"index.html"}); - context.setContextPath("/"); - context.setConfigurations(new Configuration[] {new AnnotationConfiguration(), new WebXmlConfiguration(), new WebInfConfiguration(), new MetaInfConfiguration()}); - context.setAttribute(WebInfConfiguration.CONTAINER_JAR_PATTERN, ".*/org\\.metafacture\\.fix\\.web/.*,.*\\.jar"); - context.setInitParameter("org.mortbay.jetty.servlet.Default.useFileMappedBuffer", "false"); - - final Server server = new Server(new InetSocketAddress("0.0.0.0", 8080)); - server.setHandler(context); - - final Slf4jLog log = new Slf4jLog(ServerLauncher.class.getName()); - - try { - server.start(); - - log.info("Server started " + server.getURI() + "..."); - - new Thread(() -> { - try { - log.info("Press enter to stop the server..."); - - if (System.in.read() != -1) { - server.stop(); - } - else { - log.warn("Console input is not available. In order to stop the server, you need to cancel the process manually."); - } - } - catch (final Exception e) { // checkstyle-disable-line IllegalCatch - throw Exceptions.sneakyThrow(e); - } - }).start(); - - server.join(); - } - catch (final Throwable e) { // checkstyle-disable-line IllegalCatch - if (e instanceof Exception) { - log.warn(((Exception) e).getMessage()); - System.exit(1); - } - else { - throw Exceptions.sneakyThrow(e); - } - } - } - -} diff --git a/org.metafacture.fix.web/src/main/java/org/metafacture/fix/web/FixWebModule.java b/org.metafacture.fix.web/src/main/java/org/metafacture/metafix/web/FixWebModule.java similarity index 71% rename from org.metafacture.fix.web/src/main/java/org/metafacture/fix/web/FixWebModule.java rename to org.metafacture.fix.web/src/main/java/org/metafacture/metafix/web/FixWebModule.java index 3825a191..ad61024e 100644 --- a/org.metafacture.fix.web/src/main/java/org/metafacture/fix/web/FixWebModule.java +++ b/org.metafacture.fix.web/src/main/java/org/metafacture/metafix/web/FixWebModule.java @@ -1,11 +1,10 @@ -package org.metafacture.fix.web; +package org.metafacture.metafix.web; /** - * Use this class to register additional components to be used within the web application. + * Use this class to register additional components to be used within the web + * application. */ public class FixWebModule extends AbstractFixWebModule { - public FixWebModule() { } - } diff --git a/org.metafacture.fix.web/src/main/java/org/metafacture/fix/web/FixWebSetup.java b/org.metafacture.fix.web/src/main/java/org/metafacture/metafix/web/FixWebSetup.java similarity index 71% rename from org.metafacture.fix.web/src/main/java/org/metafacture/fix/web/FixWebSetup.java rename to org.metafacture.fix.web/src/main/java/org/metafacture/metafix/web/FixWebSetup.java index 74cee0ee..ad7c397a 100644 --- a/org.metafacture.fix.web/src/main/java/org/metafacture/fix/web/FixWebSetup.java +++ b/org.metafacture.fix.web/src/main/java/org/metafacture/metafix/web/FixWebSetup.java @@ -1,8 +1,8 @@ -package org.metafacture.fix.web; +package org.metafacture.metafix.web; -import org.metafacture.fix.FixRuntimeModule; -import org.metafacture.fix.FixStandaloneSetup; -import org.metafacture.fix.ide.FixIdeModule; +import org.metafacture.metafix.FixRuntimeModule; +import org.metafacture.metafix.FixStandaloneSetup; +import org.metafacture.metafix.ide.FixIdeModule; import com.google.inject.Guice; import com.google.inject.Injector; diff --git a/org.metafacture.fix/.launch/Generate Fix (fix) Language Infrastructure.launch b/org.metafacture.fix/.launch/Generate Fix (fix) Language Infrastructure.launch index 6b079c3c..ef1daee7 100644 --- a/org.metafacture.fix/.launch/Generate Fix (fix) Language Infrastructure.launch +++ b/org.metafacture.fix/.launch/Generate Fix (fix) Language Infrastructure.launch @@ -8,7 +8,7 @@ - + - + diff --git a/org.metafacture.fix/build.gradle b/org.metafacture.fix/build.gradle index ea49e63d..71276ca2 100644 --- a/org.metafacture.fix/build.gradle +++ b/org.metafacture.fix/build.gradle @@ -42,10 +42,10 @@ test { useJUnitPlatform() } -def xtextFile = 'src/main/java/org/metafacture/fix/Fix.xtext' +def xtextFile = 'src/main/java/org/metafacture/metafix/Fix.xtext' task validateXtextLanguage(type: JavaExec) { - main = 'org.metafacture.fix.validation.XtextValidator' + main = 'org.metafacture.metafix.validation.XtextValidator' classpath = sourceSets.main.runtimeClasspath args += xtextFile } @@ -53,10 +53,10 @@ task validateXtextLanguage(type: JavaExec) { task generateXtextLanguage(type: JavaExec) { main = 'org.eclipse.emf.mwe2.launch.runtime.Mwe2Launcher' classpath = configurations.mwe2 - inputs.file 'src/main/java/org/metafacture/fix/GenerateFix.mwe2' + inputs.file 'src/main/java/org/metafacture/metafix/GenerateFix.mwe2' inputs.file xtextFile outputs.dir 'src/main/xtext-gen' - args += 'src/main/java/org/metafacture/fix/GenerateFix.mwe2' + args += 'src/main/java/org/metafacture/metafix/GenerateFix.mwe2' args += '-p' args += "rootPath=/${projectDir}/.." } diff --git a/org.metafacture.fix/src/main/java/org/metafacture/fix/Fix.xtext b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Fix.xtext similarity index 94% rename from org.metafacture.fix/src/main/java/org/metafacture/fix/Fix.xtext rename to org.metafacture.fix/src/main/java/org/metafacture/metafix/Fix.xtext index 42c12067..91c7ccc4 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/fix/Fix.xtext +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Fix.xtext @@ -1,7 +1,7 @@ -grammar org.metafacture.fix.Fix +grammar org.metafacture.metafix.Fix with org.eclipse.xtext.xbase.Xbase -generate fix "http://www.metafacture.org/fix/Fix" +generate fix "http://www.metafacture.org/metafix/Fix" Fix: elements += Expression* diff --git a/org.metafacture.fix/src/main/java/org/metafacture/fix/FixRuntimeModule.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixRuntimeModule.java similarity index 77% rename from org.metafacture.fix/src/main/java/org/metafacture/fix/FixRuntimeModule.java rename to org.metafacture.fix/src/main/java/org/metafacture/metafix/FixRuntimeModule.java index 741d4bd0..59aaca06 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/fix/FixRuntimeModule.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixRuntimeModule.java @@ -1,6 +1,6 @@ -package org.metafacture.fix; +package org.metafacture.metafix; -import org.metafacture.fix.interpreter.FixInterpreter; +import org.metafacture.metafix.interpreter.FixInterpreter; /** * Use this class to register components to be used at runtime / without the Equinox extension registry. diff --git a/org.metafacture.fix/src/main/java/org/metafacture/fix/FixStandaloneSetup.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixStandaloneSetup.java similarity index 96% rename from org.metafacture.fix/src/main/java/org/metafacture/fix/FixStandaloneSetup.java rename to org.metafacture.fix/src/main/java/org/metafacture/metafix/FixStandaloneSetup.java index a6bbec11..1197813b 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/fix/FixStandaloneSetup.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixStandaloneSetup.java @@ -1,6 +1,6 @@ -package org.metafacture.fix; +package org.metafacture.metafix; -import org.metafacture.fix.fix.Fix; +import org.metafacture.metafix.fix.Fix; import org.eclipse.emf.common.util.URI; import org.eclipse.emf.ecore.resource.Resource; diff --git a/org.metafacture.fix/src/main/java/org/metafacture/fix/GenerateFix.mwe2 b/org.metafacture.fix/src/main/java/org/metafacture/metafix/GenerateFix.mwe2 similarity index 93% rename from org.metafacture.fix/src/main/java/org/metafacture/fix/GenerateFix.mwe2 rename to org.metafacture.fix/src/main/java/org/metafacture/metafix/GenerateFix.mwe2 index d7fc73be..1658438a 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/fix/GenerateFix.mwe2 +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/GenerateFix.mwe2 @@ -1,4 +1,4 @@ -module org.metafacture.fix.GenerateFix +module org.metafacture.metafix.GenerateFix import org.eclipse.xtext.xtext.generator.* import org.eclipse.xtext.xtext.generator.model.project.* @@ -25,7 +25,7 @@ Workflow { } } language = StandardLanguage { - name = "org.metafacture.fix.Fix" + name = "org.metafacture.metafix.Fix" fileExtensions = "fix" serializer = { generateStub = false diff --git a/org.metafacture.fix/src/main/java/org/metafacture/fix/interpreter/FixInterpreter.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/interpreter/FixInterpreter.java similarity index 87% rename from org.metafacture.fix/src/main/java/org/metafacture/fix/interpreter/FixInterpreter.java rename to org.metafacture.fix/src/main/java/org/metafacture/metafix/interpreter/FixInterpreter.java index 73eaf5d4..b974ef1d 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/fix/interpreter/FixInterpreter.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/interpreter/FixInterpreter.java @@ -1,12 +1,12 @@ -package org.metafacture.fix.interpreter; - -import org.metafacture.fix.fix.Do; -import org.metafacture.fix.fix.ElsIf; -import org.metafacture.fix.fix.Else; -import org.metafacture.fix.fix.Expression; -import org.metafacture.fix.fix.Fix; -import org.metafacture.fix.fix.If; -import org.metafacture.fix.fix.MethodCall; +package org.metafacture.metafix.interpreter; + +import org.metafacture.metafix.fix.Do; +import org.metafacture.metafix.fix.ElsIf; +import org.metafacture.metafix.fix.Else; +import org.metafacture.metafix.fix.Expression; +import org.metafacture.metafix.fix.Fix; +import org.metafacture.metafix.fix.If; +import org.metafacture.metafix.fix.MethodCall; import org.metafacture.metamorph.Metafix; import org.eclipse.emf.ecore.EObject; diff --git a/org.metafacture.fix/src/main/java/org/metafacture/fix/jvmmodel/FixJvmModelInferrer.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/jvmmodel/FixJvmModelInferrer.java similarity index 95% rename from org.metafacture.fix/src/main/java/org/metafacture/fix/jvmmodel/FixJvmModelInferrer.java rename to org.metafacture.fix/src/main/java/org/metafacture/metafix/jvmmodel/FixJvmModelInferrer.java index 90eb7af8..665b89c5 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/fix/jvmmodel/FixJvmModelInferrer.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/jvmmodel/FixJvmModelInferrer.java @@ -1,6 +1,6 @@ -package org.metafacture.fix.jvmmodel; +package org.metafacture.metafix.jvmmodel; -import org.metafacture.fix.fix.Fix; +import org.metafacture.metafix.fix.Fix; import org.metafacture.metamorph.Metafix; import org.eclipse.emf.ecore.EObject; diff --git a/org.metafacture.fix/src/main/java/org/metafacture/fix/parser/antlr/FixTokenSource.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/parser/antlr/FixTokenSource.java similarity index 84% rename from org.metafacture.fix/src/main/java/org/metafacture/fix/parser/antlr/FixTokenSource.java rename to org.metafacture.fix/src/main/java/org/metafacture/metafix/parser/antlr/FixTokenSource.java index d824e398..baf82e83 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/fix/parser/antlr/FixTokenSource.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/parser/antlr/FixTokenSource.java @@ -1,6 +1,6 @@ -package org.metafacture.fix.parser.antlr; +package org.metafacture.metafix.parser.antlr; -import org.metafacture.fix.parser.antlr.internal.InternalFixParser; +import org.metafacture.metafix.parser.antlr.internal.InternalFixParser; import org.antlr.runtime.Token; import org.antlr.runtime.TokenSource; diff --git a/org.metafacture.fix/src/main/java/org/metafacture/fix/scoping/FixScopeProvider.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/scoping/FixScopeProvider.java similarity index 87% rename from org.metafacture.fix/src/main/java/org/metafacture/fix/scoping/FixScopeProvider.java rename to org.metafacture.fix/src/main/java/org/metafacture/metafix/scoping/FixScopeProvider.java index c5da0baa..14f0a75a 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/fix/scoping/FixScopeProvider.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/scoping/FixScopeProvider.java @@ -1,4 +1,4 @@ -package org.metafacture.fix.scoping; +package org.metafacture.metafix.scoping; /** * This class contains custom scoping description. diff --git a/org.metafacture.fix/src/main/java/org/metafacture/fix/validation/FixValidator.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/validation/FixValidator.java similarity index 84% rename from org.metafacture.fix/src/main/java/org/metafacture/fix/validation/FixValidator.java rename to org.metafacture.fix/src/main/java/org/metafacture/metafix/validation/FixValidator.java index 53af9a76..600112f1 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/fix/validation/FixValidator.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/validation/FixValidator.java @@ -1,4 +1,4 @@ -package org.metafacture.fix.validation; +package org.metafacture.metafix.validation; /** * This class contains custom validation rules. diff --git a/org.metafacture.fix/src/main/java/org/metafacture/fix/validation/XtextValidator.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/validation/XtextValidator.java similarity index 97% rename from org.metafacture.fix/src/main/java/org/metafacture/fix/validation/XtextValidator.java rename to org.metafacture.fix/src/main/java/org/metafacture/metafix/validation/XtextValidator.java index 793f2824..e81e1ee5 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/fix/validation/XtextValidator.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/validation/XtextValidator.java @@ -1,4 +1,4 @@ -package org.metafacture.fix.validation; +package org.metafacture.metafix.validation; import com.google.inject.Injector; import org.eclipse.emf.common.util.URI; diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/Metafix.java b/org.metafacture.fix/src/main/java/org/metafacture/metamorph/Metafix.java index f76814d8..3913ec8e 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/Metafix.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metamorph/Metafix.java @@ -18,13 +18,13 @@ package org.metafacture.metamorph; -import org.metafacture.fix.FixStandaloneSetup; -import org.metafacture.fix.fix.Expression; -import org.metafacture.fix.fix.Fix; import org.metafacture.framework.StreamPipe; import org.metafacture.framework.StreamReceiver; import org.metafacture.framework.helpers.DefaultStreamReceiver; import org.metafacture.mangling.StreamFlattener; +import org.metafacture.metafix.FixStandaloneSetup; +import org.metafacture.metafix.fix.Expression; +import org.metafacture.metafix.fix.Fix; import com.google.common.collect.LinkedListMultimap; import com.google.common.collect.Multimap; diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java b/org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java index 636f572e..1e269621 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java @@ -17,15 +17,15 @@ package org.metafacture.metamorph; import org.metafacture.commons.StringUtil; -import org.metafacture.fix.fix.Do; -import org.metafacture.fix.fix.ElsIf; -import org.metafacture.fix.fix.Else; -import org.metafacture.fix.fix.Expression; -import org.metafacture.fix.fix.Fix; -import org.metafacture.fix.fix.If; -import org.metafacture.fix.fix.MethodCall; -import org.metafacture.fix.fix.Options; -import org.metafacture.fix.fix.Unless; +import org.metafacture.metafix.fix.Do; +import org.metafacture.metafix.fix.ElsIf; +import org.metafacture.metafix.fix.Else; +import org.metafacture.metafix.fix.Expression; +import org.metafacture.metafix.fix.Fix; +import org.metafacture.metafix.fix.If; +import org.metafacture.metafix.fix.MethodCall; +import org.metafacture.metafix.fix.Options; +import org.metafacture.metafix.fix.Unless; import org.metafacture.metamorph.FixPredicate.Quantifier; import org.eclipse.emf.common.util.EList; diff --git a/org.metafacture.fix/src/test/java/org/metafacture/fix/tests/FixParsingTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/tests/FixParsingTest.java similarity index 97% rename from org.metafacture.fix/src/test/java/org/metafacture/fix/tests/FixParsingTest.java rename to org.metafacture.fix/src/test/java/org/metafacture/metafix/tests/FixParsingTest.java index e4bb4a25..f35b4481 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/fix/tests/FixParsingTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/tests/FixParsingTest.java @@ -1,6 +1,6 @@ -package org.metafacture.fix.tests; +package org.metafacture.metafix.tests; -import org.metafacture.fix.fix.Fix; +import org.metafacture.metafix.fix.Fix; import com.google.inject.Inject; import org.eclipse.emf.common.util.EList; diff --git a/org.metafacture.fix/src/test/java/org/metafacture/fix/tests/InterpreterTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/tests/InterpreterTest.java similarity index 92% rename from org.metafacture.fix/src/test/java/org/metafacture/fix/tests/InterpreterTest.java rename to org.metafacture.fix/src/test/java/org/metafacture/metafix/tests/InterpreterTest.java index ed8e3ffe..2f02eb60 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/fix/tests/InterpreterTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/tests/InterpreterTest.java @@ -1,7 +1,7 @@ -package org.metafacture.fix.tests; +package org.metafacture.metafix.tests; -import org.metafacture.fix.fix.Fix; -import org.metafacture.fix.interpreter.FixInterpreter; +import org.metafacture.metafix.fix.Fix; +import org.metafacture.metafix.interpreter.FixInterpreter; import org.metafacture.metamorph.Metafix; import com.google.inject.Inject; From 191351b973ed453408db319be03a90cbffb06862 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Thu, 19 Aug 2021 17:04:52 +0200 Subject: [PATCH 07/55] Move `metamorph` package to `metafix` (#35) --- .../org/metafacture/{metamorph => metafix}/FixMethod.java | 2 +- .../metafacture/{metamorph => metafix}/FixPredicate.java | 2 +- .../org/metafacture/{metamorph => metafix}/Metafix.java | 5 ++--- .../{metamorph => metafix}/RecordTransformer.java | 4 ++-- .../org/metafacture/metafix/interpreter/FixInterpreter.java | 2 +- .../metafacture/metafix/jvmmodel/FixJvmModelInferrer.java | 2 +- .../src/main/resources/flux-commands.properties | 2 +- .../org/metafacture/metafix/{tests => }/FixParsingTest.java | 3 ++- .../metafacture/metafix/{tests => }/InterpreterTest.java | 4 ++-- .../metafacture/{metamorph => metafix}/MetafixBindTest.java | 2 +- .../{metamorph => metafix}/MetafixFieldTest.java | 2 +- .../metafacture/{metamorph => metafix}/MetafixIfTest.java | 2 +- .../{metamorph => metafix}/MetafixLookupTest.java | 6 +++--- .../{metamorph => metafix}/MetafixMethodTest.java | 2 +- .../{metamorph => metafix}/MetafixSelectorTest.java | 2 +- .../{metamorph => metafix}/MetafixTestHelpers.java | 2 +- .../org/metafacture/{metamorph => metafix}/maps/test.csv | 0 .../org/metafacture/{metamorph => metafix}/maps/test.tsv | 0 18 files changed, 22 insertions(+), 22 deletions(-) rename org.metafacture.fix/src/main/java/org/metafacture/{metamorph => metafix}/FixMethod.java (99%) rename org.metafacture.fix/src/main/java/org/metafacture/{metamorph => metafix}/FixPredicate.java (98%) rename org.metafacture.fix/src/main/java/org/metafacture/{metamorph => metafix}/Metafix.java (97%) rename org.metafacture.fix/src/main/java/org/metafacture/{metamorph => metafix}/RecordTransformer.java (98%) rename org.metafacture.fix/src/test/java/org/metafacture/metafix/{tests => }/FixParsingTest.java (97%) rename org.metafacture.fix/src/test/java/org/metafacture/metafix/{tests => }/InterpreterTest.java (94%) rename org.metafacture.fix/src/test/java/org/metafacture/{metamorph => metafix}/MetafixBindTest.java (99%) rename org.metafacture.fix/src/test/java/org/metafacture/{metamorph => metafix}/MetafixFieldTest.java (99%) rename org.metafacture.fix/src/test/java/org/metafacture/{metamorph => metafix}/MetafixIfTest.java (99%) rename org.metafacture.fix/src/test/java/org/metafacture/{metamorph => metafix}/MetafixLookupTest.java (97%) rename org.metafacture.fix/src/test/java/org/metafacture/{metamorph => metafix}/MetafixMethodTest.java (99%) rename org.metafacture.fix/src/test/java/org/metafacture/{metamorph => metafix}/MetafixSelectorTest.java (98%) rename org.metafacture.fix/src/test/java/org/metafacture/{metamorph => metafix}/MetafixTestHelpers.java (98%) rename org.metafacture.fix/src/test/java/org/metafacture/{metamorph => metafix}/maps/test.csv (100%) rename org.metafacture.fix/src/test/java/org/metafacture/{metamorph => metafix}/maps/test.tsv (100%) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java similarity index 99% rename from org.metafacture.fix/src/main/java/org/metafacture/metamorph/FixMethod.java rename to org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index 6bac4bc8..3ca7742b 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.metafacture.metamorph; +package org.metafacture.metafix; import org.metafacture.metamorph.maps.FileMap; diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/FixPredicate.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixPredicate.java similarity index 98% rename from org.metafacture.fix/src/main/java/org/metafacture/metamorph/FixPredicate.java rename to org.metafacture.fix/src/main/java/org/metafacture/metafix/FixPredicate.java index ed57ee26..d914a3f5 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/FixPredicate.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixPredicate.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.metafacture.metamorph; +package org.metafacture.metafix; import com.google.common.collect.Multimap; diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/Metafix.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java similarity index 97% rename from org.metafacture.fix/src/main/java/org/metafacture/metamorph/Metafix.java rename to org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java index 3913ec8e..267e7916 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/Metafix.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java @@ -16,13 +16,12 @@ //TODO: move all classes here to fix package -package org.metafacture.metamorph; +package org.metafacture.metafix; import org.metafacture.framework.StreamPipe; import org.metafacture.framework.StreamReceiver; import org.metafacture.framework.helpers.DefaultStreamReceiver; import org.metafacture.mangling.StreamFlattener; -import org.metafacture.metafix.FixStandaloneSetup; import org.metafacture.metafix.fix.Expression; import org.metafacture.metafix.fix.Fix; @@ -52,7 +51,7 @@ public class Metafix implements StreamPipe { public static final String VAR_START = "$["; public static final String VAR_END = "]"; - static final Map NO_VARS = Collections.emptyMap(); + public static final Map NO_VARS = Collections.emptyMap(); private static final String ENTITIES_NOT_BALANCED = "Entity starts and ends are not balanced"; // TODO: Use SimpleRegexTrie / WildcardTrie for wildcard, alternation and character class support diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java similarity index 98% rename from org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java rename to org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java index 1e269621..7d96caa4 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metamorph/RecordTransformer.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java @@ -14,9 +14,10 @@ * limitations under the License. */ -package org.metafacture.metamorph; +package org.metafacture.metafix; import org.metafacture.commons.StringUtil; +import org.metafacture.metafix.FixPredicate.Quantifier; import org.metafacture.metafix.fix.Do; import org.metafacture.metafix.fix.ElsIf; import org.metafacture.metafix.fix.Else; @@ -26,7 +27,6 @@ import org.metafacture.metafix.fix.MethodCall; import org.metafacture.metafix.fix.Options; import org.metafacture.metafix.fix.Unless; -import org.metafacture.metamorph.FixPredicate.Quantifier; import org.eclipse.emf.common.util.EList; import com.google.common.collect.ImmutableMultimap; diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/interpreter/FixInterpreter.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/interpreter/FixInterpreter.java index b974ef1d..9ffffe9d 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/interpreter/FixInterpreter.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/interpreter/FixInterpreter.java @@ -1,5 +1,6 @@ package org.metafacture.metafix.interpreter; +import org.metafacture.metafix.Metafix; import org.metafacture.metafix.fix.Do; import org.metafacture.metafix.fix.ElsIf; import org.metafacture.metafix.fix.Else; @@ -7,7 +8,6 @@ import org.metafacture.metafix.fix.Fix; import org.metafacture.metafix.fix.If; import org.metafacture.metafix.fix.MethodCall; -import org.metafacture.metamorph.Metafix; import org.eclipse.emf.ecore.EObject; import org.eclipse.xtext.xbase.interpreter.impl.XbaseInterpreter; diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/jvmmodel/FixJvmModelInferrer.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/jvmmodel/FixJvmModelInferrer.java index 665b89c5..ae08a5cf 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/jvmmodel/FixJvmModelInferrer.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/jvmmodel/FixJvmModelInferrer.java @@ -1,7 +1,7 @@ package org.metafacture.metafix.jvmmodel; +import org.metafacture.metafix.Metafix; import org.metafacture.metafix.fix.Fix; -import org.metafacture.metamorph.Metafix; import org.eclipse.emf.ecore.EObject; import org.eclipse.xtext.common.types.JvmGenericType; diff --git a/org.metafacture.fix/src/main/resources/flux-commands.properties b/org.metafacture.fix/src/main/resources/flux-commands.properties index eeba828f..1659127a 100644 --- a/org.metafacture.fix/src/main/resources/flux-commands.properties +++ b/org.metafacture.fix/src/main/resources/flux-commands.properties @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -fix org.metafacture.metamorph.Metafix +fix org.metafacture.metafix.Metafix diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/tests/FixParsingTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/FixParsingTest.java similarity index 97% rename from org.metafacture.fix/src/test/java/org/metafacture/metafix/tests/FixParsingTest.java rename to org.metafacture.fix/src/test/java/org/metafacture/metafix/FixParsingTest.java index f35b4481..60397ff0 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/tests/FixParsingTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/FixParsingTest.java @@ -1,6 +1,7 @@ -package org.metafacture.metafix.tests; +package org.metafacture.metafix; import org.metafacture.metafix.fix.Fix; +import org.metafacture.metafix.tests.FixInjectorProvider; import com.google.inject.Inject; import org.eclipse.emf.common.util.EList; diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/tests/InterpreterTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/InterpreterTest.java similarity index 94% rename from org.metafacture.fix/src/test/java/org/metafacture/metafix/tests/InterpreterTest.java rename to org.metafacture.fix/src/test/java/org/metafacture/metafix/InterpreterTest.java index 2f02eb60..5a17bc1f 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/tests/InterpreterTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/InterpreterTest.java @@ -1,8 +1,8 @@ -package org.metafacture.metafix.tests; +package org.metafacture.metafix; import org.metafacture.metafix.fix.Fix; import org.metafacture.metafix.interpreter.FixInterpreter; -import org.metafacture.metamorph.Metafix; +import org.metafacture.metafix.tests.FixInjectorProvider; import com.google.inject.Inject; import org.eclipse.xtext.testing.InjectWith; diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixBindTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java similarity index 99% rename from org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixBindTest.java rename to org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java index 89a5c500..93c74269 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixBindTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.metafacture.metamorph; +package org.metafacture.metafix; import org.metafacture.framework.StreamReceiver; diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixFieldTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java similarity index 99% rename from org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixFieldTest.java rename to org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java index 9f9a72cc..e3085fcb 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixFieldTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.metafacture.metamorph; +package org.metafacture.metafix; import org.metafacture.framework.StreamReceiver; diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixIfTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java similarity index 99% rename from org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixIfTest.java rename to org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java index 559b09a9..5b24d95d 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixIfTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.metafacture.metamorph; +package org.metafacture.metafix; import org.metafacture.framework.StreamReceiver; diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixLookupTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java similarity index 97% rename from org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixLookupTest.java rename to org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java index 9e4c8582..8ae74d7b 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixLookupTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.metafacture.metamorph; +package org.metafacture.metafix; import org.metafacture.framework.StreamReceiver; @@ -81,7 +81,7 @@ public void inline() { @Test public void csv() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "lookup('title', 'src/test/java/org/metafacture/metamorph/maps/test.csv')"), // + "lookup('title', 'src/test/java/org/metafacture/metafix/maps/test.csv')"), // i -> { i.startRecord("1"); i.endRecord(); @@ -112,7 +112,7 @@ public void csv() { @Test public void tsv() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "lookup('title', 'src/test/java/org/metafacture/metamorph/maps/test.tsv', sep_char:'\t')"), // + "lookup('title', 'src/test/java/org/metafacture/metafix/maps/test.tsv', sep_char:'\t')"), // i -> { i.startRecord("1"); i.endRecord(); diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixMethodTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java similarity index 99% rename from org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixMethodTest.java rename to org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java index ec9c9a18..194eed9e 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixMethodTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.metafacture.metamorph; +package org.metafacture.metafix; import org.metafacture.framework.StreamReceiver; diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixSelectorTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixSelectorTest.java similarity index 98% rename from org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixSelectorTest.java rename to org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixSelectorTest.java index 95021061..ff5d9457 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixSelectorTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixSelectorTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.metafacture.metamorph; +package org.metafacture.metafix; import org.metafacture.framework.StreamReceiver; diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixTestHelpers.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java similarity index 98% rename from org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixTestHelpers.java rename to org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java index 91e8060c..b42592b2 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/MetafixTestHelpers.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.metafacture.metamorph; +package org.metafacture.metafix; import org.metafacture.framework.StreamReceiver; diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/maps/test.csv b/org.metafacture.fix/src/test/java/org/metafacture/metafix/maps/test.csv similarity index 100% rename from org.metafacture.fix/src/test/java/org/metafacture/metamorph/maps/test.csv rename to org.metafacture.fix/src/test/java/org/metafacture/metafix/maps/test.csv diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metamorph/maps/test.tsv b/org.metafacture.fix/src/test/java/org/metafacture/metafix/maps/test.tsv similarity index 100% rename from org.metafacture.fix/src/test/java/org/metafacture/metamorph/maps/test.tsv rename to org.metafacture.fix/src/test/java/org/metafacture/metafix/maps/test.tsv From 7ba8f99bf923fa72d5f08db910e65b34164efa6c Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Fri, 20 Aug 2021 11:34:08 +0200 Subject: [PATCH 08/55] Add array- and hash-related record level methods (#35) And TODOs for internal record representation as JSON-equivalent --- .../org/metafacture/metafix/FixMethod.java | 26 +++++++ .../metafacture/metafix/MetafixFieldTest.java | 68 +++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index 3ca7742b..c3808cb0 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -37,6 +37,32 @@ public void apply(final Multimap record, final List para record.replaceValues(params.get(0), Arrays.asList(params.get(1))); } }, + set_array { + public void apply(final Multimap record, final List params, + final Map options) { + record.replaceValues(params.get(0), params.subList(1, params.size())); + } + }, + set_hash { + public void apply(final Multimap record, final List params, + final Map options) { + options.keySet().forEach(k -> { + record.replaceValues(params.get(0) + "." + k, Arrays.asList(options.get(k))); + }); + } + }, + array { // array-from-hash + public void apply(final Multimap record, final List params, + final Map options) { + //if record.get(params.get(0)) instanceof Map, etc. TODO: switch internal record to JSON-equiv + } + }, + hash { // hash-from-array + public void apply(final Multimap record, final List params, + final Map options) { + //if record.get(params.get(0)) instanceof List, etc. TODO: switch internal record to JSON-equiv + } + }, add_field { public void apply(final Multimap record, final List params, final Map options) { diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java index e3085fcb..275f3c80 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java @@ -18,6 +18,7 @@ import org.metafacture.framework.StreamReceiver; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.RegisterExtension; @@ -204,4 +205,71 @@ public void remove() { o.get().endRecord(); }); } + + @Test + public void setArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "set_array('foo','a','b','c')"), // + i -> { + i.startRecord("1"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("foo", "a"); + o.get().literal("foo", "b"); + o.get().literal("foo", "c"); + o.get().endRecord(); + }); + } + + @Test + public void setHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "set_hash('foo','a': 'b','c': 'd')"), // + i -> { + i.startRecord("1"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("foo.a", "b"); + o.get().literal("foo.c", "d"); + o.get().endRecord(); + }); + } + + @Test + @Disabled // TODO: switch internal record to JSON-equiv + public void hashFromArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "set_array('foo','a','b','c','d')", // + "hash('foo')"), // + i -> { + i.startRecord("1"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("foo.a", "b"); + o.get().literal("foo.c", "d"); + o.get().endRecord(); + }); + } + + @Test + @Disabled // TODO: switch internal record to JSON-equiv + public void arrayFromHash() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "set_hash('foo','a': 'b','c': 'd')", // + "array('foo')"), // + i -> { + i.startRecord("1"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("foo", "a"); + o.get().literal("foo", "b"); + o.get().literal("foo", "c"); + o.get().literal("foo", "d"); + o.get().endRecord(); + }); + } } From 4659439c0b35b27435a56f465932980c851bd625 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Fri, 20 Aug 2021 14:55:04 +0200 Subject: [PATCH 09/55] Implement `format`, `parse_text`, and `paste` (#35) See https://github.com/LibreCat/Catmandu/wiki/Fixes-Cheat-Sheet --- .../org/metafacture/metafix/FixMethod.java | 75 ++++++++++++++++++- .../metafacture/metafix/MetafixFieldTest.java | 57 ++++++++++++++ .../metafix/MetafixMethodTest.java | 50 +++++++++++++ 3 files changed, 179 insertions(+), 3 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index c3808cb0..6753bae3 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -20,12 +20,18 @@ import com.google.common.collect.Multimap; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; enum FixMethod { @@ -54,13 +60,15 @@ public void apply(final Multimap record, final List para array { // array-from-hash public void apply(final Multimap record, final List params, final Map options) { - //if record.get(params.get(0)) instanceof Map, etc. TODO: switch internal record to JSON-equiv + // if record.get(params.get(0)) instanceof Map, etc. + // TODO: switch internal record to JSON-equiv } }, hash { // hash-from-array public void apply(final Multimap record, final List params, final Map options) { - //if record.get(params.get(0)) instanceof List, etc. TODO: switch internal record to JSON-equiv + // if record.get(params.get(0)) instanceof List, etc. + // TODO: switch internal record to JSON-equiv } }, add_field { @@ -89,10 +97,71 @@ public void apply(final Multimap record, final List para remove_field { public void apply(final Multimap record, final List params, final Map options) { - record.removeAll(params.get(0)); + params.forEach(p -> { + record.removeAll(p); + }); + } + }, + format { + public void apply(final Multimap record, final List params, + final Map options) { + final Collection oldVals = record.get(params.get(0)); + final String newVal = String.format(params.get(1), oldVals.toArray(new Object[] {})); + record.replaceValues(params.get(0), Arrays.asList(newVal)); } }, + parse_text { + public void apply(final Multimap record, final List params, + final Map options) { + record.get(params.get(0)).forEach(v -> { + final Pattern p = Pattern.compile(params.get(1)); + final Matcher m = p.matcher(v); + if (m.matches()) { + record.removeAll(params.get(0)); + final Map namedGroups = getNamedGroups(p); + if (!namedGroups.isEmpty()) { + namedGroups.keySet().forEach(k -> { + record.put(params.get(0) + "." + k, m.group(namedGroups.get(k))); + }); + } + else { + for (int i = 1; i <= m.groupCount(); i = i + 1) { + record.put(params.get(0), m.group(i)); + } + } + } + }); + } + @SuppressWarnings("unchecked") + private Map getNamedGroups(final Pattern regex) { + try { + // Not available as API, see https://stackoverflow.com/a/15596145/18154: + final Method namedGroupsMethod = Pattern.class.getDeclaredMethod("namedGroups"); + namedGroupsMethod.setAccessible(true); + Map namedGroups = null; + namedGroups = (Map) namedGroupsMethod.invoke(regex); + if (namedGroups == null) { + throw new InternalError(); + } + return Collections.unmodifiableMap(namedGroups); + } + catch (final NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { + e.printStackTrace(); + } + return Collections.emptyMap(); + } + }, + paste { + public void apply(final Multimap record, final List params, + final Map options) { + final String joinChar = options.get("join_char"); + record.put(params.get(0), + params.subList(1, params.size()).stream() + .map(k -> k.startsWith("~") ? k.substring(1) : record.get(k).iterator().next()) + .collect(Collectors.joining(joinChar != null ? joinChar : " "))); + } + }, // FIELD-LEVEL METHODS: substring { diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java index 275f3c80..b8efdcab 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java @@ -237,6 +237,63 @@ public void setHash() { }); } + @Test + public void paste() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "paste('my.string','a','b','c','d')", + "remove_field('a','b','c','d')"), // + i -> { + i.startRecord("1"); + i.literal("a", "eeny"); + i.literal("b", "meeny"); + i.literal("c", "miny"); + i.literal("d", "moe"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("my.string", "eeny meeny miny moe"); + o.get().endRecord(); + }); + } + + @Test + public void pasteWithCustomSep() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "paste('my.string','a','b','c','d','join_char': ', ')", + "remove_field('a','b','c','d')"), // + i -> { + i.startRecord("1"); + i.literal("a", "eeny"); + i.literal("b", "meeny"); + i.literal("c", "miny"); + i.literal("d", "moe"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("my.string", "eeny, meeny, miny, moe"); + o.get().endRecord(); + }); + } + + @Test + public void pasteWithLiteralStrings() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "paste('my.string','~Hi','a','~how are you?')", + "remove_field('a','b','c','d')"), // + i -> { + i.startRecord("1"); + i.literal("a", "eeny"); + i.literal("b", "meeny"); + i.literal("c", "miny"); + i.literal("d", "moe"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("my.string", "Hi eeny how are you?"); + o.get().endRecord(); + }); + } + @Test @Disabled // TODO: switch internal record to JSON-equiv public void hashFromArray() { diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java index 194eed9e..db156289 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java @@ -224,6 +224,56 @@ public void trim() { }); } + @Test + public void format() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "format(number,'%-5s: %s')"), // TODO actual number formatting with JSON-equiv record + i -> { + i.startRecord("1"); + i.literal("number", "41"); + i.literal("number", "15"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("number", "41 : 15"); + o.get().endRecord(); + }); + } + + @Test + public void parseText() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "parse_text(date, '(\\\\d{4})-(\\\\d{2})-(\\\\d{2})')"), + i -> { + i.startRecord("1"); + i.literal("date", "2015-03-07"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("date", "2015"); + o.get().literal("date", "03"); + o.get().literal("date", "07"); + o.get().endRecord(); + }); + } + + @Test + public void parseTextNamedGroups() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "parse_text(date, '(?\\\\d{4})-(?\\\\d{2})-(?\\\\d{2})')"), + i -> { + i.startRecord("1"); + i.literal("date", "2015-03-07"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("date.month", "03"); + o.get().literal("date.year", "2015"); + o.get().literal("date.day", "07"); + o.get().endRecord(); + }); + } + @Test @Disabled // Use SimpleRegexTrie/WildcardTrie public void alternation() { From 0d903b32436c6d03131467da11add0ddc55232a4 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Tue, 24 Aug 2021 09:14:53 +0200 Subject: [PATCH 10/55] Switch internal record map to `Object` values (#35) --- .../org/metafacture/metafix/FixMethod.java | 50 +++++++++---------- .../org/metafacture/metafix/FixPredicate.java | 24 ++++----- .../java/org/metafacture/metafix/Metafix.java | 6 +-- .../metafix/RecordTransformer.java | 10 ++-- 4 files changed, 45 insertions(+), 45 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index 6753bae3..5cbcb08c 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -38,19 +38,19 @@ enum FixMethod { // RECORD-LEVEL METHODS: set_field { - public void apply(final Multimap record, final List params, + public void apply(final Multimap record, final List params, final Map options) { record.replaceValues(params.get(0), Arrays.asList(params.get(1))); } }, set_array { - public void apply(final Multimap record, final List params, + public void apply(final Multimap record, final List params, final Map options) { record.replaceValues(params.get(0), params.subList(1, params.size())); } }, set_hash { - public void apply(final Multimap record, final List params, + public void apply(final Multimap record, final List params, final Map options) { options.keySet().forEach(k -> { record.replaceValues(params.get(0) + "." + k, Arrays.asList(options.get(k))); @@ -58,27 +58,27 @@ public void apply(final Multimap record, final List para } }, array { // array-from-hash - public void apply(final Multimap record, final List params, + public void apply(final Multimap record, final List params, final Map options) { // if record.get(params.get(0)) instanceof Map, etc. // TODO: switch internal record to JSON-equiv } }, hash { // hash-from-array - public void apply(final Multimap record, final List params, + public void apply(final Multimap record, final List params, final Map options) { // if record.get(params.get(0)) instanceof List, etc. // TODO: switch internal record to JSON-equiv } }, add_field { - public void apply(final Multimap record, final List params, + public void apply(final Multimap record, final List params, final Map options) { record.put(params.get(0), params.get(1)); } }, move_field { - public void apply(final Multimap record, final List params, + public void apply(final Multimap record, final List params, final Map options) { final String oldFieldName = params.get(0); final String newFieldName = params.get(1); @@ -87,7 +87,7 @@ public void apply(final Multimap record, final List para } }, copy_field { - public void apply(final Multimap record, final List params, + public void apply(final Multimap record, final List params, final Map options) { final String oldName = params.get(0); final String newName = params.get(1); @@ -95,7 +95,7 @@ public void apply(final Multimap record, final List para } }, remove_field { - public void apply(final Multimap record, final List params, + public void apply(final Multimap record, final List params, final Map options) { params.forEach(p -> { record.removeAll(p); @@ -103,19 +103,19 @@ public void apply(final Multimap record, final List para } }, format { - public void apply(final Multimap record, final List params, + public void apply(final Multimap record, final List params, final Map options) { - final Collection oldVals = record.get(params.get(0)); + final Collection oldVals = record.get(params.get(0)); final String newVal = String.format(params.get(1), oldVals.toArray(new Object[] {})); record.replaceValues(params.get(0), Arrays.asList(newVal)); } }, parse_text { - public void apply(final Multimap record, final List params, + public void apply(final Multimap record, final List params, final Map options) { record.get(params.get(0)).forEach(v -> { final Pattern p = Pattern.compile(params.get(1)); - final Matcher m = p.matcher(v); + final Matcher m = p.matcher(v.toString()); if (m.matches()) { record.removeAll(params.get(0)); final Map namedGroups = getNamedGroups(p); @@ -153,51 +153,51 @@ private Map getNamedGroups(final Pattern regex) { } }, paste { - public void apply(final Multimap record, final List params, + public void apply(final Multimap record, final List params, final Map options) { final String joinChar = options.get("join_char"); record.put(params.get(0), params.subList(1, params.size()).stream() .map(k -> k.startsWith("~") ? k.substring(1) : record.get(k).iterator().next()) - .collect(Collectors.joining(joinChar != null ? joinChar : " "))); + .map(Object::toString).collect(Collectors.joining(joinChar != null ? joinChar : " "))); } }, // FIELD-LEVEL METHODS: substring { @SuppressWarnings("checkstyle:MagicNumber") // TODO: switch to morph-style named params in general? - public void apply(final Multimap record, final List params, + public void apply(final Multimap record, final List params, final Map options) { applyToFields(record, params, s -> s.substring(Integer.parseInt(params.get(1)), Integer.parseInt(params.get(2)) - 1)); } }, trim { - public void apply(final Multimap record, final List params, + public void apply(final Multimap record, final List params, final Map options) { applyToFields(record, params, s -> s.trim()); } }, upcase { - public void apply(final Multimap record, final List params, + public void apply(final Multimap record, final List params, final Map options) { applyToFields(record, params, s -> s.toUpperCase()); } }, downcase { - public void apply(final Multimap record, final List params, + public void apply(final Multimap record, final List params, final Map options) { applyToFields(record, params, s -> s.toLowerCase()); } }, capitalize { - public void apply(final Multimap record, final List params, + public void apply(final Multimap record, final List params, final Map options) { applyToFields(record, params, s -> s.substring(0, 1).toUpperCase() + s.substring(1)); } }, lookup { - public void apply(final Multimap record, final List params, + public void apply(final Multimap record, final List params, final Map options) { applyToFields(record, params, s -> { final Map map = buildMap(options, params.size() <= 1 ? null : params.get(1)); @@ -222,17 +222,17 @@ private Map fileMap(final String location, final String separato } }; - private static void applyToFields(final Multimap record, final List params, + private static void applyToFields(final Multimap record, final List params, final Function fun) { final String key = params.get(0); if (record.containsKey(key)) { - final Collection olds = new ArrayList(record.get(key)); + final Collection olds = new ArrayList(record.get(key)); olds.forEach(old -> { record.remove(key, old); - record.put(key, fun.apply(old)); + record.put(key, fun.apply(old.toString())); }); } } - abstract void apply(Multimap record, List params, Map options); + abstract void apply(Multimap record, List params, Map options); } diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixPredicate.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixPredicate.java index d914a3f5..b3a2e935 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixPredicate.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixPredicate.java @@ -25,28 +25,28 @@ enum FixPredicate { contain { - public Predicate of(final String value) { - return v -> v.contains(value); + public Predicate of(final Object value) { + return v -> v.toString().contains(value.toString()); } }, equal { - public Predicate of(final String value) { - return v -> v.equals(value); + public Predicate of(final Object value) { + return v -> v.toString().equals(value.toString()); } }, match { - public Predicate of(final String value) { - return v -> v.matches(value); + public Predicate of(final Object value) { + return v -> v.toString().matches(value.toString()); } }; - abstract Predicate of(String value); + abstract Predicate of(Object value); enum Quantifier { all { @Override - public boolean test(final Multimap record, final FixPredicate p, + public boolean test(final Multimap record, final FixPredicate p, final List params) { return test(record, params.get(0), s -> s.allMatch(p.of(params.get(1)))); } @@ -54,14 +54,14 @@ public boolean test(final Multimap record, final FixPredicate p, }, any { @Override - public boolean test(final Multimap record, final FixPredicate p, + public boolean test(final Multimap record, final FixPredicate p, final List params) { return test(record, params.get(0), s -> s.anyMatch(p.of(params.get(1)))); } }, none { @Override - public boolean test(final Multimap record, final FixPredicate p, + public boolean test(final Multimap record, final FixPredicate p, final List params) { final String fieldName = params.get(0); final String valueToTest = params.get(1); @@ -69,10 +69,10 @@ public boolean test(final Multimap record, final FixPredicate p, } }; - boolean test(final Multimap record, final String fieldName, final Predicate> f) { + boolean test(final Multimap record, final String fieldName, final Predicate> f) { return record.containsKey(fieldName) && f.test(record.get(fieldName).stream()); } - abstract boolean test(Multimap record, FixPredicate p, List params); + abstract boolean test(Multimap record, FixPredicate p, List params); } } diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java index 267e7916..1aca51ac 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java @@ -55,7 +55,7 @@ public class Metafix implements StreamPipe { private static final String ENTITIES_NOT_BALANCED = "Entity starts and ends are not balanced"; // TODO: Use SimpleRegexTrie / WildcardTrie for wildcard, alternation and character class support - private Multimap currentRecord = LinkedListMultimap.create(); + private Multimap currentRecord = LinkedListMultimap.create(); private Fix fix; private final List expressions = new ArrayList<>(); private Map vars = NO_VARS; @@ -129,7 +129,7 @@ public void endRecord() { currentRecord = transformer.transform(); System.out.println("Sending results to " + outputStreamReceiver); currentRecord.entries().forEach(e -> { - outputStreamReceiver.literal(e.getKey(), e.getValue()); + outputStreamReceiver.literal(e.getKey(), e.getValue().toString()); // TODO: send actual entities for `nested.fields` }); outputStreamReceiver.endRecord(); @@ -186,7 +186,7 @@ public Map getVars() { return vars; } - public Multimap getCurrentRecord() { + public Multimap getCurrentRecord() { return currentRecord; } diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java index 7d96caa4..91e4e6c1 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java @@ -47,21 +47,21 @@ class RecordTransformer { private Fix fix; - private Multimap record; + private Multimap record; private Map vars; - RecordTransformer(final Multimap record, final Map vars, final Fix fix) { + RecordTransformer(final Multimap record, final Map vars, final Fix fix) { this.record = LinkedListMultimap.create(record); this.vars = vars; this.fix = fix; } - Multimap transform() { + Multimap transform() { processSubexpressions(fix.getElements()); return record; } - Multimap getRecord() { + Multimap getRecord() { return record; } @@ -90,7 +90,7 @@ else if (sub instanceof Unless) { private void processBind(final Do theDo, final EList params) { if (theDo.getName().equals("list")) { // TODO impl multiple binds via FixBind enum final Map options = options(theDo.getOptions()); - final Multimap fullRecord = LinkedListMultimap.create(record); + final Multimap fullRecord = LinkedListMultimap.create(record); record.get(options.get("path")).forEach(val -> { // for each val, bind the current record/scope/context to the given var name: record = LinkedListMultimap.create(ImmutableMultimap.of(options.get("var"), val)); From f59faf5a203499116a117fd3a12375b85df60142 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Tue, 24 Aug 2021 10:19:20 +0200 Subject: [PATCH 11/55] Emit `Map` values in record as entities (#35) --- .../org/metafacture/metafix/FixMethod.java | 33 ++++++++++++++----- .../java/org/metafacture/metafix/Metafix.java | 19 +++++++++-- .../metafacture/metafix/MetafixFieldTest.java | 15 +++++---- .../metafix/MetafixMethodTest.java | 8 +++-- 4 files changed, 55 insertions(+), 20 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index 5cbcb08c..22baf871 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -26,6 +26,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -52,23 +53,37 @@ public void apply(final Multimap record, final List para set_hash { public void apply(final Multimap record, final List params, final Map options) { - options.keySet().forEach(k -> { - record.replaceValues(params.get(0) + "." + k, Arrays.asList(options.get(k))); - }); + record.removeAll(params.get(0)); + record.put(params.get(0), options); } }, array { // array-from-hash public void apply(final Multimap record, final List params, final Map options) { - // if record.get(params.get(0)) instanceof Map, etc. - // TODO: switch internal record to JSON-equiv + final String fieldName = params.get(0); + record.get(fieldName).forEach(recordEntry -> { + if (recordEntry instanceof Map) { + record.removeAll(fieldName); + ((Map) recordEntry).entrySet().forEach(mapEntry -> { + record.put(fieldName, mapEntry.getKey()); + record.put(fieldName, mapEntry.getValue()); + }); + } + }); } }, hash { // hash-from-array public void apply(final Multimap record, final List params, final Map options) { - // if record.get(params.get(0)) instanceof List, etc. - // TODO: switch internal record to JSON-equiv + final List values = new ArrayList<>(record.get(params.get(0))); + final Map result = new HashMap<>(); + for (int i = 0; i < values.size(); i = i + 1) { + if (i % 2 == 1) { + result.put(values.get(i - 1).toString(), values.get(i)); + } + } + record.removeAll(params.get(0).toString()); + record.put(params.get(0), result); } }, add_field { @@ -120,9 +135,11 @@ public void apply(final Multimap record, final List para record.removeAll(params.get(0)); final Map namedGroups = getNamedGroups(p); if (!namedGroups.isEmpty()) { + final Map result = new HashMap<>(); namedGroups.keySet().forEach(k -> { - record.put(params.get(0) + "." + k, m.group(namedGroups.get(k))); + result.put(k, m.group(namedGroups.get(k))); }); + record.put(params.get(0), result); } else { for (int i = 1; i <= m.groupCount(); i = i + 1) { diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java index 1aca51ac..2eeee64a 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java @@ -38,6 +38,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Map.Entry; /** * Transforms a data stream sent via the {@link StreamReceiver} interface. Use @@ -129,12 +130,26 @@ public void endRecord() { currentRecord = transformer.transform(); System.out.println("Sending results to " + outputStreamReceiver); currentRecord.entries().forEach(e -> { - outputStreamReceiver.literal(e.getKey(), e.getValue().toString()); - // TODO: send actual entities for `nested.fields` + emit(e); }); outputStreamReceiver.endRecord(); } + private void emit(final Entry entry) { + final Object value = entry.getValue(); + if (value instanceof Map) { + final Map nested = (Map) value; + outputStreamReceiver.startEntity(entry.getKey().toString()); + nested.entrySet().forEach(nestedEntry -> { + emit(nestedEntry); + }); + outputStreamReceiver.endEntity(); + } + else { + outputStreamReceiver.literal(entry.getKey().toString(), value.toString()); + } + } + @Override public void startEntity(final String name) { if (name == null) { diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java index b8efdcab..b50c2056 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java @@ -18,7 +18,6 @@ import org.metafacture.framework.StreamReceiver; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.RegisterExtension; @@ -231,8 +230,10 @@ public void setHash() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("foo.a", "b"); - o.get().literal("foo.c", "d"); + o.get().startEntity("foo"); + o.get().literal("a", "b"); + o.get().literal("c", "d"); + o.get().endEntity(); o.get().endRecord(); }); } @@ -295,7 +296,6 @@ public void pasteWithLiteralStrings() { } @Test - @Disabled // TODO: switch internal record to JSON-equiv public void hashFromArray() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// "set_array('foo','a','b','c','d')", // @@ -305,14 +305,15 @@ public void hashFromArray() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("foo.a", "b"); - o.get().literal("foo.c", "d"); + o.get().startEntity("foo"); + o.get().literal("a", "b"); + o.get().literal("c", "d"); + o.get().endEntity(); o.get().endRecord(); }); } @Test - @Disabled // TODO: switch internal record to JSON-equiv public void arrayFromHash() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// "set_hash('foo','a': 'b','c': 'd')", // diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java index db156289..0e801235 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java @@ -267,9 +267,11 @@ public void parseTextNamedGroups() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("date.month", "03"); - o.get().literal("date.year", "2015"); - o.get().literal("date.day", "07"); + o.get().startEntity("date"); + o.get().literal("month", "03"); + o.get().literal("year", "2015"); + o.get().literal("day", "07"); + o.get().endEntity(); o.get().endRecord(); }); } From 92fdf53c9474227731a4be5ccddedb8929bf2f34 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Tue, 24 Aug 2021 11:16:10 +0200 Subject: [PATCH 12/55] Basic implementation of `exists` and `reject` (#35) See https://github.com/LibreCat/Catmandu/wiki/Fixes-Cheat-Sheet --- .../org/metafacture/metafix/FixMethod.java | 6 ++++++ .../java/org/metafacture/metafix/Metafix.java | 16 ++++++++++------ .../metafacture/metafix/RecordTransformer.java | 3 +++ .../metafacture/metafix/MetafixFieldTest.java | 18 ++++++++++++++++++ 4 files changed, 37 insertions(+), 6 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index 22baf871..cdba921e 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -179,6 +179,12 @@ public void apply(final Multimap record, final List para .map(Object::toString).collect(Collectors.joining(joinChar != null ? joinChar : " "))); } }, + reject { + public void apply(final Multimap record, final List params, + final Map options) { + record.put("__reject", true); + } + }, // FIELD-LEVEL METHODS: substring { diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java index 2eeee64a..59a7dd5e 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java @@ -64,6 +64,7 @@ public class Metafix implements StreamPipe { private final Deque entityCountStack = new LinkedList<>(); private int entityCount; private StreamReceiver outputStreamReceiver; + private String recordIdentifier; public Metafix() { init(); @@ -115,7 +116,7 @@ public void startRecord(final String identifier) { entityCountStack.clear(); entityCount = 0; entityCountStack.add(Integer.valueOf(entityCount)); - outputStreamReceiver.startRecord(identifier); + recordIdentifier = identifier; } @Override @@ -128,11 +129,14 @@ public void endRecord() { System.out.printf("End record, walking fix: %s\n", currentRecord); final RecordTransformer transformer = new RecordTransformer(currentRecord, vars, fix); currentRecord = transformer.transform(); - System.out.println("Sending results to " + outputStreamReceiver); - currentRecord.entries().forEach(e -> { - emit(e); - }); - outputStreamReceiver.endRecord(); + if (!currentRecord.containsEntry("__reject", true)) { + outputStreamReceiver.startRecord(recordIdentifier); + System.out.println("Sending results to " + outputStreamReceiver); + currentRecord.entries().forEach(e -> { + emit(e); + }); + outputStreamReceiver.endRecord(); + } } private void emit(final Entry entry) { diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java index 91e4e6c1..b2177f9c 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java @@ -133,6 +133,9 @@ private void processUnless(final Unless unless, final EList parameters) private boolean testConditional(final String conditional, final EList params) { System.out.printf(": %s parameters: %s\n", conditional, params); boolean result = false; + if ("exists".equals(conditional)) { + return record.containsKey(params.get(0)); + } if (!conditional.contains("_")) { throw new IllegalArgumentException("Missing quantifier prefix (all_, any_, none_) for " + conditional); } diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java index b50c2056..fc5ca34e 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java @@ -330,4 +330,22 @@ public void arrayFromHash() { o.get().endRecord(); }); } + + @Test + public void reject() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "if exists ('_metadata.error')", // + " reject()", + "end"), // + i -> { + i.startRecord("1"); + i.literal("_metadata.error", "details"); + i.endRecord(); + i.startRecord("2"); + i.endRecord(); + }, o -> { + o.get().startRecord("2"); + o.get().endRecord(); + }); + } } From 506e5fd95d42d27e347cd02cd57ca203543919ef Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Tue, 24 Aug 2021 15:20:52 +0200 Subject: [PATCH 13/55] Emit arrays as entities, support `.$append` (#35) See https://github.com/LibreCat/Catmandu/wiki/Fixes-Cheat-Sheet --- .../org/metafacture/metafix/FixMethod.java | 17 ++++- .../java/org/metafacture/metafix/Metafix.java | 37 ++++++----- .../metafacture/metafix/MetafixBindTest.java | 6 +- .../metafacture/metafix/MetafixFieldTest.java | 62 ++++++++++++++++--- .../metafacture/metafix/MetafixIfTest.java | 54 ++++++++++------ .../metafix/MetafixLookupTest.java | 25 +++++--- .../metafix/MetafixMethodTest.java | 44 ++++++++----- .../metafix/MetafixTestHelpers.java | 18 ++++-- 8 files changed, 189 insertions(+), 74 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index cdba921e..ee769278 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -47,14 +47,23 @@ public void apply(final Multimap record, final List para set_array { public void apply(final Multimap record, final List params, final Map options) { - record.replaceValues(params.get(0), params.subList(1, params.size())); + final String fieldName = params.get(0).replace(APPEND, ""); + if (fieldName.equals(params.get(0))) { // not appending, replace + record.removeAll(params.get(0)); + } + params.subList(1, params.size()).forEach(p -> { + record.put(fieldName, p); + }); } }, set_hash { public void apply(final Multimap record, final List params, final Map options) { - record.removeAll(params.get(0)); - record.put(params.get(0), options); + final String fieldName = params.get(0).replace(APPEND, ""); + if (fieldName.equals(params.get(0))) { // not appending, replace + record.removeAll(params.get(0)); + } + record.put(fieldName, options); } }, array { // array-from-hash @@ -245,6 +254,8 @@ private Map fileMap(final String location, final String separato } }; + private static final String APPEND = ".$append"; + private static void applyToFields(final Multimap record, final List params, final Function fun) { final String key = params.get(0); diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java index 59a7dd5e..b4aefdf9 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java @@ -33,12 +33,13 @@ import java.io.Reader; import java.io.StringReader; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Deque; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Map.Entry; /** * Transforms a data stream sent via the {@link StreamReceiver} interface. Use @@ -132,25 +133,33 @@ public void endRecord() { if (!currentRecord.containsEntry("__reject", true)) { outputStreamReceiver.startRecord(recordIdentifier); System.out.println("Sending results to " + outputStreamReceiver); - currentRecord.entries().forEach(e -> { - emit(e); + currentRecord.keySet().forEach(k -> { + emit(k, currentRecord.get(k)); }); outputStreamReceiver.endRecord(); } } - private void emit(final Entry entry) { - final Object value = entry.getValue(); - if (value instanceof Map) { - final Map nested = (Map) value; - outputStreamReceiver.startEntity(entry.getKey().toString()); - nested.entrySet().forEach(nestedEntry -> { - emit(nestedEntry); - }); - outputStreamReceiver.endEntity(); + private void emit(final Object key, final Collection vals) { + final boolean isMulti = vals.size() > 1; + if (isMulti) { + outputStreamReceiver.startEntity(key.toString()); } - else { - outputStreamReceiver.literal(entry.getKey().toString(), value.toString()); + vals.forEach(value -> { + if (value instanceof Map) { + final Map nested = (Map) value; + outputStreamReceiver.startEntity(isMulti ? "" : key.toString()); + nested.entrySet().forEach(nestedEntry -> { + emit(nestedEntry.getKey(), Arrays.asList(nestedEntry.getValue())); + }); + outputStreamReceiver.endEntity(); + } + else { + outputStreamReceiver.literal(isMulti ? "" : key.toString(), value.toString()); + } + }); + if (isMulti) { + outputStreamReceiver.endEntity(); } } diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java index 93c74269..cb278a79 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java @@ -63,8 +63,10 @@ public void doList() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("author", "A UNIVERSITY"); - o.get().literal("author", "MAX"); + o.get().startEntity("author"); + o.get().literal("", "A UNIVERSITY"); + o.get().literal("", "MAX"); + o.get().endEntity(); o.get().endRecord(); }); } diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java index fc5ca34e..8864d209 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java @@ -107,8 +107,10 @@ public void add() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("my.name", "max"); - o.get().literal("my.name", "nicolas"); + o.get().startEntity("my.name"); + o.get().literal("", "max"); + o.get().literal("", "nicolas"); + o.get().endEntity(); o.get().endRecord(); // o.get().startRecord("3"); @@ -214,9 +216,11 @@ public void setArray() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("foo", "a"); - o.get().literal("foo", "b"); - o.get().literal("foo", "c"); + o.get().startEntity("foo"); + o.get().literal("", "a"); + o.get().literal("", "b"); + o.get().literal("", "c"); + o.get().endEntity(); o.get().endRecord(); }); } @@ -323,10 +327,12 @@ public void arrayFromHash() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("foo", "a"); - o.get().literal("foo", "b"); - o.get().literal("foo", "c"); - o.get().literal("foo", "d"); + o.get().startEntity("foo"); + o.get().literal("", "a"); + o.get().literal("", "b"); + o.get().literal("", "c"); + o.get().literal("", "d"); + o.get().endEntity(); o.get().endRecord(); }); } @@ -348,4 +354,42 @@ public void reject() { o.get().endRecord(); }); } + + @Test + public void appendArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "set_array('nums', '1')", // + "set_array('nums.$append', '2', '3')"), // + i -> { + i.startRecord("1"); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("nums"); + o.get().literal("", "1"); + o.get().literal("", "2"); + o.get().literal("", "3"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void mixedArray() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "set_array('@context', 'https://w3id.org/kim/lrmi-profile/draft/context.jsonld')", // + "set_hash('@context.$append', '@language': 'de')"), // + i -> { + i.startRecord("1"); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("@context"); + o.get().literal("", "https://w3id.org/kim/lrmi-profile/draft/context.jsonld"); + o.get().startEntity(""); + o.get().literal("@language", "de"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } } diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java index 5b24d95d..ea6b0691 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java @@ -67,14 +67,18 @@ public void ifAny() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("name", "Mary"); - o.get().literal("name", "A University"); + o.get().startEntity("name"); + o.get().literal("", "Mary"); + o.get().literal("", "A University"); + o.get().endEntity(); o.get().literal("type", "Organization"); o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("name", "Mary"); - o.get().literal("name", "Max"); + o.get().startEntity("name"); + o.get().literal("", "Mary"); + o.get().literal("", "Max"); + o.get().endEntity(); o.get().endRecord(); // o.get().startRecord("3"); @@ -103,13 +107,17 @@ public void ifAll() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("name", "Mary"); - o.get().literal("name", "A University"); + o.get().startEntity("name"); + o.get().literal("", "Mary"); + o.get().literal("", "A University"); + o.get().endEntity(); o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("name", "Great University"); - o.get().literal("name", "A University"); + o.get().startEntity("name"); + o.get().literal("", "Great University"); + o.get().literal("", "A University"); + o.get().endEntity(); o.get().literal("type", "Organization"); o.get().endRecord(); // @@ -139,13 +147,17 @@ public void ifNone() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("name", "Mary"); - o.get().literal("name", "A University"); + o.get().startEntity("name"); + o.get().literal("", "Mary"); + o.get().literal("", "A University"); + o.get().endEntity(); o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("name", "Max"); - o.get().literal("name", "Mary"); + o.get().startEntity("name"); + o.get().literal("", "Max"); + o.get().literal("", "Mary"); + o.get().endEntity(); o.get().literal("type", "Person"); o.get().endRecord(); // @@ -266,8 +278,10 @@ public void ifAnyMatch() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("name", "Some University"); - o.get().literal("name", "Filibandrina"); + o.get().startEntity("name"); + o.get().literal("", "Some University"); + o.get().literal("", "Filibandrina"); + o.get().endEntity(); o.get().literal("type", "Organization"); o.get().endRecord(); }); @@ -291,13 +305,17 @@ public void ifAllMatch() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("name", "Max"); - o.get().literal("name", "A University"); + o.get().startEntity("name"); + o.get().literal("", "Max"); + o.get().literal("", "A University"); + o.get().endEntity(); o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("name", "Some University"); - o.get().literal("name", "University Filibandrina"); + o.get().startEntity("name"); + o.get().literal("", "Some University"); + o.get().literal("", "University Filibandrina"); + o.get().endEntity(); o.get().literal("type", "Organization"); o.get().endRecord(); }); diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java index 8ae74d7b..84815b46 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java @@ -68,9 +68,11 @@ public void inline() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("title", "Alohaeha"); - o.get().literal("title", "Moin zäme"); - o.get().literal("title", "Tach"); + o.get().startEntity("title"); + o.get().literal("", "Alohaeha"); + o.get().literal("", "Moin zäme"); + o.get().literal("", "Tach"); + o.get().endEntity(); o.get().endRecord(); // o.get().startRecord("3"); @@ -87,6 +89,7 @@ public void csv() { i.endRecord(); // i.startRecord("2"); + // i.literal("title", "Aloha"); i.literal("title", "Moin"); i.literal("title", "Hey"); @@ -99,9 +102,11 @@ public void csv() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("title", "Alohaeha"); - o.get().literal("title", "Moin zäme"); - o.get().literal("title", "Tach"); + o.get().startEntity("title"); + o.get().literal("", "Alohaeha"); + o.get().literal("", "Moin zäme"); + o.get().literal("", "Tach"); + o.get().endEntity(); o.get().endRecord(); // o.get().startRecord("3"); @@ -130,9 +135,11 @@ public void tsv() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("title", "Alohaeha"); - o.get().literal("title", "Moin zäme"); - o.get().literal("title", "Tach"); + o.get().startEntity("title"); + o.get().literal("", "Alohaeha"); + o.get().literal("", "Moin zäme"); + o.get().literal("", "Tach"); + o.get().endEntity(); o.get().endRecord(); // o.get().startRecord("3"); diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java index 0e801235..7e73dabc 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java @@ -69,8 +69,10 @@ public void upcase() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("title", "MARC"); - o.get().literal("title", "JSON"); + o.get().startEntity("title"); + o.get().literal("", "MARC"); + o.get().literal("", "JSON"); + o.get().endEntity(); o.get().endRecord(); // o.get().startRecord("3"); @@ -98,8 +100,10 @@ public void downcase() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("title", "marc"); - o.get().literal("title", "json"); + o.get().startEntity("title"); + o.get().literal("", "marc"); + o.get().literal("", "json"); + o.get().endEntity(); o.get().endRecord(); // o.get().startRecord("3"); @@ -127,8 +131,10 @@ public void capitalize() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("title", "Marc"); - o.get().literal("title", "Json"); + o.get().startEntity("title"); + o.get().literal("", "Marc"); + o.get().literal("", "Json"); + o.get().endEntity(); o.get().endRecord(); // o.get().startRecord("3"); @@ -156,8 +162,10 @@ public void substring() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("title", "m"); - o.get().literal("title", "j"); + o.get().startEntity("title"); + o.get().literal("", "m"); + o.get().literal("", "j"); + o.get().endEntity(); o.get().endRecord(); // o.get().startRecord("3"); @@ -186,8 +194,10 @@ public void substringWithVar() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("title", "ma"); - o.get().literal("title", "js"); + o.get().startEntity("title"); + o.get().literal("", "ma"); + o.get().literal("", "js"); + o.get().endEntity(); o.get().endRecord(); // o.get().startRecord("3"); @@ -215,8 +225,10 @@ public void trim() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("title", "marc"); - o.get().literal("title", "json"); + o.get().startEntity("title"); + o.get().literal("", "marc"); + o.get().literal("", "json"); + o.get().endEntity(); o.get().endRecord(); // o.get().startRecord("3"); @@ -250,9 +262,11 @@ public void parseText() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("date", "2015"); - o.get().literal("date", "03"); - o.get().literal("date", "07"); + o.get().startEntity("date"); + o.get().literal("", "2015"); + o.get().literal("", "03"); + o.get().literal("", "07"); + o.get().endEntity(); o.get().endRecord(); }); } diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java index b42592b2..f7eb5433 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java @@ -45,12 +45,22 @@ public static void assertFix(final StreamReceiver receiver, final List f assertFix(receiver, fixDef, in, (s, f) -> out.accept(s), Metafix.NO_VARS); } - public static void assertFix(final StreamReceiver receiver, final List fixDef, - final Map vars, final Consumer in, final Consumer> out) { + public static void assertFix(final StreamReceiver receiver, final List fixDef, final Consumer in, + final BiConsumer, IntFunction> out) { + assertFix(receiver, fixDef, in, (s, f) -> out.accept(s, f), Metafix.NO_VARS); + } + + public static void assertFix(final StreamReceiver receiver, final List fixDef, final Map vars, + final Consumer in, final Consumer> out) { assertFix(receiver, fixDef, in, (s, f) -> out.accept(s), vars); } - public static void assertFix(final StreamReceiver receiver, final List fixLines, final Consumer in, + public static void assertFix(final StreamReceiver receiver, final List fixDef, final Map vars, + final Consumer in, final BiConsumer, IntFunction> out) { + assertFix(receiver, fixDef, in, (s, f) -> out.accept(s, f), vars); + } + + private static void assertFix(final StreamReceiver receiver, final List fixLines, final Consumer in, final BiConsumer, IntFunction> out, final Map vars) { final String fixString = String.join("\n", fixLines); final Metafix metafix = fix(receiver, fixString, vars); @@ -67,7 +77,7 @@ public static void assertFix(final StreamReceiver receiver, final List f } } - static Metafix fix(final StreamReceiver receiver, final String fix, final Map vars) { + private static Metafix fix(final StreamReceiver receiver, final String fix, final Map vars) { System.out.println("\nFix string: " + fix); Metafix metafix = null; try { From b022b5102b29253f0778a70b417801ee5ad8d368 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Thu, 26 Aug 2021 15:26:22 +0200 Subject: [PATCH 14/55] Implement `retain` method (#35) --- .../org/metafacture/metafix/FixMethod.java | 11 +++++++++++ .../metafacture/metafix/MetafixFieldTest.java | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index ee769278..a348dc80 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -27,6 +27,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -194,6 +195,16 @@ public void apply(final Multimap record, final List para record.put("__reject", true); } }, + retain { + public void apply(final Multimap record, final List params, + final Map options) { + new HashSet<>(record.keySet()).forEach(key -> { + if (!params.contains(key)) { + record.removeAll(key); + } + }); + } + }, // FIELD-LEVEL METHODS: substring { diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java index 8864d209..21414e57 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java @@ -392,4 +392,23 @@ public void mixedArray() { o.get().endRecord(); }); } + + @Test + public void retain() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "retain('1','3')"), // + i -> { + i.startRecord("1"); + i.literal("1", "one"); + i.literal("2", "two"); + i.literal("3", "tre"); + i.literal("4", "for"); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().literal("1", "one"); + o.get().literal("3", "tre"); + o.get().endRecord(); + }); + } } From 78c9ec99e658d9d08b7e56b027710330425e1da3 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Thu, 26 Aug 2021 15:53:24 +0200 Subject: [PATCH 15/55] Skip non-existing fields in `paste` (#35) --- .../src/main/java/org/metafacture/metafix/FixMethod.java | 7 ++++++- .../java/org/metafacture/metafix/MetafixFieldTest.java | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index a348dc80..f009b7b2 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -185,9 +185,14 @@ public void apply(final Multimap record, final List para final String joinChar = options.get("join_char"); record.put(params.get(0), params.subList(1, params.size()).stream() - .map(k -> k.startsWith("~") ? k.substring(1) : record.get(k).iterator().next()) + .filter(k -> literalString(k) || record.containsKey(k)) + .map(k -> literalString(k) ? k.substring(1) : record.get(k).iterator().next()) .map(Object::toString).collect(Collectors.joining(joinChar != null ? joinChar : " "))); } + + private boolean literalString(final String s) { + return s.startsWith("~"); + } }, reject { public void apply(final Multimap record, final List params, diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java index 21414e57..63a60168 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java @@ -245,7 +245,7 @@ public void setHash() { @Test public void paste() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "paste('my.string','a','b','c','d')", + "paste('my.string','a','b','c','d','e')", "remove_field('a','b','c','d')"), // i -> { i.startRecord("1"); From 998da3d6f367b151af9ed98f50325372213b08da Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Thu, 26 Aug 2021 16:02:17 +0200 Subject: [PATCH 16/55] Implement `vacuum` method (#35) --- .../org/metafacture/metafix/FixMethod.java | 15 +++++++++++++-- .../metafacture/metafix/MetafixFieldTest.java | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index f009b7b2..b93502a5 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -48,7 +48,7 @@ public void apply(final Multimap record, final List para set_array { public void apply(final Multimap record, final List params, final Map options) { - final String fieldName = params.get(0).replace(APPEND, ""); + final String fieldName = params.get(0).replace(APPEND, EMPTY); if (fieldName.equals(params.get(0))) { // not appending, replace record.removeAll(params.get(0)); } @@ -60,7 +60,7 @@ public void apply(final Multimap record, final List para set_hash { public void apply(final Multimap record, final List params, final Map options) { - final String fieldName = params.get(0).replace(APPEND, ""); + final String fieldName = params.get(0).replace(APPEND, EMPTY); if (fieldName.equals(params.get(0))) { // not appending, replace record.removeAll(params.get(0)); } @@ -210,6 +210,16 @@ public void apply(final Multimap record, final List para }); } }, + vacuum { + public void apply(final Multimap record, final List params, + final Map options) { + new HashSet<>(record.keySet()).forEach(key -> { + if (record.containsEntry(key, EMPTY)) { + record.remove(key, EMPTY); + } + }); + } + }, // FIELD-LEVEL METHODS: substring { @@ -270,6 +280,7 @@ private Map fileMap(final String location, final String separato } }; + private static final String EMPTY = ""; private static final String APPEND = ".$append"; private static void applyToFields(final Multimap record, final List params, diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java index 63a60168..5b291178 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java @@ -411,4 +411,23 @@ public void retain() { o.get().endRecord(); }); } + + @Test + public void vacuum() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "vacuum()"), // + i -> { + i.startRecord("1"); + i.literal("1", "one"); + i.literal("2", ""); + i.literal("3", "tre"); + i.literal("4", ""); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().literal("1", "one"); + o.get().literal("3", "tre"); + o.get().endRecord(); + }); + } } From b5604b8036a8c73d50dd3acddfb836817694cfbd Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Mon, 30 Aug 2021 15:32:09 +0200 Subject: [PATCH 17/55] Add disabled tests for list binds and entity representation (#35) Flattened field names vs. actual entities, both for emitting and internal representation (apply actual JsonPath to the record?) --- .../metafacture/metafix/MetafixBindTest.java | 63 +++++++++++++++++++ .../metafacture/metafix/MetafixFieldTest.java | 53 ++++++++++++++++ 2 files changed, 116 insertions(+) diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java index cb278a79..ff2db7fb 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java @@ -71,6 +71,69 @@ public void doList() { }); } + @Test + @Disabled // implement list bind for entities / fix internal entity structure + public void doListEntitesToLiterals() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "do list('path': 'creator', 'var': 'c')", + " upcase('c.name')", + " trim('c.name')", + " copy_field('c.name', 'author')", + "end", + "remove_field('creator')"), // + i -> { + i.startRecord("1"); + i.startEntity("creator"); + i.literal("name", " A University"); + i.endEntity(); + i.startEntity("creator"); + i.literal("name", "Max "); + i.endEntity(); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().literal("author", "A UNIVERSITY"); + o.get().literal("author", "MAX"); + o.get().endRecord(); + }); + } + + @Test + @Disabled // implement list bind for entities / fix internal entity structure + public void doListEntitesToEntities() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "do list('path': 'creator', 'var': 'c')", + " copy_field('c.name', 'author.$append.name')", + " if all_contain('c.name', 'University')", // + " add_field('author.$last.type', 'Organization')", // + " else", + " add_field('author.$last.type', 'Person')", //", + " end", + "end", + "remove_field('creator')"), // + i -> { + i.startRecord("1"); + i.startEntity("creator"); + i.literal("name", "A University"); + i.endEntity(); + i.startEntity("creator"); + i.literal("name", "Max"); + i.endEntity(); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().startEntity("author"); + o.get().literal("name", "A University"); + o.get().literal("type", "Organization"); + o.get().endEntity(); + o.get().startEntity("author"); + o.get().literal("name", "Max"); + o.get().literal("type", "Person"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + @Test @Disabled // implement Fix-style binds with collectors? public void ifInCollector() { diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java index 5b291178..924f153d 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java @@ -18,6 +18,7 @@ import org.metafacture.framework.StreamReceiver; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.RegisterExtension; @@ -85,6 +86,58 @@ public void set() { }); } + @Test + @Disabled // implement internal entities representation and emit actual entities + @SuppressWarnings("checkstyle:ExecutableStatementCount") + public void setEntities() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "set_field('my.name','patrick')", + "set_field('your.name','nicolas')"), // + i -> { + i.startRecord("1"); + i.endRecord(); + // + i.startRecord("2"); + i.startEntity("my"); + i.literal("name", "max"); + i.endEntity(); + i.startEntity("your"); + i.literal("name", "mo"); + i.endEntity(); + i.endRecord(); + // + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().startEntity("my"); + o.get().literal("name", "patrick"); + o.get().endEntity(); + o.get().startEntity("your"); + o.get().literal("name", "nicolas"); + o.get().endEntity(); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().startEntity("my"); + o.get().literal("name", "patrick"); + o.get().endEntity(); + o.get().startEntity("your"); + o.get().literal("name", "nicolas"); + o.get().endEntity(); + o.get().endRecord(); + // + o.get().startRecord("3"); + o.get().startEntity("my"); + o.get().literal("name", "patrick"); + o.get().endEntity(); + o.get().startEntity("your"); + o.get().literal("name", "nicolas"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + @Test public void add() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// From db1c437ca2411ff95844141be5601fc406465a36 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Tue, 31 Aug 2021 15:13:15 +0200 Subject: [PATCH 18/55] Switch internal record from Multimap to Map (#35) --- .../org/metafacture/metafix/FixMethod.java | 130 ++++++++++-------- .../org/metafacture/metafix/FixPredicate.java | 17 ++- .../java/org/metafacture/metafix/Metafix.java | 50 +++++-- .../metafix/RecordTransformer.java | 26 ++-- 4 files changed, 131 insertions(+), 92 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index b93502a5..f04989eb 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -18,8 +18,6 @@ import org.metafacture.metamorph.maps.FileMap; -import com.google.common.collect.Multimap; - import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; @@ -40,120 +38,128 @@ enum FixMethod { // RECORD-LEVEL METHODS: set_field { - public void apply(final Multimap record, final List params, + public void apply(final Map record, final List params, final Map options) { - record.replaceValues(params.get(0), Arrays.asList(params.get(1))); + record.put(params.get(0), params.get(1)); } }, set_array { - public void apply(final Multimap record, final List params, + public void apply(final Map record, final List params, final Map options) { - final String fieldName = params.get(0).replace(APPEND, EMPTY); - if (fieldName.equals(params.get(0))) { // not appending, replace - record.removeAll(params.get(0)); + final String key = params.get(0); + final List toAdd = params.subList(1, params.size()); + if (key.contains(APPEND)) { + Metafix.addAll(record, key.replace(APPEND, EMPTY), toAdd); + } + else { + record.put(key, toAdd); } - params.subList(1, params.size()).forEach(p -> { - record.put(fieldName, p); - }); } }, set_hash { - public void apply(final Multimap record, final List params, + @SuppressWarnings("unchecked") + public void apply(final Map record, final List params, final Map options) { - final String fieldName = params.get(0).replace(APPEND, EMPTY); - if (fieldName.equals(params.get(0))) { // not appending, replace - record.removeAll(params.get(0)); + final String key = params.get(0); + final Object val = record.get(key.replace(APPEND, EMPTY)); + if (key.contains(APPEND) && val instanceof List) { + ((List) val).add(options); + } + else { + record.put(key, options); } - record.put(fieldName, options); } }, array { // array-from-hash - public void apply(final Multimap record, final List params, + public void apply(final Map record, final List params, final Map options) { final String fieldName = params.get(0); - record.get(fieldName).forEach(recordEntry -> { + Metafix.asList(record.get(fieldName)).forEach(recordEntry -> { if (recordEntry instanceof Map) { - record.removeAll(fieldName); + record.remove(fieldName); ((Map) recordEntry).entrySet().forEach(mapEntry -> { - record.put(fieldName, mapEntry.getKey()); - record.put(fieldName, mapEntry.getValue()); + Metafix.add(record, fieldName, mapEntry.getKey()); + Metafix.add(record, fieldName, mapEntry.getValue()); }); } }); } }, hash { // hash-from-array - public void apply(final Multimap record, final List params, + public void apply(final Map record, final List params, final Map options) { - final List values = new ArrayList<>(record.get(params.get(0))); + final List values = Metafix.asList(record.get(params.get(0))); final Map result = new HashMap<>(); for (int i = 0; i < values.size(); i = i + 1) { if (i % 2 == 1) { result.put(values.get(i - 1).toString(), values.get(i)); } } - record.removeAll(params.get(0).toString()); record.put(params.get(0), result); } }, add_field { - public void apply(final Multimap record, final List params, + public void apply(final Map record, final List params, final Map options) { - record.put(params.get(0), params.get(1)); + final String name = params.get(0); + final String val = params.get(1); + final Object object = record.get(name); + record.put(name, object == null ? val : Metafix.asListWith(object, val)); } + }, move_field { - public void apply(final Multimap record, final List params, + public void apply(final Map record, final List params, final Map options) { final String oldFieldName = params.get(0); final String newFieldName = params.get(1); - record.putAll(newFieldName, record.get(oldFieldName)); - record.removeAll(oldFieldName); + record.put(newFieldName, record.get(oldFieldName)); + record.remove(oldFieldName); } }, copy_field { - public void apply(final Multimap record, final List params, + public void apply(final Map record, final List params, final Map options) { final String oldName = params.get(0); final String newName = params.get(1); - record.putAll(newName, record.get(oldName)); + Metafix.add(record, newName, record.get(oldName)); } }, remove_field { - public void apply(final Multimap record, final List params, + public void apply(final Map record, final List params, final Map options) { params.forEach(p -> { - record.removeAll(p); + record.remove(p); }); } }, format { - public void apply(final Multimap record, final List params, + public void apply(final Map record, final List params, final Map options) { - final Collection oldVals = record.get(params.get(0)); + final Collection oldVals = Metafix.asList(record.get(params.get(0))); final String newVal = String.format(params.get(1), oldVals.toArray(new Object[] {})); - record.replaceValues(params.get(0), Arrays.asList(newVal)); + record.replace(params.get(0), Arrays.asList(newVal)); } }, parse_text { - public void apply(final Multimap record, final List params, + public void apply(final Map record, final List params, final Map options) { - record.get(params.get(0)).forEach(v -> { + Metafix.asList(record.get(params.get(0))).forEach(v -> { final Pattern p = Pattern.compile(params.get(1)); final Matcher m = p.matcher(v.toString()); if (m.matches()) { - record.removeAll(params.get(0)); + record.remove(params.get(0)); final Map namedGroups = getNamedGroups(p); if (!namedGroups.isEmpty()) { final Map result = new HashMap<>(); namedGroups.keySet().forEach(k -> { result.put(k, m.group(namedGroups.get(k))); }); - record.put(params.get(0), result); + Metafix.add(record, params.get(0), result); } else { for (int i = 1; i <= m.groupCount(); i = i + 1) { - record.put(params.get(0), m.group(i)); + Metafix.add(record, params.get(0), m.group(i)); } } } @@ -180,13 +186,13 @@ private Map getNamedGroups(final Pattern regex) { } }, paste { - public void apply(final Multimap record, final List params, + public void apply(final Map record, final List params, final Map options) { final String joinChar = options.get("join_char"); record.put(params.get(0), params.subList(1, params.size()).stream() .filter(k -> literalString(k) || record.containsKey(k)) - .map(k -> literalString(k) ? k.substring(1) : record.get(k).iterator().next()) + .map(k -> literalString(k) ? k.substring(1) : Metafix.asList(record.get(k)).iterator().next()) .map(Object::toString).collect(Collectors.joining(joinChar != null ? joinChar : " "))); } @@ -195,26 +201,26 @@ private boolean literalString(final String s) { } }, reject { - public void apply(final Multimap record, final List params, + public void apply(final Map record, final List params, final Map options) { - record.put("__reject", true); + record.put("__reject", null); } }, retain { - public void apply(final Multimap record, final List params, + public void apply(final Map record, final List params, final Map options) { new HashSet<>(record.keySet()).forEach(key -> { if (!params.contains(key)) { - record.removeAll(key); + record.remove(key); } }); } }, vacuum { - public void apply(final Multimap record, final List params, + public void apply(final Map record, final List params, final Map options) { new HashSet<>(record.keySet()).forEach(key -> { - if (record.containsEntry(key, EMPTY)) { + if (EMPTY.equals(record.get(key))) { record.remove(key, EMPTY); } }); @@ -224,38 +230,38 @@ public void apply(final Multimap record, final List para substring { @SuppressWarnings("checkstyle:MagicNumber") // TODO: switch to morph-style named params in general? - public void apply(final Multimap record, final List params, + public void apply(final Map record, final List params, final Map options) { applyToFields(record, params, s -> s.substring(Integer.parseInt(params.get(1)), Integer.parseInt(params.get(2)) - 1)); } }, trim { - public void apply(final Multimap record, final List params, + public void apply(final Map record, final List params, final Map options) { applyToFields(record, params, s -> s.trim()); } }, upcase { - public void apply(final Multimap record, final List params, + public void apply(final Map record, final List params, final Map options) { applyToFields(record, params, s -> s.toUpperCase()); } }, downcase { - public void apply(final Multimap record, final List params, + public void apply(final Map record, final List params, final Map options) { applyToFields(record, params, s -> s.toLowerCase()); } }, capitalize { - public void apply(final Multimap record, final List params, + public void apply(final Map record, final List params, final Map options) { applyToFields(record, params, s -> s.substring(0, 1).toUpperCase() + s.substring(1)); } }, lookup { - public void apply(final Multimap record, final List params, + public void apply(final Map record, final List params, final Map options) { applyToFields(record, params, s -> { final Map map = buildMap(options, params.size() <= 1 ? null : params.get(1)); @@ -283,17 +289,21 @@ private Map fileMap(final String location, final String separato private static final String EMPTY = ""; private static final String APPEND = ".$append"; - private static void applyToFields(final Multimap record, final List params, + private static void applyToFields(final Map record, final List params, final Function fun) { final String key = params.get(0); if (record.containsKey(key)) { - final Collection olds = new ArrayList(record.get(key)); - olds.forEach(old -> { + new ArrayList<>(Metafix.asList(record.get(key))).forEach(old -> { record.remove(key, old); - record.put(key, fun.apply(old.toString())); + final Object object = record.get(key); + if (object instanceof List) { + ((List) object).remove(old); + } + final String val = fun.apply(old.toString()); + record.put(key, object == null ? val : Metafix.asListWith(object, val)); }); } } - abstract void apply(Multimap record, List params, Map options); + abstract void apply(Map record, List params, Map options); } diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixPredicate.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixPredicate.java index b3a2e935..be722dc4 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixPredicate.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixPredicate.java @@ -16,9 +16,8 @@ package org.metafacture.metafix; -import com.google.common.collect.Multimap; - import java.util.List; +import java.util.Map; import java.util.function.Predicate; import java.util.stream.Stream; @@ -46,7 +45,7 @@ enum Quantifier { all { @Override - public boolean test(final Multimap record, final FixPredicate p, + public boolean test(final Map record, final FixPredicate p, final List params) { return test(record, params.get(0), s -> s.allMatch(p.of(params.get(1)))); } @@ -54,25 +53,25 @@ public boolean test(final Multimap record, final FixPredicate p, }, any { @Override - public boolean test(final Multimap record, final FixPredicate p, + public boolean test(final Map record, final FixPredicate p, final List params) { return test(record, params.get(0), s -> s.anyMatch(p.of(params.get(1)))); } }, none { @Override - public boolean test(final Multimap record, final FixPredicate p, + public boolean test(final Map record, final FixPredicate p, final List params) { final String fieldName = params.get(0); final String valueToTest = params.get(1); - return !record.containsKey(fieldName) || record.get(fieldName).stream().noneMatch(p.of(valueToTest)); + return !record.containsKey(fieldName) || Metafix.asList(record.get(fieldName)).stream().noneMatch(p.of(valueToTest)); } }; - boolean test(final Multimap record, final String fieldName, final Predicate> f) { - return record.containsKey(fieldName) && f.test(record.get(fieldName).stream()); + boolean test(final Map record, final String fieldName, final Predicate> f) { + return record.containsKey(fieldName) && f.test(Metafix.asList(record.get(fieldName)).stream()); } - abstract boolean test(Multimap record, FixPredicate p, List params); + abstract boolean test(Map record, FixPredicate p, List params); } } diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java index b4aefdf9..e8190636 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java @@ -25,18 +25,15 @@ import org.metafacture.metafix.fix.Expression; import org.metafacture.metafix.fix.Fix; -import com.google.common.collect.LinkedListMultimap; -import com.google.common.collect.Multimap; - import java.io.FileNotFoundException; import java.io.FileReader; import java.io.Reader; import java.io.StringReader; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.Deque; +import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -57,7 +54,7 @@ public class Metafix implements StreamPipe { private static final String ENTITIES_NOT_BALANCED = "Entity starts and ends are not balanced"; // TODO: Use SimpleRegexTrie / WildcardTrie for wildcard, alternation and character class support - private Multimap currentRecord = LinkedListMultimap.create(); + private Map currentRecord = new LinkedHashMap<>(); private Fix fix; private final List expressions = new ArrayList<>(); private Map vars = NO_VARS; @@ -98,7 +95,7 @@ private void init() { public void literal(final String name, final String value) { // TODO: set up logging System.out.printf("Putting '%s':'%s'\n", name, value); - currentRecord.put(name, value); + add(currentRecord, name, value); } }); } @@ -111,7 +108,7 @@ private void buildPipeline(final Reader fixDef, final Map theVar @Override public void startRecord(final String identifier) { - currentRecord = LinkedListMultimap.create(); + currentRecord = new LinkedHashMap<>(); System.out.printf("Start record: %s\n", currentRecord); flattener.startRecord(identifier); entityCountStack.clear(); @@ -130,7 +127,7 @@ public void endRecord() { System.out.printf("End record, walking fix: %s\n", currentRecord); final RecordTransformer transformer = new RecordTransformer(currentRecord, vars, fix); currentRecord = transformer.transform(); - if (!currentRecord.containsEntry("__reject", true)) { + if (!currentRecord.containsKey("__reject")) { outputStreamReceiver.startRecord(recordIdentifier); System.out.println("Sending results to " + outputStreamReceiver); currentRecord.keySet().forEach(k -> { @@ -140,7 +137,11 @@ public void endRecord() { } } - private void emit(final Object key, final Collection vals) { + private void emit(final Object key, final Object val) { + if (val == null) { + return; + } + final List vals = val instanceof List ? (List) val : Arrays.asList(val); final boolean isMulti = vals.size() > 1; if (isMulti) { outputStreamReceiver.startEntity(key.toString()); @@ -214,8 +215,37 @@ public Map getVars() { return vars; } - public Multimap getCurrentRecord() { + public Map getCurrentRecord() { return currentRecord; } + static void addAll(final Map record, final String fieldName, final List values) { + values.forEach(value -> { + add(record, fieldName, value); + }); + } + + static void addAll(final Map record, final Map values) { + values.entrySet().forEach(value -> { + add(record, value.getKey(), value.getValue()); + }); + } + + static void add(final Map record, final String name, final Object val) { + final Object object = record.get(name); + record.put(name, object == null ? val : asListWith(object, val)); + } + + static List asListWith(final Object object, final Object value) { + final List list = asList(object); + list.add(value); + return list; + } + + @SuppressWarnings("unchecked") + static List asList(final Object object) { + return new ArrayList<>( + object instanceof List ? (List) object : Arrays.asList(object)); + } + } diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java index b2177f9c..236121a8 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java @@ -29,11 +29,10 @@ import org.metafacture.metafix.fix.Unless; import org.eclipse.emf.common.util.EList; -import com.google.common.collect.ImmutableMultimap; -import com.google.common.collect.LinkedListMultimap; -import com.google.common.collect.Multimap; +import com.google.common.collect.ImmutableMap; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -47,21 +46,21 @@ class RecordTransformer { private Fix fix; - private Multimap record; + private Map record; private Map vars; - RecordTransformer(final Multimap record, final Map vars, final Fix fix) { - this.record = LinkedListMultimap.create(record); + RecordTransformer(final Map record, final Map vars, final Fix fix) { + this.record = new LinkedHashMap<>(record); this.vars = vars; this.fix = fix; } - Multimap transform() { + Map transform() { processSubexpressions(fix.getElements()); return record; } - Multimap getRecord() { + Map getRecord() { return record; } @@ -90,14 +89,15 @@ else if (sub instanceof Unless) { private void processBind(final Do theDo, final EList params) { if (theDo.getName().equals("list")) { // TODO impl multiple binds via FixBind enum final Map options = options(theDo.getOptions()); - final Multimap fullRecord = LinkedListMultimap.create(record); - record.get(options.get("path")).forEach(val -> { + final Map fullRecord = new LinkedHashMap<>(record); + final List values = Metafix.asList(record.get(options.get("path"))); + values.forEach(val -> { // for each val, bind the current record/scope/context to the given var name: - record = LinkedListMultimap.create(ImmutableMultimap.of(options.get("var"), val)); + record = new LinkedHashMap<>(ImmutableMap.of(options.get("var"), val)); processSubexpressions(theDo.getElements()); - record.removeAll(options.get("var")); + record.remove(options.get("var")); // and remember the things we added while bound (this probably needs some tweaking): - fullRecord.putAll(record); + Metafix.addAll(fullRecord, record); }); record = fullRecord; } From cea496e6f1d70454ecf5eca18babd21d014c0396 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Wed, 1 Sep 2021 11:15:38 +0200 Subject: [PATCH 19/55] Support dot notation for nested entities in set_field (#35) --- .../org/metafacture/metafix/FixMethod.java | 30 ++++++- .../metafacture/metafix/MetafixFieldTest.java | 85 ++++++------------- 2 files changed, 53 insertions(+), 62 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index f04989eb..b9e28a26 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -26,6 +26,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -40,7 +41,34 @@ enum FixMethod { set_field { public void apply(final Map record, final List params, final Map options) { - record.put(params.get(0), params.get(1)); + record.remove(params.get(0)); + setValue(record, params.get(0).split("\\."), params.get(1)); + } + + public Object setValue(final Map map, final String[] keys, final String value) { + final String currentKey = keys[0]; + if (keys.length == 1) { + map.put(currentKey, value); + return map; + } + final String[] remainingKeys = Arrays.copyOfRange(keys, 1, keys.length); + final Object nested = setNested(map, value, currentKey, remainingKeys); + map.put(currentKey, nested); + return map; + } + + private Object setNested(final Map map, final String value, final String currentKey, + final String[] remainingKeys) { + if (!map.containsKey(currentKey)) { + map.put(currentKey, new LinkedHashMap()); + } + final Object nested = map.get(currentKey); + if (!(nested instanceof Map)) { + throw new IllegalStateException("Nested non-map: " + nested); + } + @SuppressWarnings("unchecked") + final Object result = setValue((Map) nested, remainingKeys, value); + return result; } }, set_array { diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java index 924f153d..884e7663 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java @@ -18,7 +18,6 @@ import org.metafacture.framework.StreamReceiver; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.RegisterExtension; @@ -49,91 +48,55 @@ public MetafixFieldTest() { } @Test - public void set() { + public void setEmpty() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "set_field('my.name','patrick')", - "set_field('your.name','nicolas')"), // + "set_field('my.nested.name','patrick')", + "set_field('your.nested.name','nicolas')"), // i -> { i.startRecord("1"); i.endRecord(); - // - i.startRecord("2"); - i.startEntity("my"); - i.literal("name", "max"); - i.endEntity(); - i.startEntity("your"); - i.literal("name", "mo"); - i.endEntity(); - i.endRecord(); - // - i.startRecord("3"); - i.endRecord(); - }, o -> { + }, (o, f) -> { o.get().startRecord("1"); - o.get().literal("my.name", "patrick"); - o.get().literal("your.name", "nicolas"); - o.get().endRecord(); - // - o.get().startRecord("2"); - o.get().literal("my.name", "patrick"); - o.get().literal("your.name", "nicolas"); - o.get().endRecord(); - // - o.get().startRecord("3"); - o.get().literal("my.name", "patrick"); - o.get().literal("your.name", "nicolas"); + o.get().startEntity("my"); + o.get().startEntity("nested"); + o.get().literal("name", "patrick"); + f.apply(2).endEntity(); + o.get().startEntity("your"); + o.get().startEntity("nested"); + o.get().literal("name", "nicolas"); + f.apply(2).endEntity(); o.get().endRecord(); }); } @Test - @Disabled // implement internal entities representation and emit actual entities - @SuppressWarnings("checkstyle:ExecutableStatementCount") - public void setEntities() { + public void setExisting() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "set_field('my.name','patrick')", - "set_field('your.name','nicolas')"), // + "set_field('my.nested.name','patrick')", + "set_field('your.nested.name','nicolas')"), // i -> { i.startRecord("1"); - i.endRecord(); - // - i.startRecord("2"); i.startEntity("my"); + i.startEntity("nested"); i.literal("name", "max"); i.endEntity(); + i.endEntity(); i.startEntity("your"); + i.startEntity("nested"); i.literal("name", "mo"); i.endEntity(); + i.endEntity(); i.endRecord(); - // - i.startRecord("3"); - i.endRecord(); - }, o -> { + }, (o, f) -> { o.get().startRecord("1"); o.get().startEntity("my"); + o.get().startEntity("nested"); o.get().literal("name", "patrick"); - o.get().endEntity(); - o.get().startEntity("your"); - o.get().literal("name", "nicolas"); - o.get().endEntity(); - o.get().endRecord(); - // - o.get().startRecord("2"); - o.get().startEntity("my"); - o.get().literal("name", "patrick"); - o.get().endEntity(); - o.get().startEntity("your"); - o.get().literal("name", "nicolas"); - o.get().endEntity(); - o.get().endRecord(); - // - o.get().startRecord("3"); - o.get().startEntity("my"); - o.get().literal("name", "patrick"); - o.get().endEntity(); + f.apply(2).endEntity(); o.get().startEntity("your"); + o.get().startEntity("nested"); o.get().literal("name", "nicolas"); - o.get().endEntity(); + f.apply(2).endEntity(); o.get().endRecord(); }); } From 46719dab20e6ddcfde5f71f7bafc50c113452093 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Wed, 1 Sep 2021 14:39:44 +0200 Subject: [PATCH 20/55] Support dot notation for nested entities in add_field (#35) --- .../org/metafacture/metafix/FixMethod.java | 76 +++++++++++-------- .../metafacture/metafix/MetafixFieldTest.java | 17 +++-- .../metafacture/metafix/MetafixIfTest.java | 12 ++- 3 files changed, 64 insertions(+), 41 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index b9e28a26..d6d7db85 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -42,33 +42,7 @@ enum FixMethod { public void apply(final Map record, final List params, final Map options) { record.remove(params.get(0)); - setValue(record, params.get(0).split("\\."), params.get(1)); - } - - public Object setValue(final Map map, final String[] keys, final String value) { - final String currentKey = keys[0]; - if (keys.length == 1) { - map.put(currentKey, value); - return map; - } - final String[] remainingKeys = Arrays.copyOfRange(keys, 1, keys.length); - final Object nested = setNested(map, value, currentKey, remainingKeys); - map.put(currentKey, nested); - return map; - } - - private Object setNested(final Map map, final String value, final String currentKey, - final String[] remainingKeys) { - if (!map.containsKey(currentKey)) { - map.put(currentKey, new LinkedHashMap()); - } - final Object nested = map.get(currentKey); - if (!(nested instanceof Map)) { - throw new IllegalStateException("Nested non-map: " + nested); - } - @SuppressWarnings("unchecked") - final Object result = setValue((Map) nested, remainingKeys, value); - return result; + insert(InsertMode.REPLACE, record, params.get(0).split("\\."), params.get(1)); } }, set_array { @@ -129,10 +103,7 @@ public void apply(final Map record, final List params, add_field { public void apply(final Map record, final List params, final Map options) { - final String name = params.get(0); - final String val = params.get(1); - final Object object = record.get(name); - record.put(name, object == null ? val : Metafix.asListWith(object, val)); + insert(InsertMode.APPEND, record, params.get(0).split("\\."), params.get(1)); } }, @@ -333,5 +304,48 @@ private static void applyToFields(final Map record, final List map, final String[] keys, final String value) { + final String currentKey = keys[0]; + if (keys.length == 1) { + mode.apply(map, currentKey, value); + return map; + } + final String[] remainingKeys = Arrays.copyOfRange(keys, 1, keys.length); + final Object nested = insertNested(mode, map, value, currentKey, remainingKeys); + map.put(currentKey, nested); + return map; + } + + private static Object insertNested(final InsertMode mode, final Map map, final String value, final String currentKey, + final String[] remainingKeys) { + if (!map.containsKey(currentKey)) { + map.put(currentKey, new LinkedHashMap()); + } + final Object nested = map.get(currentKey); + if (!(nested instanceof Map)) { + throw new IllegalStateException("Nested non-map: " + nested); + } + @SuppressWarnings("unchecked") + final Object result = insert(mode, (Map) nested, remainingKeys, value); + return result; + } + + private enum InsertMode { + REPLACE { + @Override + void apply(final Map map, final String key, final String value) { + map.put(key, value); + } + }, + APPEND { + @Override + void apply(final Map map, final String key, final String value) { + final Object object = map.get(key); + map.put(key, object == null ? value : Metafix.asListWith(object, value)); + } + }; + abstract void apply(Map map, String key, String value); + } + abstract void apply(Map record, List params, Map options); } diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java index 884e7663..48af27d7 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java @@ -104,6 +104,7 @@ public void setExisting() { @Test public void add() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "add_field('my.name','patrick')", "add_field('my.name','nicolas')"), // i -> { i.startRecord("1"); @@ -117,20 +118,24 @@ public void add() { // i.startRecord("3"); i.endRecord(); - }, o -> { + }, (o, f) -> { o.get().startRecord("1"); - o.get().literal("my.name", "nicolas"); + o.get().startEntity("my"); + o.get().literal("name", "[patrick, nicolas]"); // TODO: fix list -> entity + o.get().endEntity(); o.get().endRecord(); // o.get().startRecord("2"); - o.get().startEntity("my.name"); - o.get().literal("", "max"); - o.get().literal("", "nicolas"); + o.get().literal("my.name", "max"); // TODO: fix entity -> entity + o.get().startEntity("my"); + o.get().literal("name", "[patrick, nicolas]"); // TODO: fix list -> entity o.get().endEntity(); o.get().endRecord(); // o.get().startRecord("3"); - o.get().literal("my.name", "nicolas"); + o.get().startEntity("my"); + o.get().literal("name", "[patrick, nicolas]"); // TODO: fix list -> entity + o.get().endEntity(); o.get().endRecord(); }); } diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java index ea6b0691..f49a2be7 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java @@ -231,8 +231,10 @@ public void moveAndAddIfContain() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("author.name", "A University"); - o.get().literal("author.type", "Organization"); + o.get().literal("author.name", "A University"); // TODO: fix entity -> entity + o.get().startEntity("author"); + o.get().literal("type", "Organization"); + o.get().endEntity(); o.get().endRecord(); }); } @@ -349,8 +351,10 @@ public void ifAnyMatchNested() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("author.name.label", "Some University"); - o.get().literal("author.type", "Organization"); + o.get().literal("author.name.label", "Some University"); // TODO: fix entity -> entity + o.get().startEntity("author"); + o.get().literal("type", "Organization"); + o.get().endEntity(); o.get().endRecord(); }); } From 8e104a89a1a32f2be6373a27e150ec60a062b863 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Thu, 2 Sep 2021 11:55:39 +0200 Subject: [PATCH 21/55] Rename class for testing record level methods (#35) --- .../{MetafixFieldTest.java => MetafixRecordTest.java} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename org.metafacture.fix/src/test/java/org/metafacture/metafix/{MetafixFieldTest.java => MetafixRecordTest.java} (99%) diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java similarity index 99% rename from org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java rename to org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java index 48af27d7..762953ee 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixFieldTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java @@ -29,14 +29,14 @@ import java.util.Arrays; /** - * Tests Metafix field / record level methods. Following the cheat sheet + * Tests Metafix record level methods. Following the cheat sheet * examples at https://github.com/LibreCat/Catmandu/wiki/Fixes-Cheat-Sheet * * @author Fabian Steeg */ @ExtendWith(MockitoExtension.class) @SuppressWarnings("checkstyle:MultipleStringLiterals") -public class MetafixFieldTest { +public class MetafixRecordTest { @RegisterExtension private MockitoRule mockitoRule = MockitoJUnit.rule(); @@ -44,7 +44,7 @@ public class MetafixFieldTest { @Mock private StreamReceiver streamReceiver; - public MetafixFieldTest() { + public MetafixRecordTest() { } @Test From c5343a835d6bea6e1512de9bce33cddb308d9ecf Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Mon, 6 Sep 2021 14:15:33 +0200 Subject: [PATCH 22/55] Don't flatten incoming entites, use nested maps internally (#35) WIP: 4 failing tests with TODO comments for missing entity support --- .../java/org/metafacture/metafix/Metafix.java | 21 ++++++++-- .../metafacture/metafix/MetafixIfTest.java | 13 +++++-- .../metafix/MetafixRecordTest.java | 39 +++++++++++++++---- 3 files changed, 58 insertions(+), 15 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java index e8190636..ecb4a7e5 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java @@ -63,6 +63,7 @@ public class Metafix implements StreamPipe { private int entityCount; private StreamReceiver outputStreamReceiver; private String recordIdentifier; + private List> entities = new ArrayList<>(); public Metafix() { init(); @@ -93,9 +94,8 @@ private void init() { flattener.setReceiver(new DefaultStreamReceiver() { @Override public void literal(final String name, final String value) { - // TODO: set up logging - System.out.printf("Putting '%s':'%s'\n", name, value); - add(currentRecord, name, value); + // TODO: keep flattener as option? + // add(currentRecord, name, value); } }); } @@ -170,8 +170,14 @@ public void startEntity(final String name) { throw new IllegalArgumentException("Entity name must not be null."); } ++entityCount; + final Integer currentEntityIndex = entityCountStack.peek() - 1; + final Map previousEntity = currentEntityIndex < 0 || + entities.size() <= currentEntityIndex ? null : entities.get(currentEntityIndex); entityCountStack.push(Integer.valueOf(entityCount)); flattener.startEntity(name); + final Map currentEntity = new LinkedHashMap<>(); + entities.add(currentEntity); + (previousEntity != null ? previousEntity : currentRecord).put(name, currentEntity); } @Override @@ -182,7 +188,14 @@ public void endEntity() { @Override public void literal(final String name, final String value) { - flattener.literal(name, value); + // TODO: set up logging + System.out.printf("Putting '%s':'%s'\n", name, value); + final Integer currentEntityIndex = entityCountStack.peek() - 1; + final Map currentEntity = currentEntityIndex < 0 || + entities.size() <= currentEntityIndex ? null : entities.get(currentEntityIndex); + add(currentEntity != null ? currentEntity : currentRecord, name, value); + // TODO: keep flattener as option? + // flattener.literal(name, value); } @Override diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java index f49a2be7..7bc02f36 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java @@ -325,7 +325,7 @@ public void ifAllMatch() { @Test public void ifAnyMatchNested() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// TODO: dot notation in match etc. "if any_match('author.name.label', '.*University.*')", // " add_field('author.type', 'Organization')", // "end"), // @@ -345,14 +345,19 @@ public void ifAnyMatchNested() { i.endEntity(); i.endEntity(); i.endRecord(); - }, o -> { + }, (o, f) -> { o.get().startRecord("1"); - o.get().literal("author.name.label", "Max"); + o.get().startEntity("author"); + o.get().startEntity("name"); + o.get().literal("label", "Max"); + f.apply(2).endEntity(); o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("author.name.label", "Some University"); // TODO: fix entity -> entity o.get().startEntity("author"); + o.get().startEntity("name"); + o.get().literal("label", "Some University"); + o.get().endEntity(); o.get().literal("type", "Organization"); o.get().endEntity(); o.get().endRecord(); diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java index 762953ee..016c081f 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java @@ -47,6 +47,28 @@ public class MetafixRecordTest { public MetafixRecordTest() { } + @Test + public void entitiesPassThrough() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "vacuum()"), // + i -> { + i.startRecord("1"); + i.startEntity("deep"); + i.startEntity("nested"); + i.literal("key", "val"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("deep"); + o.get().startEntity("nested"); + o.get().literal("key", "val"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + @Test public void setEmpty() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// @@ -126,9 +148,8 @@ public void add() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("my.name", "max"); // TODO: fix entity -> entity o.get().startEntity("my"); - o.get().literal("name", "[patrick, nicolas]"); // TODO: fix list -> entity + o.get().literal("name", "[max, patrick, nicolas]"); // TODO: fix list -> entity o.get().endEntity(); o.get().endRecord(); // @@ -142,7 +163,7 @@ public void add() { @Test public void move() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// TODO: dot noation in move_field "move_field('my.name','your.name')", "move_field('missing','whatever')"), // i -> { @@ -162,7 +183,9 @@ public void move() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("your.name", "max"); + o.get().startEntity("your"); + o.get().literal("name", "max"); + o.get().endEntity(); o.get().endRecord(); // o.get().startRecord("3"); @@ -172,7 +195,7 @@ public void move() { @Test public void copy() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// TODO dot notation in copy_field "copy_field('your.name','your.name2')"), // i -> { i.startRecord("1"); @@ -191,8 +214,10 @@ public void copy() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().literal("your.name", "max"); - o.get().literal("your.name2", "max"); + o.get().startEntity("your"); + o.get().literal("name", "max"); + o.get().literal("name2", "max"); + o.get().endEntity(); o.get().endRecord(); // o.get().startRecord("3"); From f08252a24224fde4404a830dc9ad1c1550fbaea6 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Tue, 7 Sep 2021 10:36:38 +0200 Subject: [PATCH 23/55] Support dot notation for nested maps in remove_field (#35) WIP: 3 failing tests with TODO comments for missing entity support --- .../org/metafacture/metafix/FixMethod.java | 33 +++++++++- .../metafix/MetafixRecordTest.java | 61 ++++++++++++++++++- 2 files changed, 90 insertions(+), 4 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index d6d7db85..d2d167d7 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -42,7 +42,7 @@ enum FixMethod { public void apply(final Map record, final List params, final Map options) { record.remove(params.get(0)); - insert(InsertMode.REPLACE, record, params.get(0).split("\\."), params.get(1)); + insert(InsertMode.REPLACE, record, split(params.get(0)), params.get(1)); } }, set_array { @@ -103,7 +103,7 @@ public void apply(final Map record, final List params, add_field { public void apply(final Map record, final List params, final Map options) { - insert(InsertMode.APPEND, record, params.get(0).split("\\."), params.get(1)); + insert(InsertMode.APPEND, record, split(params.get(0)), params.get(1)); } }, @@ -128,9 +128,32 @@ public void apply(final Map record, final List params, public void apply(final Map record, final List params, final Map options) { params.forEach(p -> { - record.remove(p); + remove(record, split(p)); }); } + + private Object remove(final Map map, final String[] keys) { + final String currentKey = keys[0]; + if (keys.length == 1) { + map.remove(currentKey); + } + if (!map.containsKey(currentKey)) { + return map; + } + final String[] remainingKeys = Arrays.copyOfRange(keys, 1, keys.length); + return removeNested(map, currentKey, remainingKeys); + } + + private Object removeNested(final Map map, final String currentKey, + final String[] remainingKeys) { + final Object nested = map.get(currentKey); + if (!(nested instanceof Map)) { + throw new IllegalStateException("Nested non-map: " + nested); + } + @SuppressWarnings("unchecked") + final Object result = remove((Map) nested, remainingKeys); + return result; + } }, format { public void apply(final Map record, final List params, @@ -330,6 +353,10 @@ private static Object insertNested(final InsertMode mode, final Map { @@ -241,6 +241,65 @@ public void remove() { // i.startRecord("3"); i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().startEntity("your"); + o.get().endEntity(); + o.get().endRecord(); + // + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + public void removeLiteralAndEntity() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "remove_field('your.name')", // + "remove_field('your')"), // + i -> { + i.startRecord("1"); + i.endRecord(); + // + i.startRecord("2"); + i.startEntity("your"); + i.literal("name", "max"); + i.endEntity(); + i.endRecord(); + // + i.startRecord("3"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().endRecord(); + // + o.get().startRecord("2"); + o.get().endRecord(); + // + o.get().startRecord("3"); + o.get().endRecord(); + }); + } + + @Test + public void removeEntity() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "remove_field('your')"), // + i -> { + i.startRecord("1"); + i.endRecord(); + // + i.startRecord("2"); + i.startEntity("your"); + i.literal("name", "max"); + i.endEntity(); + i.endRecord(); + // + i.startRecord("3"); + i.endRecord(); }, o -> { o.get().startRecord("1"); o.get().endRecord(); From c36235cbff1ab3a6cbaa3f9bd22ecd7dc43a9931 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Tue, 7 Sep 2021 11:32:53 +0200 Subject: [PATCH 24/55] Support dot notation for nested maps in copy and move (#35) WIP: 2 failing tests with TODO comments for missing entity support --- .../org/metafacture/metafix/FixMethod.java | 93 ++++++++++++------- .../metafacture/metafix/MetafixIfTest.java | 4 +- .../metafix/MetafixRecordTest.java | 2 + 3 files changed, 65 insertions(+), 34 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index d2d167d7..6cc14c51 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -105,23 +105,18 @@ public void apply(final Map record, final List params, final Map options) { insert(InsertMode.APPEND, record, split(params.get(0)), params.get(1)); } - }, move_field { public void apply(final Map record, final List params, final Map options) { - final String oldFieldName = params.get(0); - final String newFieldName = params.get(1); - record.put(newFieldName, record.get(oldFieldName)); - record.remove(oldFieldName); + copy(record, params); + remove(record, split(params.get(0))); } }, copy_field { public void apply(final Map record, final List params, final Map options) { - final String oldName = params.get(0); - final String newName = params.get(1); - Metafix.add(record, newName, record.get(oldName)); + copy(record, params); } }, remove_field { @@ -131,29 +126,6 @@ public void apply(final Map record, final List params, remove(record, split(p)); }); } - - private Object remove(final Map map, final String[] keys) { - final String currentKey = keys[0]; - if (keys.length == 1) { - map.remove(currentKey); - } - if (!map.containsKey(currentKey)) { - return map; - } - final String[] remainingKeys = Arrays.copyOfRange(keys, 1, keys.length); - return removeNested(map, currentKey, remainingKeys); - } - - private Object removeNested(final Map map, final String currentKey, - final String[] remainingKeys) { - final Object nested = map.get(currentKey); - if (!(nested instanceof Map)) { - throw new IllegalStateException("Nested non-map: " + nested); - } - @SuppressWarnings("unchecked") - final Object result = remove((Map) nested, remainingKeys); - return result; - } }, format { public void apply(final Map record, final List params, @@ -308,6 +280,7 @@ private Map fileMap(final String location, final String separato } }; + private static final String NESTED_NON_MAP = "Nested non-map: "; private static final String EMPTY = ""; private static final String APPEND = ".$append"; @@ -346,13 +319,69 @@ private static Object insertNested(final InsertMode mode, final Map) nested, remainingKeys, value); return result; } + @SuppressWarnings("checkstyle:ReturnCount") + private static String find(final Map map, final String[] keys) { + final String currentKey = keys[0]; + if (!map.containsKey(currentKey)) { + return null; + } + if (keys.length == 1) { + return map.get(currentKey).toString(); + } + final String[] remainingKeys = Arrays.copyOfRange(keys, 1, keys.length); + return findNested(map, currentKey, remainingKeys); + } + + private static String findNested(final Map map, final String currentKey, + final String[] remainingKeys) { + final Object nested = map.get(currentKey); + if (!(nested instanceof Map)) { + throw new IllegalStateException(NESTED_NON_MAP + nested); + } + @SuppressWarnings("unchecked") + final String result = find((Map) nested, remainingKeys); + return result; + } + + private static Object remove(final Map map, final String[] keys) { + final String currentKey = keys[0]; + if (keys.length == 1) { + map.remove(currentKey); + } + if (!map.containsKey(currentKey)) { + return map; + } + final String[] remainingKeys = Arrays.copyOfRange(keys, 1, keys.length); + return removeNested(map, currentKey, remainingKeys); + } + + private static Object removeNested(final Map map, final String currentKey, + final String[] remainingKeys) { + final Object nested = map.get(currentKey); + if (!(nested instanceof Map)) { + throw new IllegalStateException(NESTED_NON_MAP + nested); + } + @SuppressWarnings("unchecked") + final Object result = remove((Map) nested, remainingKeys); + return result; + } + + private static void copy(final Map record, final List params) { + final String oldName = params.get(0); + final String newName = params.get(1); + final String value = find(record, split(oldName)); + if (value != null) { + insert(InsertMode.APPEND, record, split(newName), value); + } + } + private static String[] split(final String s) { return s.split("\\."); } diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java index 7bc02f36..d3315e10 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java @@ -220,7 +220,7 @@ public void ifContainMoveField() { @Test public void moveAndAddIfContain() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// TODO: dot notation in contain etc. "move_field('name', 'author.name')", "if all_contain('author.name', 'University')", // " add_field('author.type', 'Organization')", // @@ -231,8 +231,8 @@ public void moveAndAddIfContain() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("author.name", "A University"); // TODO: fix entity -> entity o.get().startEntity("author"); + o.get().literal("name", "A University"); o.get().literal("type", "Organization"); o.get().endEntity(); o.get().endRecord(); diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java index 08fcbf07..233a6248 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java @@ -183,6 +183,8 @@ public void move() { o.get().endRecord(); // o.get().startRecord("2"); + o.get().startEntity("my"); + o.get().endEntity(); o.get().startEntity("your"); o.get().literal("name", "max"); o.get().endEntity(); From e19352cc1f4083ffadc7984981b57031460f98b5 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Tue, 7 Sep 2021 15:50:29 +0200 Subject: [PATCH 25/55] Support dot notation for nested maps in predicates (#35) Used in `if` statements with `(all|any|none)_(contain|equal|match)` --- .../org/metafacture/metafix/FixMethod.java | 16 +++++++++------- .../org/metafacture/metafix/FixPredicate.java | 7 ++++--- .../java/org/metafacture/metafix/Metafix.java | 1 + .../org/metafacture/metafix/MetafixIfTest.java | 18 ++++++++++-------- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index 6cc14c51..16a90a9b 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -327,26 +327,26 @@ private static Object insertNested(final InsertMode mode, final Map map, final String[] keys) { + static Object find(final Map map, final String[] keys) { final String currentKey = keys[0]; if (!map.containsKey(currentKey)) { return null; } if (keys.length == 1) { - return map.get(currentKey).toString(); + return map.get(currentKey); } final String[] remainingKeys = Arrays.copyOfRange(keys, 1, keys.length); return findNested(map, currentKey, remainingKeys); } - private static String findNested(final Map map, final String currentKey, + private static Object findNested(final Map map, final String currentKey, final String[] remainingKeys) { final Object nested = map.get(currentKey); if (!(nested instanceof Map)) { throw new IllegalStateException(NESTED_NON_MAP + nested); } @SuppressWarnings("unchecked") - final String result = find((Map) nested, remainingKeys); + final Object result = find((Map) nested, remainingKeys); return result; } @@ -376,13 +376,15 @@ private static Object removeNested(final Map map, final String c private static void copy(final Map record, final List params) { final String oldName = params.get(0); final String newName = params.get(1); - final String value = find(record, split(oldName)); + final Object value = find(record, split(oldName)); if (value != null) { - insert(InsertMode.APPEND, record, split(newName), value); + Metafix.asList(value).forEach(v -> { + insert(InsertMode.APPEND, record, split(newName), value.toString()); + }); } } - private static String[] split(final String s) { + static String[] split(final String s) { return s.split("\\."); } diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixPredicate.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixPredicate.java index be722dc4..6d880836 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixPredicate.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixPredicate.java @@ -62,14 +62,15 @@ public boolean test(final Map record, final FixPredicate p, @Override public boolean test(final Map record, final FixPredicate p, final List params) { - final String fieldName = params.get(0); + final Object fieldValue = FixMethod.find(record, FixMethod.split(params.get(0))); final String valueToTest = params.get(1); - return !record.containsKey(fieldName) || Metafix.asList(record.get(fieldName)).stream().noneMatch(p.of(valueToTest)); + return fieldValue == null || Metafix.asList(fieldValue).stream().noneMatch(p.of(valueToTest)); } }; boolean test(final Map record, final String fieldName, final Predicate> f) { - return record.containsKey(fieldName) && f.test(Metafix.asList(record.get(fieldName)).stream()); + final Object value = FixMethod.find(record, FixMethod.split(fieldName)); + return value != null && f.test(Metafix.asList(value).stream()); } abstract boolean test(Map record, FixPredicate p, List params); diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java index ecb4a7e5..5b9a5d94 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java @@ -115,6 +115,7 @@ public void startRecord(final String identifier) { entityCount = 0; entityCountStack.add(Integer.valueOf(entityCount)); recordIdentifier = identifier; + entities = new ArrayList<>(); } @Override diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java index d3315e10..1fd2f172 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java @@ -129,34 +129,36 @@ public void ifAll() { @Test public void ifNone() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if none_contain('name', 'University')", // + "if none_contain('author.name', 'University')", // " add_field('type', 'Person')", // "end"), // i -> { i.startRecord("1"); + i.startEntity("author"); i.literal("name", "Mary"); i.literal("name", "A University"); + i.endEntity(); i.endRecord(); // i.startRecord("2"); + i.startEntity("author"); i.literal("name", "Max"); i.literal("name", "Mary"); + i.endEntity(); i.endRecord(); // i.startRecord("3"); i.endRecord(); - }, o -> { + }, (o, f) -> { o.get().startRecord("1"); - o.get().startEntity("name"); - o.get().literal("", "Mary"); - o.get().literal("", "A University"); + o.get().startEntity("author"); + o.get().literal("name", "[Mary, A University]"); // TODO: fix list -> entity o.get().endEntity(); o.get().endRecord(); // o.get().startRecord("2"); - o.get().startEntity("name"); - o.get().literal("", "Max"); - o.get().literal("", "Mary"); + o.get().startEntity("author"); + o.get().literal("name", "[Max, Mary]"); // TODO: fix list -> entity o.get().endEntity(); o.get().literal("type", "Person"); o.get().endRecord(); From e4c440a78aa980388f88e09d4e2f4615a57fb9e8 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Mon, 20 Sep 2021 15:38:58 +0200 Subject: [PATCH 26/55] Support dot notation for nested maps in `path` option of bind (#35) --- .../metafix/RecordTransformer.java | 4 +-- .../metafacture/metafix/MetafixBindTest.java | 26 +++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java index 236121a8..de2d78d0 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java @@ -90,8 +90,8 @@ private void processBind(final Do theDo, final EList params) { if (theDo.getName().equals("list")) { // TODO impl multiple binds via FixBind enum final Map options = options(theDo.getOptions()); final Map fullRecord = new LinkedHashMap<>(record); - final List values = Metafix.asList(record.get(options.get("path"))); - values.forEach(val -> { + final Object values = FixMethod.find(record, FixMethod.split(options.get("path"))); + Metafix.asList(values).stream().filter(val -> val != null).forEach(val -> { // for each val, bind the current record/scope/context to the given var name: record = new LinkedHashMap<>(ImmutableMap.of(options.get("var"), val)); processSubexpressions(theDo.getElements()); diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java index ff2db7fb..12b09c11 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java @@ -71,6 +71,32 @@ public void doList() { }); } + @Test + public void doListPathWithDots() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "do list('path': 'some.name', 'var': 'n')", + " upcase('n')", + " trim('n')", + " copy_field('n', 'author')", + "end", + "remove_field('some')"), // + i -> { + i.startRecord("1"); + i.startEntity("some"); + i.literal("name", " A University"); + i.literal("name", "Max "); + i.endEntity(); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().startEntity("author"); + o.get().literal("", "A UNIVERSITY"); + o.get().literal("", "MAX"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + @Test @Disabled // implement list bind for entities / fix internal entity structure public void doListEntitesToLiterals() { From e1b06fbecf34ef0bec03d78c5f0d25acdfbf96c6 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Mon, 20 Sep 2021 15:54:28 +0200 Subject: [PATCH 27/55] Support dot notation for nested maps in `paste` method (#35) --- .../org/metafacture/metafix/FixMethod.java | 6 +++--- .../metafix/MetafixRecordTest.java | 20 ++++++++++++++----- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index 16a90a9b..b0a47a5c 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -183,10 +183,10 @@ private Map getNamedGroups(final Pattern regex) { public void apply(final Map record, final List params, final Map options) { final String joinChar = options.get("join_char"); - record.put(params.get(0), + insert(InsertMode.REPLACE, record, split(params.get(0)), params.subList(1, params.size()).stream() - .filter(k -> literalString(k) || record.containsKey(k)) - .map(k -> literalString(k) ? k.substring(1) : Metafix.asList(record.get(k)).iterator().next()) + .filter(k -> literalString(k) || find(record, split(k)) != null) + .map(k -> literalString(k) ? k.substring(1) : Metafix.asList(find(record, split(k))).iterator().next()) .map(Object::toString).collect(Collectors.joining(joinChar != null ? joinChar : " "))); } diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java index 233a6248..417667bd 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java @@ -352,18 +352,24 @@ public void setHash() { @Test public void paste() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "paste('my.string','a','b','c','d','e')", - "remove_field('a','b','c','d')"), // + "paste('my.string','m.n.z','m.n.a','m.n.b','m.n.c','m.n.d','m.n.e')", + "remove_field('m')"), // i -> { i.startRecord("1"); + i.startEntity("m"); + i.startEntity("n"); i.literal("a", "eeny"); i.literal("b", "meeny"); i.literal("c", "miny"); i.literal("d", "moe"); + i.endEntity(); + i.endEntity(); i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("my.string", "eeny meeny miny moe"); + o.get().startEntity("my"); + o.get().literal("string", "eeny meeny miny moe"); + o.get().endEntity(); o.get().endRecord(); }); } @@ -382,7 +388,9 @@ public void pasteWithCustomSep() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("my.string", "eeny, meeny, miny, moe"); + o.get().startEntity("my"); + o.get().literal("string", "eeny, meeny, miny, moe"); + o.get().endEntity(); o.get().endRecord(); }); } @@ -401,7 +409,9 @@ public void pasteWithLiteralStrings() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("my.string", "Hi eeny how are you?"); + o.get().startEntity("my"); + o.get().literal("string", "Hi eeny how are you?"); + o.get().endEntity(); o.get().endRecord(); }); } From 878fd8d113f2fa0f968c74f39664cde0a88841a8 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Wed, 22 Sep 2021 14:44:42 +0200 Subject: [PATCH 28/55] Merge repeated entities into single map (#35) --- .../org/metafacture/metafix/FixMethod.java | 4 +-- .../java/org/metafacture/metafix/Metafix.java | 36 ++++++++++++++----- .../metafix/MetafixRecordTest.java | 25 +++++++++++++ 3 files changed, 54 insertions(+), 11 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index b0a47a5c..d796a784 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -295,7 +295,7 @@ private static void applyToFields(final Map record, final List) object).remove(old); } final String val = fun.apply(old.toString()); - record.put(key, object == null ? val : Metafix.asListWith(object, val)); + record.put(key, object == null ? val : Metafix.merged(object, val)); }); } } @@ -399,7 +399,7 @@ void apply(final Map map, final String key, final String value) @Override void apply(final Map map, final String key, final String value) { final Object object = map.get(key); - map.put(key, object == null ? value : Metafix.asListWith(object, value)); + map.put(key, object == null ? value : Metafix.merged(object, value)); } }; abstract void apply(Map map, String key, String value); diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java index 5b9a5d94..431e6e03 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java @@ -176,9 +176,22 @@ public void startEntity(final String name) { entities.size() <= currentEntityIndex ? null : entities.get(currentEntityIndex); entityCountStack.push(Integer.valueOf(entityCount)); flattener.startEntity(name); - final Map currentEntity = new LinkedHashMap<>(); - entities.add(currentEntity); - (previousEntity != null ? previousEntity : currentRecord).put(name, currentEntity); + entities.add(currentEntity(name, previousEntity)); + } + + private Map currentEntity(final String name, final Map previousEntity) { + final Object existingValue = previousEntity != null ? previousEntity.get(name) : null; + final Map currentEntity; + if (existingValue != null && existingValue instanceof Map) { + @SuppressWarnings("unchecked") + final Map existingEntity = (Map) previousEntity.get(name); + currentEntity = existingEntity; + } + else { + currentEntity = new LinkedHashMap<>(); + add(previousEntity != null ? previousEntity : currentRecord, name, currentEntity); + } + return currentEntity; } @Override @@ -245,14 +258,19 @@ static void addAll(final Map record, final Map v }); } - static void add(final Map record, final String name, final Object val) { - final Object object = record.get(name); - record.put(name, object == null ? val : asListWith(object, val)); + static void add(final Map record, final String name, final Object newValue) { + final Object oldValue = record.get(name); + record.put(name, oldValue == null ? newValue : merged(oldValue, newValue)); } - static List asListWith(final Object object, final Object value) { - final List list = asList(object); - list.add(value); + @SuppressWarnings("unchecked") + static Object merged(final Object object1, final Object object2) { + if (object1 instanceof Map && object2 instanceof Map) { + ((Map) object1).putAll((Map) object2); + return object1; + } + final List list = asList(object1); + list.add(object2); return list; } diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java index 417667bd..05d6f929 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java @@ -69,6 +69,31 @@ public void entitiesPassThrough() { }); } + @Test + public void entitiesPassThroughRepeatEntity() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "vacuum()"), // + i -> { + i.startRecord("1"); + i.startEntity("deep"); + i.startEntity("nested"); + i.literal("key", "val1"); + i.endEntity(); + i.startEntity("nested"); + i.literal("key", "val2"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("deep"); + o.get().startEntity("nested"); + o.get().literal("key", "[val1, val2]"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + @Test public void setEmpty() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// From e5b395a9d4ec54ba82b2969426905756a9f6f798 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Thu, 23 Sep 2021 18:01:54 +0200 Subject: [PATCH 29/55] Fix output for arrays, always emit like `field[].1` (#35) --- .../java/org/metafacture/metafix/Metafix.java | 13 +++-- .../metafacture/metafix/MetafixBindTest.java | 12 ++-- .../metafacture/metafix/MetafixIfTest.java | 55 ++++++++++-------- .../metafix/MetafixLookupTest.java | 24 ++++---- .../metafix/MetafixMethodTest.java | 44 +++++++------- .../metafix/MetafixRecordTest.java | 57 +++++++++++-------- 6 files changed, 111 insertions(+), 94 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java index 431e6e03..432085bb 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java @@ -142,24 +142,25 @@ private void emit(final Object key, final Object val) { if (val == null) { return; } - final List vals = val instanceof List ? (List) val : Arrays.asList(val); + final List vals = asList(val); final boolean isMulti = vals.size() > 1; if (isMulti) { - outputStreamReceiver.startEntity(key.toString()); + outputStreamReceiver.startEntity(key.toString() + "[]"); } - vals.forEach(value -> { + for (int i = 0; i < vals.size(); ++i) { + final Object value = vals.get(i); if (value instanceof Map) { final Map nested = (Map) value; outputStreamReceiver.startEntity(isMulti ? "" : key.toString()); nested.entrySet().forEach(nestedEntry -> { - emit(nestedEntry.getKey(), Arrays.asList(nestedEntry.getValue())); + emit(nestedEntry.getKey(), asList(nestedEntry.getValue())); }); outputStreamReceiver.endEntity(); } else { - outputStreamReceiver.literal(isMulti ? "" : key.toString(), value.toString()); + outputStreamReceiver.literal(isMulti ? (i + 1) + "" : key.toString(), value.toString()); } - }); + } if (isMulti) { outputStreamReceiver.endEntity(); } diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java index 12b09c11..21d33a95 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java @@ -63,9 +63,9 @@ public void doList() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().startEntity("author"); - o.get().literal("", "A UNIVERSITY"); - o.get().literal("", "MAX"); + o.get().startEntity("author[]"); + o.get().literal("1", "A UNIVERSITY"); + o.get().literal("2", "MAX"); o.get().endEntity(); o.get().endRecord(); }); @@ -89,9 +89,9 @@ public void doListPathWithDots() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().startEntity("author"); - o.get().literal("", "A UNIVERSITY"); - o.get().literal("", "MAX"); + o.get().startEntity("author[]"); + o.get().literal("1", "A UNIVERSITY"); + o.get().literal("2", "MAX"); o.get().endEntity(); o.get().endRecord(); }); diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java index 1fd2f172..2bdfbcfd 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java @@ -67,17 +67,17 @@ public void ifAny() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().startEntity("name"); - o.get().literal("", "Mary"); - o.get().literal("", "A University"); + o.get().startEntity("name[]"); + o.get().literal("1", "Mary"); + o.get().literal("2", "A University"); o.get().endEntity(); o.get().literal("type", "Organization"); o.get().endRecord(); // o.get().startRecord("2"); - o.get().startEntity("name"); - o.get().literal("", "Mary"); - o.get().literal("", "Max"); + o.get().startEntity("name[]"); + o.get().literal("1", "Mary"); + o.get().literal("2", "Max"); o.get().endEntity(); o.get().endRecord(); // @@ -107,16 +107,16 @@ public void ifAll() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().startEntity("name"); - o.get().literal("", "Mary"); - o.get().literal("", "A University"); + o.get().startEntity("name[]"); + o.get().literal("1", "Mary"); + o.get().literal("2", "A University"); o.get().endEntity(); o.get().endRecord(); // o.get().startRecord("2"); - o.get().startEntity("name"); - o.get().literal("", "Great University"); - o.get().literal("", "A University"); + o.get().startEntity("name[]"); + o.get().literal("1", "Great University"); + o.get().literal("2", "A University"); o.get().endEntity(); o.get().literal("type", "Organization"); o.get().endRecord(); @@ -127,6 +127,7 @@ public void ifAll() { } @Test + @SuppressWarnings("checkstyle:ExecutableStatementCount") public void ifNone() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// "if none_contain('author.name', 'University')", // @@ -152,14 +153,18 @@ public void ifNone() { }, (o, f) -> { o.get().startRecord("1"); o.get().startEntity("author"); - o.get().literal("name", "[Mary, A University]"); // TODO: fix list -> entity - o.get().endEntity(); + o.get().startEntity("name[]"); + o.get().literal("1", "Mary"); + o.get().literal("2", "A University"); + f.apply(2).endEntity(); o.get().endRecord(); // o.get().startRecord("2"); o.get().startEntity("author"); - o.get().literal("name", "[Max, Mary]"); // TODO: fix list -> entity - o.get().endEntity(); + o.get().startEntity("name[]"); + o.get().literal("1", "Max"); + o.get().literal("2", "Mary"); + f.apply(2).endEntity(); o.get().literal("type", "Person"); o.get().endRecord(); // @@ -282,9 +287,9 @@ public void ifAnyMatch() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().startEntity("name"); - o.get().literal("", "Some University"); - o.get().literal("", "Filibandrina"); + o.get().startEntity("name[]"); + o.get().literal("1", "Some University"); + o.get().literal("2", "Filibandrina"); o.get().endEntity(); o.get().literal("type", "Organization"); o.get().endRecord(); @@ -309,16 +314,16 @@ public void ifAllMatch() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().startEntity("name"); - o.get().literal("", "Max"); - o.get().literal("", "A University"); + o.get().startEntity("name[]"); + o.get().literal("1", "Max"); + o.get().literal("2", "A University"); o.get().endEntity(); o.get().endRecord(); // o.get().startRecord("2"); - o.get().startEntity("name"); - o.get().literal("", "Some University"); - o.get().literal("", "University Filibandrina"); + o.get().startEntity("name[]"); + o.get().literal("1", "Some University"); + o.get().literal("2", "University Filibandrina"); o.get().endEntity(); o.get().literal("type", "Organization"); o.get().endRecord(); diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java index 84815b46..f5ad324f 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java @@ -68,10 +68,10 @@ public void inline() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().startEntity("title"); - o.get().literal("", "Alohaeha"); - o.get().literal("", "Moin zäme"); - o.get().literal("", "Tach"); + o.get().startEntity("title[]"); + o.get().literal("1", "Alohaeha"); + o.get().literal("2", "Moin zäme"); + o.get().literal("3", "Tach"); o.get().endEntity(); o.get().endRecord(); // @@ -102,10 +102,10 @@ public void csv() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().startEntity("title"); - o.get().literal("", "Alohaeha"); - o.get().literal("", "Moin zäme"); - o.get().literal("", "Tach"); + o.get().startEntity("title[]"); + o.get().literal("1", "Alohaeha"); + o.get().literal("2", "Moin zäme"); + o.get().literal("3", "Tach"); o.get().endEntity(); o.get().endRecord(); // @@ -135,10 +135,10 @@ public void tsv() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().startEntity("title"); - o.get().literal("", "Alohaeha"); - o.get().literal("", "Moin zäme"); - o.get().literal("", "Tach"); + o.get().startEntity("title[]"); + o.get().literal("1", "Alohaeha"); + o.get().literal("2", "Moin zäme"); + o.get().literal("3", "Tach"); o.get().endEntity(); o.get().endRecord(); // diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java index 7e73dabc..fbbfe097 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java @@ -69,9 +69,9 @@ public void upcase() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().startEntity("title"); - o.get().literal("", "MARC"); - o.get().literal("", "JSON"); + o.get().startEntity("title[]"); + o.get().literal("1", "MARC"); + o.get().literal("2", "JSON"); o.get().endEntity(); o.get().endRecord(); // @@ -100,9 +100,9 @@ public void downcase() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().startEntity("title"); - o.get().literal("", "marc"); - o.get().literal("", "json"); + o.get().startEntity("title[]"); + o.get().literal("1", "marc"); + o.get().literal("2", "json"); o.get().endEntity(); o.get().endRecord(); // @@ -131,9 +131,9 @@ public void capitalize() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().startEntity("title"); - o.get().literal("", "Marc"); - o.get().literal("", "Json"); + o.get().startEntity("title[]"); + o.get().literal("1", "Marc"); + o.get().literal("2", "Json"); o.get().endEntity(); o.get().endRecord(); // @@ -162,9 +162,9 @@ public void substring() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().startEntity("title"); - o.get().literal("", "m"); - o.get().literal("", "j"); + o.get().startEntity("title[]"); + o.get().literal("1", "m"); + o.get().literal("2", "j"); o.get().endEntity(); o.get().endRecord(); // @@ -194,9 +194,9 @@ public void substringWithVar() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().startEntity("title"); - o.get().literal("", "ma"); - o.get().literal("", "js"); + o.get().startEntity("title[]"); + o.get().literal("1", "ma"); + o.get().literal("2", "js"); o.get().endEntity(); o.get().endRecord(); // @@ -225,9 +225,9 @@ public void trim() { o.get().endRecord(); // o.get().startRecord("2"); - o.get().startEntity("title"); - o.get().literal("", "marc"); - o.get().literal("", "json"); + o.get().startEntity("title[]"); + o.get().literal("1", "marc"); + o.get().literal("2", "json"); o.get().endEntity(); o.get().endRecord(); // @@ -262,10 +262,10 @@ public void parseText() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().startEntity("date"); - o.get().literal("", "2015"); - o.get().literal("", "03"); - o.get().literal("", "07"); + o.get().startEntity("date[]"); + o.get().literal("1", "2015"); + o.get().literal("2", "03"); + o.get().literal("3", "07"); o.get().endEntity(); o.get().endRecord(); }); diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java index 05d6f929..56af40a5 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java @@ -70,6 +70,7 @@ public void entitiesPassThrough() { } @Test + @SuppressWarnings("checkstyle:MagicNumber") public void entitiesPassThroughRepeatEntity() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// "vacuum()"), // @@ -88,8 +89,10 @@ public void entitiesPassThroughRepeatEntity() { o.get().startRecord("1"); o.get().startEntity("deep"); o.get().startEntity("nested"); - o.get().literal("key", "[val1, val2]"); - f.apply(2).endEntity(); + o.get().startEntity("key[]"); + o.get().literal("1", "val1"); + o.get().literal("2", "val2"); + f.apply(3).endEntity(); o.get().endRecord(); }); } @@ -149,6 +152,7 @@ public void setExisting() { } @Test + @SuppressWarnings("checkstyle:ExecutableStatementCount") public void add() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// "add_field('my.name','patrick')", @@ -168,20 +172,27 @@ public void add() { }, (o, f) -> { o.get().startRecord("1"); o.get().startEntity("my"); - o.get().literal("name", "[patrick, nicolas]"); // TODO: fix list -> entity - o.get().endEntity(); + o.get().startEntity("name[]"); + o.get().literal("1", "patrick"); + o.get().literal("2", "nicolas"); + f.apply(2).endEntity(); o.get().endRecord(); // o.get().startRecord("2"); o.get().startEntity("my"); - o.get().literal("name", "[max, patrick, nicolas]"); // TODO: fix list -> entity - o.get().endEntity(); + o.get().startEntity("name[]"); + o.get().literal("1", "max"); + o.get().literal("2", "patrick"); + o.get().literal("3", "nicolas"); + f.apply(2).endEntity(); o.get().endRecord(); // o.get().startRecord("3"); o.get().startEntity("my"); - o.get().literal("name", "[patrick, nicolas]"); // TODO: fix list -> entity - o.get().endEntity(); + o.get().startEntity("name[]"); + o.get().literal("1", "patrick"); + o.get().literal("2", "nicolas"); + f.apply(2).endEntity(); o.get().endRecord(); }); } @@ -348,10 +359,10 @@ public void setArray() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().startEntity("foo"); - o.get().literal("", "a"); - o.get().literal("", "b"); - o.get().literal("", "c"); + o.get().startEntity("foo[]"); + o.get().literal("1", "a"); + o.get().literal("2", "b"); + o.get().literal("3", "c"); o.get().endEntity(); o.get().endRecord(); }); @@ -469,11 +480,11 @@ public void arrayFromHash() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().startEntity("foo"); - o.get().literal("", "a"); - o.get().literal("", "b"); - o.get().literal("", "c"); - o.get().literal("", "d"); + o.get().startEntity("foo[]"); + o.get().literal("1", "a"); + o.get().literal("2", "b"); + o.get().literal("3", "c"); + o.get().literal("4", "d"); o.get().endEntity(); o.get().endRecord(); }); @@ -507,10 +518,10 @@ public void appendArray() { i.endRecord(); }, (o, f) -> { o.get().startRecord("1"); - o.get().startEntity("nums"); - o.get().literal("", "1"); - o.get().literal("", "2"); - o.get().literal("", "3"); + o.get().startEntity("nums[]"); + o.get().literal("1", "1"); + o.get().literal("2", "2"); + o.get().literal("3", "3"); o.get().endEntity(); o.get().endRecord(); }); @@ -526,8 +537,8 @@ public void mixedArray() { i.endRecord(); }, (o, f) -> { o.get().startRecord("1"); - o.get().startEntity("@context"); - o.get().literal("", "https://w3id.org/kim/lrmi-profile/draft/context.jsonld"); + o.get().startEntity("@context[]"); + o.get().literal("1", "https://w3id.org/kim/lrmi-profile/draft/context.jsonld"); o.get().startEntity(""); o.get().literal("@language", "de"); f.apply(2).endEntity(); From b6af4a14d223a32014c4d6a1a16fecee94054bf0 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Mon, 27 Sep 2021 16:20:56 +0200 Subject: [PATCH 30/55] Fix array processing issues, work on `$append` & `$last` (#35) --- .../org/metafacture/metafix/FixMethod.java | 67 +++++++++++++---- .../java/org/metafacture/metafix/Metafix.java | 6 +- .../metafacture/metafix/MetafixBindTest.java | 39 +++++++++- .../metafix/MetafixMethodTest.java | 27 ++++++- .../metafix/MetafixRecordTest.java | 75 +++++++++++++++++++ 5 files changed, 194 insertions(+), 20 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index d796a784..cbdd55cb 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -280,7 +280,7 @@ private Map fileMap(final String location, final String separato } }; - private static final String NESTED_NON_MAP = "Nested non-map: "; + private static final String NESTED = "Nested non-map / non-list: "; private static final String EMPTY = ""; private static final String APPEND = ".$append"; @@ -312,20 +312,53 @@ private static Object insert(final InsertMode mode, final Map ma return map; } + @SuppressWarnings("unchecked") private static Object insertNested(final InsertMode mode, final Map map, final String value, final String currentKey, final String[] remainingKeys) { if (!map.containsKey(currentKey)) { map.put(currentKey, new LinkedHashMap()); } final Object nested = map.get(currentKey); - if (!(nested instanceof Map)) { - throw new IllegalStateException(NESTED_NON_MAP + nested); + final Object result; + if (nested instanceof Map) { + result = insert(mode, (Map) nested, remainingKeys, value); + } + else if (nested instanceof List) { + processList(mode, value, remainingKeys, nested); + result = map.get(currentKey); + } + else { + throw new IllegalStateException(NESTED + nested); } - @SuppressWarnings("unchecked") - final Object result = insert(mode, (Map) nested, remainingKeys, value); return result; } + @SuppressWarnings("unchecked") + private static void processList(final InsertMode mode, final String value, final String[] remainingKeys, + final Object nested) { + final List nestedList = (List) nested; + final Map nestedMap; + switch (remainingKeys[0]) { + case "$append": + nestedMap = new LinkedHashMap<>(); + nestedList.add(nestedMap); + insert(mode, nestedMap, Arrays.copyOfRange(remainingKeys, 1, remainingKeys.length), value); + break; + case "$last": + final Object last = nestedList.get(nestedList.size() - 1); + if (last instanceof Map) { + nestedMap = (Map) last; + insert(mode, nestedMap, Arrays.copyOfRange(remainingKeys, 1, remainingKeys.length), value); + } + break; + default: + nestedMap = new LinkedHashMap<>(); + nestedList.add(nestedMap); + insert(mode, nestedMap, remainingKeys, value); + break; + } + } + @SuppressWarnings("checkstyle:ReturnCount") static Object find(final Map map, final String[] keys) { final String currentKey = keys[0]; @@ -342,12 +375,17 @@ static Object find(final Map map, final String[] keys) { private static Object findNested(final Map map, final String currentKey, final String[] remainingKeys) { final Object nested = map.get(currentKey); - if (!(nested instanceof Map)) { - throw new IllegalStateException(NESTED_NON_MAP + nested); + // TODO: array of maps, like in insertNested + if (nested instanceof List) { + return ((List) nested).stream().map(o -> findNested(map, currentKey, remainingKeys)) + .collect(Collectors.toList()); } - @SuppressWarnings("unchecked") - final Object result = find((Map) nested, remainingKeys); - return result; + if (nested instanceof Map) { + @SuppressWarnings("unchecked") + final Object result = find((Map) nested, remainingKeys); + return result; + } + throw new IllegalStateException(NESTED + nested); } private static Object remove(final Map map, final String[] keys) { @@ -366,7 +404,7 @@ private static Object removeNested(final Map map, final String c final String[] remainingKeys) { final Object nested = map.get(currentKey); if (!(nested instanceof Map)) { - throw new IllegalStateException(NESTED_NON_MAP + nested); + throw new IllegalStateException(NESTED + nested); } @SuppressWarnings("unchecked") final Object result = remove((Map) nested, remainingKeys); @@ -378,9 +416,10 @@ private static void copy(final Map record, final List pa final String newName = params.get(1); final Object value = find(record, split(oldName)); if (value != null) { - Metafix.asList(value).forEach(v -> { - insert(InsertMode.APPEND, record, split(newName), value.toString()); - }); + final List vs = Metafix.asList(value); + for (int i = 0; i < vs.size(); ++i) { + insert(InsertMode.APPEND, record, split(newName), vs.get(i).toString()); + } } } diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java index 432085bb..88e67cd0 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java @@ -143,7 +143,7 @@ private void emit(final Object key, final Object val) { return; } final List vals = asList(val); - final boolean isMulti = vals.size() > 1; + final boolean isMulti = vals.size() > 1 || val instanceof List; if (isMulti) { outputStreamReceiver.startEntity(key.toString() + "[]"); } @@ -153,7 +153,7 @@ private void emit(final Object key, final Object val) { final Map nested = (Map) value; outputStreamReceiver.startEntity(isMulti ? "" : key.toString()); nested.entrySet().forEach(nestedEntry -> { - emit(nestedEntry.getKey(), asList(nestedEntry.getValue())); + emit(nestedEntry.getKey(), nestedEntry.getValue()); }); outputStreamReceiver.endEntity(); } @@ -177,7 +177,7 @@ public void startEntity(final String name) { entities.size() <= currentEntityIndex ? null : entities.get(currentEntityIndex); entityCountStack.push(Integer.valueOf(entityCount)); flattener.startEntity(name); - entities.add(currentEntity(name, previousEntity)); + entities.add(currentEntity(name, previousEntity == null && entities.size() >= 0 ? currentRecord : previousEntity)); } private Map currentEntity(final String name, final Map previousEntity) { diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java index 21d33a95..5fcbb249 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java @@ -97,6 +97,39 @@ public void doListPathWithDots() { }); } + @Test + public void doListWithAppendAndLast() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "do list('path': 'creator', 'var': 'c')", + " set_array('author')", + " copy_field('c.name', 'author.$append.name')", + " add_field('author.$last.type', 'Default')", + "end", + "remove_field('creator')"), // + i -> { + i.startRecord("1"); + i.startEntity("creator"); + i.literal("name", "A University"); + i.endEntity(); + i.startEntity("creator"); + i.literal("name", "Max"); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("author[]"); + o.get().startEntity(""); + o.get().literal("name", "A University"); + // o.get().literal("type", "Default"); // FIXME: bind scope broken + o.get().endEntity(); + o.get().startEntity(""); + o.get().literal("name", "Max"); + o.get().literal("type", "Default"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + @Test @Disabled // implement list bind for entities / fix internal entity structure public void doListEntitesToLiterals() { @@ -118,8 +151,10 @@ public void doListEntitesToLiterals() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("author", "A UNIVERSITY"); - o.get().literal("author", "MAX"); + o.get().startEntity("author[]"); + o.get().literal("1", "A UNIVERSITY"); + o.get().literal("2", "MAX"); + o.get().endEntity(); o.get().endRecord(); }); } diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java index fbbfe097..eac38cba 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java @@ -80,6 +80,29 @@ public void upcase() { }); } + @Test + @Disabled // TODO: All field-level FixMethods don't support dot notation for nested entities yet + public void upcaseDotNotationNested() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "upcase('data.title')"), // + i -> { + i.startRecord("1"); + i.startEntity("data"); + i.literal("title", "marc"); + i.literal("title", "json"); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("data"); + o.get().startEntity("title[]"); + o.get().literal("1", "MARC"); + o.get().literal("2", "JSON"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + @Test public void downcase() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// @@ -247,7 +270,9 @@ public void format() { i.endRecord(); }, o -> { o.get().startRecord("1"); - o.get().literal("number", "41 : 15"); + o.get().startEntity("number[]"); + o.get().literal("1", "41 : 15"); + o.get().endEntity(); o.get().endRecord(); }); } diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java index 56af40a5..c3b89c41 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java @@ -263,6 +263,81 @@ public void copy() { }); } + @Test + public void copyIntoArrayOfStrings() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + // "set_array('author')", <- results in separate objects/entities here + "copy_field('your.name','author.name')", + "remove_field('your')"), // + i -> { + i.startRecord("1"); + i.startEntity("your"); + i.literal("name", "maxi-mi"); + i.literal("name", "maxi-ma"); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("author"); + o.get().startEntity("name[]"); + o.get().literal("1", "maxi-mi"); + o.get().literal("2", "maxi-ma"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void copyIntoArrayOfObjects() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('author')", + "copy_field('your.name','author.name')", + "remove_field('your')"), // + i -> { + i.startRecord("1"); + i.startEntity("your"); + i.literal("name", "max"); + i.endEntity(); + i.startEntity("your"); + i.literal("name", "mo"); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("author[]"); + o.get().startEntity(""); + o.get().literal("name", "max"); + o.get().endEntity(); + o.get().startEntity(""); + o.get().literal("name", "mo"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + + @Test + public void copyIntoArrayTopLevel() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('author')", + "copy_field('your.name', 'author')", + "remove_field('your')"), // + i -> { + i.startRecord("1"); + i.startEntity("your"); + i.literal("name", "maxi-mi"); + i.literal("name", "maxi-ma"); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("author[]"); + o.get().literal("1", "maxi-mi"); + o.get().literal("2", "maxi-ma"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + @Test public void removeLiteral() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// From 3a55ab866f8590bec3905d0aec5ad397c7906579 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Thu, 30 Sep 2021 16:23:45 +0200 Subject: [PATCH 31/55] Support dot notation for nested maps in field-level methods (#35) --- .../org/metafacture/metafix/FixMethod.java | 20 ++++++++-------- .../metafacture/metafix/MetafixBindTest.java | 1 - .../metafix/MetafixLookupTest.java | 24 +++++++++++++++++++ .../metafix/MetafixMethodTest.java | 1 - 4 files changed, 34 insertions(+), 12 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index cbdd55cb..f28fcbc7 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -287,15 +287,15 @@ private Map fileMap(final String location, final String separato private static void applyToFields(final Map record, final List params, final Function fun) { final String key = params.get(0); - if (record.containsKey(key)) { - new ArrayList<>(Metafix.asList(record.get(key))).forEach(old -> { - record.remove(key, old); - final Object object = record.get(key); - if (object instanceof List) { - ((List) object).remove(old); + final Object found = find(record, split(key)); + final boolean containsKey = found != null; + if (containsKey) { + remove(record, split(key)); + new ArrayList<>(Metafix.asList(found)).forEach(old -> { + if (fun != null && old != null) { + final String val = fun.apply(old.toString()); + insert(InsertMode.APPEND, record, split(key), val); } - final String val = fun.apply(old.toString()); - record.put(key, object == null ? val : Metafix.merged(object, val)); }); } } @@ -417,8 +417,8 @@ private static void copy(final Map record, final List pa final Object value = find(record, split(oldName)); if (value != null) { final List vs = Metafix.asList(value); - for (int i = 0; i < vs.size(); ++i) { - insert(InsertMode.APPEND, record, split(newName), vs.get(i).toString()); + for (final Object v : vs.stream().filter(v -> v != null).collect(Collectors.toList())) { + insert(InsertMode.APPEND, record, split(newName), v.toString()); } } } diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java index 5fcbb249..88d75b6c 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java @@ -131,7 +131,6 @@ public void doListWithAppendAndLast() { } @Test - @Disabled // implement list bind for entities / fix internal entity structure public void doListEntitesToLiterals() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// "do list('path': 'creator', 'var': 'c')", diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java index f5ad324f..1cee81d6 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java @@ -80,6 +80,30 @@ public void inline() { }); } + @Test + public void inlineDotNotationNested() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "lookup('data.title', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)"), // + i -> { + i.startRecord("1"); + i.startEntity("data"); + i.literal("title", "Aloha"); + i.literal("title", "Moin"); + i.literal("title", "Hey"); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("data"); + o.get().startEntity("title[]"); + o.get().literal("1", "Alohaeha"); + o.get().literal("2", "Moin zäme"); + o.get().literal("3", "Tach"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + @Test public void csv() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java index eac38cba..6082f3ae 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java @@ -81,7 +81,6 @@ public void upcase() { } @Test - @Disabled // TODO: All field-level FixMethods don't support dot notation for nested entities yet public void upcaseDotNotationNested() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// "upcase('data.title')"), // From 660341798d2a0a605ba57693219f468181be536d Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Thu, 30 Sep 2021 17:06:51 +0200 Subject: [PATCH 32/55] Fix issue when merging lists (#35) --- .../java/org/metafacture/metafix/Metafix.java | 4 +++- .../metafacture/metafix/MetafixBindTest.java | 17 +++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java index 88e67cd0..f85a0262 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java @@ -271,7 +271,9 @@ static Object merged(final Object object1, final Object object2) { return object1; } final List list = asList(object1); - list.add(object2); + asList(object2).forEach(e -> { + list.add(e); + }); return list; } diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java index 88d75b6c..f6b9421c 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java @@ -159,12 +159,12 @@ public void doListEntitesToLiterals() { } @Test - @Disabled // implement list bind for entities / fix internal entity structure public void doListEntitesToEntities() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "do list('path': 'creator', 'var': 'c')", - " copy_field('c.name', 'author.$append.name')", - " if all_contain('c.name', 'University')", // + "do list('path': 'creator.name', 'var': 'c')", + " set_array('author')", + " copy_field('c', 'author.$append.name')", + " if all_contain('c', 'University')", // " add_field('author.$last.type', 'Organization')", // " else", " add_field('author.$last.type', 'Person')", //", @@ -180,16 +180,17 @@ public void doListEntitesToEntities() { i.literal("name", "Max"); i.endEntity(); i.endRecord(); - }, o -> { + }, (o, f) -> { o.get().startRecord("1"); - o.get().startEntity("author"); + o.get().startEntity("author[]"); + o.get().startEntity(""); o.get().literal("name", "A University"); o.get().literal("type", "Organization"); o.get().endEntity(); - o.get().startEntity("author"); + o.get().startEntity(""); o.get().literal("name", "Max"); o.get().literal("type", "Person"); - o.get().endEntity(); + f.apply(2).endEntity(); o.get().endRecord(); }); } From cc6ff0a5ddf7647c8e293676ca67ccb93990e9ac Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Thu, 30 Sep 2021 17:14:34 +0200 Subject: [PATCH 33/55] Add disabled failing tests for unclear entity behavior (#35) --- .../metafacture/metafix/MetafixBindTest.java | 41 +++++++++++++++++++ .../metafix/MetafixRecordTest.java | 30 +++++++++++++- 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java index f6b9421c..2633f809 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java @@ -195,6 +195,47 @@ public void doListEntitesToEntities() { }); } + @Test + @Disabled // TODO: how to handle repeated entities: turn to array vs. merge because it's the same? + public void doListEntitesWithFieldsToEntities() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "do list('path': 'creator', 'var': 'c')", + " set_array('author')", + " copy_field('c.name', 'author.$append.name')", + " if all_contain('c.type', 'corporate')", // + " add_field('author.$last.type', 'Organization')", // + " end", + " if all_contain('c.type', 'personal')", + " add_field('author.$last.type', 'Person')", //", + " end", + "end", + "remove_field('creator')"), // + i -> { + i.startRecord("1"); + i.startEntity("creator"); + i.literal("name", "A University"); + i.literal("type", "corporate"); + i.endEntity(); + i.startEntity("creator"); + i.literal("name", "Max"); + i.literal("type", "personal"); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("author[]"); + o.get().startEntity(""); + o.get().literal("name", "A University"); + o.get().literal("type", "Organization"); + o.get().endEntity(); + o.get().startEntity(""); + o.get().literal("name", "Max"); + o.get().literal("type", "Person"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + @Test @Disabled // implement Fix-style binds with collectors? public void ifInCollector() { diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java index c3b89c41..07145e60 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java @@ -18,6 +18,7 @@ import org.metafacture.framework.StreamReceiver; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.RegisterExtension; @@ -70,8 +71,35 @@ public void entitiesPassThrough() { } @Test - @SuppressWarnings("checkstyle:MagicNumber") + @Disabled // TODO: how to handle repeated entities: turn to array vs. merge because it's the same? public void entitiesPassThroughRepeatEntity() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "vacuum()"), // + i -> { + i.startRecord("1"); + i.startEntity("some"); + i.literal("key", "val1"); + i.endEntity(); + i.startEntity("some"); + i.literal("key", "val2"); + i.endEntity(); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().startEntity("some[]"); + o.get().startEntity(""); + o.get().literal("key", "val1"); + o.get().endEntity(); + o.get().startEntity(""); + o.get().literal("key", "val2"); + f.apply(2).endEntity(); + o.get().endRecord(); + }); + } + + @Test + @SuppressWarnings("checkstyle:MagicNumber") + public void entitiesPassThroughRepeatNestedEntity() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// "vacuum()"), // i -> { From 5b60a58c01d49e4b47658dc2fa8863d4e262784e Mon Sep 17 00:00:00 2001 From: Jens Wille Date: Mon, 4 Oct 2021 14:50:36 +0200 Subject: [PATCH 34/55] Upgrade Gradle wrapper to version 7.2. - `compile` -> `implementation`, `testCompile` -> `testImplementation`, `testRuntime` -> `testRuntimeOnly`: "Could not find method compile()/testCompile()/testRuntime() for arguments [...] on object of type org.gradle.api.internal.artifacts.dsl.dependencies.DefaultDependencyHandler." - `runtime` -> `runtimeOnly`: "Could not get unknown property 'runtime' for configuration container of type org.gradle.api.internal.artifacts.configurations.DefaultConfigurationContainer." - `jcenter` -> `mavenCentral`: "The RepositoryHandler.jcenter() method has been deprecated. This is scheduled to be removed in Gradle 8.0. JFrog announced JCenter's sunset in February 2021. Use mavenCentral() instead. Consult the upgrading guide for further information: https://docs.gradle.org/7.2/userguide/upgrading_version_6.html#jcenter_deprecation" - `main` -> `mainClass`: "The JavaExec.main property has been deprecated. This is scheduled to be removed in Gradle 8.0. Please use the mainClass property instead. See https://docs.gradle.org/7.2/dsl/org.gradle.api.tasks.JavaExec.html#org.gradle.api.tasks.JavaExec:main for more details." - `processResources.dependsOn(generateXtextLanguage)`: "Gradle detected a problem with the following location: '.../org.metafacture.fix/src/main/xtext-gen'. Reason: Task ':metafacture-fix:metafacture-fix:processResources' uses this output of task ':metafacture-fix:metafacture-fix:generateXtextLanguage' without declaring an explicit or implicit dependency. This can lead to incorrect results being produced, depending on what order the tasks are executed. Please refer to https://docs.gradle.org/7.2/userguide/validation_problems.html#implicit_dependency for more details about this problem. This behaviour has been deprecated and is scheduled to be removed in Gradle 8.0. Execution optimizations are disabled to ensure correctness. See https://docs.gradle.org/7.2/userguide/more_about_tasks.html#sec:up_to_date_checks for more details." - `maven` (legacy) -> `maven-publish` (+ compatibility task): "Plugin [id: 'maven'] was not found in any of the following sources: [...]" --- build.gradle | 4 ++-- gradle/wrapper/gradle-wrapper.properties | 3 ++- org.metafacture.fix.ide/build.gradle | 8 +++---- org.metafacture.fix.web/build.gradle | 18 +++++++------- org.metafacture.fix/build.gradle | 30 ++++++++++++++---------- 5 files changed, 35 insertions(+), 28 deletions(-) diff --git a/build.gradle b/build.gradle index 315a47ec..8600dd0a 100644 --- a/build.gradle +++ b/build.gradle @@ -52,12 +52,12 @@ subprojects { targetCompatibility = '1.8' repositories { - jcenter() + mavenCentral() maven githubPackage.invoke("metafacture") } dependencies { - compile platform("org.eclipse.xtext:xtext-dev-bom:${versions.xtext}") + implementation platform("org.eclipse.xtext:xtext-dev-bom:${versions.xtext}") } configurations.all { diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 44e7c4d1..a2e01c0d 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-5.2.1-bin.zip +distributionSha256Sum=f581709a9c35e9cb92e16f585d2c4bc99b2b1a5f85d2badbd3dc6bff59e1e6dd +distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/org.metafacture.fix.ide/build.gradle b/org.metafacture.fix.ide/build.gradle index b438ace3..652bac62 100644 --- a/org.metafacture.fix.ide/build.gradle +++ b/org.metafacture.fix.ide/build.gradle @@ -3,10 +3,10 @@ plugins { } dependencies { - compile project(':metafacture-fix') + implementation project(':metafacture-fix') - compile "org.eclipse.xtext:org.eclipse.xtext.ide:${versions.xtext}" - compile "org.eclipse.xtext:org.eclipse.xtext.xbase.ide:${versions.xtext}" + implementation "org.eclipse.xtext:org.eclipse.xtext.ide:${versions.xtext}" + implementation "org.eclipse.xtext:org.eclipse.xtext.xbase.ide:${versions.xtext}" } apply plugin: 'application' @@ -17,7 +17,7 @@ applicationName = 'xtext-server' shadowJar { from(project.convention.getPlugin(JavaPluginConvention).sourceSets.main.output) - configurations = [project.configurations.runtime] + configurations = [project.configurations.runtimeOnly] exclude( '*.html', diff --git a/org.metafacture.fix.web/build.gradle b/org.metafacture.fix.web/build.gradle index 0c4bdcc8..1f73c7ed 100644 --- a/org.metafacture.fix.web/build.gradle +++ b/org.metafacture.fix.web/build.gradle @@ -3,15 +3,15 @@ plugins { } dependencies { - compile project(':metafacture-fix') - compile project(':org.metafacture.fix.ide') + implementation project(':metafacture-fix') + implementation project(':org.metafacture.fix.ide') - compile "org.eclipse.xtend:org.eclipse.xtend.lib:${versions.xtext}" - compile "org.eclipse.xtext:org.eclipse.xtext.web.servlet:${versions.xtext}" - compile "org.eclipse.xtext:org.eclipse.xtext.xbase.web:${versions.xtext}" - compile "org.webjars:ace:${versions.ace}" - compile "org.webjars:jquery:${versions.jquery}" - compile "org.webjars:requirejs:${versions.requirejs}" + implementation "org.eclipse.xtend:org.eclipse.xtend.lib:${versions.xtext}" + implementation "org.eclipse.xtext:org.eclipse.xtext.web.servlet:${versions.xtext}" + implementation "org.eclipse.xtext:org.eclipse.xtext.xbase.web:${versions.xtext}" + implementation "org.webjars:ace:${versions.ace}" + implementation "org.webjars:jquery:${versions.jquery}" + implementation "org.webjars:requirejs:${versions.requirejs}" providedCompile "org.eclipse.jetty:jetty-annotations:${versions.jetty}" providedCompile "org.slf4j:slf4j-simple:${versions.slf4j}" @@ -27,7 +27,7 @@ dependencies { task jettyRun(type: JavaExec) { dependsOn(sourceSets.main.runtimeClasspath) classpath = sourceSets.main.runtimeClasspath.filter { it.exists() } - main = 'org.metafacture.metafix.web.ServerLauncher' + mainClass = 'org.metafacture.metafix.web.ServerLauncher' standardInput = System.in group = 'run' description = 'Starts an example Jetty server with your language' diff --git a/org.metafacture.fix/build.gradle b/org.metafacture.fix/build.gradle index 71276ca2..6402cd98 100644 --- a/org.metafacture.fix/build.gradle +++ b/org.metafacture.fix/build.gradle @@ -1,19 +1,19 @@ plugins { id 'com.adarshr.test-logger' version '1.7.0' - id 'maven' + id 'maven-publish' } dependencies { - compile "org.eclipse.xtext:org.eclipse.xtext:${versions.xtext}" - compile "org.eclipse.xtext:org.eclipse.xtext.xbase:${versions.xtext}" - compile "com.google.guava:guava:${versions.guava}" + implementation "org.eclipse.xtext:org.eclipse.xtext:${versions.xtext}" + implementation "org.eclipse.xtext:org.eclipse.xtext.xbase:${versions.xtext}" + implementation "com.google.guava:guava:${versions.guava}" - testCompile "org.junit.jupiter:junit-jupiter-api:${versions.junit_jupiter}" - testCompile "org.junit.platform:junit-platform-launcher:${versions.junit_platform}" - testCompile "org.eclipse.xtext:org.eclipse.xtext.testing:${versions.xtext}" - testCompile "org.eclipse.xtext:org.eclipse.xtext.xbase.testing:${versions.xtext}" + testImplementation "org.junit.jupiter:junit-jupiter-api:${versions.junit_jupiter}" + testImplementation "org.junit.platform:junit-platform-launcher:${versions.junit_platform}" + testImplementation "org.eclipse.xtext:org.eclipse.xtext.testing:${versions.xtext}" + testImplementation "org.eclipse.xtext:org.eclipse.xtext.xbase.testing:${versions.xtext}" - testRuntime "org.junit.jupiter:junit-jupiter-engine:${versions.junit_jupiter}" + testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${versions.junit_jupiter}" implementation "org.metafacture:metafacture-commons:${versions.metafacture}" implementation "org.metafacture:metafacture-mangling:${versions.metafacture}" @@ -27,7 +27,7 @@ dependencies { configurations { mwe2 { - extendsFrom compile + extendsFrom implementation } } @@ -42,16 +42,21 @@ test { useJUnitPlatform() } +task install(dependsOn: publishToMavenLocal, + description: "Installs the 'archives' artifacts into the local Maven repository. [deprecated]") { + doFirst { println "This task is deprecated; use 'publishToMavenLocal' instead." } +} + def xtextFile = 'src/main/java/org/metafacture/metafix/Fix.xtext' task validateXtextLanguage(type: JavaExec) { - main = 'org.metafacture.metafix.validation.XtextValidator' + mainClass = 'org.metafacture.metafix.validation.XtextValidator' classpath = sourceSets.main.runtimeClasspath args += xtextFile } task generateXtextLanguage(type: JavaExec) { - main = 'org.eclipse.emf.mwe2.launch.runtime.Mwe2Launcher' + mainClass = 'org.eclipse.emf.mwe2.launch.runtime.Mwe2Launcher' classpath = configurations.mwe2 inputs.file 'src/main/java/org/metafacture/metafix/GenerateFix.mwe2' inputs.file xtextFile @@ -61,6 +66,7 @@ task generateXtextLanguage(type: JavaExec) { args += "rootPath=/${projectDir}/.." } +processResources.dependsOn(generateXtextLanguage) generateXtext.dependsOn(generateXtextLanguage) compileJava.dependsOn(generateXtextLanguage) clean.dependsOn(cleanGenerateXtextLanguage) From f8aab3773de2c9e75852b461e45ed1b7539b6178 Mon Sep 17 00:00:00 2001 From: Jens Wille Date: Tue, 5 Oct 2021 14:23:14 +0200 Subject: [PATCH 35/55] Replace unresolvable `runtimeOnly` configuration with resolvable `runtimeClasspath` configuration for shadowJar task. (5b60a58) "Resolving dependency configuration 'runtimeOnly' is not allowed as it is defined as 'canBeResolved=false'. Instead, a resolvable ('canBeResolved=true') dependency configuration that extends 'runtimeOnly' should be resolved." --- org.metafacture.fix.ide/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/org.metafacture.fix.ide/build.gradle b/org.metafacture.fix.ide/build.gradle index 652bac62..7dac4199 100644 --- a/org.metafacture.fix.ide/build.gradle +++ b/org.metafacture.fix.ide/build.gradle @@ -17,7 +17,7 @@ applicationName = 'xtext-server' shadowJar { from(project.convention.getPlugin(JavaPluginConvention).sourceSets.main.output) - configurations = [project.configurations.runtimeOnly] + configurations = [project.configurations.runtimeClasspath] exclude( '*.html', From f310be809c3fdaf7c0b92fab44c12f2b4d471cd6 Mon Sep 17 00:00:00 2001 From: Jens Wille Date: Wed, 6 Oct 2021 18:52:37 +0200 Subject: [PATCH 36/55] Initial logging setup. --- org.metafacture.fix/build.gradle | 2 ++ .../main/java/org/metafacture/metafix/Metafix.java | 14 +++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/org.metafacture.fix/build.gradle b/org.metafacture.fix/build.gradle index 71276ca2..6c27ec8b 100644 --- a/org.metafacture.fix/build.gradle +++ b/org.metafacture.fix/build.gradle @@ -7,6 +7,7 @@ dependencies { compile "org.eclipse.xtext:org.eclipse.xtext:${versions.xtext}" compile "org.eclipse.xtext:org.eclipse.xtext.xbase:${versions.xtext}" compile "com.google.guava:guava:${versions.guava}" + compile "org.slf4j:slf4j-api:${versions.slf4j}" testCompile "org.junit.jupiter:junit-jupiter-api:${versions.junit_jupiter}" testCompile "org.junit.platform:junit-platform-launcher:${versions.junit_platform}" @@ -14,6 +15,7 @@ dependencies { testCompile "org.eclipse.xtext:org.eclipse.xtext.xbase.testing:${versions.xtext}" testRuntime "org.junit.jupiter:junit-jupiter-engine:${versions.junit_jupiter}" + testRuntime "org.slf4j:slf4j-simple:${versions.slf4j}" implementation "org.metafacture:metafacture-commons:${versions.metafacture}" implementation "org.metafacture:metafacture-mangling:${versions.metafacture}" diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java index f85a0262..38000c8c 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java @@ -25,6 +25,9 @@ import org.metafacture.metafix.fix.Expression; import org.metafacture.metafix.fix.Fix; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.FileNotFoundException; import java.io.FileReader; import java.io.Reader; @@ -53,6 +56,8 @@ public class Metafix implements StreamPipe { public static final Map NO_VARS = Collections.emptyMap(); private static final String ENTITIES_NOT_BALANCED = "Entity starts and ends are not balanced"; + private static final Logger LOG = LoggerFactory.getLogger(Metafix.class); + // TODO: Use SimpleRegexTrie / WildcardTrie for wildcard, alternation and character class support private Map currentRecord = new LinkedHashMap<>(); private Fix fix; @@ -109,7 +114,7 @@ private void buildPipeline(final Reader fixDef, final Map theVar @Override public void startRecord(final String identifier) { currentRecord = new LinkedHashMap<>(); - System.out.printf("Start record: %s\n", currentRecord); + LOG.debug("Start record: {}", identifier); flattener.startRecord(identifier); entityCountStack.clear(); entityCount = 0; @@ -125,12 +130,12 @@ public void endRecord() { throw new IllegalStateException(ENTITIES_NOT_BALANCED); } flattener.endRecord(); - System.out.printf("End record, walking fix: %s\n", currentRecord); + LOG.debug("End record, walking fix: {}", currentRecord); final RecordTransformer transformer = new RecordTransformer(currentRecord, vars, fix); currentRecord = transformer.transform(); if (!currentRecord.containsKey("__reject")) { outputStreamReceiver.startRecord(recordIdentifier); - System.out.println("Sending results to " + outputStreamReceiver); + LOG.debug("Sending results to {}", outputStreamReceiver); currentRecord.keySet().forEach(k -> { emit(k, currentRecord.get(k)); }); @@ -203,8 +208,7 @@ public void endEntity() { @Override public void literal(final String name, final String value) { - // TODO: set up logging - System.out.printf("Putting '%s':'%s'\n", name, value); + LOG.debug("Putting '{}': '{}'", name, value); final Integer currentEntityIndex = entityCountStack.peek() - 1; final Map currentEntity = currentEntityIndex < 0 || entities.size() <= currentEntityIndex ? null : entities.get(currentEntityIndex); From 24d72f04a0516f15f5171b4d6e2ff7250a0790fb Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Mon, 11 Oct 2021 13:46:56 +0200 Subject: [PATCH 37/55] Make internal ID available in `_id`, don't emit `_` fields (#35) --- .../main/java/org/metafacture/metafix/Metafix.java | 4 +++- .../org/metafacture/metafix/MetafixRecordTest.java | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java index f85a0262..59081830 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/Metafix.java @@ -18,6 +18,7 @@ package org.metafacture.metafix; +import org.metafacture.framework.StandardEventNames; import org.metafacture.framework.StreamPipe; import org.metafacture.framework.StreamReceiver; import org.metafacture.framework.helpers.DefaultStreamReceiver; @@ -116,6 +117,7 @@ public void startRecord(final String identifier) { entityCountStack.add(Integer.valueOf(entityCount)); recordIdentifier = identifier; entities = new ArrayList<>(); + literal(StandardEventNames.ID, identifier); } @Override @@ -131,7 +133,7 @@ public void endRecord() { if (!currentRecord.containsKey("__reject")) { outputStreamReceiver.startRecord(recordIdentifier); System.out.println("Sending results to " + outputStreamReceiver); - currentRecord.keySet().forEach(k -> { + currentRecord.keySet().stream().filter(k -> !k.startsWith("_")).forEach(k -> { emit(k, currentRecord.get(k)); }); outputStreamReceiver.endRecord(); diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java index 07145e60..4fb3c033 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java @@ -70,6 +70,20 @@ public void entitiesPassThrough() { }); } + @Test + public void internalIdUsage() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + "copy_field('_id', id)"), + i -> { + i.startRecord("1"); + i.endRecord(); + }, (o, f) -> { + o.get().startRecord("1"); + o.get().literal("id", "1"); + o.get().endRecord(); + }); + } + @Test @Disabled // TODO: how to handle repeated entities: turn to array vs. merge because it's the same? public void entitiesPassThroughRepeatEntity() { From 1df43615e0f05d74b053ff2427225d6ecd7d5ea5 Mon Sep 17 00:00:00 2001 From: Jens Wille Date: Mon, 11 Oct 2021 14:55:56 +0200 Subject: [PATCH 38/55] Replace print statements with logging. (#58) --- .../org/metafacture/metafix/FixStandaloneSetup.java | 6 +++++- .../org/metafacture/metafix/RecordTransformer.java | 8 ++++++-- .../metafacture/metafix/validation/XtextValidator.java | 10 ++++++---- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixStandaloneSetup.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixStandaloneSetup.java index 1197813b..7eb4c8a1 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixStandaloneSetup.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixStandaloneSetup.java @@ -12,6 +12,8 @@ import org.eclipse.xtext.validation.Issue; import com.google.common.io.CharStreams; import com.google.inject.Injector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.FileWriter; @@ -23,6 +25,8 @@ */ public class FixStandaloneSetup extends FixStandaloneSetupGenerated { + private static final Logger LOG = LoggerFactory.getLogger(FixStandaloneSetup.class); + public FixStandaloneSetup() { } @@ -41,7 +45,7 @@ public static Fix parseFix(final Reader fixDef) { final IResourceValidator validator = ((XtextResource) resource).getResourceServiceProvider().getResourceValidator(); for (final Issue issue : validator.validate(resource, CheckMode.ALL, CancelIndicator.NullImpl)) { - System.err.println(issue.getMessage()); + LOG.warn(issue.getMessage()); } return (Fix) resource.getContents().get(0); diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java index de2d78d0..128ae0d4 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java @@ -30,6 +30,8 @@ import org.eclipse.emf.common.util.EList; import com.google.common.collect.ImmutableMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.HashMap; import java.util.LinkedHashMap; @@ -45,6 +47,8 @@ */ class RecordTransformer { + private static final Logger LOG = LoggerFactory.getLogger(RecordTransformer.class); + private Fix fix; private Map record; private Map vars; @@ -102,7 +106,7 @@ record = new LinkedHashMap<>(ImmutableMap.of(options.get("var"), val)); record = fullRecord; } else { - System.out.println("Unprocessed bind: " + theDo); + LOG.warn("Unprocessed bind: {}", theDo); // TODO, possibly: use morph collectors here // final CollectFactory collectFactory = new CollectFactory(); // final Map attributes = resolvedAttributeMap(params, theDo.getOptions()); @@ -131,7 +135,7 @@ private void processUnless(final Unless unless, final EList parameters) } private boolean testConditional(final String conditional, final EList params) { - System.out.printf(": %s parameters: %s\n", conditional, params); + LOG.debug(": {} parameters: {}", conditional, params); boolean result = false; if ("exists".equals(conditional)) { return record.containsKey(params.get(0)); diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/validation/XtextValidator.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/validation/XtextValidator.java index e81e1ee5..860fff28 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/validation/XtextValidator.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/validation/XtextValidator.java @@ -9,12 +9,16 @@ import org.eclipse.xtext.validation.CheckMode; import org.eclipse.xtext.validation.IResourceValidator; import org.eclipse.xtext.validation.Issue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.List; public class XtextValidator { + private static final Logger LOG = LoggerFactory.getLogger(XtextValidator.class); + private XtextValidator() { throw new IllegalAccessError("Utility class"); } @@ -29,12 +33,10 @@ public static boolean validate(final URI uri) throws IOException { final int count = issues.size(); if (count > 0) { - System.out.println(String.format("The Xtext file '%s' has %d issue%s:\n", - resource.getURI(), count, count > 1 ? "s" : "")); + LOG.warn("The Xtext file '{}' has {} issue{}:", resource.getURI(), count, count > 1 ? "s" : ""); for (final Issue issue : issues) { - System.out.println(String.format("- %s: %s (%d:%d)", - issue.getSeverity(), issue.getMessage(), issue.getLineNumber(), issue.getColumn())); + LOG.warn("- {}: {} ({}:{})", issue.getSeverity(), issue.getMessage(), issue.getLineNumber(), issue.getColumn()); } return false; From adf3f7d08b989d0b7d489beacd74650be33d355d Mon Sep 17 00:00:00 2001 From: Jens Wille Date: Mon, 11 Oct 2021 14:57:24 +0200 Subject: [PATCH 39/55] Let Checkstyle enforce alphabetical sorting of imports. --- config/checkstyle/checkstyle.xml | 1 + .../main/java/org/metafacture/metafix/FixStandaloneSetup.java | 4 ++-- .../main/java/org/metafacture/metafix/RecordTransformer.java | 2 +- .../test/java/org/metafacture/metafix/MetafixMethodTest.java | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/config/checkstyle/checkstyle.xml b/config/checkstyle/checkstyle.xml index 5b5bb2c9..7d7b827f 100644 --- a/config/checkstyle/checkstyle.xml +++ b/config/checkstyle/checkstyle.xml @@ -29,6 +29,7 @@ + diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixStandaloneSetup.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixStandaloneSetup.java index 7eb4c8a1..ca5a5b61 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixStandaloneSetup.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixStandaloneSetup.java @@ -2,6 +2,8 @@ import org.metafacture.metafix.fix.Fix; +import com.google.common.io.CharStreams; +import com.google.inject.Injector; import org.eclipse.emf.common.util.URI; import org.eclipse.emf.ecore.resource.Resource; import org.eclipse.xtext.resource.XtextResource; @@ -10,8 +12,6 @@ import org.eclipse.xtext.validation.CheckMode; import org.eclipse.xtext.validation.IResourceValidator; import org.eclipse.xtext.validation.Issue; -import com.google.common.io.CharStreams; -import com.google.inject.Injector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java index 128ae0d4..cef90916 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java @@ -28,8 +28,8 @@ import org.metafacture.metafix.fix.Options; import org.metafacture.metafix.fix.Unless; -import org.eclipse.emf.common.util.EList; import com.google.common.collect.ImmutableMap; +import org.eclipse.emf.common.util.EList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java index 6082f3ae..7717f9cf 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java @@ -18,6 +18,7 @@ import org.metafacture.framework.StreamReceiver; +import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -26,7 +27,6 @@ import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; import org.mockito.junit.jupiter.MockitoExtension; -import com.google.common.collect.ImmutableMap; import java.util.Arrays; From 6393b4d8a41e3ee8d09f30db50fe2c99b35556dc Mon Sep 17 00:00:00 2001 From: Jens Wille Date: Fri, 29 Oct 2021 12:27:50 +0200 Subject: [PATCH 40/55] Consolidate Xtext/Fix file validation. - Fixes validation logging (see 1df4361). - Introduces Gradle task to validate Fix files. --- org.metafacture.fix/build.gradle | 7 ++- .../metafix/FixStandaloneSetup.java | 38 ++++-------- .../metafix/validation/XtextValidator.java | 59 +++++++++++++------ 3 files changed, 57 insertions(+), 47 deletions(-) diff --git a/org.metafacture.fix/build.gradle b/org.metafacture.fix/build.gradle index e6fd2113..17764cf2 100644 --- a/org.metafacture.fix/build.gradle +++ b/org.metafacture.fix/build.gradle @@ -15,7 +15,7 @@ dependencies { testImplementation "org.eclipse.xtext:org.eclipse.xtext.xbase.testing:${versions.xtext}" testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${versions.junit_jupiter}" - testRuntimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}" + runtimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}" implementation "org.metafacture:metafacture-commons:${versions.metafacture}" implementation "org.metafacture:metafacture-mangling:${versions.metafacture}" @@ -49,6 +49,11 @@ task install(dependsOn: publishToMavenLocal, doFirst { println "This task is deprecated; use 'publishToMavenLocal' instead." } } +task validateFixFile(type: JavaExec) { + mainClass = 'org.metafacture.metafix.FixStandaloneSetup' + classpath = sourceSets.main.runtimeClasspath +} + def xtextFile = 'src/main/java/org/metafacture/metafix/Fix.xtext' task validateXtextLanguage(type: JavaExec) { diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixStandaloneSetup.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixStandaloneSetup.java index ca5a5b61..0743bb6e 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixStandaloneSetup.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixStandaloneSetup.java @@ -1,19 +1,9 @@ package org.metafacture.metafix; import org.metafacture.metafix.fix.Fix; +import org.metafacture.metafix.validation.XtextValidator; import com.google.common.io.CharStreams; -import com.google.inject.Injector; -import org.eclipse.emf.common.util.URI; -import org.eclipse.emf.ecore.resource.Resource; -import org.eclipse.xtext.resource.XtextResource; -import org.eclipse.xtext.resource.XtextResourceSet; -import org.eclipse.xtext.util.CancelIndicator; -import org.eclipse.xtext.validation.CheckMode; -import org.eclipse.xtext.validation.IResourceValidator; -import org.eclipse.xtext.validation.Issue; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.File; import java.io.FileWriter; @@ -25,35 +15,29 @@ */ public class FixStandaloneSetup extends FixStandaloneSetupGenerated { - private static final Logger LOG = LoggerFactory.getLogger(FixStandaloneSetup.class); - public FixStandaloneSetup() { } - public static void doSetup() { - new FixStandaloneSetup().createInjectorAndDoEMFRegistration(); + public static void main(final String[] args) { + if (args != null && args.length == 1) { + System.exit(XtextValidator.validate(args[0], new FixStandaloneSetup()) ? 0 : 1); + } + + throw new IllegalArgumentException(String.format("Usage: %s ", FixStandaloneSetup.class.getName())); } public static Fix parseFix(final Reader fixDef) { - // TODO: do this only once per application - final Injector injector = new FixStandaloneSetup().createInjectorAndDoEMFRegistration(); - FixStandaloneSetup.doSetup(); + final String path; try { - final URI uri = URI.createFileURI(absPathToTempFile(fixDef, ".fix")); - final Resource resource = injector.getInstance(XtextResourceSet.class).getResource(uri, true); - final IResourceValidator validator = ((XtextResource) resource).getResourceServiceProvider().getResourceValidator(); - - for (final Issue issue : validator.validate(resource, CheckMode.ALL, CancelIndicator.NullImpl)) { - LOG.warn(issue.getMessage()); - } - - return (Fix) resource.getContents().get(0); + path = absPathToTempFile(fixDef, ".fix"); } catch (final IOException e) { e.printStackTrace(); return null; } + + return (Fix) XtextValidator.getValidatedResource(path, new FixStandaloneSetup()).getContents().get(0); } public static String absPathToTempFile(final Reader fixDef, final String suffix) throws IOException { diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/validation/XtextValidator.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/validation/XtextValidator.java index 860fff28..a8de336b 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/validation/XtextValidator.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/validation/XtextValidator.java @@ -1,17 +1,17 @@ package org.metafacture.metafix.validation; -import com.google.inject.Injector; import org.eclipse.emf.common.util.URI; -import org.eclipse.emf.ecore.resource.Resource; -import org.eclipse.emf.ecore.resource.ResourceSet; +import org.eclipse.xtext.ISetup; import org.eclipse.xtext.XtextStandaloneSetup; +import org.eclipse.xtext.resource.XtextResource; +import org.eclipse.xtext.resource.XtextResourceSet; import org.eclipse.xtext.util.CancelIndicator; import org.eclipse.xtext.validation.CheckMode; -import org.eclipse.xtext.validation.IResourceValidator; import org.eclipse.xtext.validation.Issue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; import java.util.List; @@ -23,21 +23,18 @@ private XtextValidator() { throw new IllegalAccessError("Utility class"); } - public static boolean validate(final URI uri) throws IOException { - final Injector injector = new XtextStandaloneSetup().createInjectorAndDoEMFRegistration(); - final Resource resource = injector.getInstance(ResourceSet.class).getResource(uri, true); - resource.load(null); + private static boolean validate(final XtextResource resource, final ISetup setup) { + final List issues = resource.getResourceServiceProvider() + .getResourceValidator().validate(resource, CheckMode.ALL, CancelIndicator.NullImpl); - final List issues = injector.getInstance(IResourceValidator.class) - .validate(resource, CheckMode.ALL, CancelIndicator.NullImpl); final int count = issues.size(); if (count > 0) { - LOG.warn("The Xtext file '{}' has {} issue{}:", resource.getURI(), count, count > 1 ? "s" : ""); + LOG.warn("The {} file '{}' has {} issue{}:", + setup.getClass().getSimpleName(), resource.getURI().toFileString(), count, count > 1 ? "s" : ""); - for (final Issue issue : issues) { - LOG.warn("- {}: {} ({}:{})", issue.getSeverity(), issue.getMessage(), issue.getLineNumber(), issue.getColumn()); - } + issues.forEach(i -> LOG.warn("- {}: {} ({}:{})", + i.getSeverity(), i.getMessage(), i.getLineNumber(), i.getColumn())); return false; } @@ -45,13 +42,37 @@ public static boolean validate(final URI uri) throws IOException { return true; } - public static void main(final String[] args) throws IOException { - if (args == null || args.length != 1) { - throw new IllegalArgumentException(String.format("Usage: %s ", - XtextValidator.class.getName())); + public static boolean validate(final String path, final ISetup setup) { + return validate(getResource(path, setup), setup); + } + + private static XtextResource getResource(final String path, final ISetup setup) { + final File file = new File(path); + String absolutePath; + + try { + absolutePath = file.getCanonicalPath(); + } + catch (final IOException e) { + absolutePath = file.getAbsolutePath(); + } + + return (XtextResource) setup.createInjectorAndDoEMFRegistration() + .getInstance(XtextResourceSet.class).getResource(URI.createFileURI(absolutePath), true); + } + + public static XtextResource getValidatedResource(final String path, final ISetup setup) { + final XtextResource resource = getResource(path, setup); + validate(resource, setup); + return resource; + } + + public static void main(final String[] args) { + if (args != null && args.length == 1) { + System.exit(validate(args[0], new XtextStandaloneSetup()) ? 0 : 1); } - System.exit(validate(URI.createURI(args[0])) ? 0 : 1); + throw new IllegalArgumentException(String.format("Usage: %s ", XtextValidator.class.getName())); } } From 9c8a0b8e82c97865c2f5c9ad8016ab291e20b60b Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Tue, 2 Nov 2021 10:30:39 +0100 Subject: [PATCH 41/55] Wrap and rethrow exceptions as MetafactureException To allow handling exceptions in clients See https://github.com/metafacture/metafacture-playground/issues/47 --- .../src/main/java/org/metafacture/metafix/FixMethod.java | 4 ++-- .../java/org/metafacture/metafix/FixStandaloneSetup.java | 4 ++-- .../main/java/org/metafacture/metafix/RecordTransformer.java | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index f28fcbc7..65ebdfdd 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -16,6 +16,7 @@ package org.metafacture.metafix; +import org.metafacture.framework.MetafactureException; import org.metafacture.metamorph.maps.FileMap; import java.lang.reflect.InvocationTargetException; @@ -174,9 +175,8 @@ private Map getNamedGroups(final Pattern regex) { return Collections.unmodifiableMap(namedGroups); } catch (final NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { - e.printStackTrace(); + throw new MetafactureException(e); } - return Collections.emptyMap(); } }, paste { diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixStandaloneSetup.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixStandaloneSetup.java index ca5a5b61..9c4fdc85 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixStandaloneSetup.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixStandaloneSetup.java @@ -1,5 +1,6 @@ package org.metafacture.metafix; +import org.metafacture.framework.MetafactureException; import org.metafacture.metafix.fix.Fix; import com.google.common.io.CharStreams; @@ -51,8 +52,7 @@ public static Fix parseFix(final Reader fixDef) { return (Fix) resource.getContents().get(0); } catch (final IOException e) { - e.printStackTrace(); - return null; + throw new MetafactureException(e); } } diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java index cef90916..500423ee 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/RecordTransformer.java @@ -17,6 +17,7 @@ package org.metafacture.metafix; import org.metafacture.commons.StringUtil; +import org.metafacture.framework.MetafactureException; import org.metafacture.metafix.FixPredicate.Quantifier; import org.metafacture.metafix.fix.Do; import org.metafacture.metafix.fix.ElsIf; @@ -150,7 +151,7 @@ private boolean testConditional(final String conditional, final EList pa result = quantifier.test(record, predicate, params); } catch (final IllegalArgumentException e) { - e.printStackTrace(); + throw new MetafactureException(e); } // TODO, possibly: use morph functions here (& in processFunction): // final FunctionFactory functionFactory = new FunctionFactory(); @@ -169,7 +170,7 @@ private void processFunction(final Expression expression, final List par method.apply(record, resolvedParams, options); } catch (final IllegalArgumentException e) { - e.printStackTrace(); + throw new MetafactureException(e); } } From 171ab7d203c74bf4aea9dd437f71bedebe81b5b5 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Wed, 3 Nov 2021 09:05:06 +0100 Subject: [PATCH 42/55] Avoid redundant wrapping of BiConsumer (#35) Co-authored-by: Jens Wille --- .../test/java/org/metafacture/metafix/MetafixTestHelpers.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java index f7eb5433..b3c5f9b9 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java @@ -47,7 +47,7 @@ public static void assertFix(final StreamReceiver receiver, final List f public static void assertFix(final StreamReceiver receiver, final List fixDef, final Consumer in, final BiConsumer, IntFunction> out) { - assertFix(receiver, fixDef, in, (s, f) -> out.accept(s, f), Metafix.NO_VARS); + assertFix(receiver, fixDef, in, out, Metafix.NO_VARS); } public static void assertFix(final StreamReceiver receiver, final List fixDef, final Map vars, @@ -57,7 +57,7 @@ public static void assertFix(final StreamReceiver receiver, final List f public static void assertFix(final StreamReceiver receiver, final List fixDef, final Map vars, final Consumer in, final BiConsumer, IntFunction> out) { - assertFix(receiver, fixDef, in, (s, f) -> out.accept(s, f), vars); + assertFix(receiver, fixDef, in, out, vars); } private static void assertFix(final StreamReceiver receiver, final List fixLines, final Consumer in, From 9009c39441ce7e3d00e1a3c63d034170b6663c93 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Wed, 3 Nov 2021 09:11:15 +0100 Subject: [PATCH 43/55] Simplify map manipulation (#35) Co-authored-by: Jens Wille --- .../main/java/org/metafacture/metafix/FixMethod.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index 65ebdfdd..e5d7f168 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -203,21 +203,13 @@ public void apply(final Map record, final List params, retain { public void apply(final Map record, final List params, final Map options) { - new HashSet<>(record.keySet()).forEach(key -> { - if (!params.contains(key)) { - record.remove(key); - } - }); + record.keySet().retainAll(params); } }, vacuum { public void apply(final Map record, final List params, final Map options) { - new HashSet<>(record.keySet()).forEach(key -> { - if (EMPTY.equals(record.get(key))) { - record.remove(key, EMPTY); - } - }); + record.values().removeIf(EMPTY::equals); } }, // FIELD-LEVEL METHODS: From efa22235d39863944e3d2171d614bb5e7f2fbfa2 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Wed, 3 Nov 2021 09:15:16 +0100 Subject: [PATCH 44/55] Remove unused import (#35) --- .../src/main/java/org/metafacture/metafix/FixMethod.java | 1 - 1 file changed, 1 deletion(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index e5d7f168..af1a77d8 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -26,7 +26,6 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; From 5a3bd03463f2dd4b630816bd4e457270d9960a24 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Wed, 3 Nov 2021 09:19:10 +0100 Subject: [PATCH 45/55] Change logging dependency to `runtimeOnly` (#35) To fix logging in `./gradlew validateXtextLanguage` Co-authored-by: Jens Wille --- org.metafacture.fix/build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/org.metafacture.fix/build.gradle b/org.metafacture.fix/build.gradle index e6fd2113..7186f4a3 100644 --- a/org.metafacture.fix/build.gradle +++ b/org.metafacture.fix/build.gradle @@ -15,7 +15,8 @@ dependencies { testImplementation "org.eclipse.xtext:org.eclipse.xtext.xbase.testing:${versions.xtext}" testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${versions.junit_jupiter}" - testRuntimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}" + + runtimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}" implementation "org.metafacture:metafacture-commons:${versions.metafacture}" implementation "org.metafacture:metafacture-mangling:${versions.metafacture}" From 43d13253072eb2de824c0f4387860e58d345aceb Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Wed, 3 Nov 2021 10:07:18 +0100 Subject: [PATCH 46/55] Replace `contains` with `endsWith` (#35) Co-authored-by: Jens Wille --- .../src/main/java/org/metafacture/metafix/FixMethod.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index af1a77d8..2ae25d21 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -50,7 +50,7 @@ public void apply(final Map record, final List params, final Map options) { final String key = params.get(0); final List toAdd = params.subList(1, params.size()); - if (key.contains(APPEND)) { + if (key.endsWith(APPEND)) { Metafix.addAll(record, key.replace(APPEND, EMPTY), toAdd); } else { @@ -64,7 +64,7 @@ public void apply(final Map record, final List params, final Map options) { final String key = params.get(0); final Object val = record.get(key.replace(APPEND, EMPTY)); - if (key.contains(APPEND) && val instanceof List) { + if (key.endsWith(APPEND) && val instanceof List) { ((List) val).add(options); } else { From 111e554ede1839f759b37ef6bfb9090c2152ef20 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Wed, 3 Nov 2021 10:25:14 +0100 Subject: [PATCH 47/55] Use constants for `$append` and `$last` (#35) --- .../java/org/metafacture/metafix/FixMethod.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index 2ae25d21..65ea9444 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -50,8 +50,8 @@ public void apply(final Map record, final List params, final Map options) { final String key = params.get(0); final List toAdd = params.subList(1, params.size()); - if (key.endsWith(APPEND)) { - Metafix.addAll(record, key.replace(APPEND, EMPTY), toAdd); + if (key.endsWith(DOT_APPEND)) { + Metafix.addAll(record, key.replace(DOT_APPEND, EMPTY), toAdd); } else { record.put(key, toAdd); @@ -63,8 +63,8 @@ public void apply(final Map record, final List params, public void apply(final Map record, final List params, final Map options) { final String key = params.get(0); - final Object val = record.get(key.replace(APPEND, EMPTY)); - if (key.endsWith(APPEND) && val instanceof List) { + final Object val = record.get(key.replace(DOT_APPEND, EMPTY)); + if (key.endsWith(DOT_APPEND) && val instanceof List) { ((List) val).add(options); } else { @@ -273,7 +273,9 @@ private Map fileMap(final String location, final String separato private static final String NESTED = "Nested non-map / non-list: "; private static final String EMPTY = ""; - private static final String APPEND = ".$append"; + private static final String APPEND = "$append"; + private static final String DOT_APPEND = "." + APPEND; + private static final String LAST = "$last"; private static void applyToFields(final Map record, final List params, final Function fun) { @@ -330,12 +332,12 @@ private static void processList(final InsertMode mode, final String value, final final List nestedList = (List) nested; final Map nestedMap; switch (remainingKeys[0]) { - case "$append": + case APPEND: nestedMap = new LinkedHashMap<>(); nestedList.add(nestedMap); insert(mode, nestedMap, Arrays.copyOfRange(remainingKeys, 1, remainingKeys.length), value); break; - case "$last": + case LAST: final Object last = nestedList.get(nestedList.size() - 1); if (last instanceof Map) { nestedMap = (Map) last; From 15977a4e4581399ad542a7ab0295bf5ee34b3418 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Wed, 3 Nov 2021 13:26:19 +0100 Subject: [PATCH 48/55] Remove trailing empty comments (#35) Those were a hack to avoid joining lines by the Eclipse formatter (should use formatter on-off tags where/when needed). --- .../metafacture/metafix/MetafixBindTest.java | 88 ++++---- .../metafacture/metafix/MetafixIfTest.java | 196 +++++++++--------- .../metafix/MetafixLookupTest.java | 42 ++-- .../metafix/MetafixMethodTest.java | 118 +++++------ .../metafix/MetafixRecordTest.java | 156 +++++++------- .../metafix/MetafixSelectorTest.java | 16 +- 6 files changed, 308 insertions(+), 308 deletions(-) diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java index 2633f809..11c910bb 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java @@ -49,13 +49,13 @@ public MetafixBindTest() { @Test public void doList() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "do list('path': 'name', 'var': 'n')", " upcase('n')", " trim('n')", " copy_field('n', 'author')", "end", - "remove_field('name')"), // + "remove_field('name')"), i -> { i.startRecord("1"); i.literal("name", " A University"); @@ -73,13 +73,13 @@ public void doList() { @Test public void doListPathWithDots() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "do list('path': 'some.name', 'var': 'n')", " upcase('n')", " trim('n')", " copy_field('n', 'author')", "end", - "remove_field('some')"), // + "remove_field('some')"), i -> { i.startRecord("1"); i.startEntity("some"); @@ -99,13 +99,13 @@ public void doListPathWithDots() { @Test public void doListWithAppendAndLast() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "do list('path': 'creator', 'var': 'c')", " set_array('author')", " copy_field('c.name', 'author.$append.name')", " add_field('author.$last.type', 'Default')", "end", - "remove_field('creator')"), // + "remove_field('creator')"), i -> { i.startRecord("1"); i.startEntity("creator"); @@ -132,13 +132,13 @@ public void doListWithAppendAndLast() { @Test public void doListEntitesToLiterals() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "do list('path': 'creator', 'var': 'c')", " upcase('c.name')", " trim('c.name')", " copy_field('c.name', 'author')", "end", - "remove_field('creator')"), // + "remove_field('creator')"), i -> { i.startRecord("1"); i.startEntity("creator"); @@ -160,17 +160,17 @@ public void doListEntitesToLiterals() { @Test public void doListEntitesToEntities() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "do list('path': 'creator.name', 'var': 'c')", " set_array('author')", " copy_field('c', 'author.$append.name')", - " if all_contain('c', 'University')", // - " add_field('author.$last.type', 'Organization')", // + " if all_contain('c', 'University')", + " add_field('author.$last.type', 'Organization')", " else", " add_field('author.$last.type', 'Person')", //", " end", "end", - "remove_field('creator')"), // + "remove_field('creator')"), i -> { i.startRecord("1"); i.startEntity("creator"); @@ -198,18 +198,18 @@ public void doListEntitesToEntities() { @Test @Disabled // TODO: how to handle repeated entities: turn to array vs. merge because it's the same? public void doListEntitesWithFieldsToEntities() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "do list('path': 'creator', 'var': 'c')", " set_array('author')", " copy_field('c.name', 'author.$append.name')", - " if all_contain('c.type', 'corporate')", // - " add_field('author.$last.type', 'Organization')", // + " if all_contain('c.type', 'corporate')", + " add_field('author.$last.type', 'Organization')", " end", " if all_contain('c.type', 'personal')", " add_field('author.$last.type', 'Person')", //", " end", "end", - "remove_field('creator')"), // + "remove_field('creator')"), i -> { i.startRecord("1"); i.startEntity("creator"); @@ -239,13 +239,13 @@ public void doListEntitesWithFieldsToEntities() { @Test @Disabled // implement Fix-style binds with collectors? public void ifInCollector() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "do entity('author')", " map('name')", - " if all_contain('name', 'University')", // - " add_field('type', 'Organization')", // + " if all_contain('name', 'University')", + " add_field('type', 'Organization')", " end", - "end"), // + "end"), i -> { i.startRecord("1"); i.literal("name", "A University"); @@ -263,22 +263,22 @@ public void ifInCollector() { @Test @Disabled // implement Fix-style binds with collectors? public void ifInCollectorMultiRecords() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "do entity('author')", " map('name')", - " if all_contain('name', 'University')", // - " add_field('type', 'Organization')", // + " if all_contain('name', 'University')", + " add_field('type', 'Organization')", " end", - "end"), // + "end"), i -> { i.startRecord("1"); i.literal("name", "Max"); i.endRecord(); - // + i.startRecord("2"); i.literal("name", "A University"); i.endRecord(); - // + i.startRecord("3"); i.literal("name", "Mary"); i.endRecord(); @@ -288,14 +288,14 @@ public void ifInCollectorMultiRecords() { o.get().literal("name", "Max"); o.get().endEntity(); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().startEntity("author"); o.get().literal("type", "Organization"); o.get().literal("name", "A University"); o.get().endEntity(); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().startEntity("author"); o.get().literal("name", "Mary"); @@ -307,17 +307,17 @@ public void ifInCollectorMultiRecords() { @Test @Disabled // implement Fix-style binds with collectors? public void ifInCollectorChoose() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "do choose(flushWith: 'record')", - " if all_contain('name', 'University')", // - " add_field('type', 'Organization')", // - " end", // - "end"), // + " if all_contain('name', 'University')", + " add_field('type', 'Organization')", + " end", + "end"), i -> { i.startRecord("1"); i.literal("name", "Max University"); i.endRecord(); - // + i.startRecord("2"); i.literal("name", "Max Musterman"); i.endRecord(); @@ -325,7 +325,7 @@ public void ifInCollectorChoose() { o.get().startRecord("1"); o.get().literal("type", "Organization"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().endRecord(); }); @@ -334,20 +334,20 @@ public void ifInCollectorChoose() { @Test @Disabled // implement Fix-style binds with collectors? public void ifInCollectorCombine() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "do combine(name: 'fullName', value: '${first} ${last}')", // - " if all_contain('author.type', 'Person')", // - " map('author.first', 'first')", // - " map('author.last', 'last')", // - " end", // - "end"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "do combine(name: 'fullName', value: '${first} ${last}')", + " if all_contain('author.type', 'Person')", + " map('author.first', 'first')", + " map('author.last', 'last')", + " end", + "end"), i -> { i.startRecord("1"); i.startEntity("author"); i.literal("type", "Organization"); i.endEntity(); i.endRecord(); - // + i.startRecord("2"); i.startEntity("author"); i.literal("first", "Max"); @@ -358,7 +358,7 @@ public void ifInCollectorCombine() { }, o -> { o.get().startRecord("1"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().literal("fullName", "Max Musterman"); o.get().endRecord(); diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java index 2bdfbcfd..52da1754 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixIfTest.java @@ -48,21 +48,21 @@ public MetafixIfTest() { @Test public void ifAny() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if any_contain('name', 'University')", // - " add_field('type', 'Organization')", // - "end"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if any_contain('name', 'University')", + " add_field('type', 'Organization')", + "end"), i -> { i.startRecord("1"); i.literal("name", "Mary"); i.literal("name", "A University"); i.endRecord(); - // + i.startRecord("2"); i.literal("name", "Mary"); i.literal("name", "Max"); i.endRecord(); - // + i.startRecord("3"); i.endRecord(); }, o -> { @@ -73,14 +73,14 @@ public void ifAny() { o.get().endEntity(); o.get().literal("type", "Organization"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().startEntity("name[]"); o.get().literal("1", "Mary"); o.get().literal("2", "Max"); o.get().endEntity(); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().endRecord(); }); @@ -88,21 +88,21 @@ public void ifAny() { @Test public void ifAll() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if all_contain('name', 'University')", // - " add_field('type', 'Organization')", // - "end"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if all_contain('name', 'University')", + " add_field('type', 'Organization')", + "end"), i -> { i.startRecord("1"); i.literal("name", "Mary"); i.literal("name", "A University"); i.endRecord(); - // + i.startRecord("2"); i.literal("name", "Great University"); i.literal("name", "A University"); i.endRecord(); - // + i.startRecord("3"); i.endRecord(); }, o -> { @@ -112,7 +112,7 @@ public void ifAll() { o.get().literal("2", "A University"); o.get().endEntity(); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().startEntity("name[]"); o.get().literal("1", "Great University"); @@ -120,7 +120,7 @@ public void ifAll() { o.get().endEntity(); o.get().literal("type", "Organization"); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().endRecord(); }); @@ -129,10 +129,10 @@ public void ifAll() { @Test @SuppressWarnings("checkstyle:ExecutableStatementCount") public void ifNone() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if none_contain('author.name', 'University')", // - " add_field('type', 'Person')", // - "end"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if none_contain('author.name', 'University')", + " add_field('type', 'Person')", + "end"), i -> { i.startRecord("1"); i.startEntity("author"); @@ -140,14 +140,14 @@ public void ifNone() { i.literal("name", "A University"); i.endEntity(); i.endRecord(); - // + i.startRecord("2"); i.startEntity("author"); i.literal("name", "Max"); i.literal("name", "Mary"); i.endEntity(); i.endRecord(); - // + i.startRecord("3"); i.endRecord(); }, (o, f) -> { @@ -158,7 +158,7 @@ public void ifNone() { o.get().literal("2", "A University"); f.apply(2).endEntity(); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().startEntity("author"); o.get().startEntity("name[]"); @@ -167,7 +167,7 @@ public void ifNone() { f.apply(2).endEntity(); o.get().literal("type", "Person"); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().literal("type", "Person"); o.get().endRecord(); @@ -176,9 +176,9 @@ public void ifNone() { @Test public void ifEqual() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if all_equal('name', 'University')", // - " add_field('type', 'Organization')", // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if all_equal('name', 'University')", + " add_field('type', 'Organization')", "end"), i -> { i.startRecord("1"); @@ -194,19 +194,19 @@ public void ifEqual() { @Test public void ifContainMoveField() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if all_contain('name', 'University')", // - " move_field('name', 'orgName')", // - "end"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if all_contain('name', 'University')", + " move_field('name', 'orgName')", + "end"), i -> { i.startRecord("1"); i.literal("name", "Max"); i.endRecord(); - // + i.startRecord("2"); i.literal("name", "A University"); i.endRecord(); - // + i.startRecord("3"); i.literal("name", "Mary"); i.endRecord(); @@ -214,11 +214,11 @@ public void ifContainMoveField() { o.get().startRecord("1"); o.get().literal("name", "Max"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().literal("orgName", "A University"); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().literal("name", "Mary"); o.get().endRecord(); @@ -229,9 +229,9 @@ public void ifContainMoveField() { public void moveAndAddIfContain() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// TODO: dot notation in contain etc. "move_field('name', 'author.name')", - "if all_contain('author.name', 'University')", // - " add_field('author.type', 'Organization')", // - "end"), // + "if all_contain('author.name', 'University')", + " add_field('author.type', 'Organization')", + "end"), i -> { i.startRecord("1"); i.literal("name", "A University"); @@ -248,11 +248,11 @@ public void moveAndAddIfContain() { @Test public void ifContainMultipleAddField() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if all_contain('name', 'University')", // - " add_field('type', 'Organization')", // - " add_field('comment', 'type was guessed')", // - "end"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if all_contain('name', 'University')", + " add_field('type', 'Organization')", + " add_field('comment', 'type was guessed')", + "end"), i -> { i.startRecord("1"); i.literal("name", "A University"); @@ -268,15 +268,15 @@ public void ifContainMultipleAddField() { @Test public void ifAnyMatch() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if any_match('name', '.*University.*')", // - " add_field('type', 'Organization')", // - "end"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if any_match('name', '.*University.*')", + " add_field('type', 'Organization')", + "end"), i -> { i.startRecord("1"); i.literal("name", "Max"); i.endRecord(); - // + i.startRecord("2"); i.literal("name", "Some University"); i.literal("name", "Filibandrina"); @@ -285,7 +285,7 @@ public void ifAnyMatch() { o.get().startRecord("1"); o.get().literal("name", "Max"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().startEntity("name[]"); o.get().literal("1", "Some University"); @@ -298,16 +298,16 @@ public void ifAnyMatch() { @Test public void ifAllMatch() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if all_match('name', '.*University.*')", // - " add_field('type', 'Organization')", // - "end"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if all_match('name', '.*University.*')", + " add_field('type', 'Organization')", + "end"), i -> { i.startRecord("1"); i.literal("name", "Max"); i.literal("name", "A University"); i.endRecord(); - // + i.startRecord("2"); i.literal("name", "Some University"); i.literal("name", "University Filibandrina"); @@ -319,7 +319,7 @@ public void ifAllMatch() { o.get().literal("2", "A University"); o.get().endEntity(); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().startEntity("name[]"); o.get().literal("1", "Some University"); @@ -333,9 +333,9 @@ public void ifAllMatch() { @Test public void ifAnyMatchNested() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// TODO: dot notation in match etc. - "if any_match('author.name.label', '.*University.*')", // - " add_field('author.type', 'Organization')", // - "end"), // + "if any_match('author.name.label', '.*University.*')", + " add_field('author.type', 'Organization')", + "end"), i -> { i.startRecord("1"); i.startEntity("author"); @@ -344,7 +344,7 @@ public void ifAnyMatchNested() { i.endEntity(); i.endEntity(); i.endRecord(); - // + i.startRecord("2"); i.startEntity("author"); i.startEntity("name"); @@ -359,7 +359,7 @@ public void ifAnyMatchNested() { o.get().literal("label", "Max"); f.apply(2).endEntity(); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().startEntity("author"); o.get().startEntity("name"); @@ -373,15 +373,15 @@ public void ifAnyMatchNested() { @Test public void ifAnyMatchFirstRecord() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if any_match('name', '.*University.*')", // - " add_field('type', 'Organization')", // - "end"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if any_match('name', '.*University.*')", + " add_field('type', 'Organization')", + "end"), i -> { i.startRecord("1"); i.literal("name", "Some University"); i.endRecord(); - // + i.startRecord("2"); i.literal("name", "Max"); i.endRecord(); @@ -390,7 +390,7 @@ public void ifAnyMatchFirstRecord() { o.get().literal("name", "Some University"); o.get().literal("type", "Organization"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().literal("name", "Max"); o.get().endRecord(); @@ -399,15 +399,15 @@ public void ifAnyMatchFirstRecord() { @Test public void ifAnyMatchLastRecord() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if any_match('name', '.*University.*')", // - " add_field('type', 'Organization')", // - "end"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if any_match('name', '.*University.*')", + " add_field('type', 'Organization')", + "end"), i -> { i.startRecord("1"); i.literal("name", "Max"); i.endRecord(); - // + i.startRecord("2"); i.literal("name", "Some University"); i.endRecord(); @@ -415,7 +415,7 @@ public void ifAnyMatchLastRecord() { o.get().startRecord("1"); o.get().literal("name", "Max"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().literal("name", "Some University"); o.get().literal("type", "Organization"); @@ -425,15 +425,15 @@ public void ifAnyMatchLastRecord() { @Test public void unlessAnyMatch() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "unless any_match('name', '.*University.*')", // - " add_field('type', 'Person')", // - "end"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "unless any_match('name', '.*University.*')", + " add_field('type', 'Person')", + "end"), i -> { i.startRecord("1"); i.literal("name", "Max"); i.endRecord(); - // + i.startRecord("2"); i.literal("name", "Some University"); i.endRecord(); @@ -442,7 +442,7 @@ public void unlessAnyMatch() { o.get().literal("name", "Max"); o.get().literal("type", "Person"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().literal("name", "Some University"); o.get().endRecord(); @@ -451,17 +451,17 @@ public void unlessAnyMatch() { @Test public void ifAnyMatchElse() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if any_match('name', '.*University.*')", // - " add_field('type', 'Organization')", // - "else", // - " add_field('type', 'Person')", // - "end"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if any_match('name', '.*University.*')", + " add_field('type', 'Organization')", + "else", + " add_field('type', 'Person')", + "end"), i -> { i.startRecord("1"); i.literal("name", "Max"); i.endRecord(); - // + i.startRecord("2"); i.literal("name", "Some University"); i.endRecord(); @@ -470,7 +470,7 @@ public void ifAnyMatchElse() { o.get().literal("name", "Max"); o.get().literal("type", "Person"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().literal("name", "Some University"); o.get().literal("type", "Organization"); @@ -480,23 +480,23 @@ public void ifAnyMatchElse() { @Test public void ifAnyMatchElsif() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if any_match('name', '.*University.*')", // - " add_field('type', 'Organization')", // - "elsif any_match('name', '[^ ]* [^ ]*')", // - " add_field('type', 'Person')", // - "else", // - " add_field('type', 'Unknown')", // - "end"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if any_match('name', '.*University.*')", + " add_field('type', 'Organization')", + "elsif any_match('name', '[^ ]* [^ ]*')", + " add_field('type', 'Person')", + "else", + " add_field('type', 'Unknown')", + "end"), i -> { i.startRecord("1"); i.literal("name", "Max Power"); i.endRecord(); - // + i.startRecord("2"); i.literal("name", "Some University"); i.endRecord(); - // + i.startRecord("3"); i.literal("name", "Filibandrina"); i.endRecord(); @@ -505,12 +505,12 @@ public void ifAnyMatchElsif() { o.get().literal("name", "Max Power"); o.get().literal("type", "Person"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().literal("name", "Some University"); o.get().literal("type", "Organization"); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().literal("name", "Filibandrina"); o.get().literal("type", "Unknown"); diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java index 1cee81d6..967c4585 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java @@ -49,24 +49,24 @@ public MetafixLookupTest() { @Test public void inline() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "lookup('title', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "lookup('title', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)"), i -> { i.startRecord("1"); i.endRecord(); - // + i.startRecord("2"); i.literal("title", "Aloha"); i.literal("title", "Moin"); i.literal("title", "Hey"); i.endRecord(); - // + i.startRecord("3"); i.endRecord(); }, o -> { o.get().startRecord("1"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().startEntity("title[]"); o.get().literal("1", "Alohaeha"); @@ -74,7 +74,7 @@ public void inline() { o.get().literal("3", "Tach"); o.get().endEntity(); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().endRecord(); }); @@ -82,8 +82,8 @@ public void inline() { @Test public void inlineDotNotationNested() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "lookup('data.title', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "lookup('data.title', Aloha: Alohaeha, 'Moin': 'Moin zäme', __default: Tach)"), i -> { i.startRecord("1"); i.startEntity("data"); @@ -106,25 +106,25 @@ public void inlineDotNotationNested() { @Test public void csv() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "lookup('title', 'src/test/java/org/metafacture/metafix/maps/test.csv')"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "lookup('title', 'src/test/java/org/metafacture/metafix/maps/test.csv')"), i -> { i.startRecord("1"); i.endRecord(); - // + i.startRecord("2"); - // + i.literal("title", "Aloha"); i.literal("title", "Moin"); i.literal("title", "Hey"); i.endRecord(); - // + i.startRecord("3"); i.endRecord(); }, o -> { o.get().startRecord("1"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().startEntity("title[]"); o.get().literal("1", "Alohaeha"); @@ -132,7 +132,7 @@ public void csv() { o.get().literal("3", "Tach"); o.get().endEntity(); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().endRecord(); }); @@ -140,24 +140,24 @@ public void csv() { @Test public void tsv() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "lookup('title', 'src/test/java/org/metafacture/metafix/maps/test.tsv', sep_char:'\t')"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "lookup('title', 'src/test/java/org/metafacture/metafix/maps/test.tsv', sep_char:'\t')"), i -> { i.startRecord("1"); i.endRecord(); - // + i.startRecord("2"); i.literal("title", "Aloha"); i.literal("title", "Moin"); i.literal("title", "Hey"); i.endRecord(); - // + i.startRecord("3"); i.endRecord(); }, o -> { o.get().startRecord("1"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().startEntity("title[]"); o.get().literal("1", "Alohaeha"); @@ -165,7 +165,7 @@ public void tsv() { o.get().literal("3", "Tach"); o.get().endEntity(); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().endRecord(); }); diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java index 7717f9cf..b8074fd6 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java @@ -51,30 +51,30 @@ public MetafixMethodTest() { @Test public void upcase() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "upcase('title')"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "upcase('title')"), i -> { i.startRecord("1"); i.endRecord(); - // + i.startRecord("2"); i.literal("title", "marc"); i.literal("title", "json"); i.endRecord(); - // + i.startRecord("3"); i.endRecord(); }, o -> { o.get().startRecord("1"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().startEntity("title[]"); o.get().literal("1", "MARC"); o.get().literal("2", "JSON"); o.get().endEntity(); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().endRecord(); }); @@ -82,8 +82,8 @@ public void upcase() { @Test public void upcaseDotNotationNested() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "upcase('data.title')"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "upcase('data.title')"), i -> { i.startRecord("1"); i.startEntity("data"); @@ -104,30 +104,30 @@ public void upcaseDotNotationNested() { @Test public void downcase() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "downcase('title')"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "downcase('title')"), i -> { i.startRecord("1"); i.endRecord(); - // + i.startRecord("2"); i.literal("title", "MARC"); i.literal("title", "Json"); i.endRecord(); - // + i.startRecord("3"); i.endRecord(); }, o -> { o.get().startRecord("1"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().startEntity("title[]"); o.get().literal("1", "marc"); o.get().literal("2", "json"); o.get().endEntity(); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().endRecord(); }); @@ -135,30 +135,30 @@ public void downcase() { @Test public void capitalize() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "capitalize('title')"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "capitalize('title')"), i -> { i.startRecord("1"); i.endRecord(); - // + i.startRecord("2"); i.literal("title", "marc"); i.literal("title", "json"); i.endRecord(); - // + i.startRecord("3"); i.endRecord(); }, o -> { o.get().startRecord("1"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().startEntity("title[]"); o.get().literal("1", "Marc"); o.get().literal("2", "Json"); o.get().endEntity(); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().endRecord(); }); @@ -166,30 +166,30 @@ public void capitalize() { @Test public void substring() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "substring('title', '0', '2')"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "substring('title', '0', '2')"), i -> { i.startRecord("1"); i.endRecord(); - // + i.startRecord("2"); i.literal("title", "marc"); i.literal("title", "json"); i.endRecord(); - // + i.startRecord("3"); i.endRecord(); }, o -> { o.get().startRecord("1"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().startEntity("title[]"); o.get().literal("1", "m"); o.get().literal("2", "j"); o.get().endEntity(); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().endRecord(); }); @@ -197,31 +197,31 @@ public void substring() { @Test public void substringWithVar() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "substring('title', '0', '$[end]')"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "substring('title', '0', '$[end]')"), ImmutableMap.of("end", "3"), i -> { i.startRecord("1"); i.endRecord(); - // + i.startRecord("2"); i.literal("title", "marc"); i.literal("title", "json"); i.endRecord(); - // + i.startRecord("3"); i.endRecord(); }, o -> { o.get().startRecord("1"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().startEntity("title[]"); o.get().literal("1", "ma"); o.get().literal("2", "js"); o.get().endEntity(); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().endRecord(); }); @@ -229,30 +229,30 @@ public void substringWithVar() { @Test public void trim() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "trim('title')"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "trim('title')"), i -> { i.startRecord("1"); i.endRecord(); - // + i.startRecord("2"); i.literal("title", " marc "); i.literal("title", " json "); i.endRecord(); - // + i.startRecord("3"); i.endRecord(); }, o -> { o.get().startRecord("1"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().startEntity("title[]"); o.get().literal("1", "marc"); o.get().literal("2", "json"); o.get().endEntity(); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().endRecord(); }); @@ -260,7 +260,7 @@ public void trim() { @Test public void format() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "format(number,'%-5s: %s')"), // TODO actual number formatting with JSON-equiv record i -> { i.startRecord("1"); @@ -278,7 +278,7 @@ public void format() { @Test public void parseText() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "parse_text(date, '(\\\\d{4})-(\\\\d{2})-(\\\\d{2})')"), i -> { i.startRecord("1"); @@ -297,7 +297,7 @@ public void parseText() { @Test public void parseTextNamedGroups() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "parse_text(date, '(?\\\\d{4})-(?\\\\d{2})-(?\\\\d{2})')"), i -> { i.startRecord("1"); @@ -317,28 +317,28 @@ public void parseTextNamedGroups() { @Test @Disabled // Use SimpleRegexTrie/WildcardTrie public void alternation() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "trim('title-1|title-2')"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "trim('title-1|title-2')"), i -> { i.startRecord("1"); i.endRecord(); - // + i.startRecord("2"); i.literal("title-1", " marc "); i.literal("title-2", " json "); i.endRecord(); - // + i.startRecord("3"); i.endRecord(); }, o -> { o.get().startRecord("1"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().literal("title-2", "marc"); o.get().literal("title-1", "json"); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().endRecord(); }); @@ -347,28 +347,28 @@ public void alternation() { @Test @Disabled // Use SimpleRegexTrie/WildcardTrie public void wildcard() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "trim('title-?')"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "trim('title-?')"), i -> { i.startRecord("1"); i.endRecord(); - // + i.startRecord("2"); i.literal("title-1", " marc "); i.literal("title-2", " json "); i.endRecord(); - // + i.startRecord("3"); i.endRecord(); }, o -> { o.get().startRecord("1"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().literal("title-2", "marc"); o.get().literal("title-1", "json"); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().endRecord(); }); @@ -377,28 +377,28 @@ public void wildcard() { @Test @Disabled // Use SimpleRegexTrie public void characterClass() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "trim('title-[12]')"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "trim('title-[12]')"), i -> { i.startRecord("1"); i.endRecord(); - // + i.startRecord("2"); i.literal("title-1", " marc "); i.literal("title-2", " json "); i.endRecord(); - // + i.startRecord("3"); i.endRecord(); }, o -> { o.get().startRecord("1"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().literal("title-2", "marc"); o.get().literal("title-1", "json"); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().endRecord(); }); diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java index 4fb3c033..8180e2ae 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java @@ -50,8 +50,8 @@ public MetafixRecordTest() { @Test public void entitiesPassThrough() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "vacuum()"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "vacuum()"), i -> { i.startRecord("1"); i.startEntity("deep"); @@ -72,7 +72,7 @@ public void entitiesPassThrough() { @Test public void internalIdUsage() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "copy_field('_id', id)"), i -> { i.startRecord("1"); @@ -87,8 +87,8 @@ public void internalIdUsage() { @Test @Disabled // TODO: how to handle repeated entities: turn to array vs. merge because it's the same? public void entitiesPassThroughRepeatEntity() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "vacuum()"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "vacuum()"), i -> { i.startRecord("1"); i.startEntity("some"); @@ -114,8 +114,8 @@ public void entitiesPassThroughRepeatEntity() { @Test @SuppressWarnings("checkstyle:MagicNumber") public void entitiesPassThroughRepeatNestedEntity() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "vacuum()"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "vacuum()"), i -> { i.startRecord("1"); i.startEntity("deep"); @@ -141,9 +141,9 @@ public void entitiesPassThroughRepeatNestedEntity() { @Test public void setEmpty() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "set_field('my.nested.name','patrick')", - "set_field('your.nested.name','nicolas')"), // + "set_field('your.nested.name','nicolas')"), i -> { i.startRecord("1"); i.endRecord(); @@ -163,9 +163,9 @@ public void setEmpty() { @Test public void setExisting() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "set_field('my.nested.name','patrick')", - "set_field('your.nested.name','nicolas')"), // + "set_field('your.nested.name','nicolas')"), i -> { i.startRecord("1"); i.startEntity("my"); @@ -196,19 +196,19 @@ public void setExisting() { @Test @SuppressWarnings("checkstyle:ExecutableStatementCount") public void add() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "add_field('my.name','patrick')", - "add_field('my.name','nicolas')"), // + "add_field('my.name','nicolas')"), i -> { i.startRecord("1"); i.endRecord(); - // + i.startRecord("2"); i.startEntity("my"); i.literal("name", "max"); i.endEntity(); i.endRecord(); - // + i.startRecord("3"); i.endRecord(); }, (o, f) -> { @@ -219,7 +219,7 @@ public void add() { o.get().literal("2", "nicolas"); f.apply(2).endEntity(); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().startEntity("my"); o.get().startEntity("name[]"); @@ -228,7 +228,7 @@ public void add() { o.get().literal("3", "nicolas"); f.apply(2).endEntity(); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().startEntity("my"); o.get().startEntity("name[]"); @@ -243,23 +243,23 @@ public void add() { public void move() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// TODO: dot noation in move_field "move_field('my.name','your.name')", - "move_field('missing','whatever')"), // + "move_field('missing','whatever')"), i -> { i.startRecord("1"); i.endRecord(); - // + i.startRecord("2"); i.startEntity("my"); i.literal("name", "max"); i.endEntity(); i.endRecord(); - // + i.startRecord("3"); i.endRecord(); }, o -> { o.get().startRecord("1"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().startEntity("my"); o.get().endEntity(); @@ -267,7 +267,7 @@ public void move() { o.get().literal("name", "max"); o.get().endEntity(); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().endRecord(); }); @@ -276,30 +276,30 @@ public void move() { @Test public void copy() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// TODO dot notation in copy_field - "copy_field('your.name','your.name2')"), // + "copy_field('your.name','your.name2')"), i -> { i.startRecord("1"); i.endRecord(); - // + i.startRecord("2"); i.startEntity("your"); i.literal("name", "max"); i.endEntity(); i.endRecord(); - // + i.startRecord("3"); i.endRecord(); }, o -> { o.get().startRecord("1"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().startEntity("your"); o.get().literal("name", "max"); o.get().literal("name2", "max"); o.get().endEntity(); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().endRecord(); }); @@ -310,7 +310,7 @@ public void copyIntoArrayOfStrings() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( // "set_array('author')", <- results in separate objects/entities here "copy_field('your.name','author.name')", - "remove_field('your')"), // + "remove_field('your')"), i -> { i.startRecord("1"); i.startEntity("your"); @@ -334,7 +334,7 @@ public void copyIntoArrayOfObjects() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "set_array('author')", "copy_field('your.name','author.name')", - "remove_field('your')"), // + "remove_field('your')"), i -> { i.startRecord("1"); i.startEntity("your"); @@ -362,7 +362,7 @@ public void copyIntoArrayTopLevel() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "set_array('author')", "copy_field('your.name', 'author')", - "remove_field('your')"), // + "remove_field('your')"), i -> { i.startRecord("1"); i.startEntity("your"); @@ -382,29 +382,29 @@ public void copyIntoArrayTopLevel() { @Test public void removeLiteral() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "remove_field('your.name')"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "remove_field('your.name')"), i -> { i.startRecord("1"); i.endRecord(); - // + i.startRecord("2"); i.startEntity("your"); i.literal("name", "max"); i.endEntity(); i.endRecord(); - // + i.startRecord("3"); i.endRecord(); }, o -> { o.get().startRecord("1"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().startEntity("your"); o.get().endEntity(); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().endRecord(); }); @@ -412,28 +412,28 @@ public void removeLiteral() { @Test public void removeLiteralAndEntity() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "remove_field('your.name')", // - "remove_field('your')"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "remove_field('your.name')", + "remove_field('your')"), i -> { i.startRecord("1"); i.endRecord(); - // + i.startRecord("2"); i.startEntity("your"); i.literal("name", "max"); i.endEntity(); i.endRecord(); - // + i.startRecord("3"); i.endRecord(); }, o -> { o.get().startRecord("1"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().endRecord(); }); @@ -441,27 +441,27 @@ public void removeLiteralAndEntity() { @Test public void removeEntity() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "remove_field('your')"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "remove_field('your')"), i -> { i.startRecord("1"); i.endRecord(); - // + i.startRecord("2"); i.startEntity("your"); i.literal("name", "max"); i.endEntity(); i.endRecord(); - // + i.startRecord("3"); i.endRecord(); }, o -> { o.get().startRecord("1"); o.get().endRecord(); - // + o.get().startRecord("2"); o.get().endRecord(); - // + o.get().startRecord("3"); o.get().endRecord(); }); @@ -469,8 +469,8 @@ public void removeEntity() { @Test public void setArray() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "set_array('foo','a','b','c')"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('foo','a','b','c')"), i -> { i.startRecord("1"); i.endRecord(); @@ -487,8 +487,8 @@ public void setArray() { @Test public void setHash() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "set_hash('foo','a': 'b','c': 'd')"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_hash('foo','a': 'b','c': 'd')"), i -> { i.startRecord("1"); i.endRecord(); @@ -504,9 +504,9 @@ public void setHash() { @Test public void paste() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "paste('my.string','m.n.z','m.n.a','m.n.b','m.n.c','m.n.d','m.n.e')", - "remove_field('m')"), // + "remove_field('m')"), i -> { i.startRecord("1"); i.startEntity("m"); @@ -529,9 +529,9 @@ public void paste() { @Test public void pasteWithCustomSep() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "paste('my.string','a','b','c','d','join_char': ', ')", - "remove_field('a','b','c','d')"), // + "remove_field('a','b','c','d')"), i -> { i.startRecord("1"); i.literal("a", "eeny"); @@ -550,9 +550,9 @@ public void pasteWithCustomSep() { @Test public void pasteWithLiteralStrings() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "paste('my.string','~Hi','a','~how are you?')", - "remove_field('a','b','c','d')"), // + "remove_field('a','b','c','d')"), i -> { i.startRecord("1"); i.literal("a", "eeny"); @@ -571,9 +571,9 @@ public void pasteWithLiteralStrings() { @Test public void hashFromArray() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "set_array('foo','a','b','c','d')", // - "hash('foo')"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('foo','a','b','c','d')", + "hash('foo')"), i -> { i.startRecord("1"); i.endRecord(); @@ -589,9 +589,9 @@ public void hashFromArray() { @Test public void arrayFromHash() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "set_hash('foo','a': 'b','c': 'd')", // - "array('foo')"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_hash('foo','a': 'b','c': 'd')", + "array('foo')"), i -> { i.startRecord("1"); i.endRecord(); @@ -609,10 +609,10 @@ public void arrayFromHash() { @Test public void reject() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if exists ('_metadata.error')", // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if exists ('_metadata.error')", " reject()", - "end"), // + "end"), i -> { i.startRecord("1"); i.literal("_metadata.error", "details"); @@ -627,9 +627,9 @@ public void reject() { @Test public void appendArray() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "set_array('nums', '1')", // - "set_array('nums.$append', '2', '3')"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('nums', '1')", + "set_array('nums.$append', '2', '3')"), i -> { i.startRecord("1"); i.endRecord(); @@ -646,9 +646,9 @@ public void appendArray() { @Test public void mixedArray() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "set_array('@context', 'https://w3id.org/kim/lrmi-profile/draft/context.jsonld')", // - "set_hash('@context.$append', '@language': 'de')"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "set_array('@context', 'https://w3id.org/kim/lrmi-profile/draft/context.jsonld')", + "set_hash('@context.$append', '@language': 'de')"), i -> { i.startRecord("1"); i.endRecord(); @@ -665,8 +665,8 @@ public void mixedArray() { @Test public void retain() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "retain('1','3')"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "retain('1','3')"), i -> { i.startRecord("1"); i.literal("1", "one"); @@ -684,8 +684,8 @@ public void retain() { @Test public void vacuum() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "vacuum()"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "vacuum()"), i -> { i.startRecord("1"); i.literal("1", "one"); diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixSelectorTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixSelectorTest.java index ff5d9457..04b9b6cc 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixSelectorTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixSelectorTest.java @@ -49,13 +49,13 @@ public MetafixSelectorTest() { } @Test public void reject() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "reject exists(error)"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "reject exists(error)"), i -> { i.startRecord("1"); i.literal("error", "details"); i.endRecord(); - // + i.startRecord("2"); i.endRecord(); }, o -> { @@ -67,15 +67,15 @@ public void reject() { @Test public void rejectWithExplicitConditional() { - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(// - "if exists(error)", // - " reject()", // - "end"), // + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "if exists(error)", + " reject()", + "end"), i -> { i.startRecord("1"); i.literal("error", "details"); i.endRecord(); - // + i.startRecord("2"); i.endRecord(); }, o -> { From 4f06e3ae2b3a9ba359c63a9f039bbb468c3d37db Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Wed, 3 Nov 2021 14:06:19 +0100 Subject: [PATCH 49/55] Move comments into `@Disabled` arguments (#35) --- .../java/org/metafacture/metafix/MetafixBindTest.java | 10 +++++----- .../org/metafacture/metafix/MetafixMethodTest.java | 6 +++--- .../org/metafacture/metafix/MetafixRecordTest.java | 2 +- .../org/metafacture/metafix/MetafixSelectorTest.java | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java index 11c910bb..b55e81ea 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixBindTest.java @@ -196,7 +196,7 @@ public void doListEntitesToEntities() { } @Test - @Disabled // TODO: how to handle repeated entities: turn to array vs. merge because it's the same? + @Disabled("TODO: how to handle repeated entities: turn to array vs. merge because it's the same?") public void doListEntitesWithFieldsToEntities() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "do list('path': 'creator', 'var': 'c')", @@ -237,7 +237,7 @@ public void doListEntitesWithFieldsToEntities() { } @Test - @Disabled // implement Fix-style binds with collectors? + @Disabled("implement Fix-style binds with collectors?") public void ifInCollector() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "do entity('author')", @@ -261,7 +261,7 @@ public void ifInCollector() { } @Test - @Disabled // implement Fix-style binds with collectors? + @Disabled("implement Fix-style binds with collectors?") public void ifInCollectorMultiRecords() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "do entity('author')", @@ -305,7 +305,7 @@ public void ifInCollectorMultiRecords() { } @Test - @Disabled // implement Fix-style binds with collectors? + @Disabled("implement Fix-style binds with collectors?") public void ifInCollectorChoose() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "do choose(flushWith: 'record')", @@ -332,7 +332,7 @@ public void ifInCollectorChoose() { } @Test - @Disabled // implement Fix-style binds with collectors? + @Disabled("implement Fix-style binds with collectors?") public void ifInCollectorCombine() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "do combine(name: 'fullName', value: '${first} ${last}')", diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java index b8074fd6..479a44ef 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java @@ -315,7 +315,7 @@ public void parseTextNamedGroups() { } @Test - @Disabled // Use SimpleRegexTrie/WildcardTrie + @Disabled("Use SimpleRegexTrie/WildcardTrie") public void alternation() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "trim('title-1|title-2')"), @@ -345,7 +345,7 @@ public void alternation() { } @Test - @Disabled // Use SimpleRegexTrie/WildcardTrie + @Disabled("Use SimpleRegexTrie/WildcardTrie") public void wildcard() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "trim('title-?')"), @@ -375,7 +375,7 @@ public void wildcard() { } @Test - @Disabled // Use SimpleRegexTrie + @Disabled("Use SimpleRegexTrie") public void characterClass() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "trim('title-[12]')"), diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java index 8180e2ae..7d431db9 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java @@ -85,7 +85,7 @@ public void internalIdUsage() { } @Test - @Disabled // TODO: how to handle repeated entities: turn to array vs. merge because it's the same? + @Disabled("TODO: how to handle repeated entities: turn to array vs. merge because it's the same?") public void entitiesPassThroughRepeatEntity() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "vacuum()"), diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixSelectorTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixSelectorTest.java index 04b9b6cc..934412ad 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixSelectorTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixSelectorTest.java @@ -36,7 +36,7 @@ * */ @ExtendWith(MockitoExtension.class) -@Disabled // TODO: support Fix-style selectors https://github.com/LibreCat/Catmandu/wiki/Selectors +@Disabled("TODO: support Fix-style selectors https://github.com/LibreCat/Catmandu/wiki/Selectors") public final class MetafixSelectorTest { @RegisterExtension From ea54a0ec96a2bb90a47f057f70c05c83d563fd82 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Wed, 3 Nov 2021 14:50:12 +0100 Subject: [PATCH 50/55] Switch `checkstyle-disable-line` to `@SuppressWarnings` (#35) --- .../org/metafacture/metafix/interpreter/FixInterpreter.java | 3 ++- .../src/test/java/org/metafacture/metafix/InterpreterTest.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/interpreter/FixInterpreter.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/interpreter/FixInterpreter.java index 9ffffe9d..9a8f8960 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/interpreter/FixInterpreter.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/interpreter/FixInterpreter.java @@ -32,7 +32,8 @@ public void run(final Metafix metafixParam, final EObject program) { } } - private void process(final Expression expression) { // checkstyle-disable-line CyclomaticComplexity|NPathComplexity + @SuppressWarnings({"checkstyle:CyclomaticComplexity", "checkstyle:NPathComplexity"}) + private void process(final Expression expression) { metafix.getExpressions().add(expression); boolean matched = false; diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/InterpreterTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/InterpreterTest.java index 5a17bc1f..588452f7 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/InterpreterTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/InterpreterTest.java @@ -37,8 +37,9 @@ public void shouldInterpretSimple() throws Exception { } @Test + @SuppressWarnings("checkstyle:MagicNumber") public void shouldInterpretNested() throws Exception { - interpret(3, // checkstyle-disable-line MagicNumber + interpret(3, "do marc_each()", "\tif marc_has(f700)", "\t\tmarc_map(f700a,authors.$append)", From 5395e907ee69a9d61f3a5dcb705d30cfa7b2e840 Mon Sep 17 00:00:00 2001 From: Jens Wille Date: Thu, 4 Nov 2021 13:18:18 +0100 Subject: [PATCH 51/55] Extract named groups for `parse_text`. - Scans the supplied pattern without relying on reflection (which is incompatible with the Java module system [1]). - Preserves order of named groups in the output record. [1] java.lang.reflect.InaccessibleObjectException: Unable to make java.util.Map java.util.regex.Pattern.namedGroups() accessible: module java.base does not "opens java.util.regex" to unnamed module [...] --- .../org/metafacture/metafix/FixMethod.java | 47 ++++++++----------- .../metafix/MetafixMethodTest.java | 22 ++++++++- 2 files changed, 40 insertions(+), 29 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index 65ea9444..9e973317 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -16,15 +16,11 @@ package org.metafacture.metafix; -import org.metafacture.framework.MetafactureException; import org.metafacture.metamorph.maps.FileMap; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -143,12 +139,23 @@ public void apply(final Map record, final List params, final Matcher m = p.matcher(v.toString()); if (m.matches()) { record.remove(params.get(0)); - final Map namedGroups = getNamedGroups(p); - if (!namedGroups.isEmpty()) { - final Map result = new HashMap<>(); - namedGroups.keySet().forEach(k -> { - result.put(k, m.group(namedGroups.get(k))); - }); + + /** + * {@code Pattern.namedGroups()} not available as API, + * see https://stackoverflow.com/a/65012527. + * + * Assumptions: 1. Named groups are not escaped/quoted; + * 2. Named groups are not mixed with unnamed groups. + */ + final Matcher groupMatcher = NAMED_GROUP_PATTERN.matcher(p.pattern()); + final Map result = new LinkedHashMap<>(); + int count = 0; + + while (groupMatcher.find()) { + result.put(groupMatcher.group(1), m.group(++count)); + } + + if (count > 0) { Metafix.add(record, params.get(0), result); } else { @@ -159,24 +166,6 @@ public void apply(final Map record, final List params, } }); } - - @SuppressWarnings("unchecked") - private Map getNamedGroups(final Pattern regex) { - try { - // Not available as API, see https://stackoverflow.com/a/15596145/18154: - final Method namedGroupsMethod = Pattern.class.getDeclaredMethod("namedGroups"); - namedGroupsMethod.setAccessible(true); - Map namedGroups = null; - namedGroups = (Map) namedGroupsMethod.invoke(regex); - if (namedGroups == null) { - throw new InternalError(); - } - return Collections.unmodifiableMap(namedGroups); - } - catch (final NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { - throw new MetafactureException(e); - } - } }, paste { public void apply(final Map record, final List params, @@ -271,6 +260,8 @@ private Map fileMap(final String location, final String separato } }; + private static final Pattern NAMED_GROUP_PATTERN = Pattern.compile("\\(\\?<(.+?)>"); + private static final String NESTED = "Nested non-map / non-list: "; private static final String EMPTY = ""; private static final String APPEND = "$append"; diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java index 479a44ef..4a9e934a 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java @@ -306,14 +306,34 @@ public void parseTextNamedGroups() { }, o -> { o.get().startRecord("1"); o.get().startEntity("date"); - o.get().literal("month", "03"); o.get().literal("year", "2015"); + o.get().literal("month", "03"); o.get().literal("day", "07"); o.get().endEntity(); o.get().endRecord(); }); } + @Test + public void parseTextNestedNamedGroups() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "parse_text(data, '.(?.(?.).(?.).).(?.).')"), + i -> { + i.startRecord("1"); + i.literal("data", "abcdefghi"); + i.endRecord(); + }, o -> { + o.get().startRecord("1"); + o.get().startEntity("data"); + o.get().literal("outer1", "bcdef"); + o.get().literal("inner1", "c"); + o.get().literal("inner2", "e"); + o.get().literal("outer2", "h"); + o.get().endEntity(); + o.get().endRecord(); + }); + } + @Test @Disabled("Use SimpleRegexTrie/WildcardTrie") public void alternation() { From 26d2a9e549e64272e24350b3bebdecefbd358ee8 Mon Sep 17 00:00:00 2001 From: Jens Wille Date: Thu, 4 Nov 2021 17:05:58 +0100 Subject: [PATCH 52/55] Fix named group access in `parse_text`. (5395e90) Access group by name instead of index (which may be off when used in combination with unnamed groups). --- .../org/metafacture/metafix/FixMethod.java | 6 +- .../metafix/MetafixMethodTest.java | 59 +++++++++++++++++++ 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java index 9e973317..6b8f1ff4 100644 --- a/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/org.metafacture.fix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -149,13 +149,13 @@ public void apply(final Map record, final List params, */ final Matcher groupMatcher = NAMED_GROUP_PATTERN.matcher(p.pattern()); final Map result = new LinkedHashMap<>(); - int count = 0; while (groupMatcher.find()) { - result.put(groupMatcher.group(1), m.group(++count)); + final String group = groupMatcher.group(1); + result.put(group, m.group(group)); } - if (count > 0) { + if (!result.isEmpty()) { Metafix.add(record, params.get(0), result); } else { diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java index 4a9e934a..7a5df333 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java @@ -16,9 +16,11 @@ package org.metafacture.metafix; +import org.metafacture.framework.MetafactureException; import org.metafacture.framework.StreamReceiver; import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -334,6 +336,63 @@ public void parseTextNestedNamedGroups() { }); } + @Test + public void parseTextMixedGroups() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "parse_text(data, '(?.)(.)(?.)')" + ), + i -> { + i.startRecord("1"); + i.literal("data", "abc"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("data"); + o.get().literal("a", "a"); + o.get().literal("c", "c"); + o.get().endEntity(); + o.get().endRecord(); + } + ); + } + + @Test + public void parseTextEscapedGroups() { + Assertions.assertThrows(MetafactureException.class, () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "parse_text(data, '(?.)(.)\\\\(?.\\\\)')" + ), + i -> { + i.startRecord("1"); + i.literal("data", "ab(c)"); + i.endRecord(); + }, + o -> { + } + ), + "No group with name " + ); + } + + @Test + public void parseTextQuotedGroups() { + Assertions.assertThrows(MetafactureException.class, () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "parse_text(data, '(?.)(.)\\\\Q(?.)\\\\E')" + ), + i -> { + i.startRecord("1"); + i.literal("data", "ab(?.)"); + i.endRecord(); + }, + o -> { + } + ), + "No group with name " + ); + } + @Test @Disabled("Use SimpleRegexTrie/WildcardTrie") public void alternation() { From bdc43eeb72e1a65878b648b73e3887bc85c9b96b Mon Sep 17 00:00:00 2001 From: Jens Wille Date: Thu, 4 Nov 2021 17:14:37 +0100 Subject: [PATCH 53/55] Fix Checkstyle violations in MetafixMethodTest. (26d2a9e) --- .../metafix/MetafixMethodTest.java | 74 +++++++++---------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java index 7a5df333..2469f9d9 100644 --- a/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java +++ b/org.metafacture.fix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java @@ -339,57 +339,57 @@ public void parseTextNestedNamedGroups() { @Test public void parseTextMixedGroups() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "parse_text(data, '(?.)(.)(?.)')" - ), - i -> { - i.startRecord("1"); - i.literal("data", "abc"); - i.endRecord(); - }, - o -> { - o.get().startRecord("1"); - o.get().startEntity("data"); - o.get().literal("a", "a"); - o.get().literal("c", "c"); - o.get().endEntity(); - o.get().endRecord(); - } + "parse_text(data, '(?.)(.)(?.)')" + ), + i -> { + i.startRecord("1"); + i.literal("data", "abc"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("data"); + o.get().literal("a", "a"); + o.get().literal("c", "c"); + o.get().endEntity(); + o.get().endRecord(); + } ); } @Test public void parseTextEscapedGroups() { Assertions.assertThrows(MetafactureException.class, () -> - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "parse_text(data, '(?.)(.)\\\\(?.\\\\)')" - ), - i -> { - i.startRecord("1"); - i.literal("data", "ab(c)"); - i.endRecord(); - }, - o -> { - } + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "parse_text(data, '(?.)(.)\\\\(?.\\\\)')" ), - "No group with name " + i -> { + i.startRecord("1"); + i.literal("data", "ab(c)"); + i.endRecord(); + }, + o -> { + } + ), + "No group with name " ); } @Test public void parseTextQuotedGroups() { Assertions.assertThrows(MetafactureException.class, () -> - MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "parse_text(data, '(?.)(.)\\\\Q(?.)\\\\E')" - ), - i -> { - i.startRecord("1"); - i.literal("data", "ab(?.)"); - i.endRecord(); - }, - o -> { - } + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "parse_text(data, '(?.)(.)\\\\Q(?.)\\\\E')" ), - "No group with name " + i -> { + i.startRecord("1"); + i.literal("data", "ab(?.)"); + i.endRecord(); + }, + o -> { + } + ), + "No group with name " ); } From e4dec3aef20ed025150f43cea4fe7cba582f6a0d Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Fri, 5 Nov 2021 13:56:41 +0100 Subject: [PATCH 54/55] Restore FixServlet and ServerLauncher (#35) Accidentally deleted during refactoring in a786f73 --- .../metafacture/metafix/web/FixServlet.java | 112 ++++++++++++++++++ .../metafix/web/ServerLauncher.java | 74 ++++++++++++ .../src/main/webapp/index.html | 2 +- 3 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 org.metafacture.fix.web/src/main/java/org/metafacture/metafix/web/FixServlet.java create mode 100644 org.metafacture.fix.web/src/main/java/org/metafacture/metafix/web/ServerLauncher.java diff --git a/org.metafacture.fix.web/src/main/java/org/metafacture/metafix/web/FixServlet.java b/org.metafacture.fix.web/src/main/java/org/metafacture/metafix/web/FixServlet.java new file mode 100644 index 00000000..16866e70 --- /dev/null +++ b/org.metafacture.fix.web/src/main/java/org/metafacture/metafix/web/FixServlet.java @@ -0,0 +1,112 @@ +package org.metafacture.metafix.web; + +import org.metafacture.metafix.FixStandaloneSetup; +import org.metafacture.runner.Flux; + +import org.antlr.runtime.RecognitionException; +import org.eclipse.xtext.util.DisposableRegistry; +import org.eclipse.xtext.web.servlet.XtextServlet; +import org.eclipse.xtext.xbase.lib.InputOutput; + +import java.io.IOException; +import java.io.StringReader; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Map; +import javax.servlet.ServletException; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * Deploy this class into a servlet container to enable DSL-specific services. + */ +@WebServlet(name = "XtextServices", urlPatterns = "/xtext-service/*") +public class FixServlet extends XtextServlet { + + private static final String COMMAND_FIX = "fix"; + + private static final String PARAM_DATA = "data"; + private static final String PARAM_FIX = "fix"; + private static final String PARAM_FLUX = "flux"; + + private DisposableRegistry disposableRegistry; + + public FixServlet() { + } + + @Override + public void init() throws ServletException { + disposableRegistry = new FixWebSetup().createInjectorAndDoEMFRegistration().getInstance(DisposableRegistry.class); + } + + @Override + public void destroy() { + if (disposableRegistry != null) { + disposableRegistry.dispose(); + disposableRegistry = null; + } + + super.destroy(); + } + + @Override + public void doPost(final HttpServletRequest request, final HttpServletResponse response) throws IOException, ServletException { + InputOutput.println("POST Request: " + request); + + if (!request.getPathInfo().endsWith("/run") || !process(request, response)) { + super.doPost(request, response); + } + } + + @Override + public void doGet(final HttpServletRequest request, final HttpServletResponse response) throws IOException, ServletException { + InputOutput.println("GET Request: " + request); + + if (!process(request, response)) { + super.doGet(request, response); + } + } + + private boolean process(final HttpServletRequest request, final HttpServletResponse response) throws IOException { + final Map params = request.getParameterMap(); + + if (!params.containsKey(PARAM_DATA) || !params.containsKey(PARAM_FLUX) || !params.containsKey(PARAM_FIX)) { + return false; + } + + final StringBuilder builder = new StringBuilder(); + + final String inData = request.getParameter(PARAM_DATA); + builder.append(inData == null || inData.isEmpty() ? "" : + "\"" + absPathToTempFile(inData, ".txt") + "\"|open-file|"); + + final String fixFile = absPathToTempFile(request.getParameter(PARAM_FIX), ".fix"); + final String outFile = absPathToTempFile("", ".txt"); + + builder.append(request.getParameter(PARAM_FLUX).replaceAll("\\s?\\|\\s?", "|").replace( + "|" + COMMAND_FIX + "|", + "|org.metafacture.metafix.Metafix(\"" + fixFile + "\")|")); + builder.append("|write(\""); + builder.append(outFile); + builder.append("\");"); + + final String fullFlux = builder.toString(); + InputOutput.println("full flux: " + fullFlux); + + try { + Flux.main(new String[] {absPathToTempFile(fullFlux, ".flux")}); + } + catch (final RecognitionException e) { + throw new RuntimeException(e); + } + + Files.copy(Paths.get(outFile), response.getOutputStream()); + return true; + } + + private String absPathToTempFile(final String content, final String suffix) throws IOException { + return FixStandaloneSetup.absPathToTempFile(new StringReader(content), suffix); + } + +} diff --git a/org.metafacture.fix.web/src/main/java/org/metafacture/metafix/web/ServerLauncher.java b/org.metafacture.fix.web/src/main/java/org/metafacture/metafix/web/ServerLauncher.java new file mode 100644 index 00000000..93893f7c --- /dev/null +++ b/org.metafacture.fix.web/src/main/java/org/metafacture/metafix/web/ServerLauncher.java @@ -0,0 +1,74 @@ +package org.metafacture.metafix.web; + +import org.eclipse.jetty.annotations.AnnotationConfiguration; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.util.log.Slf4jLog; +import org.eclipse.jetty.webapp.Configuration; +import org.eclipse.jetty.webapp.MetaInfConfiguration; +import org.eclipse.jetty.webapp.WebAppContext; +import org.eclipse.jetty.webapp.WebInfConfiguration; +import org.eclipse.jetty.webapp.WebXmlConfiguration; +import org.eclipse.xtext.xbase.lib.Exceptions; + +import java.net.InetSocketAddress; + +/** + * This program starts an HTTP server for testing the web integration of your DSL. + * Just execute it and point a web browser to http://localhost:8080/ + */ +@SuppressWarnings({"checkstyle:ClassDataAbstractionCoupling", "checkstyle:IllegalCatch"}) +public class ServerLauncher { + + private ServerLauncher() { + throw new IllegalAccessError("Utility class"); + } + + public static void main(final String[] args) { + final WebAppContext context = new WebAppContext(); + context.setResourceBase("src/main/webapp"); + context.setWelcomeFiles(new String[] {"index.html"}); + context.setContextPath("/"); + context.setConfigurations(new Configuration[] {new AnnotationConfiguration(), new WebXmlConfiguration(), new WebInfConfiguration(), new MetaInfConfiguration()}); + context.setAttribute(WebInfConfiguration.CONTAINER_JAR_PATTERN, ".*/org\\.metafacture\\.fix\\.web/.*,.*\\.jar"); + context.setInitParameter("org.mortbay.jetty.servlet.Default.useFileMappedBuffer", "false"); + + final Server server = new Server(new InetSocketAddress("0.0.0.0", 8080)); + server.setHandler(context); + + final Slf4jLog log = new Slf4jLog(ServerLauncher.class.getName()); + + try { + server.start(); + + log.info("Server started " + server.getURI() + "..."); + + new Thread(() -> { + try { + log.info("Press enter to stop the server..."); + + if (System.in.read() != -1) { + server.stop(); + } + else { + log.warn("Console input is not available. In order to stop the server, you need to cancel the process manually."); + } + } + catch (final Exception e) { + throw Exceptions.sneakyThrow(e); + } + }).start(); + + server.join(); + } + catch (final Throwable e) { + if (e instanceof Exception) { + log.warn(((Exception) e).getMessage()); + System.exit(1); + } + else { + throw Exceptions.sneakyThrow(e); + } + } + } + +} diff --git a/org.metafacture.fix.web/src/main/webapp/index.html b/org.metafacture.fix.web/src/main/webapp/index.html index 55f347f2..88eea196 100644 --- a/org.metafacture.fix.web/src/main/webapp/index.html +++ b/org.metafacture.fix.web/src/main/webapp/index.html @@ -34,7 +34,7 @@