Discard/replace "null" values by pattern.

Fixes metafacture/metafacture-fix#34.
metafacture · Dec 3, 2020 · 36bcec9 · 36bcec9
1 parent fdd7035
commit 36bcec9
Show file tree

Hide file tree

Showing 2 changed files with 71 additions and 3 deletions.
diff --git a/metafacture-mangling/src/main/java/org/metafacture/mangling/NullFilter.java b/metafacture-mangling/src/main/java/org/metafacture/mangling/NullFilter.java
@@ -21,9 +21,12 @@
 import org.metafacture.framework.annotations.Out;
 import org.metafacture.framework.helpers.ForwardingStreamPipe;
 
+import java.util.function.Predicate;
+import java.util.regex.Pattern;
+
 /**
- * Replaces null values with replacement string, or, if replacement
- * string is null (default), discards null values entirely.
+ * Replaces "null" values with replacement string, or, if replacement
+ * string is null (default), discards them entirely.
  *
  * @author Jens Wille
  *
@@ -33,8 +36,21 @@
 @Out(StreamReceiver.class)
 public final class NullFilter extends ForwardingStreamPipe {
 
+    private String pattern = null;
+
+    private Predicate<String> predicate = v -> false;
+
     private String replacement = null;
 
+    public void setPattern(final String pattern) {
+        this.pattern = pattern;
+        this.predicate = Pattern.compile(pattern).asPredicate();
+    }
+
+    public String getPattern() {
+        return pattern;
+    }
+
     public void setReplacement(final String replacement) {
         this.replacement = replacement;
     }
@@ -45,7 +61,7 @@ public String getReplacement() {
 
     @Override
     public void literal(final String name, final String value) {
-        if (value != null) {
+        if (value != null && !predicate.test(value)) {
             getReceiver().literal(name, value);
         } else if (replacement != null) {
             getReceiver().literal(name, replacement);

diff --git a/metafacture-mangling/src/test/java/org/metafacture/mangling/NullFilterTest.java b/metafacture-mangling/src/test/java/org/metafacture/mangling/NullFilterTest.java
@@ -16,6 +16,7 @@
 package org.metafacture.mangling;
 
 import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 
 import org.junit.After;
@@ -37,6 +38,7 @@ public final class NullFilterTest {
     private static final String RECORD_ID = "id";
     private static final String ENTITY_NAME = "entity-name";
     private static final String LITERAL_NAME = "literal-name";
+    private static final String LITERAL_NULL = "literal-NULL";
     private static final String LITERAL_VALUE = "literal-value";
 
     private NullFilter nullFilter;
@@ -79,6 +81,7 @@ public void shouldDiscardNullValues() {
         nullFilter.startRecord(RECORD_ID);
         nullFilter.startEntity(ENTITY_NAME);
         nullFilter.literal(LITERAL_NAME, LITERAL_VALUE);
+        nullFilter.literal(LITERAL_NAME, LITERAL_NULL);
         nullFilter.literal(LITERAL_NAME, null);
         nullFilter.endEntity();
         nullFilter.endRecord();
@@ -87,6 +90,7 @@ public void shouldDiscardNullValues() {
         ordered.verify(receiver).startRecord(RECORD_ID);
         ordered.verify(receiver).startEntity(ENTITY_NAME);
         ordered.verify(receiver).literal(LITERAL_NAME, LITERAL_VALUE);
+        ordered.verify(receiver).literal(LITERAL_NAME, LITERAL_NULL);
         ordered.verify(receiver).endEntity();
         ordered.verify(receiver).endRecord();
 
@@ -100,6 +104,7 @@ public void shouldReplaceNullValues() {
         nullFilter.startRecord(RECORD_ID);
         nullFilter.startEntity(ENTITY_NAME);
         nullFilter.literal(LITERAL_NAME, LITERAL_VALUE);
+        nullFilter.literal(LITERAL_NAME, LITERAL_NULL);
         nullFilter.literal(LITERAL_NAME, null);
         nullFilter.endEntity();
         nullFilter.endRecord();
@@ -108,11 +113,58 @@ public void shouldReplaceNullValues() {
         ordered.verify(receiver).startRecord(RECORD_ID);
         ordered.verify(receiver).startEntity(ENTITY_NAME);
         ordered.verify(receiver).literal(LITERAL_NAME, LITERAL_VALUE);
+        ordered.verify(receiver).literal(LITERAL_NAME, LITERAL_NULL);
         ordered.verify(receiver).literal(LITERAL_NAME, "replacement");
         ordered.verify(receiver).endEntity();
         ordered.verify(receiver).endRecord();
 
         verifyNoMoreInteractions(receiver);
     }
 
+    @Test
+    public void shouldDiscardNullValuesByPattern() {
+        nullFilter.setPattern("NULL");
+
+        nullFilter.startRecord(RECORD_ID);
+        nullFilter.startEntity(ENTITY_NAME);
+        nullFilter.literal(LITERAL_NAME, LITERAL_VALUE);
+        nullFilter.literal(LITERAL_NAME, LITERAL_NULL);
+        nullFilter.literal(LITERAL_NAME, null);
+        nullFilter.endEntity();
+        nullFilter.endRecord();
+
+        final InOrder ordered = inOrder(receiver);
+        ordered.verify(receiver).startRecord(RECORD_ID);
+        ordered.verify(receiver).startEntity(ENTITY_NAME);
+        ordered.verify(receiver).literal(LITERAL_NAME, LITERAL_VALUE);
+        ordered.verify(receiver).endEntity();
+        ordered.verify(receiver).endRecord();
+
+        verifyNoMoreInteractions(receiver);
+    }
+
+    @Test
+    public void shouldReplaceNullValuesByPattern() {
+        nullFilter.setPattern("NULL");
+        nullFilter.setReplacement("replacement");
+
+        nullFilter.startRecord(RECORD_ID);
+        nullFilter.startEntity(ENTITY_NAME);
+        nullFilter.literal(LITERAL_NAME, LITERAL_VALUE);
+        nullFilter.literal(LITERAL_NAME, LITERAL_NULL);
+        nullFilter.literal(LITERAL_NAME, null);
+        nullFilter.endEntity();
+        nullFilter.endRecord();
+
+        final InOrder ordered = inOrder(receiver);
+        ordered.verify(receiver).startRecord(RECORD_ID);
+        ordered.verify(receiver).startEntity(ENTITY_NAME);
+        ordered.verify(receiver).literal(LITERAL_NAME, LITERAL_VALUE);
+        ordered.verify(receiver, times(2)).literal(LITERAL_NAME, "replacement");
+        ordered.verify(receiver).endEntity();
+        ordered.verify(receiver).endRecord();
+
+        verifyNoMoreInteractions(receiver);
+    }
+
 }