Merge pull request #84 from metafacture/63-emitUnderscoreFields

Emit all underscore fields. (#63)

metafix/src/main/java/org/metafacture/metafix/Metafix.java

@@ -113,14 +113,14 @@ private void buildPipeline(final Reader fixDef, final Map<String, String> theVar
     @Override
     public void startRecord(final String identifier) {
         currentRecord = new Record();
+        currentRecord.putVirtualField(StandardEventNames.ID, new Value(identifier));
         LOG.debug("Start record: {}", identifier);
         flattener.startRecord(identifier);
         entityCountStack.clear();
         entityCount = 0;
         entityCountStack.add(Integer.valueOf(entityCount));
         recordIdentifier = identifier;
         entities = new ArrayList<>();
-        literal(StandardEventNames.ID, identifier);
     }
 
     @Override
@@ -136,11 +136,7 @@ public void endRecord() {
         if (!currentRecord.getReject()) {
             outputStreamReceiver.startRecord(recordIdentifier);
             LOG.debug("Sending results to {}", outputStreamReceiver);
-            currentRecord.forEach((f, v) -> {
-                if (!f.startsWith("_")) {
-                    emit(f, v);
-                }
-            });
+            currentRecord.forEach(this::emit);
             outputStreamReceiver.endRecord();
         }
     }

metafix/src/main/java/org/metafacture/metafix/Record.java

@@ -16,12 +16,18 @@
 
 package org.metafacture.metafix;
 
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
 /**
  * Represents a metadata record, i.e., a {@link Value.Hash Hash} of fields
  * and values.
  */
 public class Record extends Value.Hash {
 
+    private final Map<String, Value> virtualFields = new LinkedHashMap<>();
+
     private boolean reject;
 
     /**
@@ -40,6 +46,7 @@ public Record shallowClone() {
 
         clone.setReject(reject);
         forEach(clone::put);
+        virtualFields.forEach(clone::putVirtualField);
 
         return clone;
     }
@@ -62,10 +69,88 @@ public boolean getReject() {
         return reject;
     }
 
+    /**
+     * Checks whether this record contains the <i>virtual</i> field.
+     *
+     * @param field the field name
+     * @return true if this record contains the <i>virtual</i> field, false otherwise
+     */
+    public boolean containsVirtualField(final String field) {
+        return virtualFields.containsKey(field);
+    }
+
+    /**
+     * Adds a <i>virtual</i> field/value pair to this record, provided it's not
+     * {@link Value#isNull(Value) null}. Virtual fields can be
+     * {@link #get(String) accessed} like regular metadata fields, but aren't
+     * {@link #forEach(BiConsumer) emitted} by default.
+     *
+     * @param field the field name
+     * @param value the metadata value
+     *
+     * @see #retainFields(Collection)
+     */
+    public void putVirtualField(final String field, final Value value) {
+        if (!Value.isNull(value)) {
+            virtualFields.put(field, value);
+        }
+    }
+
     @Override
     public String toString() {
-        // TODO: Improve string representation? Include reject status, etc.?
+        // TODO: Improve string representation? Include reject status, virtual fields, etc.?
         return super.toString();
     }
 
+    /**
+     * Retrieves the field value from this record. Falls back to retrieving the
+     * <i>virtual</i> field if the field name is not already
+     * {@link #containsField(String) present}.
+     *
+     * @param field the field name
+     * @return the metadata value
+     */
+    @Override
+    public Value get(final String field) {
+        return containsField(field) ? super.get(field) : virtualFields.get(field);
+    }
+
+    /**
+     * {@link #put(String, Value) Adds} a field/value pair to this record. Turns
+     * <i>virtual</i> fields into regular metadata fields if they're not already
+     * {@link #containsField(String) present}.
+     *
+     * @param field the field name
+     * @param newValue the new metadata value
+     */
+    @Override
+    public void add(final String field, final Value newValue) {
+        if (containsField(field)) {
+            super.add(field, newValue);
+        }
+        else {
+            put(field, newValue);
+        }
+    }
+
+    /**
+     * Retains only the given field/value pairs in this record. Turns
+     * <i>virtual</i> fields into regular metadata fields if they're not already
+     * {@link #containsField(String) present}.
+     *
+     * @param fields the field names
+     */
+    @Override
+    public void retainFields(final Collection<String> fields) {
+        virtualFields.keySet().retainAll(fields);
+
+        virtualFields.forEach((f, v) -> {
+            if (!containsField(f)) {
+                put(f, v);
+            }
+        });
+
+        super.retainFields(fields);
+    }
+
 }

metafix/src/main/java/org/metafacture/metafix/Value.java

@@ -206,10 +206,6 @@ public Value asList(final Consumer<Array> consumer) {
         }
     }
 
-    public Value merge(final Value value) {
-        return asList(a1 -> value.asList(a2 -> a2.forEach(a1::add)));
-    }
-
     @Override
     public String toString() {
         final String result;
@@ -266,9 +262,7 @@ void apply(final Hash hash, final String field, final String value) {
             APPEND {
                 @Override
                 void apply(final Hash hash, final String field, final String value) {
-                    final Value oldValue = hash.get(field);
-                    final Value newValue = new Value(value);
-                    hash.put(field, oldValue == null ? newValue : oldValue.merge(newValue));
+                    hash.add(field, new Value(value));
                 }
             };
 
@@ -472,7 +466,7 @@ public int size() {
         }
 
         /**
-         * Adds a field/value pair to this hash, provided it's not {@code null}.
+         * Adds a field/value pair to this hash, provided it's not {@link #isNull(Value) null}.
          *
          * @param field the field name
          * @param value the metadata value
@@ -566,9 +560,16 @@ public void addAll(final Hash hash) {
             hash.forEach(this::add);
         }
 
+        /**
+         * {@link #put(String, Value) Adds} a field/value pair to this hash,
+         * potentially merging with an existing value.
+         *
+         * @param field the field name
+         * @param newValue the new metadata value
+         */
         public void add(final String field, final Value newValue) {
             final Value oldValue = get(field);
-            put(field, oldValue == null ? newValue : oldValue.merge(newValue));
+            put(field, oldValue == null ? newValue : oldValue.asList(a1 -> newValue.asList(a2 -> a2.forEach(a1::add))));
         }
 
         public Value insert(final InsertMode mode, final String fieldPath, final String newValue) {

metafix/src/test/java/org/metafacture/metafix/HashValueTest.java

@@ -19,9 +19,7 @@
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.List;
 
 public class HashValueTest {
 
@@ -200,21 +198,19 @@ public void shouldRemoveEmptyValues() {
 
     @Test
     public void shouldIterateOverFieldValuePairs() {
+        final Value emptyValue = new Value("");
+        final Value specialValue = new Value("1");
+
         final Value.Hash hash = newHash();
         hash.put(FIELD, VALUE);
         hash.put(OTHER_FIELD, OTHER_VALUE);
-        hash.put("empty field", new Value(""));
-        hash.put("_special field", new Value("1"));
-
-        final List<String> fields = new ArrayList<>();
-        final List<String> values = new ArrayList<>();
-        hash.forEach((f, v) -> {
-            fields.add(f);
-            values.add(v.asString());
-        });
-
-        Assertions.assertEquals(Arrays.asList(FIELD, OTHER_FIELD, "empty field", "_special field"), fields);
-        Assertions.assertEquals(Arrays.asList(VALUE.asString(), OTHER_VALUE.asString(), "", "1"), values);
+        hash.put("empty field", emptyValue);
+        hash.put("_special field", specialValue);
+
+        MetafixTestHelpers.assertEmittedFields(hash,
+                Arrays.asList(FIELD, OTHER_FIELD, "empty field", "_special field"),
+                Arrays.asList(VALUE, OTHER_VALUE, emptyValue, specialValue)
+        );
     }
 
     private Value.Hash newHash() {

metafix/src/test/java/org/metafacture/metafix/MetafixRecordTest.java

@@ -70,17 +70,70 @@ public void entitiesPassThrough() {
     }
 
     @Test
-    public void internalIdUsage() {
+    public void shouldNotEmitVirtualFieldsByDefault() {
         MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(
-                "copy_field('_id', id)"),
+                "vacuum()"
+            ),
             i -> {
                 i.startRecord("1");
                 i.endRecord();
-            }, o -> {
+            },
+            o -> {
+                o.get().startRecord("1");
+                o.get().endRecord();
+            }
+        );
+    }
+
+    @Test
+    public void shouldEmitVirtualFieldsWhenRetained() {
+        MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(
+                "retain('_id')"
+            ),
+            i -> {
+                i.startRecord("1");
+                i.endRecord();
+            },
+            o -> {
+                o.get().startRecord("1");
+                o.get().literal("_id", "1");
+                o.get().endRecord();
+            }
+        );
+    }
+
+    @Test
+    public void shouldEmitVirtualFieldsWhenCopied() {
+        MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(
+                "copy_field('_id', id)"
+            ),
+            i -> {
+                i.startRecord("1");
+                i.endRecord();
+            },
+            o -> {
                 o.get().startRecord("1");
                 o.get().literal("id", "1");
                 o.get().endRecord();
-            });
+            }
+        );
+    }
+
+    @Test
+    public void shouldEmitVirtualFieldsWhenAdded() {
+        MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(
+                "add_field('_id', 'id')"
+            ),
+            i -> {
+                i.startRecord("1");
+                i.endRecord();
+            },
+            o -> {
+                o.get().startRecord("1");
+                o.get().literal("_id", "id");
+                o.get().endRecord();
+            }
+        );
     }
 
     @Test

metafix/src/test/java/org/metafacture/metafix/MetafixTestHelpers.java

@@ -18,11 +18,13 @@
 
 import org.metafacture.framework.StreamReceiver;
 
+import org.junit.jupiter.api.Assertions;
 import org.mockito.InOrder;
 import org.mockito.Mockito;
 import org.mockito.exceptions.base.MockitoAssertionError;
 
 import java.io.FileNotFoundException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.function.BiConsumer;
@@ -38,7 +40,8 @@
  */
 public final class MetafixTestHelpers {
 
-    private MetafixTestHelpers() { }
+    private MetafixTestHelpers() {
+    }
 
     public static void assertFix(final StreamReceiver receiver, final List<String> fixDef, final Consumer<Metafix> in,
             final Consumer<Supplier<StreamReceiver>> out) {
@@ -90,4 +93,17 @@ private static Metafix fix(final StreamReceiver receiver, final String fix, fina
         return metafix;
     }
 
+    public static void assertEmittedFields(final Value.Hash hash, final List<String> expectedFields, final List<Value> expectedValues) {
+        final List<String> actualFields = new ArrayList<>();
+        final List<Value> actualValues = new ArrayList<>();
+
+        hash.forEach((f, v) -> {
+            actualFields.add(f);
+            actualValues.add(v);
+        });
+
+        Assertions.assertEquals(expectedFields, actualFields);
+        Assertions.assertEquals(expectedValues, actualValues);
+    }
+
 }