Skip to content
This repository has been archived by the owner on Jan 27, 2025. It is now read-only.

Consolidate path traversal #170

Merged
merged 63 commits into from
Apr 8, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
63 commits
Select commit Hold shift + click to select a range
425dc49
Unify ReservedField usage, align structure of insert methods (#102)
fsteeg Feb 18, 2022
1a3278a
Replace existing values in set_array and set_hash (#102, #111)
fsteeg Feb 22, 2022
8c42874
Remove appendTo method, hide FixPath behind record (#102, #92, #89)
fsteeg Feb 22, 2022
c25ede3
Implement transformIn using findIn & insertInto (#102, #121, #147)
fsteeg Feb 28, 2022
692b4f9
Move transform methods to Record, avoid FixPath in FixMethod (#102)
fsteeg Feb 28, 2022
bc59bf4
Merge remote-tracking branch 'origin/master' into 102-path
fsteeg Mar 4, 2022
5ec283d
Exclude .project and metafix.err files in editorconfigChecker
fsteeg Mar 4, 2022
61e3748
Update passing integration tests (#102, #92, #121, #149, #155)
fsteeg Mar 4, 2022
34c96f1
Don't use `path` in equals, hashCode and toString methods (#102)
fsteeg Mar 4, 2022
e9dd4ab
Tweak `lookup` behavior, partial `delete` support (#102, #149)
fsteeg Mar 4, 2022
1bcc5ca
Integrate feedback from #170 (see #102)
fsteeg Mar 4, 2022
be764e4
Throw exception on transform of non-string field (#102, #100, #133)
fsteeg Mar 7, 2022
8b27a52
Integrate feedback from #170 (see #102)
fsteeg Mar 7, 2022
93aaf2e
Some cleanup as suggested in #170 (see #102)
fsteeg Mar 8, 2022
6f455eb
Fix test to actually use internal map (#170, #102)
fsteeg Mar 10, 2022
b376291
Remove unused `MetafixToDo.Extension` (#170, #102)
fsteeg Mar 10, 2022
c479c26
Merge remote-tracking branch 'origin/master' into 102-path
fsteeg Mar 10, 2022
50c45f7
Enable passing integration tests (#102, #113, #130, #156)
fsteeg Mar 10, 2022
238e840
Handle null results directly in transform (#102, #149, #170)
fsteeg Mar 10, 2022
63bf98a
Add failing test case for asterisk in `replace_all()`. (#170)
blackwinter Mar 10, 2022
006e3ac
Remove unnecessary parens, add unit test (#102, #149, #170)
fsteeg Mar 10, 2022
44dc0be
Merge remote-tracking branch 'origin/102-path' into 102-path
fsteeg Mar 10, 2022
5494f42
Add test cases from review (#102, #170)
fsteeg Mar 21, 2022
1e8ac6e
Remove untested $append, $first, $last support on hashes (#102)
fsteeg Mar 18, 2022
31e844b
Merge remote-tracking branch 'origin/master' into 102-path
fsteeg Mar 22, 2022
158c4cf
Add unit tests for failing integration test, mark as TODO (#170)
fsteeg Mar 22, 2022
e027e1c
Update passing integration tests (#102, #149, #170)
fsteeg Mar 22, 2022
b189d26
Merge remote-tracking branch 'origin/master' into 102-path
fsteeg Mar 24, 2022
96f65f4
Fix typo in integration test folder name (#170, #205)
fsteeg Mar 24, 2022
cbcde34
Add unit tests for failing integration test, mark as TODO (#205)
fsteeg Mar 24, 2022
64d8d98
Only create new path if old value's path doesn't match (#102, #170)
fsteeg Mar 25, 2022
8cfe1a8
Add unit test for record transformation. (#102, #170)
blackwinter Mar 25, 2022
e2cf74c
Fix nested removal from array (#102, #113, #170)
fsteeg Mar 25, 2022
2378455
Add unit test for record transformation. (#102, #170)
blackwinter Mar 25, 2022
16ac0a0
Merge remote-tracking branch 'origin/master' into 102-path
fsteeg Mar 28, 2022
2063b07
Update passing integration test, mark failing as TODO (#206)
fsteeg Mar 28, 2022
f210dd3
Preserve order in Record#transform (#102, #170)
fsteeg Mar 29, 2022
a4d557e
Explicitly set path in record test (#102, #170)
fsteeg Mar 29, 2022
a816a73
Add failing test case for unsuccessful lookup. (f210dd3, #170)
blackwinter Mar 30, 2022
ec9f8ef
Delete reversed null results after ordered transform (#102, #170)
fsteeg Mar 31, 2022
725097a
Add failing test case for asterisk in `replace_all()` with repeated f…
blackwinter Mar 31, 2022
2bd779e
Merge remote-tracking branch 'origin/master' into 102-path
fsteeg Mar 31, 2022
40ab054
Mark failing strictness tests as TODOs (#102, #170)
fsteeg Mar 31, 2022
d8cc4f8
Extract try-catch to avoid Eclipse compiler bug (#210)
fsteeg Mar 31, 2022
f4c4faf
Merge remote-tracking branch 'origin/102-path' into 102-path
fsteeg Mar 31, 2022
4324378
Update path for repeated fields to indexed array (#102, #170)
fsteeg Apr 1, 2022
cb4f27e
Add failing integration test for asterisk in `replace_all()` with rep…
blackwinter Apr 1, 2022
1d495ab
Add unit tests for new integration test from cb4f27e (#170)
fsteeg Apr 1, 2022
ca1cba1
Update path when adding, rename on copy/move field (#102, #170)
fsteeg Apr 1, 2022
82bee32
Add failing test case for multiple `replace_all()` on same subfield. …
blackwinter Apr 1, 2022
049c709
Set correct path after inserting transformed value (#102, #170)
fsteeg Apr 4, 2022
7b74937
Add failing test case for `prepend()` on array with wildcard. (#170)
blackwinter Apr 5, 2022
4961b95
Tweak test to produce correct order (#170)
fsteeg Apr 5, 2022
31cf8e9
Replace matchType.ifString.orElseThrow with asString (#102, #170)
fsteeg Apr 5, 2022
eafca9e
Revert "Tweak test to produce correct order (#170)"
blackwinter Apr 5, 2022
0dfcfb8
Update strictness tests for changed behaviour. (#170, 4961b95)
blackwinter Apr 5, 2022
2b476e2
Replace path heuristic with actual value check in lookup (#170)
fsteeg Apr 7, 2022
5e5bf5f
Update strictness integration tests to correspond to unit tests
fsteeg Apr 8, 2022
540f43f
Update TODOs on implicit append in unit & integration tests (#170)
fsteeg Apr 8, 2022
6779544
Improve `*` wildcard behavior, additional tests (#102, #170)
fsteeg Apr 8, 2022
746fcf4
Wrap NumberFormatException into IllegalStateException (#170)
fsteeg Apr 8, 2022
9b8fa39
Tweak shouldNotTrimIndexedArray, add shouldNotTrimHash test (#170)
fsteeg Apr 8, 2022
5a3fa9b
Handle NumberFormatException in `apply` (#170)
fsteeg Apr 8, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 7 additions & 14 deletions metafix/src/main/java/org/metafacture/metafix/FixPath.java
Original file line number Diff line number Diff line change
Expand Up @@ -45,16 +45,20 @@ private FixPath(final String[] path) {
}

/*package-private*/ Value findIn(final Hash hash) {
return findIn(hash, false);
}

/*package-private*/ Value findIn(final Hash hash, final boolean enforceStringValue) {
final String currentSegment = path[0];
final FixPath remainingPath = new FixPath(tail(path));
if (currentSegment.equals(ASTERISK)) {
// TODO: search in all elements of value.asHash()?
return remainingPath.findIn(hash);
return remainingPath.findIn(hash, enforceStringValue);
}
final Value value = hash.get(currentSegment);
final Value value = hash.get(currentSegment, enforceStringValue && path.length == 1);
return value == null || path.length == 1 ? value : value.extractType((m, c) -> m
.ifArray(a -> c.accept(remainingPath.findIn(a)))
.ifHash(h -> c.accept(remainingPath.findIn(h)))
.ifHash(h -> c.accept(remainingPath.findIn(h, enforceStringValue)))
.orElseThrow()
);
}
Expand Down Expand Up @@ -118,17 +122,6 @@ public String toString() {
// try to replace this with consistent usage of Value#getPath
// (e.g. take care of handling repeated fields and their paths)

/*package-private*/ void throwIfNonString(final Value value) {
final boolean isNonString = value != null &&
// basic idea: path is only set on literals/strings
value.getPath() == null &&
// but with wildcards, we might still point to literals/strings
!hasWildcard();
if (isNonString) {
value.asString();
}
}

/*package-private*/ FixPath to(final Value value, final int i) {
final FixPath result;
// One *, no matching path: replace with index of current result
Expand Down
3 changes: 1 addition & 2 deletions metafix/src/main/java/org/metafacture/metafix/Record.java
Original file line number Diff line number Diff line change
Expand Up @@ -200,8 +200,7 @@ public void retainFields(final Collection<String> fields) {
*/
public void transform(final String field, final UnaryOperator<String> operator) {
final FixPath findPath = new FixPath(field);
final Value found = findPath.findIn(this);
findPath.throwIfNonString(found);
final Value found = findPath.findIn(this, true);
Value.asList(found, results -> {
final Deque<FixPath> toDelete = new LinkedList<>();
for (int i = 0; i < results.size(); ++i) {
blackwinter marked this conversation as resolved.
Show resolved Hide resolved
Expand Down
12 changes: 11 additions & 1 deletion metafix/src/main/java/org/metafacture/metafix/Value.java
Original file line number Diff line number Diff line change
Expand Up @@ -555,8 +555,18 @@ public void replace(final String field, final Value value) {
* @return the metadata value
*/
public Value get(final String field) {
return get(field, false);
}

/*package-private*/ Value get(final String field, final boolean enforceStringValue) { // TODO use Type.String etc.?
// TODO: special treatment (only) for exact matches?
final List<Value> list = findFields(field).map(this::getField).collect(Collectors.toList());
final List<Value> list = findFields(field).map(actualField -> {
final Value value = getField(actualField);
if (enforceStringValue) {
value.asString();
}
return value;
}).collect(Collectors.toList());
return list.isEmpty() ? null : list.size() == 1 ? list.get(0) : newArray(a -> list.forEach(v -> v.matchType()
.ifArray(b -> b.forEach(a::add))
.orElse(a::add)));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -305,10 +305,32 @@ public void shouldNotTrimRepeatedField() {
);
}

@Test
public void shouldNotTrimIndexedArray() {
blackwinter marked this conversation as resolved.
Show resolved Hide resolved
MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected String, got Hash", () ->
MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(
"trim('data.title')"
),
i -> {
i.startRecord("1");
i.startEntity("data");
i.startEntity("title");
i.literal("1", " marc ");
i.literal("2", " json ");
i.endEntity();
i.endEntity();
i.endRecord();
},
o -> {
}
)
);
}

@Test
// See https://github.com/metafacture/metafacture-fix/pull/133
public void shouldNotTrimStringInImplicitArrayOfHashes() {
MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected String, got Array", () ->
MetafixTestHelpers.assertExecutionException(NumberFormatException.class, "For input string: \"title\"", () ->
blackwinter marked this conversation as resolved.
Show resolved Hide resolved
MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(
"trim('data.title')"
),
Expand Down Expand Up @@ -550,6 +572,39 @@ public void wildcard() {
});
}

@Test
public void wildcardNested() {
MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(
"trim('work.title-?')"),
i -> {
i.startRecord("1");
i.endRecord();

i.startRecord("2");
i.startEntity("work");
i.literal("title-1", " marc ");
i.literal("title-2", " json ");
i.endEntity();
i.endRecord();

i.startRecord("3");
i.endRecord();
}, o -> {
o.get().startRecord("1");
o.get().endRecord();

o.get().startRecord("2");
o.get().startEntity("work");
o.get().literal("title-1", "marc");
o.get().literal("title-2", "json");
o.get().endEntity();
o.get().endRecord();

o.get().startRecord("3");
o.get().endRecord();
});
}

@Test
public void characterClass() {
MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(
Expand Down Expand Up @@ -1171,7 +1226,6 @@ public void shouldNotPrependValueToArray() {
}

@Test
@MetafixToDo("See https://github.com/metafacture/metafacture-fix/pull/170")
public void shouldNotPrependValueToArrayWithWildcard() {
MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected String, got Array", () ->
MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(
Expand All @@ -1192,6 +1246,29 @@ public void shouldNotPrependValueToArrayWithWildcard() {
);
}

@Test
public void shouldNotPrependValueToArrayWithWildcardNested() {
MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected String, got Array", () ->
MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(
"prepend('some.animal?[]', 'the first one')"
),
i -> {
i.startRecord("1");
i.startEntity("some");
i.startEntity("animals[]");
i.literal("1", "dog");
i.literal("2", "cat");
i.literal("3", "zebra");
i.endEntity();
i.endEntity();
i.endRecord();
},
o -> {
}
)
);
}

@Test
public void shouldReplaceAllRegexes() {
MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(
Expand Down