From 81f1e2dbcc72dc239da396ca73825634bd44d2ec Mon Sep 17 00:00:00 2001 From: Daniel Worley Date: Mon, 14 Dec 2020 17:08:19 -0500 Subject: [PATCH] Es7 10 0 (#348) * update gradle props * fix for #340 * wip part2, gradlew test is failing for reworked LoggingFetchSubPhase * Fix testConventions error with gradle clean check * Fix doLog test * Fix other failing test * Update note around sketchy code * Remove commented code * Add support for FetchSubPhaseProcessor and inject IndexNameExpressionResolver Co-authored-by: Nathan Day Co-authored-by: David Causse --- .gitignore | 2 + CONTRIBUTING.md | 4 +- build.gradle | 12 +++ gradle.properties | 8 +- gradle/wrapper/gradle-wrapper.properties | 2 +- .../es/ltr/LtrQueryClientYamlTestSuiteIT.java | 0 .../java/com/o19s/es/ltr/NodeSettingsIT.java | 0 .../ltr/action/AddFeaturesToSetActionIT.java | 0 .../es/ltr/action/BaseIntegrationTest.java | 0 .../o19s/es/ltr/action/LTRStatsActionIT.java | 0 .../es/ltr/action/ListStoresActionIT.java | 0 .../ValidatingFeatureStoreActionIT.java | 0 .../com/o19s/es/ltr/logging/LoggingIT.java | 0 .../o19s/es/ltr/query/StoredLtrQueryIT.java | 0 .../com/o19s/es/ltr/LtrQueryParserPlugin.java | 8 +- .../store/index/IndexFeatureStore.java | 2 +- .../es/ltr/logging/LoggingFetchSubPhase.java | 101 +++++++----------- .../suppliers/PluginHealthStatusSupplier.java | 6 +- .../stats/suppliers/StoreStatsSupplier.java | 4 +- .../logging/LoggingFetchSubPhaseTests.java | 24 +++-- .../ranker/parser/XGBoostJsonParserTests.java | 2 +- .../PluginHealthStatusSupplierTests.java | 5 +- 22 files changed, 92 insertions(+), 88 deletions(-) rename src/{test => javaRestTest}/java/com/o19s/es/ltr/LtrQueryClientYamlTestSuiteIT.java (100%) rename src/{test => javaRestTest}/java/com/o19s/es/ltr/NodeSettingsIT.java (100%) rename src/{test => javaRestTest}/java/com/o19s/es/ltr/action/AddFeaturesToSetActionIT.java (100%) rename src/{test => javaRestTest}/java/com/o19s/es/ltr/action/BaseIntegrationTest.java (100%) rename src/{test => javaRestTest}/java/com/o19s/es/ltr/action/LTRStatsActionIT.java (100%) rename src/{test => javaRestTest}/java/com/o19s/es/ltr/action/ListStoresActionIT.java (100%) rename src/{test => javaRestTest}/java/com/o19s/es/ltr/action/ValidatingFeatureStoreActionIT.java (100%) rename src/{test => javaRestTest}/java/com/o19s/es/ltr/logging/LoggingIT.java (100%) rename src/{test => javaRestTest}/java/com/o19s/es/ltr/query/StoredLtrQueryIT.java (100%) diff --git a/.gitignore b/.gitignore index 9d292640..c07e1002 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,5 @@ eclipse-build !/.settings/org.eclipse.core.resources.prefs !/.settings/org.eclipse.jdt.core.prefs !/.settings/org.eclipse.jdt.ui.prefs + +bin/* diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7fb1d8cf..04075414 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,7 +73,7 @@ As a practice, we maintain a branch per Elasticsearch version, prefixed by ES. S ### Upgrading to latest Elasticsearch major/minor version (ie `es_6_7`) -Upgrading to the latest ES version is a great way to help us out, and get your feet wet. The first step is to open build.gradle, and change the ES version numbers in the dependencies to the version you wish to upgrade to. We recommend trying to build, (it'll likely fail) making a branch name with the es version number (ie `es_6_7`), and then sending us a "Work in Progress" PR to master. This will let us rally around the (sometimes annoyingly painful) upgrade as a team. +Upgrading to the latest ES version is a great way to help us out, and get your feet wet. The first step is to open `gradle.properties`, and change the ES/Lucene version numbers in the dependencies to the version you wish to upgrade to. We recommend trying to build, (it'll likely fail) making a branch name with the es version number (ie `es_6_7`), and then sending us a "Work in Progress" PR to master. This will let us rally around the (sometimes annoyingly painful) upgrade as a team. #### Gradlew Wrapper and Java Upgrade @@ -132,4 +132,4 @@ Most people come to this plugin, unfamiliar with the basic Learning to Rank work # Other questions? Get in touch! -Please [email Doug Turnbull](mailto:dturnbull@opensourceconnections.com) to ask any questions about contributing not covered by this document. If there's a "bug" in this document, please feel free to file an issue. +Please open an issue to ask any questions about contributing not covered by this document. If there's a "bug" in this document, please feel free to file PR, typos are the wosrt. diff --git a/build.gradle b/build.gradle index e1d8d656..6f5636e9 100644 --- a/build.gradle +++ b/build.gradle @@ -29,6 +29,7 @@ allprojects { apply plugin: 'idea' apply plugin: 'elasticsearch.esplugin' +apply plugin: 'elasticsearch.java-rest-test' apply plugin: 'elasticsearch.rest-resources' // license of this project @@ -69,6 +70,13 @@ dependencyLicenses { mapping from: /compiler-.*/, to: 'lucene' } +sourceSets { + javaRestTest { + compileClasspath += sourceSets["main"].output + sourceSets["test"].output + configurations["testRuntimeClasspath"] + runtimeClasspath += output + compileClasspath + } +} + // Set to false to not use elasticsearch checkstyle rules checkstyleMain.enabled = true checkstyleTest.enabled = true @@ -84,3 +92,7 @@ licenseHeaders.enabled = false // No need to validate POM, as we do not upload to sonatype validateNebulaPom.enabled = false + +// Elastic tried to remove the logging requirement for plugins, but didn't get it quite right so this is a short term fix until 7.11 +// https://github.com/elastic/elasticsearch/issues/65247 +loggerUsageCheck.enabled = false diff --git a/gradle.properties b/gradle.properties index 839fd5a9..01d6bf79 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ -ltrVersion = 1.5.3 -elasticsearchVersion = 7.9.3 -luceneVersion = 8.6.2 -ow2Version = 7.2 +ltrVersion = 1.5.4 +elasticsearchVersion = 7.10.0 +luceneVersion = 8.7.0 +ow2Version = 8.0.1 antlrVersion=4.5.1-1 diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 622ab64a..12d38de6 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.5-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/src/test/java/com/o19s/es/ltr/LtrQueryClientYamlTestSuiteIT.java b/src/javaRestTest/java/com/o19s/es/ltr/LtrQueryClientYamlTestSuiteIT.java similarity index 100% rename from src/test/java/com/o19s/es/ltr/LtrQueryClientYamlTestSuiteIT.java rename to src/javaRestTest/java/com/o19s/es/ltr/LtrQueryClientYamlTestSuiteIT.java diff --git a/src/test/java/com/o19s/es/ltr/NodeSettingsIT.java b/src/javaRestTest/java/com/o19s/es/ltr/NodeSettingsIT.java similarity index 100% rename from src/test/java/com/o19s/es/ltr/NodeSettingsIT.java rename to src/javaRestTest/java/com/o19s/es/ltr/NodeSettingsIT.java diff --git a/src/test/java/com/o19s/es/ltr/action/AddFeaturesToSetActionIT.java b/src/javaRestTest/java/com/o19s/es/ltr/action/AddFeaturesToSetActionIT.java similarity index 100% rename from src/test/java/com/o19s/es/ltr/action/AddFeaturesToSetActionIT.java rename to src/javaRestTest/java/com/o19s/es/ltr/action/AddFeaturesToSetActionIT.java diff --git a/src/test/java/com/o19s/es/ltr/action/BaseIntegrationTest.java b/src/javaRestTest/java/com/o19s/es/ltr/action/BaseIntegrationTest.java similarity index 100% rename from src/test/java/com/o19s/es/ltr/action/BaseIntegrationTest.java rename to src/javaRestTest/java/com/o19s/es/ltr/action/BaseIntegrationTest.java diff --git a/src/test/java/com/o19s/es/ltr/action/LTRStatsActionIT.java b/src/javaRestTest/java/com/o19s/es/ltr/action/LTRStatsActionIT.java similarity index 100% rename from src/test/java/com/o19s/es/ltr/action/LTRStatsActionIT.java rename to src/javaRestTest/java/com/o19s/es/ltr/action/LTRStatsActionIT.java diff --git a/src/test/java/com/o19s/es/ltr/action/ListStoresActionIT.java b/src/javaRestTest/java/com/o19s/es/ltr/action/ListStoresActionIT.java similarity index 100% rename from src/test/java/com/o19s/es/ltr/action/ListStoresActionIT.java rename to src/javaRestTest/java/com/o19s/es/ltr/action/ListStoresActionIT.java diff --git a/src/test/java/com/o19s/es/ltr/action/ValidatingFeatureStoreActionIT.java b/src/javaRestTest/java/com/o19s/es/ltr/action/ValidatingFeatureStoreActionIT.java similarity index 100% rename from src/test/java/com/o19s/es/ltr/action/ValidatingFeatureStoreActionIT.java rename to src/javaRestTest/java/com/o19s/es/ltr/action/ValidatingFeatureStoreActionIT.java diff --git a/src/test/java/com/o19s/es/ltr/logging/LoggingIT.java b/src/javaRestTest/java/com/o19s/es/ltr/logging/LoggingIT.java similarity index 100% rename from src/test/java/com/o19s/es/ltr/logging/LoggingIT.java rename to src/javaRestTest/java/com/o19s/es/ltr/logging/LoggingIT.java diff --git a/src/test/java/com/o19s/es/ltr/query/StoredLtrQueryIT.java b/src/javaRestTest/java/com/o19s/es/ltr/query/StoredLtrQueryIT.java similarity index 100% rename from src/test/java/com/o19s/es/ltr/query/StoredLtrQueryIT.java rename to src/javaRestTest/java/com/o19s/es/ltr/query/StoredLtrQueryIT.java diff --git a/src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java b/src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java index 1ed17935..ca67d203 100644 --- a/src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java +++ b/src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java @@ -256,17 +256,17 @@ public Collection createComponents(Client client, } } }); - return asList(caches, parserFactory, getStats(client, clusterService)); + return asList(caches, parserFactory, getStats(client, clusterService, indexNameExpressionResolver)); } - private LTRStats getStats(Client client, ClusterService clusterService) { + private LTRStats getStats(Client client, ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver) { Map stats = new HashMap<>(); stats.put(StatName.CACHE.getName(), new LTRStat(false, new CacheStatsOnNodeSupplier(caches))); stats.put(StatName.STORES.getName(), - new LTRStat(true, new StoreStatsSupplier(client, clusterService))); + new LTRStat(true, new StoreStatsSupplier(client, clusterService, indexNameExpressionResolver))); stats.put(StatName.PLUGIN_STATUS.getName(), - new LTRStat(true, new PluginHealthStatusSupplier(clusterService))); + new LTRStat(true, new PluginHealthStatusSupplier(clusterService, indexNameExpressionResolver))); return new LTRStats(unmodifiableMap(stats)); } diff --git a/src/main/java/com/o19s/es/ltr/feature/store/index/IndexFeatureStore.java b/src/main/java/com/o19s/es/ltr/feature/store/index/IndexFeatureStore.java index fd8a0251..b8130585 100644 --- a/src/main/java/com/o19s/es/ltr/feature/store/index/IndexFeatureStore.java +++ b/src/main/java/com/o19s/es/ltr/feature/store/index/IndexFeatureStore.java @@ -37,7 +37,6 @@ import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.Streams; import org.apache.logging.log4j.LogManager; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; @@ -48,6 +47,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.core.internal.io.Streams; import java.io.ByteArrayOutputStream; import java.io.IOException; diff --git a/src/main/java/com/o19s/es/ltr/logging/LoggingFetchSubPhase.java b/src/main/java/com/o19s/es/ltr/logging/LoggingFetchSubPhase.java index 795d16d5..48d61a5e 100644 --- a/src/main/java/com/o19s/es/ltr/logging/LoggingFetchSubPhase.java +++ b/src/main/java/com/o19s/es/ltr/logging/LoggingFetchSubPhase.java @@ -23,7 +23,6 @@ import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; @@ -31,17 +30,15 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; -import org.elasticsearch.search.fetch.subphase.InnerHitsContext; -import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.rescore.QueryRescorer; import org.elasticsearch.search.rescore.RescoreContext; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -49,18 +46,18 @@ public class LoggingFetchSubPhase implements FetchSubPhase { @Override - public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOException { + public FetchSubPhaseProcessor getProcessor(FetchContext context) throws IOException { LoggingSearchExtBuilder ext = (LoggingSearchExtBuilder) context.getSearchExt(LoggingSearchExtBuilder.NAME); if (ext == null) { - return; + return null; } - // Use a boolean query with all the models to log - // This way we reuse existing code to advance through multiple scorers/iterators BooleanQuery.Builder builder = new BooleanQuery.Builder(); List loggers = new ArrayList<>(); Map namedQueries = context.parsedQuery().namedFilters(); - if (!(context instanceof InnerHitsContext.InnerHitSubContext)) { + + + if (namedQueries.size() > 0) { ext.logSpecsStream().filter((l) -> l.getNamedQuery() != null).forEach((l) -> { Tuple query = extractQuery(l, namedQueries); builder.add(new BooleanClause(query.v1(), BooleanClause.Occur.MUST)); @@ -74,58 +71,15 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept }); } - doLog(builder.build(), loggers, context.searcher(), hits); - - } - - void doLog(Query query, List loggers, IndexSearcher searcher, SearchHit[] hits) throws IOException { - // Reorder hits by id so we can scan all the docs belonging to the same - // segment by reusing the same scorer. - SearchHit[] reordered = new SearchHit[hits.length]; - System.arraycopy(hits, 0, reordered, 0, hits.length); - Arrays.sort(reordered, Comparator.comparingInt(SearchHit::docId)); - int hitUpto = 0; - int readerUpto = -1; - int endDoc = 0; - int docBase = 0; - Scorer scorer = null; - Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE, 1F); - // Loop logic borrowed from lucene QueryRescorer - while (hitUpto < reordered.length) { - SearchHit hit = reordered[hitUpto]; - int docID = hit.docId(); - loggers.forEach((l) -> l.nextDoc(hit)); - LeafReaderContext readerContext = null; - while (docID >= endDoc) { - readerUpto++; - readerContext = searcher.getTopReaderContext().leaves().get(readerUpto); - endDoc = readerContext.docBase + readerContext.reader().maxDoc(); - } - if (readerContext != null) { - // We advanced to another segment: - docBase = readerContext.docBase; - scorer = weight.scorer(readerContext); - } - - if (scorer != null) { - int targetDoc = docID - docBase; - int actualDoc = scorer.docID(); - if (actualDoc < targetDoc) { - actualDoc = scorer.iterator().advance(targetDoc); - } - if (actualDoc == targetDoc) { - // Scoring will trigger log collection - scorer.score(); - } - } + Weight w = context.searcher().rewrite(builder.build()).createWeight(context.searcher(), ScoreMode.COMPLETE, 1.0F); - hitUpto++; - } + return new LoggingFetchSubPhaseProcessor(w, loggers); } - private Tuple extractQuery(LoggingSearchExtBuilder.LogSpec logSpec, Map namedQueries) { + private Tuple extractQuery(LoggingSearchExtBuilder.LogSpec + logSpec, Map namedQueries) { Query q = namedQueries.get(logSpec.getNamedQuery()); if (q == null) { throw new IllegalArgumentException("No query named [" + logSpec.getNamedQuery() + "] found"); @@ -172,6 +126,31 @@ private Tuple toLogger(LoggingSearchExtBuilder.LogS query = query.toLoggerQuery(consumer); return new Tuple<>(query, consumer); } + static class LoggingFetchSubPhaseProcessor implements FetchSubPhaseProcessor { + private final Weight weight; + private final List loggers; + private Scorer scorer; + + LoggingFetchSubPhaseProcessor(Weight weight, List loggers) { + this.weight = weight; + this.loggers = loggers; + } + + + @Override + public void setNextReader(LeafReaderContext readerContext) throws IOException { + scorer = weight.scorer(readerContext); + } + + @Override + public void process(HitContext hitContext) throws IOException { + if (scorer != null && scorer.iterator().advance(hitContext.docId()) == hitContext.docId()) { + loggers.forEach((l) -> l.nextDoc(hitContext.hit())); + // Scoring will trigger log collection + scorer.score(); + } + } + } static class HitLogConsumer implements LogLtrRanker.LogConsumer { private static final String FIELD_NAME = "_ltrlog"; @@ -191,7 +170,7 @@ static class HitLogConsumer implements LogLtrRanker.LogConsumer { // ] private List> currentLog; private SearchHit currentHit; - private Map extraLogging; + private Map extraLogging; HitLogConsumer(String name, FeatureSet set, boolean missingAsZero) { @@ -226,14 +205,14 @@ public void accept(int featureOrdinal, float score) { /** * Return Map to store additional logging information returned with the feature values. - * + *

* The Map is created on first access. */ @Override - public Map getExtraLoggingMap() { + public Map getExtraLoggingMap() { if (extraLogging == null) { extraLogging = new HashMap<>(); - Map logEntry = new HashMap<>(); + Map logEntry = new HashMap<>(); logEntry.put("name", EXTRA_LOGGING_NAME); logEntry.put("value", extraLogging); currentLog.add(logEntry); diff --git a/src/main/java/com/o19s/es/ltr/stats/suppliers/PluginHealthStatusSupplier.java b/src/main/java/com/o19s/es/ltr/stats/suppliers/PluginHealthStatusSupplier.java index 8a76292b..82daa208 100644 --- a/src/main/java/com/o19s/es/ltr/stats/suppliers/PluginHealthStatusSupplier.java +++ b/src/main/java/com/o19s/es/ltr/stats/suppliers/PluginHealthStatusSupplier.java @@ -5,6 +5,7 @@ import org.elasticsearch.cluster.health.ClusterIndexHealth; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.ThreadContext; import java.util.Arrays; import java.util.Locale; @@ -21,9 +22,10 @@ public class PluginHealthStatusSupplier implements Supplier { private final ClusterService clusterService; private final IndexNameExpressionResolver indexNameExpressionResolver; - public PluginHealthStatusSupplier(ClusterService clusterService) { + public PluginHealthStatusSupplier(ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver) { this.clusterService = clusterService; - this.indexNameExpressionResolver = new IndexNameExpressionResolver(); + ThreadContext threadContext = new ThreadContext(clusterService.getSettings()); + this.indexNameExpressionResolver = indexNameExpressionResolver; } // currently it combines the store statuses to get the overall health diff --git a/src/main/java/com/o19s/es/ltr/stats/suppliers/StoreStatsSupplier.java b/src/main/java/com/o19s/es/ltr/stats/suppliers/StoreStatsSupplier.java index f7f1cd3d..3c2587dc 100644 --- a/src/main/java/com/o19s/es/ltr/stats/suppliers/StoreStatsSupplier.java +++ b/src/main/java/com/o19s/es/ltr/stats/suppliers/StoreStatsSupplier.java @@ -58,10 +58,10 @@ public String getName() { } } - public StoreStatsSupplier(Client client, ClusterService clusterService) { + public StoreStatsSupplier(Client client, ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver) { this.client = client; this.clusterService = clusterService; - this.indexNameExpressionResolver = new IndexNameExpressionResolver(); + this.indexNameExpressionResolver = indexNameExpressionResolver; } @Override diff --git a/src/test/java/com/o19s/es/ltr/logging/LoggingFetchSubPhaseTests.java b/src/test/java/com/o19s/es/ltr/logging/LoggingFetchSubPhaseTests.java index fc7a64f1..8c0f6b1b 100644 --- a/src/test/java/com/o19s/es/ltr/logging/LoggingFetchSubPhaseTests.java +++ b/src/test/java/com/o19s/es/ltr/logging/LoggingFetchSubPhaseTests.java @@ -19,6 +19,7 @@ import com.o19s.es.ltr.feature.PrebuiltFeature; import com.o19s.es.ltr.feature.PrebuiltFeatureSet; import com.o19s.es.ltr.feature.PrebuiltLtrModel; +import com.o19s.es.ltr.logging.LoggingFetchSubPhase.LoggingFetchSubPhaseProcessor; import com.o19s.es.ltr.query.RankerQuery; import com.o19s.es.ltr.ranker.LtrRanker; import com.o19s.es.ltr.ranker.linear.LinearRankerTests; @@ -39,6 +40,7 @@ import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.SimpleCollector; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.Weight; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; @@ -46,16 +48,17 @@ import org.elasticsearch.common.lucene.search.function.FieldValueFactorFunction; import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery; import org.elasticsearch.common.text.Text; -import org.elasticsearch.index.Index; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.fetch.FetchSubPhase; +import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.lookup.SourceLookup; import org.junit.AfterClass; import org.junit.BeforeClass; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -113,8 +116,11 @@ public void testLogging() throws IOException { .add(new BooleanClause(query2, BooleanClause.Occur.MUST)) .build(); LoggingFetchSubPhase subPhase = new LoggingFetchSubPhase(); - SearchHit[] hits = selectRandomHits(); - subPhase.doLog(query, Arrays.asList(logger1, logger2), searcher, hits); + Weight weight = searcher.createWeight(query, ScoreMode.COMPLETE, 1.0F); + List loggers = Arrays.asList(logger1, logger2); + LoggingFetchSubPhaseProcessor processor = new LoggingFetchSubPhaseProcessor(weight, loggers); + + SearchHit[] hits = preprocessRandomHits(processor); for (SearchHit hit : hits) { assertTrue(docs.containsKey(hit.getId())); Document d = docs.get(hit.getId()); @@ -146,7 +152,7 @@ public void testLogging() throws IOException { } } - public SearchHit[] selectRandomHits() throws IOException { + public SearchHit[] preprocessRandomHits(FetchSubPhaseProcessor processor) throws IOException { int minHits = TestUtil.nextInt(random(), 5, 10); int maxHits = TestUtil.nextInt(random(), minHits, minHits+10); List hits = new ArrayList<>(maxHits); @@ -165,6 +171,7 @@ public ScoreMode scoreMode() { protected void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); this.context = context; + processor.setNextReader(context); } @Override @@ -173,19 +180,19 @@ public void collect(int doc) throws IOException { Document d = context.reader().document(doc); String id = d.get("id"); SearchHit hit = new SearchHit( - doc+context.docBase, + doc, id, new Text("text"), random().nextBoolean() ? new HashMap<>() : null, null ); + processor.process(new FetchSubPhase.HitContext(hit, context, doc, new SourceLookup(), new HashMap<>())); hits.add(hit); } } }); assert hits.size() >= minHits; - Collections.shuffle(hits, random()); return hits.toArray(new SearchHit[0]); } @@ -210,8 +217,7 @@ public RankerQuery buildQuery(String text) { public Query buildFunctionScore() { FieldValueFactorFunction fieldValueFactorFunction = new FieldValueFactorFunction("score", FACTOR, LN2P, 0D, - new SortedNumericIndexFieldData(new Index("test", "123"), - "score", FLOAT)); + new SortedNumericIndexFieldData("score", FLOAT)); return new FunctionScoreQuery(new MatchAllDocsQuery(), fieldValueFactorFunction, CombineFunction.MULTIPLY, 0F, Float.MAX_VALUE); } diff --git a/src/test/java/com/o19s/es/ltr/ranker/parser/XGBoostJsonParserTests.java b/src/test/java/com/o19s/es/ltr/ranker/parser/XGBoostJsonParserTests.java index 17eebf21..777b8368 100644 --- a/src/test/java/com/o19s/es/ltr/ranker/parser/XGBoostJsonParserTests.java +++ b/src/test/java/com/o19s/es/ltr/ranker/parser/XGBoostJsonParserTests.java @@ -26,7 +26,7 @@ import com.o19s.es.ltr.ranker.linear.LinearRankerTests; import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.io.Streams; +import org.elasticsearch.core.internal.io.Streams; import org.hamcrest.CoreMatchers; import java.io.ByteArrayOutputStream; diff --git a/src/test/java/com/o19s/es/ltr/stats/suppliers/PluginHealthStatusSupplierTests.java b/src/test/java/com/o19s/es/ltr/stats/suppliers/PluginHealthStatusSupplierTests.java index 8c750415..ec349f25 100644 --- a/src/test/java/com/o19s/es/ltr/stats/suppliers/PluginHealthStatusSupplierTests.java +++ b/src/test/java/com/o19s/es/ltr/stats/suppliers/PluginHealthStatusSupplierTests.java @@ -1,6 +1,9 @@ package com.o19s.es.ltr.stats.suppliers; import com.o19s.es.ltr.feature.store.index.IndexFeatureStore; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.test.ESIntegTestCase; import org.junit.Before; @@ -10,7 +13,7 @@ public class PluginHealthStatusSupplierTests extends ESIntegTestCase { @Before public void setup() { pluginHealthStatusSupplier = - new PluginHealthStatusSupplier(clusterService()); + new PluginHealthStatusSupplier(clusterService(), new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY))); } public void testPluginHealthStatusNoLtrStore() {