diff --git a/.github/workflows/performance-test.yml b/.github/workflows/performance-test.yml index 5e55e158a6f..d4376c242c5 100644 --- a/.github/workflows/performance-test.yml +++ b/.github/workflows/performance-test.yml @@ -91,7 +91,7 @@ jobs: cp otp-shaded/target/otp-shaded-*-SNAPSHOT.jar otp.jar java -Xmx32G -jar otp.jar --build --save test/performance/${{ matrix.location }}/ - - name: Run speed test + - name: Run RAPTOR speed test if: matrix.profile == 'core' || github.ref == 'refs/heads/dev-2.x' env: PERFORMANCE_INFLUX_DB_PASSWORD: ${{ secrets.PERFORMANCE_INFLUX_DB_PASSWORD }} @@ -113,3 +113,12 @@ jobs: with: name: ${{ matrix.location }}-flight-recorder path: application/${{ matrix.location }}-speed-test.jfr + + - name: Run transfer cache speed test + if: matrix.profile == 'core' || github.ref == 'refs/heads/dev-2.x' + env: + PERFORMANCE_INFLUX_DB_PASSWORD: ${{ secrets.PERFORMANCE_INFLUX_DB_PASSWORD }} + SPEEDTEST_LOCATION: ${{ matrix.location }} + MAVEN_OPTS: "-Xmx50g -Dmaven.repo.local=/home/lenni/.m2/repository/" + run: | + mvn --projects application exec:java -Dexec.mainClass="org.opentripplanner.transit.speed_test.TransferCacheTest" -Dexec.classpathScope=test -Dexec.args="--dir=test/performance/${{ matrix.location }}" diff --git a/application/src/test/java/org/opentripplanner/transit/speed_test/LoadModel.java b/application/src/test/java/org/opentripplanner/transit/speed_test/LoadModel.java new file mode 100644 index 00000000000..5e5823c842b --- /dev/null +++ b/application/src/test/java/org/opentripplanner/transit/speed_test/LoadModel.java @@ -0,0 +1,7 @@ +package org.opentripplanner.transit.speed_test; + +import org.opentripplanner.routing.graph.Graph; +import org.opentripplanner.standalone.config.BuildConfig; +import org.opentripplanner.transit.service.TimetableRepository; + +record LoadModel(Graph graph, TimetableRepository timetableRepository, BuildConfig buildConfig) {} diff --git a/application/src/test/java/org/opentripplanner/transit/speed_test/SetupHelper.java b/application/src/test/java/org/opentripplanner/transit/speed_test/SetupHelper.java new file mode 100644 index 00000000000..86383d6d94e --- /dev/null +++ b/application/src/test/java/org/opentripplanner/transit/speed_test/SetupHelper.java @@ -0,0 +1,41 @@ +package org.opentripplanner.transit.speed_test; + +import java.io.File; +import java.net.URI; +import javax.annotation.Nullable; +import org.opentripplanner.datastore.OtpDataStore; +import org.opentripplanner.routing.graph.Graph; +import org.opentripplanner.routing.graph.SerializedGraphObject; +import org.opentripplanner.standalone.config.ConfigModel; +import org.opentripplanner.standalone.config.OtpConfigLoader; +import org.opentripplanner.transit.service.TimetableRepository; +import org.opentripplanner.transit.speed_test.options.SpeedTestCmdLineOpts; + +/** + * A package-private helper class for setting up speed tests. + */ +class SetupHelper { + + static LoadModel loadGraph(File baseDir, @Nullable URI path) { + File file = path == null + ? OtpDataStore.graphFile(baseDir) + : path.isAbsolute() ? new File(path) : new File(baseDir, path.getPath()); + SerializedGraphObject serializedGraphObject = SerializedGraphObject.load(file); + Graph graph = serializedGraphObject.graph; + + if (graph == null) { + throw new IllegalStateException( + "Could not find graph at %s".formatted(file.getAbsolutePath()) + ); + } + + TimetableRepository timetableRepository = serializedGraphObject.timetableRepository; + timetableRepository.index(); + graph.index(timetableRepository.getSiteRepository()); + return new LoadModel(graph, timetableRepository, serializedGraphObject.buildConfig); + } + + static void loadOtpFeatures(SpeedTestCmdLineOpts opts) { + ConfigModel.initializeOtpFeatures(new OtpConfigLoader(opts.rootDir()).loadOtpConfig()); + } +} diff --git a/application/src/test/java/org/opentripplanner/transit/speed_test/SpeedTest.java b/application/src/test/java/org/opentripplanner/transit/speed_test/SpeedTest.java index ca4e85eed84..2a3add223e8 100644 --- a/application/src/test/java/org/opentripplanner/transit/speed_test/SpeedTest.java +++ b/application/src/test/java/org/opentripplanner/transit/speed_test/SpeedTest.java @@ -27,7 +27,6 @@ import org.opentripplanner.service.vehiclerental.internal.DefaultVehicleRentalService; import org.opentripplanner.standalone.OtpStartupInfo; import org.opentripplanner.standalone.api.OtpServerRequestContext; -import org.opentripplanner.standalone.config.BuildConfig; import org.opentripplanner.standalone.config.ConfigModel; import org.opentripplanner.standalone.config.DebugUiConfig; import org.opentripplanner.standalone.config.OtpConfigLoader; @@ -153,8 +152,8 @@ public static void main(String[] args) { // Given the following setup SpeedTestCmdLineOpts opts = new SpeedTestCmdLineOpts(args); var config = SpeedTestConfig.config(opts.rootDir()); - loadOtpFeatures(opts); - var model = loadGraph(opts.rootDir(), config.graph); + SetupHelper.loadOtpFeatures(opts); + var model = SetupHelper.loadGraph(opts.rootDir(), config.graph); var timetableRepository = model.timetableRepository(); var buildConfig = model.buildConfig(); var graph = model.graph(); @@ -192,6 +191,9 @@ public void runTest() { } updateTimersWithGlobalCounters(); + + timer.finishUp(); + printProfileStatistics(); saveTestCasesToResultFile(); System.err.println("\nSpeedTest done! " + projectInfo().getVersionString()); @@ -267,27 +269,6 @@ private RoutingResponse performRouting(TestCase testCase) { /* setup helper methods */ - private static void loadOtpFeatures(SpeedTestCmdLineOpts opts) { - ConfigModel.initializeOtpFeatures(new OtpConfigLoader(opts.rootDir()).loadOtpConfig()); - } - - private static LoadModel loadGraph(File baseDir, URI path) { - File file = path == null - ? OtpDataStore.graphFile(baseDir) - : path.isAbsolute() ? new File(path) : new File(baseDir, path.getPath()); - SerializedGraphObject serializedGraphObject = SerializedGraphObject.load(file); - Graph graph = serializedGraphObject.graph; - - if (graph == null) { - throw new IllegalStateException(); - } - - TimetableRepository timetableRepository = serializedGraphObject.timetableRepository; - timetableRepository.index(); - graph.index(timetableRepository.getSiteRepository()); - return new LoadModel(graph, timetableRepository, serializedGraphObject.buildConfig); - } - private void initProfileStatistics() { for (SpeedTestProfile key : opts.profiles()) { workerResults.put(key, new ArrayList<>()); @@ -352,7 +333,6 @@ private void updateTimersWithGlobalCounters() { timer.globalCount("jvm_max_memory", runtime.maxMemory()); timer.globalCount("jvm_total_memory", runtime.totalMemory()); timer.globalCount("jvm_used_memory", runtime.totalMemory() - runtime.freeMemory()); - timer.finishUp(); } /** @@ -368,8 +348,4 @@ private List trimItineraries(RoutingResponse routingResponse) { } return stream.limit(opts.numOfItineraries()).toList(); } - - /* inline classes */ - - record LoadModel(Graph graph, TimetableRepository timetableRepository, BuildConfig buildConfig) {} } diff --git a/application/src/test/java/org/opentripplanner/transit/speed_test/TransferCacheTest.java b/application/src/test/java/org/opentripplanner/transit/speed_test/TransferCacheTest.java new file mode 100644 index 00000000000..e6c8de67688 --- /dev/null +++ b/application/src/test/java/org/opentripplanner/transit/speed_test/TransferCacheTest.java @@ -0,0 +1,68 @@ +package org.opentripplanner.transit.speed_test; + +import static org.opentripplanner.standalone.configure.ConstructApplication.creatTransitLayerForRaptor; +import static org.opentripplanner.transit.speed_test.support.AssertSpeedTestSetup.assertTestDateHasData; + +import java.util.stream.IntStream; +import org.opentripplanner.framework.application.OtpAppException; +import org.opentripplanner.routing.api.request.RouteRequest; +import org.opentripplanner.standalone.OtpStartupInfo; +import org.opentripplanner.transit.service.TimetableRepository; +import org.opentripplanner.transit.speed_test.model.timer.SpeedTestTimer; +import org.opentripplanner.transit.speed_test.options.SpeedTestCmdLineOpts; +import org.opentripplanner.transit.speed_test.options.SpeedTestConfig; + +/** + * Test how long it takes to compute the transfer cache. + */ +public class TransferCacheTest { + + public static void main(String[] args) { + try { + OtpStartupInfo.logInfo("Run transfer cache test"); + // Given the following setup + SpeedTestCmdLineOpts opts = new SpeedTestCmdLineOpts(args); + var config = SpeedTestConfig.config(opts.rootDir()); + SetupHelper.loadOtpFeatures(opts); + var model = SetupHelper.loadGraph(opts.rootDir(), config.graph); + var timetableRepository = model.timetableRepository(); + var buildConfig = model.buildConfig(); + + var timer = new SpeedTestTimer(); + timer.setUp(false); + + // Creating transitLayerForRaptor should be integrated into the TimetableRepository, but for now + // we do it manually here + creatTransitLayerForRaptor(timetableRepository, config.transitRoutingParams); + + assertTestDateHasData(timetableRepository, config, buildConfig); + + measureTransferCacheComputation(timer, timetableRepository); + + timer.finishUp(); + } catch (Exception e) { + System.err.println(e.getMessage()); + e.printStackTrace(System.err); + System.exit(1); + } + } + + /** + * Measure how long it takes to compute the transfer cache. + */ + private static void measureTransferCacheComputation( + SpeedTestTimer timer, + TimetableRepository timetableRepository + ) { + IntStream + .range(1, 7) + .forEach(reluctance -> { + RouteRequest routeRequest = new RouteRequest(); + routeRequest.withPreferences(b -> b.withWalk(c -> c.withReluctance(reluctance))); + timer.recordTimer( + "transfer_cache_computation", + () -> timetableRepository.getTransitLayer().initTransferCacheForRequest(routeRequest) + ); + }); + } +} diff --git a/application/src/test/java/org/opentripplanner/transit/speed_test/model/timer/SpeedTestTimer.java b/application/src/test/java/org/opentripplanner/transit/speed_test/model/timer/SpeedTestTimer.java index 48a0548d28e..80970eaad0a 100644 --- a/application/src/test/java/org/opentripplanner/transit/speed_test/model/timer/SpeedTestTimer.java +++ b/application/src/test/java/org/opentripplanner/transit/speed_test/model/timer/SpeedTestTimer.java @@ -39,6 +39,7 @@ public class SpeedTestTimer { List.of(loggerRegistry) ); private final MeterRegistry uploadRegistry = MeterRegistrySetup.getRegistry().orElse(null); + private boolean groupResultByTestCaseCategory = false; public static int nanosToMillisecond(long nanos) { @@ -136,6 +137,18 @@ public void globalCount(String meterName, long count) { } } + /** + * Execute the runnable and record its runtime in the meter name passed in. + */ + public void recordTimer(String meterName, Runnable runnable) { + if (uploadRegistry != null) { + registry.add(uploadRegistry); + var timer = registry.timer(meterName); + timer.record(runnable); + registry.remove(uploadRegistry); + } + } + /** * Calculate the total time mean for the given timer. If the timer is not * found {@link #NOT_AVAILABLE} is returned. This can be the case in unit tests, @@ -175,7 +188,7 @@ public String name(String name, Meter.Type type, String unit) { } private String capitalize(String name) { - if (name.length() != 0 && !Character.isUpperCase(name.charAt(0))) { + if (!name.isEmpty() && !Character.isUpperCase(name.charAt(0))) { char[] chars = name.toCharArray(); chars[0] = Character.toUpperCase(chars[0]); return new String(chars); @@ -208,8 +221,8 @@ public static Result merge(Collection results) { for (Result it : results) { any = it; - min = it.min < min ? it.min : min; - max = it.max > max ? it.max : max; + min = Math.min(it.min, min); + max = Math.max(it.max, max); totTime += it.totTime; count += it.count; }