From 12c7f8c3a41cea88b31e98f1d40007a2c1de3122 Mon Sep 17 00:00:00 2001 From: tatiana-yan Date: Wed, 14 Apr 2021 17:46:06 +0300 Subject: [PATCH] [transit] Generate trivial shapes. --- .../gtfs_converter/gtfs_converter.cpp | 8 +- transit/world_feed/world_feed.cpp | 78 +++++++++++++++---- transit/world_feed/world_feed.hpp | 7 +- .../world_feed_integration_tests.cpp | 12 +-- 4 files changed, 76 insertions(+), 29 deletions(-) diff --git a/transit/world_feed/gtfs_converter/gtfs_converter.cpp b/transit/world_feed/gtfs_converter/gtfs_converter.cpp index df0762a0083..89bdd6e5ef9 100644 --- a/transit/world_feed/gtfs_converter/gtfs_converter.cpp +++ b/transit/world_feed/gtfs_converter/gtfs_converter.cpp @@ -25,6 +25,8 @@ DEFINE_string(path_json, "", "Output directory for dumping json files"); DEFINE_string(path_resources, "", "MAPS.ME resources directory"); DEFINE_string(start_feed, "", "Optional. Feed directory from which the process continues"); DEFINE_string(stop_feed, "", "Optional. Feed directory on which to stop the process"); +DEFINE_bool(generate_trivial_shapes, false, + "Optional. Generate trivial shapes for trips without shapes."); // Finds subdirectories with feeds. Platform::FilesList GetGtfsFeedsInDirectory(std::string const & path) @@ -103,13 +105,13 @@ FeedStatus ReadFeed(gtfs::Feed & feed) { // First we read shapes. If there are no shapes in feed we do not need to read all the required // files - agencies, stops, etc. - if (auto res = feed.read_shapes(); res != gtfs::ResultCode::OK) + if (auto res = feed.read_shapes(); res != gtfs::ResultCode::OK && !FLAGS_generate_trivial_shapes) { LOG(LWARNING, ("Could not get shapes.", res.message)); return FeedStatus::NO_SHAPES; } - if (feed.get_shapes().empty()) + if (feed.get_shapes().empty() && !FLAGS_generate_trivial_shapes) return FeedStatus::NO_SHAPES; // We try to parse required for json files and return error in case of invalid file content. @@ -220,7 +222,7 @@ bool ConvertFeeds(transit::IdGenerator & generator, transit::IdGenerator & gener transit::WorldFeed globalFeed(generator, generatorEdges, colorPicker, mwmMatcher); - if (!globalFeed.SetFeed(std::move(feed))) + if (!globalFeed.SetFeed(std::move(feed), FLAGS_generate_trivial_shapes)) { LOG(LINFO, ("Error transforming feed for json representation.")); ++feedsNotDumpedCount; diff --git a/transit/world_feed/world_feed.cpp b/transit/world_feed/world_feed.cpp index 3c655d15a22..2061b86be51 100644 --- a/transit/world_feed/world_feed.cpp +++ b/transit/world_feed/world_feed.cpp @@ -679,7 +679,7 @@ bool WorldFeed::FillStopsEdges() return !m_edges.m_data.empty(); } -bool WorldFeed::FillLinesAndShapes() +bool WorldFeed::FillLinesAndShapes(bool generateTrivialShapes) { std::unordered_map shapes; for (const auto & shape : m_feed.get_shapes()) @@ -705,28 +705,48 @@ bool WorldFeed::FillLinesAndShapes() base::LessBy(>fs::StopTime::stop_sequence)); } + size_t generatedId = 0; + std::unordered_map generatedShapeIds; for (const auto & trip : m_feed.get_trips()) { + bool generate = false; + // We skip routes filtered on the route preparation stage. auto const itRoute = m_gtfsIdToHash[FieldIdx::RoutesIdx].find(trip.route_id); if (itRoute == m_gtfsIdToHash[FieldIdx::RoutesIdx].end()) continue; + std::string stopIds; + for (auto const & stopTime : stopTimes[trip.trip_id]) + stopIds += stopTime.stop_id + kDelimiter; + + auto tripShapeId = trip.shape_id; // Skip trips with corrupted shapes. - if (trip.shape_id.empty()) + if (trip.shape_id.empty() && !generateTrivialShapes) continue; - std::string const & routeHash = itRoute->second; - - std::string stopIds; + if (trip.shape_id.empty()) + { + auto const it = generatedShapeIds.find(stopIds); + if (it != generatedShapeIds.end()) + { + tripShapeId = it->second; + } + else + { + generate = true; + tripShapeId = "generated" + strings::to_string(generatedId); + generatedShapeIds[stopIds] = tripShapeId; + ++generatedId; + } + } - for (auto const & stopTime : stopTimes[trip.trip_id]) - stopIds += stopTime.stop_id + kDelimiter; + std::string const & routeHash = itRoute->second; - std::string const & lineHash = BuildHash(routeHash, trip.shape_id, stopIds); + std::string const & lineHash = BuildHash(routeHash, tripShapeId, stopIds); auto const lineId = m_idGenerator.MakeId(lineHash); - auto [itShape, insertedShape] = m_gtfsIdToHash[ShapesIdx].emplace(trip.shape_id, ""); + auto [itShape, insertedShape] = m_gtfsIdToHash[ShapesIdx].emplace(tripShapeId, ""); // Skip invalid shape. if (!insertedShape && itShape->second.empty()) @@ -734,15 +754,39 @@ bool WorldFeed::FillLinesAndShapes() if (insertedShape) { - auto const & shapeItems = getShape(trip.shape_id); - // Skip trips with corrupted shapes. - if (shapeItems.size() < 2) + auto const & shapeItems = getShape(tripShapeId); + if (generate) + { + gtfs::Shape shape; + size_t sequence = 1; + for (auto const & stopTime : stopTimes[trip.trip_id]) + { + auto const stop = m_feed.get_stop(stopTime.stop_id); + if (!stop) + break; + + gtfs::ShapePoint point; + point.shape_id = tripShapeId; + point.shape_pt_lat = stop->stop_lat; + point.shape_pt_lon = stop->stop_lon; + point.shape_pt_sequence = sequence; + ++sequence; + shape.push_back(point); + } + if (shape.size() != stopTimes[trip.trip_id].size()) + continue; + AddShape(itShape, shape, lineId); + } + else if (shapeItems.size() < 2) { - LOG(LINFO, ("Invalid shape. Length:", shapeItems.size(), "Shape id", trip.shape_id)); + // Skip trips with corrupted shapes. + LOG(LINFO, ("Invalid shape. Length:", shapeItems.size(), "Shape id", tripShapeId)); continue; } - - AddShape(itShape, shapeItems, lineId); + else + { + AddShape(itShape, shapeItems, lineId); + } } auto [it, inserted] = m_lines.m_data.emplace(lineId, LineData()); @@ -1416,7 +1460,7 @@ bool WorldFeed::UpdateEdgeWeights() return true; } -bool WorldFeed::SetFeed(gtfs::Feed && feed) +bool WorldFeed::SetFeed(gtfs::Feed && feed, bool generateTrivialShapes) { m_feed = std::move(feed); m_gtfsIdToHash.resize(FieldIdx::IdxCount); @@ -1442,7 +1486,7 @@ bool WorldFeed::SetFeed(gtfs::Feed && feed) } LOG(LINFO, ("Filled routes.")); - if (!FillLinesAndShapes()) + if (!FillLinesAndShapes(generateTrivialShapes)) { LOG(LWARNING, ("Could not fill lines.", m_lines.m_data.size())); return false; diff --git a/transit/world_feed/world_feed.hpp b/transit/world_feed/world_feed.hpp index efaea7e665e..7757e55307d 100644 --- a/transit/world_feed/world_feed.hpp +++ b/transit/world_feed/world_feed.hpp @@ -285,7 +285,7 @@ class WorldFeed WorldFeed(IdGenerator & generator, IdGenerator & generatorEdges, ColorPicker & colorPicker, feature::CountriesFilesAffiliation & mwmMatcher); // Transforms GTFS feed into the global feed. - bool SetFeed(gtfs::Feed && feed); + bool SetFeed(gtfs::Feed && feed, bool generateTrivialShapes); // Dumps global feed to |world_feed_path|. bool Save(std::string const & worldFeedDir, bool overwrite); @@ -302,8 +302,9 @@ class WorldFeed bool FillNetworks(); // Fills routes from GTFS routes data. bool FillRoutes(); - // Fills lines and corresponding shapes from GTFS trips and shapes. - bool FillLinesAndShapes(); + // Fills lines and corresponding shapes from GTFS trips and shapes. Iff |generateTrivialShapes| is + // set to true generates trivial shapes for trips without shape id. + bool FillLinesAndShapes(bool generateTrivialShapes); // Deletes shapes which are sub-shapes and refreshes corresponding links in lines. void ModifyLinesAndShapes(); // Gets service monthday open/closed ranges, weekdays and exceptions in schedule. diff --git a/transit/world_feed/world_feed_integration_tests/world_feed_integration_tests.cpp b/transit/world_feed/world_feed_integration_tests/world_feed_integration_tests.cpp index 28bc703c4d2..b53f814dfe3 100644 --- a/transit/world_feed/world_feed_integration_tests/world_feed_integration_tests.cpp +++ b/transit/world_feed/world_feed_integration_tests/world_feed_integration_tests.cpp @@ -43,7 +43,7 @@ class WorldFeedIntegrationTests { gtfs::Feed feed(base::JoinPath(m_testPath, "minimalistic_feed")); TEST_EQUAL(feed.read_feed().code, gtfs::ResultCode::OK, ()); - TEST(m_globalFeed.SetFeed(std::move(feed)), ()); + TEST(m_globalFeed.SetFeed(std::move(feed), false /* generateTrivialShapes */), ()); TEST_EQUAL(m_globalFeed.m_networks.m_data.size(), 1, ()); TEST_EQUAL(m_globalFeed.m_routes.m_data.size(), 2, ()); @@ -59,7 +59,7 @@ class WorldFeedIntegrationTests { gtfs::Feed feed(base::JoinPath(m_testPath, "real_life_feed")); TEST_EQUAL(feed.read_feed().code, gtfs::ResultCode::OK, ()); - TEST(m_globalFeed.SetFeed(std::move(feed)), ()); + TEST(m_globalFeed.SetFeed(std::move(feed), false /* generateTrivialShapes */), ()); TEST_EQUAL(m_globalFeed.m_networks.m_data.size(), 21, ()); TEST_EQUAL(m_globalFeed.m_routes.m_data.size(), 87, ()); @@ -78,7 +78,7 @@ class WorldFeedIntegrationTests { gtfs::Feed feed(base::JoinPath(m_testPath, "feed_with_multiple_shape_projections")); TEST_EQUAL(feed.read_feed().code, gtfs::ResultCode::OK, ()); - TEST(m_globalFeed.SetFeed(std::move(feed)), ()); + TEST(m_globalFeed.SetFeed(std::move(feed), false /* generateTrivialShapes */), ()); TEST_EQUAL(m_globalFeed.m_networks.m_data.size(), 1, ()); TEST_EQUAL(m_globalFeed.m_routes.m_data.size(), 1, ()); @@ -96,14 +96,14 @@ class WorldFeedIntegrationTests gtfs::Feed feed(base::JoinPath(m_testPath, "feed_with_wrong_stops_order")); TEST_EQUAL(feed.read_feed().code, gtfs::ResultCode::OK, ()); // Feed has wrong stops order (impossible for trip shape) and should be rejected. - TEST(!m_globalFeed.SetFeed(std::move(feed)), ()); + TEST(!m_globalFeed.SetFeed(std::move(feed), false /* generateTrivialShapes */), ()); } void ReadFeedWithBackwardOrder() { gtfs::Feed feed(base::JoinPath(m_testPath, "feed_with_backward_order")); TEST_EQUAL(feed.read_feed().code, gtfs::ResultCode::OK, ()); - TEST(m_globalFeed.SetFeed(std::move(feed)), ()); + TEST(m_globalFeed.SetFeed(std::move(feed), false /* generateTrivialShapes */), ()); TEST_EQUAL(m_globalFeed.m_networks.m_data.size(), 1, ()); TEST_EQUAL(m_globalFeed.m_routes.m_data.size(), 1, ()); @@ -124,7 +124,7 @@ class WorldFeedIntegrationTests { gtfs::Feed feed(base::JoinPath(m_testPath, "feed_long_itinerary")); TEST_EQUAL(feed.read_feed().code, gtfs::ResultCode::OK, ()); - TEST(m_globalFeed.SetFeed(std::move(feed)), ()); + TEST(m_globalFeed.SetFeed(std::move(feed), false /* generateTrivialShapes */), ()); TEST_EQUAL(m_globalFeed.m_lines.m_data.size(), 1, ()); TEST_EQUAL(m_globalFeed.m_stops.m_data.size(), 4, ());