[indexer] Lazy metadata read.

drape_frontend/rule_drawer.cpp

@@ -52,18 +52,17 @@ double GetBuildingHeightInMeters(FeatureType & f)
   double constexpr kDefaultHeightInMeters = 3.0;
   double constexpr kMetersPerLevel = 3.0;
 
-  feature::Metadata const & md = f.GetMetadata();
   double heightInMeters = kDefaultHeightInMeters;
 
-  std::string value = md.Get(feature::Metadata::FMD_HEIGHT);
+  std::string value = f.GetMetadata(feature::Metadata::FMD_HEIGHT);
   if (!value.empty())
   {
     if (!strings::to_double(value, heightInMeters))
       heightInMeters = kDefaultHeightInMeters;
   }
   else
   {
-    value = md.Get(feature::Metadata::FMD_BUILDING_LEVELS);
+    value = f.GetMetadata(feature::Metadata::FMD_BUILDING_LEVELS);
     if (!value.empty())
     {
       if (strings::to_double(value, heightInMeters))
@@ -75,8 +74,7 @@ double GetBuildingHeightInMeters(FeatureType & f)
 
 double GetBuildingMinHeightInMeters(FeatureType & f)
 {
-  feature::Metadata const & md = f.GetMetadata();
-  std::string value = md.Get(feature::Metadata::FMD_MIN_HEIGHT);
+  std::string value = f.GetMetadata(feature::Metadata::FMD_MIN_HEIGHT);
   if (value.empty())
     return 0.0;
 
@@ -92,12 +90,11 @@ df::BaseApplyFeature::HotelData ExtractHotelData(FeatureType & f)
   df::BaseApplyFeature::HotelData result;
   if (ftypes::IsBookingChecker::Instance()(f))
   {
-    auto const & metadata = f.GetMetadata();
     result.m_isHotel = true;
-    result.m_rating = metadata.Get(feature::Metadata::FMD_RATING);
-    if (!strings::to_int(metadata.Get(feature::Metadata::FMD_STARS), result.m_stars))
+    result.m_rating = f.GetMetadata(feature::Metadata::FMD_RATING);
+    if (!strings::to_int(f.GetMetadata(feature::Metadata::FMD_STARS), result.m_stars))
       result.m_stars = 0;
-    if (!strings::to_int(metadata.Get(feature::Metadata::FMD_PRICE_RATE), result.m_priceCategory))
+    if (!strings::to_int(f.GetMetadata(feature::Metadata::FMD_PRICE_RATE), result.m_priceCategory))
       result.m_priceCategory = 0;
   }
   return result;

feature_list/feature_list.cpp

@@ -205,9 +205,10 @@ class Processor
   {
     f.ParseBeforeStatistic();
     string const & category = GetReadableType(f);
+    auto const & meta = f.GetMetadata();
     // "operator" is a reserved word, hence "operatr". This word is pretty
     // common in C++ projects.
-    string const & operatr = f.GetMetadata().Get(feature::Metadata::FMD_OPERATOR);
+    string const & operatr = meta.Get(feature::Metadata::FMD_OPERATOR);
     auto const & osmIt = ft2osm.find(f.GetID().m_index);
     if ((!f.HasName() && operatr.empty()) ||
         (f.GetGeomType() == feature::GeomType::Line && category != "highway-pedestrian") ||
@@ -238,7 +239,7 @@ class Processor
     {
       // For sponsored types, adding invented sponsored ids (booking = 00) to the id tail.
       if (ftypes::IsBookingChecker::Instance()(f))
-        osmId = f.GetMetadata().Get(feature::Metadata::FMD_SPONSORED_ID) + "00";
+        osmId = meta.Get(feature::Metadata::FMD_SPONSORED_ID) + "00";
     }
     string const & uid = BuildUniqueId(ll, name);
     string const & lat = strings::to_string_with_digits_after_comma(ll.m_lat, 6);
@@ -261,17 +262,17 @@ class Processor
         addrHouse = addr.GetHouseNumber();
       }
     }
-    string const & phone = f.GetMetadata().Get(feature::Metadata::FMD_PHONE_NUMBER);
-    string const & website = f.GetMetadata().Get(feature::Metadata::FMD_WEBSITE);
-    string cuisine = f.GetMetadata().Get(feature::Metadata::FMD_CUISINE);
+    string const & phone = meta.Get(feature::Metadata::FMD_PHONE_NUMBER);
+    string const & website = meta.Get(feature::Metadata::FMD_WEBSITE);
+    string cuisine = meta.Get(feature::Metadata::FMD_CUISINE);
     replace(cuisine.begin(), cuisine.end(), ';', ',');
-    string const & stars = f.GetMetadata().Get(feature::Metadata::FMD_STARS);
-    string const & internet = f.GetMetadata().Get(feature::Metadata::FMD_INTERNET);
-    string const & denomination = f.GetMetadata().Get(feature::Metadata::FMD_DENOMINATION);
+    string const & stars = meta.Get(feature::Metadata::FMD_STARS);
+    string const & internet = meta.Get(feature::Metadata::FMD_INTERNET);
+    string const & denomination = meta.Get(feature::Metadata::FMD_DENOMINATION);
     string const & wheelchair = GetWheelchairType(f);
-    string const & opening_hours = f.GetMetadata().Get(feature::Metadata::FMD_OPEN_HOURS);
-    string const & wikipedia = f.GetMetadata().GetWikiURL();
-    string const & floor = f.GetMetadata().Get(feature::Metadata::FMD_LEVEL);
+    string const & opening_hours = meta.Get(feature::Metadata::FMD_OPEN_HOURS);
+    string const & wikipedia = meta.GetWikiURL();
+    string const & floor = meta.Get(feature::Metadata::FMD_LEVEL);
     string const & fee = strings::EndsWith(category, "-fee") ? "yes" : "";
     string const & atm = HasAtm(f) ? "yes" : "";
 

generator/generator_tests_support/test_feature.cpp

@@ -89,7 +89,7 @@ void TestFeature::Init()
 
 bool TestFeature::Matches(FeatureType & feature) const
 {
-  istringstream is(feature.GetMetadata().Get(Metadata::FMD_TEST_ID));
+  istringstream is(feature.GetMetadata(Metadata::FMD_TEST_ID));
   uint64_t id;
   is >> id;
   return id == m_id;

generator/search_index_builder.cpp

@@ -247,7 +247,7 @@ bool InsertPostcodes(FeatureType & f, function<void(strings::UniString const &)>
   using namespace search;
 
   auto const & postBoxChecker = ftypes::IsPostBoxChecker::Instance();
-  string const postcode = f.GetMetadata().Get(feature::Metadata::FMD_POSTCODE);
+  string const postcode = f.GetMetadata(feature::Metadata::FMD_POSTCODE);
   vector<string> postcodes;
   if (!postcode.empty())
     postcodes.push_back(postcode);
@@ -332,17 +332,17 @@ class FeatureInserter
 
     if (ftypes::IsAirportChecker::Instance()(types))
     {
-      string const iata = f.GetMetadata().Get(feature::Metadata::FMD_AIRPORT_IATA);
+      string const iata = f.GetMetadata(feature::Metadata::FMD_AIRPORT_IATA);
       if (!iata.empty())
         inserter(StringUtf8Multilang::kDefaultCode, iata);
     }
 
     // Index operator to support "Sberbank ATM" for objects with amenity=atm and operator=Sberbank.
-    string const op = f.GetMetadata().Get(feature::Metadata::FMD_OPERATOR);
+    string const op = f.GetMetadata(feature::Metadata::FMD_OPERATOR);
     if (!op.empty())
       inserter(StringUtf8Multilang::kDefaultCode, op);
 
-    string const brand = f.GetMetadata().Get(feature::Metadata::FMD_BRAND);
+    string const brand = f.GetMetadata(feature::Metadata::FMD_BRAND);
     if (!brand.empty())
     {
       auto const & brands = indexer::GetDefaultBrands();

generator/utils.cpp

@@ -117,8 +117,7 @@ bool ParseFeatureIdToTestIdMapping(std::string const & path,
 {
   bool success = true;
   feature::ForEachFeature(path, [&](FeatureType & feature, uint32_t fid) {
-    auto const & metatada = feature.GetMetadata();
-    auto const testIdStr = metatada.Get(feature::Metadata::FMD_TEST_ID);
+    auto const testIdStr = feature.GetMetadata(feature::Metadata::FMD_TEST_ID);
     uint64_t testId;
     if (!strings::to_uint64(testIdStr, testId))
     {

indexer/drules_selector.cpp

@@ -160,7 +160,7 @@ bool GetRating(FeatureType & ft, double & rating)
 {
   double constexpr kDefaultRating = 0.0;
 
-  string ratingStr = ft.GetMetadata().Get(feature::Metadata::FMD_RATING);
+  string ratingStr = ft.GetMetadata(feature::Metadata::FMD_RATING);
   if (ratingStr.empty() || !strings::to_double(ratingStr, rating))
     rating = kDefaultRating;
   return true;

indexer/feature.cpp

@@ -244,6 +244,7 @@ FeatureType::FeatureType(osm::MapObject const & emo)
 
   m_metadata = emo.GetMetadata();
   m_parsed.m_metadata = true;
+  m_parsed.m_metaIds = true;
 
   CHECK_LESS_OR_EQUAL(emo.GetTypes().Size(), feature::kMaxTypesCount, ());
   copy(emo.GetTypes().begin(), emo.GetTypes().end(), m_types.begin());
@@ -586,6 +587,28 @@ void FeatureType::ParseMetadata()
   m_parsed.m_metadata = true;
 }
 
+void FeatureType::ParseMetaIds()
+{
+  if (m_parsed.m_metaIds)
+    return;
+
+  CHECK(m_loadInfo, ());
+  try
+  {
+    auto const format = m_loadInfo->GetMWMFormat();
+    if (format >= version::Format::v11)
+      UNUSED_VALUE(m_metadataDeserializer->GetIds(m_id.m_index, m_metaIds));
+    else
+      ParseMetadata();
+  }
+  catch (Reader::OpenException const &)
+  {
+    // now ignore exception because not all mwm have needed sections
+  }
+
+  m_parsed.m_metaIds = true;
+}
+
 StringUtf8Multilang const & FeatureType::GetNames()
 {
   ParseCommon();
@@ -803,8 +826,33 @@ string FeatureType::GetRoadNumber()
   return m_params.ref;
 }
 
-feature::Metadata & FeatureType::GetMetadata()
+feature::Metadata const & FeatureType::GetMetadata()
 {
   ParseMetadata();
   return m_metadata;
 }
+
+std::string FeatureType::GetMetadata(feature::Metadata::EType type)
+{
+  ParseMetaIds();
+  if (m_metadata.Has(type))
+    return m_metadata.Get(type);
+
+  auto const it = base::FindIf(m_metaIds, [&type](auto const & v) { return v.first == type; });
+  if (it == m_metaIds.end())
+    return {};
+
+  auto const value = m_metadataDeserializer->GetMetaById(it->second);
+  m_metadata.Set(type, value);
+  return value;
+}
+
+bool FeatureType::HasMetadata(feature::Metadata::EType type)
+{
+  ParseMetaIds();
+  if (m_metadata.Has(type))
+    return true;
+
+  return base::FindIf(m_metaIds, [&type](auto const & v) { return v.first == type; }) !=
+         m_metaIds.end();
+}

indexer/feature.hpp

@@ -157,7 +157,11 @@ class FeatureType
   uint64_t GetPopulation();
   std::string GetRoadNumber();
 
-  feature::Metadata & GetMetadata();
+  feature::Metadata const & GetMetadata();
+
+  // Gets single metadata string. Does not parse all metadata.
+  std::string GetMetadata(feature::Metadata::EType type);
+  bool HasMetadata(feature::Metadata::EType type);
 
   /// @name Statistic functions.
   //@{
@@ -195,8 +199,12 @@ class FeatureType
     bool m_points = false;
     bool m_triangles = false;
     bool m_metadata = false;
+    bool m_metaIds = false;
 
-    void Reset() { m_types = m_common = m_header2 = m_points = m_triangles = m_metadata = false; }
+    void Reset()
+    {
+      m_types = m_common = m_header2 = m_points = m_triangles = m_metadata = m_metaIds = false;
+    }
   };
 
   struct Offsets
@@ -218,6 +226,7 @@ class FeatureType
   void ParseCommon();
   void ParseHeader2();
   void ParseMetadata();
+  void ParseMetaIds();
   void ParseGeometryAndTriangles(int scale);
 
   uint8_t m_header = 0;
@@ -235,6 +244,7 @@ class FeatureType
   using Points = buffer_vector<m2::PointD, kStaticBufferSize>;
   Points m_points, m_triangles;
   feature::Metadata m_metadata;
+  indexer::MetadataDeserializer::MetaIds m_metaIds;
 
   // Non-owning pointer to shared load info. SharedLoadInfo created once per FeaturesVector.
   feature::SharedLoadInfo const * m_loadInfo = nullptr;

indexer/indexer_tests/features_vector_test.cpp

@@ -65,7 +65,7 @@ UNIT_TEST(FeaturesVectorTest_ParseMetadata)
   map<string, int> actual;
   fv.ForEach([&](FeatureType & ft, uint32_t index)
              {
-               string postcode = ft.GetMetadata().Get(feature::Metadata::FMD_POSTCODE);
+               string postcode = ft.GetMetadata(feature::Metadata::FMD_POSTCODE);
                if (!postcode.empty())
                  ++actual[postcode];
              });

indexer/metadata_serdes.cpp

@@ -25,10 +25,10 @@ void MetadataDeserializer::Header::Read(Reader & reader)
   m_metadataMapSize = ReadPrimitiveFromSource<uint32_t>(source);
 }
 
-bool MetadataDeserializer::Get(uint32_t id, feature::MetadataBase & meta)
+bool MetadataDeserializer::Get(uint32_t featureId, feature::MetadataBase & meta)
 {
   MetaIds metaIds;
-  if (!m_map->GetThreadsafe(id, metaIds))
+  if (!m_map->GetThreadsafe(featureId, metaIds))
     return false;
 
   lock_guard<mutex> guard(m_stringsMutex);
@@ -40,6 +40,17 @@ bool MetadataDeserializer::Get(uint32_t id, feature::MetadataBase & meta)
   return true;
 }
 
+bool MetadataDeserializer::GetIds(uint32_t featureId, MetaIds & metaIds) const
+{
+  return m_map->GetThreadsafe(featureId, metaIds);
+}
+
+std::string MetadataDeserializer::GetMetaById(uint8_t id)
+{
+  lock_guard<mutex> guard(m_stringsMutex);
+  return m_strings.ExtractString(*m_stringsSubreader, id);
+}
+
 // static
 unique_ptr<MetadataDeserializer> MetadataDeserializer::Load(Reader & reader)
 {

indexer/metadata_serdes.hpp

@@ -66,6 +66,14 @@ class MetadataDeserializer
   // This method is threadsafe.
   WARN_UNUSED_RESULT bool Get(uint32_t featureId, feature::MetadataBase & meta);
 
+  // Tries to get string ids for metagata of the feature with id |featureId|. Returns false
+  // if table does not have entry for the feature.
+  // This method is threadsafe.
+  WARN_UNUSED_RESULT bool GetIds(uint32_t featureId, MetaIds & metaIds) const;
+
+  // Gets single metadata string from text storage. This method is threadsafe.
+  std::string GetMetaById(uint8_t id);
+
 private:
   using Map = MapUint32ToValue<MetaIds>;
 

map/booking_availability_filter.cpp

@@ -211,7 +211,7 @@ void PrepareData(DataSource const & dataSource, search::Results const & results,
     if (!ftypes::IsBookingChecker::Instance()(*ft))
       continue;
 
-    auto hotelId = ft->GetMetadata().Get(feature::Metadata::FMD_SPONSORED_ID);
+    auto hotelId = ft->GetMetadata(feature::Metadata::FMD_SPONSORED_ID);
     auto info = cache.Get(hotelId);
     auto const status = info.m_status;
 
@@ -252,7 +252,7 @@ void PrepareData(DataSource const & dataSource, std::vector<FeatureID> const & f
     if (!ftypes::IsBookingChecker::Instance()(*ft))
       continue;
 
-    auto const hotelId = ft->GetMetadata().Get(feature::Metadata::FMD_SPONSORED_ID);
+    auto const hotelId = ft->GetMetadata(feature::Metadata::FMD_SPONSORED_ID);
     auto info = cache.Get(hotelId);
     auto const status = info.m_status;
 
@@ -377,7 +377,7 @@ void AvailabilityFilter::GetFeaturesFromCache(search::Results const & results,
       continue;
     }
 
-    auto const & hotelId = ft->GetMetadata().Get(feature::Metadata::FMD_SPONSORED_ID);
+    auto const & hotelId = ft->GetMetadata(feature::Metadata::FMD_SPONSORED_ID);
 
     auto info = m_cache->Get(hotelId);
     if (info.m_status == availability::Cache::HotelStatus::Available)

map/map_tests/booking_filter_test.cpp

@@ -191,7 +191,7 @@ UNIT_CLASS_TEST(TestMwmEnvironment, BookingFilter_ProcessorSmoke)
       search::Result result(ft.GetID(), ft.GetCenter(), name, "", 0, details);
       InsertResult(result, results);
 
-      auto const sponsoredId = ft.GetMetadata().Get(feature::Metadata::FMD_SPONSORED_ID);
+      auto const sponsoredId = ft.GetMetadata(feature::Metadata::FMD_SPONSORED_ID);
 
       if (kAvailableHotels.find(sponsoredId) != kAvailableHotels.cend())
         InsertResult(result, expectedAvailabilityResults);
@@ -323,7 +323,7 @@ UNIT_CLASS_TEST(TestMwmEnvironment, BookingFilter_ApplyFilterOntoWithFeatureIds)
 
       allFeatureIds.push_back(ft.GetID());
 
-      auto hotelId = ft.GetMetadata().Get(feature::Metadata::FMD_SPONSORED_ID);
+      auto hotelId = ft.GetMetadata(feature::Metadata::FMD_SPONSORED_ID);
       if (kHotelIds.cend() != std::find(kHotelIds.cbegin(), kHotelIds.cend(), hotelId))
         expectedFeatureIds.push_back(ft.GetID());
     },

routing/geometry.cpp

@@ -210,7 +210,7 @@ void RoadGeometry::Load(VehicleModelInterface const & vehicleModel, FeatureType
 
   if (m_routingOptions.Has(RoutingOptions::Road::Ferry))
   {
-    auto const durationHours = feature.GetMetadata().Get(feature::Metadata::FMD_DURATION);
+    auto const durationHours = feature.GetMetadata(feature::Metadata::FMD_DURATION);
     auto const roadLenKm = GetRoadLengthM() / 1000.0;
     double const durationH = CalcFerryDurationHours(durationHours, roadLenKm);