[indexer] Use text storage for metadata.

generator/feature_sorter.cpp

@@ -69,7 +69,6 @@ class FeaturesCollector2 : public FeaturesCollector
       m_trgFile.push_back(make_unique<TmpFile>(filename + TRIANGLE_FILE_TAG + postfix));
     }
 
-    m_metadataFile = make_unique<TmpFile>(filename + METADATA_FILE_TAG);
     m_addrFile = make_unique<FileWriter>(filename + TEMP_ADDR_FILENAME);
   }
 
@@ -138,17 +137,10 @@ class FeaturesCollector2 : public FeaturesCollector
       finalizeFn(move(m_trgFile[i]), GetTagForIndex(TRIANGLE_FILE_TAG, i));
     }
 
-    finalizeFn(move(m_metadataFile), METADATA_FILE_TAG);
-
     {
       FilesContainerW writer(m_filename, FileWriter::OP_WRITE_EXISTING);
-      auto w = writer.GetWriter(METADATA_INDEX_FILE_TAG);
-
-      MetadataIndexBuilder metaIdxBuilder;
-      for (auto const & v : m_metadataOffset)
-        metaIdxBuilder.Put(v.first, v.second);
-
-      metaIdxBuilder.Freeze(*w);
+      auto w = writer.GetWriter(METADATA_FILE_TAG);
+      m_metadataBuilder.Freeze(*w);
     }
 
     if (m_header.GetType() == DataHeader::MapType::Country ||
@@ -246,16 +238,10 @@ class FeaturesCollector2 : public FeaturesCollector
 
       featureId = WriteFeatureBase(buffer.m_buffer, fb);
 
-      fb.GetAddressData().Serialize(*m_addrFile);
+      fb.GetAddressData().SerializeForMwmTmp(*m_addrFile);
 
       if (!fb.GetMetadata().Empty())
-      {
-        uint64_t const offset = m_metadataFile->Pos();
-        ASSERT_LESS_OR_EQUAL(offset, numeric_limits<uint32_t>::max(), ());
-
-        m_metadataOffset.emplace_back(featureId, static_cast<uint32_t>(offset));
-        fb.GetMetadata().Serialize(*m_metadataFile);
-      }
+        m_metadataBuilder.Put(featureId, fb.GetMetadata());
 
       if (fb.HasOsmIds())
         m_osm2ft.AddIds(generator::MakeCompositeId(fb), featureId);
@@ -311,11 +297,9 @@ class FeaturesCollector2 : public FeaturesCollector
   unique_ptr<FileWriter> m_addrFile;
 
   // Temporary files for sections.
-  unique_ptr<TmpFile> m_metadataFile;
   TmpFiles m_geoFile, m_trgFile;
 
-  // Mapping from feature id to offset in file section with the correspondent metadata.
-  vector<pair<uint32_t, uint32_t>> m_metadataOffset;
+  indexer::MetadataBuilder m_metadataBuilder;
 
   DataHeader m_header;
   RegionData m_regionData;

generator/postcodes_section_builder.cpp

@@ -40,7 +40,7 @@ bool BuildPostcodesSection(std::string const & path, std::string const & country
     while (src.Size() > 0)
     {
       addrs.push_back({});
-      addrs.back().Deserialize(src);
+      addrs.back().DeserializeFromMwmTmp(src);
     }
   }
 

generator/search_index_builder.cpp

@@ -392,7 +392,7 @@ void ReadAddressData(string const & filename, vector<feature::AddressData> & add
   while (src.Size() > 0)
   {
     addrs.push_back({});
-    addrs.back().Deserialize(src);
+    addrs.back().DeserializeFromMwmTmp(src);
   }
 }
 

indexer/CMakeLists.txt

@@ -102,6 +102,8 @@ set(
   map_style_reader.hpp
   meta_idx.cpp
   meta_idx.hpp
+  metadata_serdes.cpp
+  metadata_serdes.hpp
   mwm_set.cpp
   mwm_set.hpp
   postcodes.cpp

indexer/feature.cpp

@@ -186,11 +186,18 @@ uint8_t ReadByte(TSource & src)
 }  // namespace
 
 FeatureType::FeatureType(SharedLoadInfo const * loadInfo, vector<uint8_t> && buffer,
-                         MetadataIndex const * metadataIndex)
-  : m_loadInfo(loadInfo), m_data(buffer), m_metadataIndex(metadataIndex)
+                         MetadataIndex const * metadataIndex,
+                         indexer::MetadataDeserializer * metadataDeserializer)
+  : m_loadInfo(loadInfo)
+  , m_data(buffer)
+  , m_metadataIndex(metadataIndex)
+  , m_metadataDeserializer(metadataDeserializer)
 {
   CHECK(loadInfo, ());
-  ASSERT(m_loadInfo->GetMWMFormat() < version::Format::v10 || m_metadataIndex,
+
+  ASSERT(m_loadInfo->GetMWMFormat() < version::Format::v10 ||
+             m_loadInfo->GetMWMFormat() == version::Format::v10 && m_metadataIndex ||
+             m_loadInfo->GetMWMFormat() > version::Format::v10 && m_metadataDeserializer,
          (m_loadInfo->GetMWMFormat()));
 
   m_header = Header(m_data);
@@ -522,15 +529,20 @@ void FeatureType::ParseMetadata()
   try
   {
     auto const format = m_loadInfo->GetMWMFormat();
-    if (format >= version::Format::v10)
+    if (format >= version::Format::v11)
+    {
+      UNUSED_VALUE(m_metadataDeserializer->Get(m_id.m_index, m_metadata));
+    }
+    else if (format == version::Format::v10)
     {
       uint32_t offset;
       CHECK(m_metadataIndex, ("metadata index should be set for mwm format >= v10"));
       if (m_metadataIndex->Get(m_id.m_index, offset))
       {
         ReaderSource<FilesContainerR::TReader> src(m_loadInfo->GetMetadataReader());
         src.Skip(offset);
-        m_metadata.Deserialize(src);
+        // Before v11 we used the same metadata serialization for mwm and mwm.tmp
+        m_metadata.DeserializeFromMwmTmp(src);
       }
     }
     else
@@ -552,7 +564,8 @@ void FeatureType::ParseMetadata()
         src.Skip(it->value);
         CHECK_GREATER_OR_EQUAL(m_loadInfo->GetMWMFormat(), version::Format::v8,
                                ("Unsupported mwm format"));
-        m_metadata.Deserialize(src);
+        // Before v11 we used the same metadata serialization for mwm and mwm.tmp
+        m_metadata.DeserializeFromMwmTmp(src);
       }
     }
   }

indexer/feature.hpp

@@ -35,7 +35,8 @@ class FeatureType
   using GeometryOffsets = buffer_vector<uint32_t, feature::DataHeader::kMaxScalesCount>;
 
   FeatureType(feature::SharedLoadInfo const * loadInfo, std::vector<uint8_t> && buffer,
-              feature::MetadataIndex const * metadataIndex);
+              feature::MetadataIndex const * metadataIndex,
+              indexer::MetadataDeserializer * metadataDeserializer);
   FeatureType(osm::MapObject const & emo);
 
   feature::GeomType GetGeomType() const;
@@ -246,8 +247,11 @@ class FeatureType
   // Non-owning pointer to shared load info. SharedLoadInfo created once per FeaturesVector.
   feature::SharedLoadInfo const * m_loadInfo = nullptr;
   std::vector<uint8_t> m_data;
-  // Pointer to shared metadata index. Must be set for mwm format >= Format::v10
+
+  // Pointer to shared metadata index. Must be set for mwm format == Format::v10
   feature::MetadataIndex const * m_metadataIndex = nullptr;
+  // Pointer to shared metedata deserializer. Must be set for mwm format >= Format::v11
+  indexer::MetadataDeserializer * m_metadataDeserializer = nullptr;
 
   ParsedFlags m_parsed;
   Offsets m_offsets;

indexer/feature_data.hpp

@@ -323,16 +323,16 @@ class FeatureBuilderParams : public FeatureParams
   void Write(Sink & sink) const
   {
     FeatureParams::Write(sink);
-    m_metadata.Serialize(sink);
-    m_addrTags.Serialize(sink);
+    m_metadata.SerializeForMwmTmp(sink);
+    m_addrTags.SerializeForMwmTmp(sink);
   }
 
   template <class Source>
   void Read(Source & src)
   {
     FeatureParams::Read(src);
-    m_metadata.Deserialize(src);
-    m_addrTags.Deserialize(src);
+    m_metadata.DeserializeFromMwmTmp(src);
+    m_addrTags.DeserializeFromMwmTmp(src);
   }
 
   bool GetReversedGeometry() const { return m_reversedGeometry; }

indexer/feature_meta.hpp

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "indexer/metadata_serdes.hpp"
+
 #include "coding/reader.hpp"
 #include "coding/string_utf8_multilang.hpp"
 
@@ -47,8 +49,8 @@ class MetadataBase
   inline bool Empty() const { return m_metadata.empty(); }
   inline size_t Size() const { return m_metadata.size(); }
 
-  template <class TSink>
-  void Serialize(TSink & sink) const
+  template <class Sink>
+  void SerializeForMwmTmp(Sink & sink) const
   {
     auto const sz = static_cast<uint32_t>(m_metadata.size());
     WriteVarUint(sink, sz);
@@ -59,8 +61,8 @@ class MetadataBase
     }
   }
 
-  template <class TSource>
-  void Deserialize(TSource & src)
+  template <class Source>
+  void DeserializeFromMwmTmp(Source & src)
   {
     auto const sz = ReadVarUint<uint32_t>(src);
     for (size_t i = 0; i < sz; ++i)
@@ -76,6 +78,8 @@ class MetadataBase
   }
 
 protected:
+  friend bool indexer::MetadataDeserializer::Get(uint32_t id, MetadataBase & meta);
+
   // TODO: Change uint8_t to appropriate type when FMD_COUNT reaches 256.
   void Set(uint8_t type, std::string const & value)
   {
@@ -205,6 +209,18 @@ class RegionData : public MetadataBase
     PH_CANADA_DAY = 23
   };
 
+  template <class Sink>
+  void Serialize(Sink & sink) const
+  {
+    MetadataBase::SerializeForMwmTmp(sink);
+  }
+
+  template <class Source>
+  void Deserialize(Source & src)
+  {
+    MetadataBase::DeserializeFromMwmTmp(src);
+  }
+
   void Set(Type type, std::string const & s)
   {
     CHECK_NOT_EQUAL(type, Type::RD_LANGUAGES, ("Please use RegionData::SetLanguages method"));

indexer/features_vector.cpp

@@ -9,7 +9,7 @@ std::unique_ptr<FeatureType> FeaturesVector::GetByIndex(uint32_t index) const
 {
   auto const ftOffset = m_table ? m_table->GetFeatureOffset(index) : index;
   return std::make_unique<FeatureType>(&m_loadInfo, m_recordReader->ReadRecord(ftOffset),
-                                       m_metaidx.get());
+                                       m_metaidx.get(), m_metaDeserializer.get());
 }
 
 size_t FeaturesVector::GetNumFeatures() const

indexer/features_vector.hpp

@@ -3,6 +3,7 @@
 #include "indexer/dat_section_header.hpp"
 #include "indexer/feature.hpp"
 #include "indexer/meta_idx.hpp"
+#include "indexer/metadata_serdes.hpp"
 #include "indexer/shared_load_info.hpp"
 
 #include "coding/var_record_reader.hpp"
@@ -24,7 +25,22 @@ class FeaturesVector
                  feature::FeaturesOffsetsTable const * table)
     : m_loadInfo(cont, header), m_table(table)
   {
-    if (m_loadInfo.GetMWMFormat() >= version::Format::v10)
+    if (m_loadInfo.GetMWMFormat() >= version::Format::v11)
+    {
+      FilesContainerR::TReader reader = m_loadInfo.GetDataReader();
+
+      feature::DatSectionHeader header;
+      header.Read(*reader.GetPtr());
+      CHECK(header.m_version == feature::DatSectionHeader::Version::V0,
+            (base::Underlying(header.m_version)));
+      m_recordReader = std::make_unique<RecordReader>(
+          reader.SubReader(header.m_featuresOffset, header.m_featuresSize));
+
+      auto metaReader = m_loadInfo.GetMetadataReader();
+      m_metaDeserializer = indexer::MetadataDeserializer::Load(*metaReader.GetPtr());
+      CHECK(m_metaDeserializer, ());
+    }
+    else if (m_loadInfo.GetMWMFormat() == version::Format::v10)
     {
       FilesContainerR::TReader reader = m_loadInfo.GetDataReader();
 
@@ -54,7 +70,7 @@ class FeaturesVector
   {
     uint32_t index = 0;
     m_recordReader->ForEachRecord([&](uint32_t pos, std::vector<uint8_t> && data) {
-      FeatureType ft(&m_loadInfo, std::move(data), m_metaidx.get());
+      FeatureType ft(&m_loadInfo, std::move(data), m_metaidx.get(), m_metaDeserializer.get());
 
       // We can't properly set MwmId here, because FeaturesVector
       // works with FileContainerR, not with MwmId/MwmHandle/MwmValue.
@@ -80,6 +96,7 @@ class FeaturesVector
   std::unique_ptr<RecordReader> m_recordReader;
   feature::FeaturesOffsetsTable const * m_table;
   std::unique_ptr<feature::MetadataIndex> m_metaidx;
+  std::unique_ptr<indexer::MetadataDeserializer> m_metaDeserializer;
 };
 
 /// Test features vector (reader) that combines all the needed data for stand-alone work.

indexer/indexer_tests/feature_metadata_test.cpp

@@ -68,7 +68,7 @@ UNIT_TEST(Feature_Metadata_PresentTypes)
     TEST_EQUAL(m.Get(type), kKeyValues.find(static_cast<Metadata::EType>(type))->second, ());
 }
 
-UNIT_TEST(Feature_Serialization)
+UNIT_TEST(Feature_MwmTmpSerialization)
 {
   Metadata original;
   for (auto const & value : kKeyValues)
@@ -79,11 +79,11 @@ UNIT_TEST(Feature_Serialization)
     Metadata serialized;
     vector<char> buffer;
     MemWriter<decltype(buffer)> writer(buffer);
-    original.Serialize(writer);
+    original.SerializeForMwmTmp(writer);
 
     MemReader reader(buffer.data(), buffer.size());
     ReaderSource<MemReader> src(reader);
-    serialized.Deserialize(src);
+    serialized.DeserializeFromMwmTmp(src);
 
     for (auto const & value : kKeyValues)
       TEST_EQUAL(serialized.Get(value.first), value.second, ());

indexer/meta_idx.cpp

@@ -63,7 +63,6 @@ bool MetadataIndex::Init(unique_ptr<Reader> reader)
 {
   m_indexSubreader = move(reader);
 
-  // Decodes block encoded by writeBlockCallback from MetadataIndexBuilder::Freeze.
   auto const readBlockCallback = [&](NonOwningReaderSource & source, uint32_t blockSize,
                                      vector<uint32_t> & values) {
     ASSERT_NOT_EQUAL(blockSize, 0, ());
@@ -80,45 +79,6 @@ bool MetadataIndex::Init(unique_ptr<Reader> reader)
   return m_map != nullptr;
 }
 
-// MetadataIndexBuilder -----------------------------------------------------------------------
-void MetadataIndexBuilder::Put(uint32_t featureId, uint32_t offset)
-{
-  m_builder.Put(featureId, offset);
-}
-
-void MetadataIndexBuilder::Freeze(Writer & writer) const
-{
-  size_t startOffset = writer.Pos();
-  CHECK(coding::IsAlign8(startOffset), ());
-
-  MetadataIndex::Header header;
-  header.Serialize(writer);
-
-  uint64_t bytesWritten = writer.Pos();
-  coding::WritePadding(writer, bytesWritten);
-
-  auto const writeBlockCallback = [](auto & w, auto begin, auto end) {
-    WriteVarUint(w, *begin);
-    auto prevIt = begin;
-    for (auto it = begin + 1; it != end; ++it)
-    {
-      CHECK_GREATER_OR_EQUAL(*it, *prevIt, ());
-      WriteVarUint(w, *it - *prevIt);
-      prevIt = it;
-    }
-  };
-
-  header.m_indexOffset = base::asserted_cast<uint32_t>(writer.Pos() - startOffset);
-  m_builder.Freeze(writer, writeBlockCallback);
-  header.m_indexSize =
-      base::asserted_cast<uint32_t>(writer.Pos() - header.m_indexOffset - startOffset);
-
-  auto const endOffset = writer.Pos();
-  writer.Seek(startOffset);
-  header.Serialize(writer);
-  writer.Seek(endOffset);
-}
-
 std::string DebugPrint(MetadataIndex::Version v)
 {
   CHECK(v == MetadataIndex::Version::V0, (base::Underlying(v)));