[map][tracking] Archieve tracker - bicycles, pedestrians, cars.

3party/bsdiff-courgette/bsdiff/bsdiff_tests/CMakeLists.txt

@@ -14,6 +14,7 @@ omim_link_libraries(
   coding
   base
   stats_client
+  oauthcpp
   ${LIBZ}
 )
 

coding/coding_tests/CMakeLists.txt

@@ -60,6 +60,7 @@ omim_link_libraries(
   minizip
   succinct
   stats_client
+  oauthcpp
   ${Qt5Widgets_LIBRARIES}
   ${LIBZ}
 )

defines.hpp

@@ -13,6 +13,8 @@
 #define LOC_IDX_FILE_EXTENSION ".locidx"
 #define LOC_DATA_FILE_EXTENSION ".locdata"
 #define OSM_DATA_FILE_EXTENSION ".osm"
+#define ARCHIVE_TRACKS_FILE_EXTENSION ".track"
+#define ARCHIVE_TRACKS_ZIPPED_FILE_EXTENSION ".track.zip"
 
 #define NODES_FILE "nodes.dat"
 #define WAYS_FILE "ways.dat"

descriptions/descriptions_tests/CMakeLists.txt

@@ -16,6 +16,7 @@ omim_link_libraries(
   base
   jansson
   stats_client
+  oauthcpp
   ${LIBZ}
 )
 

drape/drape_tests/CMakeLists.txt

@@ -52,6 +52,7 @@ omim_link_libraries(
   icu
   expat
   stats_client
+  oauthcpp
   freetype
   gmock
   stb_image

drape_frontend/drape_frontend_tests/CMakeLists.txt

@@ -28,6 +28,7 @@ omim_link_libraries(
   base
   expat
   stats_client
+  oauthcpp
   freetype
   stb_image
   sdf_image

generator/mwm_diff/mwm_diff_tests/CMakeLists.txt

@@ -15,6 +15,7 @@ omim_link_libraries(
   coding
   base
   stats_client
+  oauthcpp
   ${LIBZ}
 )
 

generator/mwm_diff/pymwm_diff/CMakeLists.txt

@@ -14,6 +14,7 @@ omim_link_libraries(
   mwm_diff
   bsdiff
   platform
+  oauthcpp
   coding
   base
   stats_client

local_ads/statistics.cpp

@@ -8,7 +8,6 @@
 
 #include "coding/file_writer.hpp"
 #include "coding/point_coding.hpp"
-#include "coding/sha1.hpp"
 #include "coding/url_encode.hpp"
 #include "coding/write_to_sink.hpp"
 #include "coding/zlib.hpp"
@@ -128,11 +127,6 @@ local_ads::Timestamp GetMaxTimestamp(std::list<local_ads::Event> const & events,
   return maxTimestamp;
 }
 
-std::string GetClientIdHash()
-{
-  return coding::SHA1::CalculateBase64ForString(GetPlatform().UniqueClientId());
-}
-
 std::string GetPath(std::string const & fileName)
 {
   return base::JoinPath(GetPlatform().SettingsDir(), kStatisticsFolderName, fileName);
@@ -243,7 +237,7 @@ bool CanUpload()
 namespace local_ads
 {
 Statistics::Statistics()
-  : m_userId(GetClientIdHash())
+  : m_userId(GetPlatform().UniqueIdHash())
 {}
 
 void Statistics::Startup()

map/extrapolation_benchmark/CMakeLists.txt

@@ -23,6 +23,7 @@ omim_link_libraries(
   coding
   base
   stats_client
+  oauthcpp
   gflags
   ${LIBZ}
 )

map/mwm_tests/CMakeLists.txt

@@ -33,6 +33,7 @@ omim_link_libraries(
   succinct
   pugixml
   stats_client
+  oauthcpp
   ${LIBZ}
 )
 

map/routing_manager.cpp

@@ -314,6 +314,7 @@ RoutingManager::RoutingManager(Callbacks && callbacks, Delegate & delegate)
   , m_delegate(delegate)
   , m_trackingReporter(platform::CreateSocket(), TRACKING_REALTIME_HOST, TRACKING_REALTIME_PORT,
                        tracking::Reporter::kPushDelayMs)
+  , m_trackingReporterArchive(TRACKING_HISTORICAL_HOST)
   , m_extrapolator(
         [this](location::GpsInfo const & gpsInfo) { this->OnExtrapolatedLocationUpdate(gpsInfo); })
 {
@@ -481,6 +482,14 @@ void RoutingManager::OnRoutePointPassed(RouteMarkType type, size_t intermediateI
 void RoutingManager::OnLocationUpdate(location::GpsInfo const & info)
 {
   m_extrapolator.OnLocationUpdate(info);
+
+  if (IsTrackingReporterArchiveEnabled())
+  {
+    location::GpsInfo gpsInfo(info);
+    auto routeMatchingInfo = GetRouteMatchingInfo(gpsInfo);
+    m_trackingReporterArchive.Insert(m_currentRouterType, info,
+                                     m_routingSession.MatchTraffic(routeMatchingInfo));
+  }
 }
 
 RouterType RoutingManager::GetBestRouter(m2::PointD const & startPoint,
@@ -1262,6 +1271,23 @@ bool RoutingManager::IsTrackingReporterEnabled() const
   return enableTracking;
 }
 
+bool RoutingManager::IsTrackingReporterArchiveEnabled() const
+{
+  if (m_currentRouterType != RouterType::Vehicle && m_currentRouterType != RouterType::Bicycle &&
+      m_currentRouterType != RouterType::Pedestrian)
+  {
+    return false;
+  }
+
+  if (!m_routingSession.IsFollowing())
+    return false;
+
+  bool enableTracking = true;
+  settings::TryGet(tracking::Reporter::kEnableTrackingKey, enableTracking);
+
+  return enableTracking;
+}
+
 void RoutingManager::SetRouter(RouterType type)
 {
   CHECK_THREAD_CHECKER(m_threadChecker, ("SetRouter"));

map/routing_manager.hpp

@@ -18,6 +18,7 @@
 
 #include "drape/pointers.hpp"
 
+#include "tracking/archival_reporter.hpp"
 #include "tracking/reporter.hpp"
 
 #include "geometry/point2d.hpp"
@@ -333,6 +334,7 @@ class RoutingManager final
   void CreateRoadWarningMarks(RoadWarningsCollection && roadWarnings);
 
   bool IsTrackingReporterEnabled() const;
+  bool IsTrackingReporterArchiveEnabled() const;
   void MatchLocationToRoute(location::GpsInfo & info,
                             location::RouteMatchingInfo & routeMatchingInfo) const;
   location::RouteMatchingInfo GetRouteMatchingInfo(location::GpsInfo & info);
@@ -365,6 +367,7 @@ class RoutingManager final
   routing::RoutingSession m_routingSession;
   Delegate & m_delegate;
   tracking::Reporter m_trackingReporter;
+  tracking::ArchivalReporter m_trackingReporterArchive;
   BookmarkManager * m_bmManager = nullptr;
   extrapolation::Extrapolator m_extrapolator;
 

metrics/metrics_tests/CMakeLists.txt

@@ -15,6 +15,7 @@ omim_link_libraries(
   geometry
   base
   stats_client
+  oauthcpp
   jansson
   ${LIBZ})
 

platform/platform.cpp

@@ -4,6 +4,7 @@
 
 #include "coding/base64.hpp"
 #include "coding/internal/file_data.hpp"
+#include "coding/sha1.hpp"
 #include "coding/writer.hpp"
 
 #include "base/file_name_utils.hpp"
@@ -83,6 +84,11 @@ Platform::EError Platform::ErrnoToError()
   }
 }
 
+std::string Platform::UniqueIdHash() const
+{
+  return coding::SHA1::CalculateBase64ForString(UniqueClientId());
+}
+
 // static
 bool Platform::RmDirRecursively(string const & dirName)
 {

platform/platform.hpp

@@ -270,6 +270,8 @@ class Platform
 
   std::string UniqueClientId() const;
 
+  std::string UniqueIdHash() const;
+
   std::string AdvertisingId() const;
 
   std::string MacAddress(bool md5Decoded) const;

private_default.h

@@ -31,7 +31,6 @@
 #define TRACKING_REALTIME_HOST ""
 #define TRACKING_REALTIME_PORT 0
 #define TRACKING_HISTORICAL_HOST ""
-#define TRACKING_HISTORICAL_PORT 0
 #define TRAFFIC_DATA_BASE_URL ""
 #define LOCAL_ADS_SERVER_URL ""
 #define LOCAL_ADS_STATISTICS_SERVER_URL ""

shaders/shaders_tests/CMakeLists.txt

@@ -39,6 +39,7 @@ omim_link_libraries(
   base
   expat
   stats_client
+  oauthcpp
   freetype
   stb_image
   sdf_image

track_generator/pytrack_generator/CMakeLists.txt

@@ -20,6 +20,7 @@ omim_link_libraries(
   storage
   indexer
   platform
+  oauthcpp
   coding
   geometry
   base

tracking/CMakeLists.txt

@@ -8,6 +8,14 @@ set(
   protocol.hpp
   reporter.cpp
   reporter.hpp
+  archival_file.cpp
+  archival_file.hpp
+  archival_manager.cpp
+  archival_manager.hpp
+  archival_reporter.cpp
+  archival_reporter.hpp
+  archive.cpp
+  archive.hpp
 )
 
 omim_add_library(${PROJECT_NAME} ${SRC})

tracking/archival_file.cpp

@@ -0,0 +1,139 @@
+#include "tracking/archival_file.hpp"
+
+#include "platform/platform.hpp"
+
+#include "coding/zip_creator.hpp"
+
+#include "base/assert.hpp"
+#include "base/file_name_utils.hpp"
+#include "base/logging.hpp"
+
+#include <exception>
+
+#include "defines.hpp"
+
+namespace
+{
+#ifdef DEBUG
+size_t constexpr kMaxFilesTotalSizeToSendBytes = 100 * 1024;  // 0.1 Mb
+#else
+size_t constexpr kMaxFilesTotalSizeToSendBytes = 1000 * 1024;  // 1 Mb
+#endif
+
+char constexpr kDelimiter = '_';
+}  // namespace
+
+namespace tracking
+{
+namespace archival_file
+{
+void FilesAccumulator::HandleFile(std::string const & fileName)
+{
+  uint64_t fileSize = 0;
+  if (!Platform::GetFileSizeByFullPath(fileName, fileSize))
+  {
+    LOG(LDEBUG, ("File does not exist", fileName));
+    return;
+  }
+  if (fileSize == 0)
+  {
+    LOG(LDEBUG, ("File is empty", fileName));
+    base::DeleteFileX(fileName);
+    return;
+  }
+  if (fileSize > kMaxFilesTotalSizeToSendBytes)
+  {
+    LOG(LDEBUG, ("File is too large", fileName, fileSize));
+    base::DeleteFileX(fileName);
+    return;
+  }
+
+  FileInfo const meta = ParseArchiveFilename(fileName);
+  auto const insData = m_filesByType.emplace(meta.m_trackType, FilesBatch());
+  auto const it = insData.first;
+  auto & fileBatch = it->second;
+  if (!insData.second)
+  {
+    if (fileBatch.m_totalSize + fileSize > kMaxFilesTotalSizeToSendBytes)
+      return;
+  }
+
+  fileBatch.m_totalSize += fileSize;
+  fileBatch.m_files.push_back(fileName);
+}
+
+std::vector<std::string> FilesAccumulator::PrepareArchives(std::string const & path)
+{
+  std::vector<std::string> archives;
+  try
+  {
+    for (auto const & it : m_filesByType)
+    {
+      if (it.second.m_files.empty())
+        continue;
+      std::string const archivePath =
+          base::JoinPath(path, base::GetNameFromFullPathWithoutExt(it.second.m_files[0]) +
+                                   ARCHIVE_TRACKS_ZIPPED_FILE_EXTENSION);
+
+      if (CreateZipFromFiles(it.second.m_files, archivePath, CompressionLevel::NoCompression))
+        archives.emplace_back(archivePath);
+    }
+  }
+  catch (std::exception const & e)
+  {
+    LOG(LWARNING, ("Error while archiving files", e.what()));
+  }
+  return archives;
+}
+
+void FilesAccumulator::DeleteProcessedFiles()
+{
+  for (auto const & it : m_filesByType)
+  {
+    for (auto const & file : it.second.m_files)
+      base::DeleteFileX(file);
+  }
+}
+
+std::string GetArchiveFilename(uint8_t protocolVersion, std::chrono::seconds timestamp,
+                               routing::RouterType const & trackType)
+{
+  std::string filename;
+  size_t constexpr kTrackFilenameSize = 20;
+  filename.reserve(kTrackFilenameSize);  // All filename parts have fixed length.
+  filename = std::to_string(protocolVersion) + kDelimiter + std::to_string(timestamp.count()) +
+             kDelimiter + std::to_string(static_cast<uint8_t>(trackType)) +
+             ARCHIVE_TRACKS_FILE_EXTENSION;
+  CHECK_EQUAL(filename.size(), kTrackFilenameSize, ());
+  return filename;
+}
+
+FileInfo ParseArchiveFilename(std::string const & fileName)
+{
+  std::string const metaData = base::GetNameFromFullPathWithoutExt(fileName);
+  size_t const indexFirstDelim = metaData.find(kDelimiter);
+  size_t const indexLastDelim = metaData.rfind(kDelimiter);
+  if (indexFirstDelim != 1 || indexLastDelim != 12)
+  {
+    LOG(LWARNING, ("Could not find delimiters in filename", fileName));
+    return {};
+  }
+
+  try
+  {
+    FileInfo res;
+    res.m_protocolVersion = static_cast<uint32_t>(std::stoul(metaData.substr(0, indexFirstDelim)));
+    res.m_timestamp =
+        std::stoul(metaData.substr(indexFirstDelim + 1, indexLastDelim - indexFirstDelim - 1));
+    res.m_trackType =
+        static_cast<routing::RouterType>(std::stoul(metaData.substr(indexLastDelim + 1)));
+    return res;
+  }
+  catch (std::exception const & e)
+  {
+    LOG(LWARNING, ("Error while parsing filename", e.what()));
+  }
+  return {};
+}
+}  // namespace archival_file
+}  // namespace tracking