Refactoring/reverse geocoding data normalization

app/controllers/map_controller.rb

@@ -8,8 +8,20 @@ def index
 
     @countries_and_cities = CountriesAndCities.new(@points).call
     @coordinates =
-      @points.pluck(:latitude, :longitude, :battery, :altitude, :timestamp, :velocity, :id, :country)
-             .map { [_1.to_f, _2.to_f, _3.to_s, _4.to_s, _5.to_s, _6.to_s, _7.to_s, _8.to_s] }
+      @points
+        .select(:latitude, :longitude, :battery, :altitude, :timestamp, :velocity, :id)
+        .map do |point|
+          [
+            point.latitude.to_f,
+            point.longitude.to_f,
+            point.battery.to_s,
+            point.altitude.to_s,
+            point.timestamp.to_s,
+            point.velocity.to_s,
+            point.id.to_s,
+            point.country_name.to_s
+          ]
+        end
     @distance = distance
     @start_at = Time.zone.at(start_at)
     @end_at = Time.zone.at(end_at)

app/controllers/trips_controller.rb

@@ -10,11 +10,6 @@ def index
   end
 
   def show
-    @coordinates = @trip.points.pluck(
-      :latitude, :longitude, :battery, :altitude, :timestamp, :velocity, :id,
-      :country
-    ).map { [_1.to_f, _2.to_f, _3.to_s, _4.to_s, _5.to_s, _6.to_s, _7.to_s, _8.to_s] }
-
     @photo_previews = Rails.cache.fetch("trip_photos_#{@trip.id}", expires_in: 1.day) do
       @trip.photo_previews
     end
@@ -58,10 +53,11 @@ def set_trip
   end
 
   def set_coordinates
-    @coordinates = @trip.points.pluck(
-      :latitude, :longitude, :battery, :altitude, :timestamp, :velocity, :id,
-      :country
-    ).map { [_1.to_f, _2.to_f, _3.to_s, _4.to_s, _5.to_s, _6.to_s, _7.to_s, _8.to_s] }
+    @coordinates =
+      @trip
+        .points
+        .select(:latitude, :longitude, :timestamp)
+        .map { |point| [point.latitude.to_f, point.longitude.to_f, point.timestamp.to_s] }
   end
 
   def trip_params

app/jobs/data_migrations/migrate_points_data_to_columns_and_tables_job.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class DataMigrations::MigratePointsDataToColumnsAndTablesJob < ApplicationJob
+  queue_as :default
+
+  def perform(point_ids)
+    Rails.logger.info(
+      "=====Migrating points data for #{point_ids.size} ( #{point_ids.first} - #{point_ids.last} ) points====="
+    )
+    points = Point.where(id: point_ids)
+
+    points.each { DataMigrations::MigratePoint.new(_1).call }
+
+    Rails.logger.info(
+      "=====Migrated #{point_ids.size} ( #{point_ids.first} - #{point_ids.last} ) points====="
+    )
+  end
+end

app/models/city.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class City < ApplicationRecord
+  belongs_to :country
+  belongs_to :state, optional: true
+  belongs_to :county, optional: true
+
+  validates :name, :country, presence: true
+end

app/models/country.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class Country < ApplicationRecord
+  has_many :cities, dependent: :destroy
+  has_many :states, dependent: :destroy
+  has_many :counties, dependent: :destroy
+  validates :name, :iso2_code, presence: true
+end

app/models/county.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class County < ApplicationRecord
+  belongs_to :country
+  belongs_to :state, optional: true
+
+  validates :name, :country, presence: true
+end

app/models/point.rb

@@ -7,8 +7,18 @@ class Point < ApplicationRecord
   belongs_to :visit, optional: true
   belongs_to :user
 
+  belongs_to :country, optional: true
+  belongs_to :state, optional: true
+  belongs_to :county, optional: true
+  belongs_to :city, optional: true
+
   validates :latitude, :longitude, :timestamp, presence: true
 
+  delegate :name, to: :city, prefix: true, allow_nil: true
+  delegate :name, to: :country, prefix: true, allow_nil: true
+  delegate :name, to: :state, prefix: true, allow_nil: true
+  delegate :name, to: :county, prefix: true, allow_nil: true
+
   enum :battery_status, { unknown: 0, unplugged: 1, charging: 2, full: 3 }, suffix: true
   enum :trigger, {
     unknown: 0, background_event: 1, circular_region_event: 2, beacon_event: 3,
@@ -57,7 +67,7 @@ def broadcast_coordinates
         timestamp.to_s,
         velocity.to_s,
         id.to_s,
-        country.to_s
+        country_name.to_s
       ]
     )
   end

app/models/state.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class State < ApplicationRecord
+  belongs_to :country
+  has_many :cities, dependent: :destroy
+  has_many :counties, dependent: :destroy
+
+  validates :name, :country, presence: true
+end

app/models/trip.rb

@@ -14,7 +14,8 @@ def points
   end
 
   def countries
-    points.pluck(:country).uniq.compact
+    ids = points.pluck(:country_id).uniq.compact
+    Country.where(id: ids).pluck(:name)
   end
 
   def photo_previews

app/serializers/point_serializer.rb

@@ -1,17 +1,28 @@
 # frozen_string_literal: true
 
 class PointSerializer
-  EXCLUDED_ATTRIBUTES = %w[created_at updated_at visit_id id import_id user_id raw_data].freeze
+  EXCLUDED_ATTRIBUTES = %w[
+    created_at updated_at visit_id id import_id user_id raw_data city_id
+    country_id state_id county_id
+  ].freeze
 
   def initialize(point)
     @point = point
   end
 
   def call
-    point.attributes.except(*EXCLUDED_ATTRIBUTES)
+    attributes
   end
 
   private
 
   attr_reader :point
+
+  def attributes
+    attrs = point.attributes.except(*EXCLUDED_ATTRIBUTES)
+    attrs[:city] = point.city_name
+    attrs[:country] = point.country_name
+
+    attrs
+  end
 end

app/services/countries_and_cities.rb

@@ -10,8 +10,8 @@ def initialize(points)
 
   def call
     points
-      .reject { |point| point.country.nil? || point.city.nil? }
-      .group_by(&:country)
+      .reject { |point| point.country_name.nil? || point.city_name.nil? }
+      .group_by(&:country_name)
       .transform_values { |country_points| process_country_points(country_points) }
       .map { |country, cities| CountryData.new(country: country, cities: cities) }
   end
@@ -22,7 +22,7 @@ def call
 
   def process_country_points(country_points)
     country_points
-      .group_by(&:city)
+      .group_by(&:city_name)
       .transform_values { |city_points| create_city_data_if_valid(city_points) }
       .values
       .compact
@@ -31,7 +31,7 @@ def process_country_points(country_points)
   def create_city_data_if_valid(city_points)
     timestamps = city_points.pluck(:timestamp)
     duration = calculate_duration_in_minutes(timestamps)
-    city = city_points.first.city
+    city = city_points.first.city_name
     points_count = city_points.size
 
     build_city_data(city, points_count, timestamps, duration)

app/services/data_migrations/migrate_point.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+class DataMigrations::MigratePoint
+  def initialize(point)
+    @point = point
+  end
+
+  def call
+    country = find_or_create_country(geodata['country'], geodata['countrycode'])
+    state   = find_or_create_state(geodata['state'], country)
+    county  = find_or_create_county(geodata['county'], state)
+    city    = find_or_create_city(geodata['city'], county)
+
+    point.update(
+      country: country,
+      state: state,
+      county: county,
+      city: city,
+
+      osm_id: geodata['osm_id'],
+      osm_type: geodata['osm_type'],
+      osm_key: geodata['osm_key'],
+      osm_value: geodata['osm_value'],
+
+      post_code: geodata['postcode'],
+      street: geodata['street'],
+      house_number: geodata['housenumber'],
+      type: geodata['type'],
+      name: geodata['name'],
+      district: geodata['district'],
+      locality: geodata['locality'],
+      importance: geodata['importance'],
+      object_type: geodata['objecttype'],
+      classification: geodata['classification']
+    )
+  end
+
+  private
+
+  def geodata
+    @geodata ||= @point.geodata['properties']
+  end
+
+  def find_or_create_country(country_name, country_code)
+    Country.find_or_create_by(name: country_name, iso2_code: country_code)
+  end
+
+  def find_or_create_state(state_name, country)
+    State.find_or_create_by(name: state_name, country: country)
+  end
+
+  def find_or_create_county(county_name, state)
+    County.find_or_create_by(name: county_name, state: state)
+  end
+
+  def find_or_create_city(city_name, county)
+    City.find_or_create_by(name: city_name, county: county)
+  end
+end

app/services/reverse_geocoding/places/fetch_data.rb

@@ -12,10 +12,7 @@ def initialize(place_id)
   end
 
   def call
-    if ::PHOTON_API_HOST.blank?
-      Rails.logger.warn('PHOTON_API_HOST is not set')
-      return
-    end
+    Rails.logger.warn('PHOTON_API_HOST is not set') and return if ::PHOTON_API_HOST.blank?
 
     first_place = reverse_geocoded_places.shift
     update_place(first_place)
@@ -34,8 +31,8 @@ def update_place(reverse_geocoded_place)
       name:       place_name(data),
       latitude:   data['geometry']['coordinates'][1],
       longitude:  data['geometry']['coordinates'][0],
-      city:       data['properties']['city'],
-      country:    data['properties']['country'],
+      city:       city(data),
+      country:    country(data),
       geodata:    data,
       source:     Place.sources[:photon],
       reverse_geocoded_at: Time.current
@@ -47,8 +44,8 @@ def fetch_and_create_place(reverse_geocoded_place)
     new_place = find_place(data)
 
     new_place.name = place_name(data)
-    new_place.city = data['properties']['city']
-    new_place.country = data['properties']['country']
+    new_place.city = city(data)
+    new_place.country = country(data)
     new_place.geodata = data
     new_place.source = :photon
 
@@ -103,4 +100,18 @@ def reverse_geocoded_places
         place.data['properties']['osm_key'].in?(IGNORED_OSM_KEYS)
     end
   end
+
+  def country(data)
+    Country.find_or_create_by(
+      name: data['properties']['country'],
+      iso2_code: data['properties']['countrycode']
+    )
+  end
+
+  def city(data)
+    City.find_or_create_by(
+      name: data['properties']['city'],
+      country: country(data)
+    )
+  end
 end

app/services/reverse_geocoding/points/fetch_data.rb

@@ -21,9 +21,16 @@ def update_point_with_geocoding_data
     response = Geocoder.search([point.latitude, point.longitude]).first
     return if response.blank? || response.data['error'].present?
 
+    country = Country.find_or_create_by(name: response.country, iso2_code: response.countrycode)
+    city = City.find_or_create_by(name: response.city, country: country)
+    county = County.find_or_create_by(name: response.county, country: country)
+    state = State.find_or_create_by(name: response.state, country: country)
+
     point.update!(
-      city: response.city,
-      country: response.country,
+      city: city,
+      country: country,
+      county: county,
+      state: state,
       geodata: response.data,
       reverse_geocoded_at: Time.current
     )

app/services/stats/calculate_month.rb

@@ -46,7 +46,8 @@ def points
               .tracked_points
               .without_raw_data
               .where(timestamp: start_timestamp..end_timestamp)
-              .select(:latitude, :longitude, :timestamp, :city, :country)
+              .select(:latitude, :longitude, :timestamp)
+              .includes(:city, :country)
               .order(timestamp: :asc)
   end
 

app/views/trips/_trip.html.erb

@@ -13,7 +13,7 @@
             class="rounded-lg z-0"
             data-controller="trip-map"
             data-trip-map-trip-id-value="<%= trip.id %>"
-            data-trip-map-coordinates-value="<%= trip.points.pluck(:latitude, :longitude, :battery, :altitude, :timestamp, :velocity, :id, :country).to_json %>"
+            data-trip-map-coordinates-value="<%= trip.points.pluck(:latitude, :longitude, :battery, :altitude, :timestamp, :velocity, :id).to_json %>"
             data-trip-map-api-key-value="<%= current_user.api_key %>"
             data-trip-map-user-settings-value="<%= current_user.settings.to_json %>"
             data-trip-map-timezone-value="<%= Rails.configuration.time_zone %>"

app/views/trips/index.html.erb

@@ -30,6 +30,12 @@
           <%= render 'trip', trip: trip %>
         <% end %>
       </div>
+
+      <div class="flex justify-center my-5">
+        <div class='flex'>
+          <%= paginate @trips %>
+        </div>
+      </div>
     <% end %>
   </div>
 </div>

config/environment.rb

@@ -6,8 +6,11 @@
 # Initialize the Rails application.
 Rails.application.initialize!
 
-# Clear the cache
-Cache::CleaningJob.perform_later
+# Only run cache jobs when starting the server
+if defined?(Rails::Server)
+  # Clear the cache
+  Cache::CleaningJob.perform_later
 
-# Preheat the cache
-Cache::PreheatingJob.perform_later
+  # Preheat the cache
+  Cache::PreheatingJob.perform_later
+end