Merge pull request #51 from datacite/feature_parametrise_comrpessed_r…

…eports

Feature parametrise compressed reports

app/controllers/application_controller.rb

@@ -7,7 +7,7 @@ class ApplicationController < ActionController::API
 
   attr_accessor :current_user
 
-  before_action :default_format_json, :transform_params
+  before_action :default_format_json, :transform_params, :permit_all_params
   after_action :set_jsonp_format, :set_consumer_header
   after_action :set_jsonp_format
 
@@ -49,6 +49,10 @@ def default_format_json
     request.format = :json if request.format.html?
   end
 
+  def permit_all_params
+    ActionController::Parameters.permit_all_parameters = true
+  end
+
   # convert parameters with hyphen to parameters with underscore.
   #  https://stackoverflow.com/questions/35812277/fields-parameters-with-hyphen-in-ruby-on-rails
   def transform_params

app/controllers/reports_controller.rb

@@ -91,7 +91,6 @@ def update
   end
 
   def create
-
     @report = Report.where(created_by: safe_params[:created_by])
     .where(month: get_month(safe_params.dig("reporting_period","begin_date")))
     .where(year: get_year(safe_params.dig("reporting_period","begin_date")))
@@ -132,38 +131,71 @@ def validate_monthly_report
 
   def safe_params
 
+    fail JSON::ParserError, "You need to provide a payload following the SUSHI specification" unless params[:report_header].present?
+    x = usage_report_params        if params[:report_datasets].present?  && params[:report_header].fetch(:release) == "rd1" && params[:encoding] != "gzip" 
+    x = resolution_report_params   if params[:report_header].fetch(:release) == "drl" && params[:encoding] == "gzip" && x.nil?
+    x = compressed_report_params   if params[:compressed].present?  && params[:encoding] == "gzip" && params[:report_header].fetch(:release) == "rd1" && x.nil?
+    x = decompressed_report_params if params[:encoding] == "gzip" && params[:compressed].nil? && params[:report_header].fetch(:release) == "rd1" && x.nil?
+    x
+  end
+
+  def usage_report_params
     fail JSON::ParserError, "You need to provide a payload following the SUSHI specification" unless params[:report_datasets].present? and params[:report_header].present? 
 
     header, datasets = params.require([:report_header, :report_datasets])
-    header.merge!({report_datasets: datasets}) 
+    header[:report_datasets] = datasets
+
     nested_names = [:name, :value]
     nested_types = [:type, :value]
     codes =  IsoCountryCodes.for_select.map {|code| code.last.downcase}
+
     header.permit(
       :report_name, :report_id, :release, :created, :created_by, 
       report_attributes: nested_names, 
       report_filters: nested_names, 
-      reporting_period: [:end_date, :begin_date], 
-      exceptions: [:message, :severity, :data, :code, :help_url], 
+      reporting_period: ["end_date", "begin_date"], 
+      exceptions: [:message, :severity, :data, :code, "help_url"], 
       report_datasets: [
-        :dataset_title, 
+        "dataset-title", 
         :yop,
         :uri,
         :platform,
-        :data_type, 
+        "data-type", 
         :publisher,
-        :access_method,
-        publisher_id: nested_types, 
-        dataset_dates:nested_types, 
+        "publisher-id": nested_types, 
+        "dataset-dates": nested_types, 
         performance: [
-          period: [:end_date, :begin_date],
-          instance: [:access_method, :metric_type, :count, country_counts: codes]
+          period: ["end-date", "begin-date"],
+          instance: ["access-method", "metric-type", :count, "country-counts": codes]
         ],
-        dataset_contributors: nested_types,
-        dataset_attributes: nested_types,
-        dataset_id: nested_types
+        "dataset-contributors": nested_types,
+        "dataset-attributes": nested_types,
+        "dataset-id": nested_types
       ]
     )
+  end
 
+  def resolution_report_params
+    puts "Resolutions!!!!"
+    fail  fail JSON::ParserError, "Resolution Reports need to be compressed" unless params[:compressed].present? and params[:encoding] == "gzip" 
+    header, report = params.require([:report_header, :compressed])
+    # header[:report_datasets] = []
+    header[:compressed] = report.string
+    header
+  end
+
+  def compressed_report_params
+    fail JSON::ParserError, "You need to provide a payload following the SUSHI specification and int compressed" unless params[:compressed].present? and params[:report_header].present? 
+    header, report = params.require([:report_header, :compressed])
+    # header[:report_datasets] = []
+    header[:compressed] = report.string
+    header
+  end
+
+  def decompressed_report_params
+    fail JSON::ParserError, "You need to provide a payload following the SUSHI specification" unless params[:report_datasets].present? and params[:report_header].present? 
+    header, datasets = params.require([:report_header, :report_datasets])
+    header[:report_datasets] = datasets
+    header
   end
 end

app/models/concerns/metadatable.rb

@@ -8,22 +8,44 @@ module Metadatable
   included do
 
     def validate_sushi 
+      puts "this is being validated"
       schema = load_schema
-      report = self.attributes.except("compressed").deep_transform_keys { |key| key.tr('_', '-') }
+      report = self.attributes.except("compressed")
+      report.transform_keys! { |key| key.tr('_', '-') }
+
+      JSON::Validator.fully_validate(schema, report.to_json, :errors_as_objects => true)
+    end
+
+    def validate_sample_sushi
+      puts "this is being sampled validated"
+      schema = load_schema
+      report = self.attributes.except("compressed")
+      report.transform_keys! { |key| key.tr('_', '-') }
+      size = report["report-datasets"].length
+      if (size/8) > 0  
+        sample =  (size/8) > 100 ? 100 : size
+      else
+        sample = 1
+      end
+      report["report-datasets"] = report["report-datasets"].sample(sample)
       JSON::Validator.fully_validate(schema, report.to_json, :errors_as_objects => true)
     end
 
     def is_valid_sushi? 
       schema = load_schema
-      report = self.attributes.except("compressed").deep_transform_keys { |key| key.tr('_', '-') }
+      # report = self.attributes.except("compressed").deep_transform_keys { |key| key.tr('_', '-') }
+      report = self.attributes.except("compressed")
+      report.transform_keys! { |key| key.tr('_', '-') }
       JSON::Validator.validate(schema, report.to_json)
     end
 
     USAGE_SCHEMA_FILE = "lib/sushi_schema/sushi_usage_schema.json"
     RESOLUTION_SCHEMA_FILE = "lib/sushi_schema/sushi_resolution_schema.json"
 
+
     def load_schema
-      report = self.attributes.except("compressed").deep_transform_keys { |key| key.tr('_', '-') }
+      report = self.attributes.except("compressed")
+      report.transform_keys! { |key| key.tr('_', '-') }
       file = report.dig("report-name") == "resolution report" && report.dig("created-by") == "datacite" ? RESOLUTION_SCHEMA_FILE : USAGE_SCHEMA_FILE
       begin
         File.read(file)

app/models/report.rb

@@ -10,6 +10,9 @@
 =end
 
 require 'base32/url'
+require 'digest'
+require 'yajl'
+
 
 class Report < ApplicationRecord
 
@@ -22,15 +25,18 @@ class Report < ApplicationRecord
   include Queueable 
 
   # attr_accessor :month, :year, :compressed
-  validates_presence_of :report_id, :created_by, :report_datasets, :client_id, :provider_id, :created, :reporting_period
+  validates_presence_of :report_id, :created_by, :client_id, :provider_id, :created, :reporting_period 
+  validates_presence_of :report_datasets, if: :normal_report?
+  #, :report_datasets
   validates :uid, uniqueness: true
-  validates :validate_sushi, sushi: {presence: true}
+  validates :validate_sushi, sushi: {presence: true}, if: :normal_report?
   attr_readonly :created_by, :month, :year, :client_id
 
   # serialize :exceptions, Array
-  # before_create :to_compress
-  before_create :set_id
+  before_validation :to_compress 
   before_validation :set_uid, on: :create
+  after_validation :clean_datasets
+  before_create :set_id
   after_commit :push_report
 
   def push_report
@@ -58,19 +64,58 @@ def compress
     ActiveSupport::Gzip.compress(json_report.to_json)
   end
 
+  def encode_compressed
+    return nil if self.compressed.nil?
+    Base64.strict_encode64(self.compressed)
+  end
+
+  def checksum
+    return nil if self.compressed.nil?
+     Digest::SHA256.hexdigest(self.compressed)
+  end
+
   private
 
   # random number that fits into MySQL bigint field (8 bytes)
   def set_id
     self.id = SecureRandom.random_number(9223372036854775807)
   end
 
-  # def to_compress
-  #   write_attribute(:compressed, compress)
-  #   # puts params
-  #   self.report_datasets = {empty:"too large"}
-  #   write_attribute(:report_datasets, {empty:"too large"})
-  # end
+  def to_compress
+    if  self.compressed.nil?
+      write_attribute(:compressed, compress) 
+    else
+      # parser = Yajl::Parser.new
+      # self.report_datasets = parser.parse(ActiveSupport::Gzip.decompress(self.compressed))
+    end
+  end
+
+  def clean_datasets
+    write_attribute(:report_datasets, compressed_message) if compressed_report?
+  end
+
+  def compressed_message
+    {
+      empty: "too large",
+      checksum: checksum,
+    }
+  end
+
+  def normal_report?
+    return nil if compressed_report?
+    return nil if self.report_datasets.nil?
+    true
+  end
+
+  def compressed_report?
+    return nil if self.exceptions.empty? 
+    return nil if self.compressed.nil?
+    code = self.exceptions.dig(0).fetch("code",nil)
+    return nil if code != 69
+    true
+  end
+
+
 
   def set_uid
     return ActionController::ParameterMissing if self.reporting_period.nil?
@@ -80,10 +125,5 @@ def set_uid
     year = Date.strptime(self.reporting_period["begin_date"],"%Y-%m-%d").year.to_s 
     write_attribute(:month,  month ) 
     write_attribute(:year,  year) 
-    write_attribute(:compressed, compress)
-    puts self.compressed
-    puts self.compressed.nil?
-
-    # write_attribute(:report_datasets, {empty:"too large"}) unless self.compressed.nil?
   end
 end

app/serializers/report_serializer.rb

@@ -1,12 +1,20 @@
 class ReportSerializer < ActiveModel::Serializer
   type 'report'
 
-  attributes  :id, :report_header, :report_datasets
+  attributes  :id, :report_header, :report_datasets, :gzip, :checksum
 
   def id
     object.uid
   end
 
+  def gzip
+    object.encode_compressed
+  end
+
+  def checksum
+      object.checksum
+  end
+
   def report_header
     {
       :report_name => object.report_name,
@@ -19,5 +27,9 @@ def report_header
       :report_attributes => object.report_attributes,
       :exceptions => object.exceptions, 
     }
+
+
+
+
   end
 end

app/validators/sushi_validator.rb

@@ -1,7 +1,9 @@
 class SushiValidator < ActiveModel::EachValidator
 
   def validate_each(record, _schema, _value)
-    unless  record.is_valid_sushi? 
+    # unless  record.is_valid_sushi? 
+    valid = record.validate_sushi
+    unless  valid == true
 
       def dig_errors(errors)
         return [] if errors.nil?
@@ -12,7 +14,8 @@ def dig_errors(errors)
           errors.map { |error| {error.dig(:fragment) => error.dig(:message)}  }
         end
       end
-      nice_erros = dig_errors record.validate_sushi
+      # nice_erros = dig_errors record.validate_sushi
+      nice_erros = dig_errors valid
 
       record.errors["errors"] << (nice_erros || "Your SUSHI is wrong mate!!")
     end

config/initializers/json_param_key_transform.rb

@@ -6,7 +6,11 @@
   data = {:_json => data} unless data.is_a?(Hash)
 
   # Transform camelCase param keys to snake_case:
-  data.deep_transform_keys!(&:underscore)
-  data.deep_transform_keys! { |key| key.underscore }
+  # data.deep_transform_keys!(&:underscore)
+  # data.deep_transform_keys! { |key| key.underscore }
 
+  data.transform_keys! { |key| key.underscore }
+  header = data.fetch("report_header").deep_transform_keys { |key| key.underscore }
+  data["report_header"] = header
+  data
 }

lib/middleware/compressed_requests.rb

@@ -26,14 +26,16 @@ def call(env)
       request.update_param('report_header', hsh.fetch("report-header",{}).deep_transform_keys { |key| key.tr('-', '_') } )
       puts "This is header"
 
-      request.update_param('report_datasets', hsh.fetch("report-datasets",{}).map { |dataset| dataset.deep_transform_keys { |key| key.tr('-', '_') } }  )
+      # request.update_param('report_datasets', hsh.fetch("report-datasets",[]))
+      # puts "This is body"
+      request.update_param('compressed', env['rack.input'])
       puts "This is body"
 
       request.update_param('encoding', env['HTTP_CONTENT_ENCODING'])
 
       env.delete('HTTP_CONTENT_ENCODING')
-      env['CONTENT_LENGTH'] = extracted.length
-      env['rack.input'] = StringIO.new(extracted)
+      # env['CONTENT_LENGTH'] = extracted.length
+      # env['rack.input'] = StringIO.new(extracted)
     end