Merge pull request #138 from datacite/revert-then-fix-issue-135

Revert then fix issue 135

Gemfile

@@ -34,7 +34,6 @@ gem 'iso_country_codes'
 gem 'sentry-raven', '~> 2.9'
 gem 'git', '~> 1.5'
 gem "sprockets", "~> 3.7", ">= 3.7.2"
-gem "aasm", "~> 5.0", ">= 5.0.1"
 
 group :development, :test do
   gem 'rspec-rails', '~> 3.8'

Gemfile.lock

@@ -1,8 +1,6 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    aasm (5.0.6)
-      concurrent-ruby (~> 1.0)
     actioncable (5.2.4.1)
       actionpack (= 5.2.4.1)
       nio4r (~> 2.0)
@@ -333,7 +331,6 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  aasm (~> 5.0, >= 5.0.1)
   active_model_serializers (~> 0.10.10)
   api-pagination
   aws-sdk-s3
@@ -391,4 +388,4 @@ DEPENDENCIES
   webmock (~> 3.1)
 
 BUNDLED WITH
-   2.1.4
+   2.0.2

README.md

@@ -252,18 +252,6 @@ POST /reports
 POST /reports
 ```
 
-### HTTP Response Codes
-
-| Status Code | Message                                                 |
-|-------------|---------------------------------------------------------|
-| 200         | Report has been updated                                 |
-| 201         | Report has been CREATED and validated correctly         |
-| 202         | Report has been ACCEPTED and its waiting for validation |
-| 404         | Report does not exists                                  |
-| 422         | Report or subreport has failed validation               |
-
-One can use the filter paramaters `?incorrect=true` (422s) and `?queued=true` (202s) to list all the reports with validation errors or queued for validation, respectvely. 
-
 ### Metadata Validation
 
 The validation of the metadata in the reports its a two-step process. The controller takes care of checking presence of fields. Then the Schema validation is performed before saving the report. We use json-schema validation for this.

app/controllers/reports_controller.rb

@@ -29,19 +29,15 @@ def index
            end
 
     collection = if params[:id].present?
-                   Report.correct.where(uid: params[:id].split(","))
-                 elsif params[:incorrect].present?
-                   Report.incorrect.any? ? Report.incorrect.where(user_id: params[:client_id]) : []
-                 elsif params[:queued].present?
-                   Report.queued.any? ? Report.queued.where(user_id: params[:client_id] ) : []
+                   Report.where(uid: params[:id].split(","))
                  elsif params[:created_by].present?
-                   Report.correct.where(created_by: params[:created_by])
+                   Report.where(created_by: params[:created_by])
                  elsif params[:year].present?
-                   Report.correct.where(year: params[:year])
+                   Report.where(year: params[:year])
                  elsif params[:client_id].present?
-                   Report.correct.where(user_id: params[:client_id])
+                   Report.where(user_id: params[:client_id])
                  else
-                   Report.correct
+                   Report.all
                  end
 
     total = collection.size
@@ -67,17 +63,7 @@ def destroy
   end
 
   def show
-    case true
-    when @report.correct?
-      render json: @report, status: :ok
-    when @report.queued?
-      render json: @report, status: :accepted
-    when @report.incorrect?
-      render json: @report, status: :unprocessable_entity
-    else
-      Rails.logger.warn @report.errors.inspect
-      render json: serialize(@report.errors), status: :unprocessable_entity
-    end
+    render json: @report
   end
 
   def update
@@ -89,23 +75,14 @@ def update
     if exists && params[:compressed].present?
       @report.report_subsets.destroy_all
       @report.report_subsets << ReportSubset.new(compressed: safe_params[:compressed])
-      authorize! :delete_all, @report.report_subsets
+      # authorize! :delete_all, @report.report_subsets
     end
     # create report if it doesn't exist already
     @report = Report.new(safe_params.merge(uid: params[:id])) if @report.blank?
-    authorize! :update, @report
+    # authorize! :update, @report
 
     if @report.update(safe_params.merge(@user_hash))
-      updated = true
-    end
-
-    case true 
-    when updated && @report.correct?
       render json: @report, status: exists ? :ok : :created
-    when updated && @report.queued?
-      render json: @report, status: :accepted
-    when updated && @report.incorrect?
-      render json: @report, status: :unprocessable_entity
     else
       Rails.logger.warn @report.errors.inspect
       render json: serialize(@report.errors), status: :unprocessable_entity
@@ -124,19 +101,10 @@ def create
     # add_subsets
 
     @report = Report.new(safe_params.merge(@user_hash)) if @report.blank?
-    authorize! :create, @report
+    # authorize! :create, @report
 
     if @report.save
-      saved = true
-    end
-
-    case true
-    when saved && @report.correct?
       render json: @report, status: :created
-    when saved && @report.queued?
-      render json: @report, status: :accepted
-    when saved && @report.incorrect?
-      render json: @report, status: :unprocessable_entity
     else
       Rails.logger.error @report.errors.inspect
       render json: @report.errors, status: :unprocessable_entity
@@ -147,6 +115,7 @@ def create
 
   def set_report
     @report = Report.where(uid: params[:id]).first
+
     fail ActiveRecord::RecordNotFound if @report.blank?
   end
 

app/jobs/validation_job.rb

@@ -1,10 +1,50 @@
-require "benchmark"
+require 'benchmark'
 
 class ValidationJob < ActiveJob::Base
   queue_as :sashimi
 
-  def perform(id, options = {})
-    subset = ReportSubset.find(id)
-    subset.validate_compressed(options)
+  def perform(id, options={})
+    subset = nil
+    full_report = nil
+    subset = nil
+    header = nil
+    parsed = nil
+    is_valid = nil
+
+    bm = Benchmark.ms {
+      subset = ReportSubset.where(id: id).first
+      full_report = ActiveSupport::Gzip.decompress(subset.compressed)
+    }
+    Rails.logger.warn message: "[ValidationJob] Decompress", duration: bm
+
+    bm = Benchmark.ms {
+      parsed = JSON.parse(full_report)
+    }
+    Rails.logger.warn message: "[ValidationJob] Parse", duration: bm
+
+    header = parsed.dig("report-header")
+    header["report-datasets"] = parsed.dig("report-datasets")
+
+    bm = Benchmark.ms {
+      # validate subset of usage report, raise error with bug tracker otherwise
+      is_valid = subset.validate_this_sushi_with_error(header)
+    }
+    Rails.logger.warn message: "[ValidationJob] Validation", duration: bm
+
+    if is_valid
+      message = "[ValidationJob] Subset #{id} of Usage Report #{subset.report.uid} successfully validated."
+      subset.update_column(:aasm, "valid")
+      subset.push_report unless options[:validate_only]
+      Rails.logger.info message
+      true
+    else
+      # store error details in database
+      validation_errors =  subset.validate_this_sushi(header)
+
+      message = "[ValidationJobError] Subset #{id} of Usage Report #{subset.report.uid} failed validation. There are #{validation_errors.size} errors, starting with \"#{validation_errors.first[:message]}\"."
+      subset.update_columns(aasm: "not_valid", exceptions: validation_errors)
+      Rails.logger.error message
+      false
+    end
   end
 end

app/models/ability.rb

@@ -11,6 +11,7 @@ def initialize(user)
       can :manage, :all
     elsif user.role_id == "client_admin" && user.uid.present?
       can [:create, :update, :read], Report, :user_id => user.uid
+      can :manage, ReportSubset
     end
   end
 end

app/models/concerns/metadatable.rb

@@ -1,70 +1,70 @@
 module Metadatable
   extend ActiveSupport::Concern
 
-  require "json-schema"
-  require "fileutils"
-  require "json"
+  require 'json-schema'
+  require 'fileutils'
+  require 'json' 
 
   included do
-    def validate_sushi
+    def validate_sushi 
       schema = load_schema
-      report = attributes.except("compressed", "aasm_state")
-      report.transform_keys! { |key| key.tr("_", "-") }
-
-      JSON::Validator.fully_validate(schema, report.to_json, errors_as_objects: true)
+      report = self.attributes.except("compressed")
+      report.transform_keys! { |key| key.tr('_', '-') }
+   
+      JSON::Validator.fully_validate(schema, report.to_json, :errors_as_objects => true)
     end
 
     def validate_this_sushi(sushi)
-      schema = load_schema
-      JSON::Validator.fully_validate(schema, sushi.to_json, errors_as_objects: true)
+      schema = load_schema 
+      JSON::Validator.fully_validate(schema, sushi.to_json, :errors_as_objects => true)
     end
 
     def validate_this_sushi_with_error(sushi)
-      schema = load_schema
+      schema = load_schema 
       JSON::Validator.validate!(schema, sushi.to_json)
-    rescue JSON::Schema::ValidationError => e
-      Raven.capture_exception(e)
+    rescue JSON::Schema::ValidationError => exception
+      Raven.capture_exception(exception)
       false
     end
 
     def validate_sample_sushi
       schema = load_schema
-      report = attributes.except("compressed", "aasm_state")
-      report.transform_keys! { |key| key.tr("_", "-") }
+      report = self.attributes.except("compressed")
+      report.transform_keys! { |key| key.tr('_', '-') }
       size = report["report-datasets"].length
-      sample = if (size / 8) > 0
-                 (size / 8) > 100 ? 100 : size
-               else
-                 1
-               end
+      if (size/8) > 0  
+        sample = (size/8) > 100 ? 100 : size
+      else
+        sample = 1
+      end
       report["report-datasets"] = report["report-datasets"].sample(sample)
-      JSON::Validator.fully_validate(schema, report.to_json, errors_as_objects: true)
+      JSON::Validator.fully_validate(schema, report.to_json, :errors_as_objects => true)
     end
 
-    def is_valid_sushi?
+    def is_valid_sushi? 
       schema = load_schema
       report = attributes.except("compressed", "aasm_state")
       report.transform_keys! { |key| key.tr("_", "-") }
       JSON::Validator.validate(schema, report.to_json)
     end
 
     def load_schema
-      if is_a?(ReportSubset)
-        release = report.release
+      if self.is_a?(ReportSubset)
+        release = self.report.release
       else
         report = attributes.except("compressed", "aasm_state")
         release = report.dig("release")
       end
-
+      
       file = case release
-             when "rd1" then "lib/sushi_schema/sushi_usage_schema.json"
-             when "drl" then "lib/sushi_schema/sushi_resolution_schema.json"
+             when 'rd1' then "lib/sushi_schema/sushi_usage_schema.json"
+             when 'drl' then "lib/sushi_schema/sushi_resolution_schema.json"
              end
 
       begin
         File.read(file)
-      rescue StandardError
-        Rails.logger.error "must redo the JSON schema file"
+      rescue
+        Rails.logger.error 'must redo the JSON schema file'
         {} # return an empty hash
       end
     end

app/models/report.rb

@@ -15,26 +15,6 @@ class Report < ApplicationRecord
   # include validation methods for sushi
   include Queueable
 
-  include AASM
-
-  aasm whiny_transitions: false do
-    state :queued, initial: true
-    state :correct, :incorrect
-
-    event :accept do
-      transitions from: [:queued, :incorrect], to: :correct
-    end
-
-    event :reject do
-      transitions from: [:queued, :correct], to: :incorrect
-    end
-
-    event :validating do
-      transitions from: [:queued, :correct, :incorrect], to: :queued
-    end
-
-  end
-
   # attr_accessor :month, :year, :compressed
   validates_presence_of :report_id, :created_by, :user_id, :created, :reporting_period
   validates_presence_of :report_datasets, if: :normal_report?
@@ -58,10 +38,6 @@ class Report < ApplicationRecord
   # def validate_report_job
   #   ValidationJob.perform_later(self)
   # end
-
-  scope :queued, -> { where aasm_state: "queued" }
-  scope :incorrect, -> { where aasm_state: "incorrect" }
-  scope :correct, -> { where aasm_state: "correct" }
 
   def destroy_report_events
     DestroyEventsJob.perform_later(uid)
@@ -92,21 +68,6 @@ def push_report
     send_message(body) if ENV["AWS_REGION"].present?
   end
 
-  def update_state
-    return null if report_subsets.empty?
-
-    statuses = report_subsets.map &:aasm
-
-    case true
-    when statuses.all? { |s| s == "valid" }
-      accept!
-    when statuses.any? { |s| s == "queued" }
-      validating!
-    else
-      reject!
-    end
-  end
-
   def compress
     json_report = {
       "report-header":