diff --git a/db/migrate/20250107195202_add_err.rb b/db/migrate/20250107195202_add_err.rb new file mode 100644 index 000000000..546710309 --- /dev/null +++ b/db/migrate/20250107195202_add_err.rb @@ -0,0 +1,12 @@ +class AddErr < ActiveRecord::Migration[8.0] + def change + Scorer.where(name: 'ERR@10').first_or_create( + scale: (0..3).to_a, + scale_with_labels: {"0":"Poor","1":"Fair","2":"Good","3":"Perfect"}, + show_scale_labels: true, + code: File.readlines('./db/scorers/err@10.js','\n').join('\n'), + name: 'ERR@10', + communal: true + ) + end +end diff --git a/db/schema.rb b/db/schema.rb index 6e585e0b7..12533d079 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2024_12_18_173819) do +ActiveRecord::Schema[8.0].define(version: 2025_01_07_195202) do create_table "active_storage_attachments", charset: "utf8mb4", collation: "utf8mb4_bin", force: :cascade do |t| t.string "name", null: false t.string "record_type", null: false diff --git a/db/scorers/err@10.js b/db/scorers/err@10.js new file mode 100644 index 000000000..5d72a5cdc --- /dev/null +++ b/db/scorers/err@10.js @@ -0,0 +1,31 @@ +let k = 10 // @Rank +// if less than K results, need to reduce K now or final score is too low +k = numReturned() < k ? numReturned() : k +const scores = Array(k); +const missing_rating = 0; // pessimistic assumption +let i = 0 +for (i = 0; i < k; i++) { + if (hasDocRating(i)) { + scores[i] = (docRating(i)); + } else { + scores[i] = missing_rating; + } +} + +function gain(grade, maxGrade) { + return (Math.pow(2,grade) - 1.0)/Math.pow(2,maxGrade) +} +function err(lst) { + let ERR = 0.0 + let trust = 1.0 + for (i = 0; i < lst.length; i++) { + const rank = i + 1.0 + // max is the maximum possible rating for the scorer. + const pUseful = gain(lst[i], max) + const disc = pUseful/rank + ERR = ERR + trust * disc + trust = trust * (1.0 - pUseful) + } + return ERR +} +setScore(err(scores)) diff --git a/db/seeds.rb b/db/seeds.rb index 4010d1f3c..9e40ac120 100755 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -60,6 +60,15 @@ communal: true ) +Scorer.where(name: 'ERR@10').first_or_create( + scale: (0..3).to_a, + scale_with_labels: {"0":"Poor","1":"Fair","2":"Good","3":"Perfect"}, + show_scale_labels: true, + code: File.readlines('./db/scorers/err@10.js','\n').join('\n'), + name: 'ERR@10', + communal: true +) + SelectionStrategy.where(name: 'Single Rater').first_or_create( name: 'Single Rater', description: 'A single rating for each query/doc pair is all that is required. The fastest way to get a lot of ratings, with lower quality.'