From 3f63549cc2da2886d3a4d188a7399a4ffd239dcf Mon Sep 17 00:00:00 2001 From: David Fisher Date: Thu, 28 Mar 2024 10:12:20 -0400 Subject: [PATCH 01/11] Compute p@10 using 0 == irrel, 1+ == rel, so that P@10 is in the expected range [0..1] --- db/scorers/p@10.js | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/db/scorers/p@10.js b/db/scorers/p@10.js index e8a37cbc4..1bb645bfc 100644 --- a/db/scorers/p@10.js +++ b/db/scorers/p@10.js @@ -1,4 +1,11 @@ -var k = 10 // @Rank -var score = avgRating(k); - +const k = 10; // @Rank +// k may be > length list, so count up the total number of documents processed. +let count = 0, total = 0; +eachDoc(function(doc, i) { + if (hasDocRating(i) && (docRating(i)) > 0) { // map 0 -> irrel, 1+ ->rel + count = count i+1 + } + total = total + 1 +}, k); +const score = rank / total : 0.0; setScore(score); From 55e368f9c770e97ec661e4d27bebe5ff27f160b2 Mon Sep 17 00:00:00 2001 From: David Fisher Date: Thu, 28 Mar 2024 10:16:04 -0400 Subject: [PATCH 02/11] Compute count, not rank divided by total. --- db/scorers/p@10.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/scorers/p@10.js b/db/scorers/p@10.js index 1bb645bfc..7b92b7fc2 100644 --- a/db/scorers/p@10.js +++ b/db/scorers/p@10.js @@ -7,5 +7,5 @@ eachDoc(function(doc, i) { } total = total + 1 }, k); -const score = rank / total : 0.0; +const score = count / total : 0.0; setScore(score); From 3df1a9c22698e5f63cb49cc7f44b83b169534c44 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 26 Jun 2024 14:29:30 -0400 Subject: [PATCH 03/11] Trigger a migration for existing Quepid users to get the new P@10 javascript code --- db/migrate/20240626181338_reindex_p_at10_code.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 db/migrate/20240626181338_reindex_p_at10_code.rb diff --git a/db/migrate/20240626181338_reindex_p_at10_code.rb b/db/migrate/20240626181338_reindex_p_at10_code.rb new file mode 100644 index 000000000..b7bf0bba9 --- /dev/null +++ b/db/migrate/20240626181338_reindex_p_at10_code.rb @@ -0,0 +1,12 @@ +class ReindexPAt10Code < ActiveRecord::Migration[7.1] + # force existing P@10 scorers to reload using the latest scorer information. + def change + scorers_to_update = ['P@10'] + scorers_to_update.each do |scorer_name| + scorer = Scorer.where(name: scorer_name, communal: true).first + name = scorer.name + scorer.code = File.readlines("./db/scorers/#{name.downcase}.js",'\n').join('\n') + scorer.save! + end + end +end From 0800e1f00c7a307a6ba8475172c0c05b3ae00808 Mon Sep 17 00:00:00 2001 From: David Fisher Date: Wed, 26 Jun 2024 15:08:32 -0400 Subject: [PATCH 04/11] Do that math in a mathy way --- db/scorers/p@10.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/scorers/p@10.js b/db/scorers/p@10.js index 7b92b7fc2..539529f2c 100644 --- a/db/scorers/p@10.js +++ b/db/scorers/p@10.js @@ -3,9 +3,9 @@ const k = 10; // @Rank let count = 0, total = 0; eachDoc(function(doc, i) { if (hasDocRating(i) && (docRating(i)) > 0) { // map 0 -> irrel, 1+ ->rel - count = count i+1 + count = count + 1 } - total = total + 1 + total = total + 1.0 }, k); -const score = count / total : 0.0; +const score = total ? count / total : 0.0; setScore(score); From 9e48483dbd3b6eaf5ceae70ca2f4d0920d0c97cf Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 26 Jun 2024 15:12:31 -0400 Subject: [PATCH 05/11] lint! --- db/schema.rb | 2 +- db/scorers/p@10.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/db/schema.rb b/db/schema.rb index 0a19e24b7..116dc63f7 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_06_26_173902) do +ActiveRecord::Schema[7.1].define(version: 2024_06_26_181338) do create_table "active_storage_attachments", charset: "utf8mb4", collation: "utf8mb4_bin", force: :cascade do |t| t.string "name", null: false t.string "record_type", null: false diff --git a/db/scorers/p@10.js b/db/scorers/p@10.js index 539529f2c..f094fbdb8 100644 --- a/db/scorers/p@10.js +++ b/db/scorers/p@10.js @@ -3,9 +3,9 @@ const k = 10; // @Rank let count = 0, total = 0; eachDoc(function(doc, i) { if (hasDocRating(i) && (docRating(i)) > 0) { // map 0 -> irrel, 1+ ->rel - count = count + 1 + count = count + 1; } - total = total + 1.0 + total = total + 1.0; }, k); const score = total ? count / total : 0.0; setScore(score); From c0ebb98158c2dae47d9bfcfea48d07c3a03d52be Mon Sep 17 00:00:00 2001 From: David Fisher Date: Fri, 3 Jan 2025 10:49:08 -0500 Subject: [PATCH 06/11] Add a scorer for Expected Reciprocal Rank (ERR). ERR is the multi-valued judgment variant of reciprocal rank. --- db/scorers/err@10.js | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 db/scorers/err@10.js diff --git a/db/scorers/err@10.js b/db/scorers/err@10.js new file mode 100644 index 000000000..26bba9441 --- /dev/null +++ b/db/scorers/err@10.js @@ -0,0 +1,29 @@ +var k = 10 // @Rank +var scores = Array(k); +var missing_rating = 0; // pessimistic assumption +// if less than K results, need to reduce K now or final score is too low +k = numReturned() < k ? numReturned() : k + +for (var i = 0; i < k; i++) { + if (hasDocRating(i)) { + scores[i] = (docRating(i)); + } else { + scores[i] = missing_rating; + } +} + +function gain(grade, maxGrade=3.0) { + return (Math.pow(2,grade) - 1.0)/Math.pow(2,maxGrade) +} +function err(lst, maxG=3.0) { + trust = 1.0 + for (i = 0; i < lst.length; i++) { + rank = i + 1.0 + pUseful = gain(lst, maxG) + disc = pUseful/rank + ERR = ERR + trust * disc + trust = trust * (1.0 - pUseful) + } + return ERR +} +setScore(err(scores)) From 98724f0b671dfa74931c1d09dfe3f335758159bc Mon Sep 17 00:00:00 2001 From: David Fisher Date: Fri, 3 Jan 2025 11:00:10 -0500 Subject: [PATCH 07/11] Address javascript deepsource warnings. --- db/scorers/err@10.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/db/scorers/err@10.js b/db/scorers/err@10.js index 26bba9441..4dd91dcb9 100644 --- a/db/scorers/err@10.js +++ b/db/scorers/err@10.js @@ -1,10 +1,10 @@ -var k = 10 // @Rank -var scores = Array(k); -var missing_rating = 0; // pessimistic assumption +let k = 10 // @Rank +let scores = Array(k); +let missing_rating = 0; // pessimistic assumption // if less than K results, need to reduce K now or final score is too low k = numReturned() < k ? numReturned() : k - -for (var i = 0; i < k; i++) { +var i +for (i = 0; i < k; i++) { if (hasDocRating(i)) { scores[i] = (docRating(i)); } else { @@ -16,12 +16,12 @@ function gain(grade, maxGrade=3.0) { return (Math.pow(2,grade) - 1.0)/Math.pow(2,maxGrade) } function err(lst, maxG=3.0) { - trust = 1.0 + var trust = 1.0 for (i = 0; i < lst.length; i++) { - rank = i + 1.0 - pUseful = gain(lst, maxG) - disc = pUseful/rank - ERR = ERR + trust * disc + var rank = i + 1.0 + var pUseful = gain(lst, maxG) + var disc = pUseful/rank + var ERR = ERR + trust * disc trust = trust * (1.0 - pUseful) } return ERR From 00fe9da2627261e6baabe612139289989a2345cd Mon Sep 17 00:00:00 2001 From: David Fisher Date: Fri, 3 Jan 2025 11:39:16 -0500 Subject: [PATCH 08/11] Address javascript deepsource warnings. --- db/scorers/err@10.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/db/scorers/err@10.js b/db/scorers/err@10.js index 4dd91dcb9..d654f3bcf 100644 --- a/db/scorers/err@10.js +++ b/db/scorers/err@10.js @@ -1,10 +1,10 @@ let k = 10 // @Rank -let scores = Array(k); -let missing_rating = 0; // pessimistic assumption // if less than K results, need to reduce K now or final score is too low k = numReturned() < k ? numReturned() : k -var i -for (i = 0; i < k; i++) { +let scores = Array(k); +let missing_rating = 0; // pessimistic assumption + +for (var i = 0; i < k; i++) { if (hasDocRating(i)) { scores[i] = (docRating(i)); } else { @@ -16,12 +16,13 @@ function gain(grade, maxGrade=3.0) { return (Math.pow(2,grade) - 1.0)/Math.pow(2,maxGrade) } function err(lst, maxG=3.0) { + var ERR = 0.0 var trust = 1.0 - for (i = 0; i < lst.length; i++) { + for (var i = 0; i < lst.length; i++) { var rank = i + 1.0 - var pUseful = gain(lst, maxG) + var pUseful = gain(lst[i], maxG) var disc = pUseful/rank - var ERR = ERR + trust * disc + ERR = ERR + trust * disc trust = trust * (1.0 - pUseful) } return ERR From 2d8e54d4c0d114fd344e5a8f59297847b7662f3c Mon Sep 17 00:00:00 2001 From: David Fisher Date: Fri, 3 Jan 2025 11:44:32 -0500 Subject: [PATCH 09/11] Address javascript deepsource warnings. --- db/scorers/err@10.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/db/scorers/err@10.js b/db/scorers/err@10.js index d654f3bcf..f241d39f1 100644 --- a/db/scorers/err@10.js +++ b/db/scorers/err@10.js @@ -1,10 +1,10 @@ let k = 10 // @Rank // if less than K results, need to reduce K now or final score is too low k = numReturned() < k ? numReturned() : k -let scores = Array(k); -let missing_rating = 0; // pessimistic assumption - -for (var i = 0; i < k; i++) { +const scores = Array(k); +const missing_rating = 0; // pessimistic assumption +let i = 0 +for (i = 0; i < k; i++) { if (hasDocRating(i)) { scores[i] = (docRating(i)); } else { @@ -16,12 +16,12 @@ function gain(grade, maxGrade=3.0) { return (Math.pow(2,grade) - 1.0)/Math.pow(2,maxGrade) } function err(lst, maxG=3.0) { - var ERR = 0.0 - var trust = 1.0 - for (var i = 0; i < lst.length; i++) { - var rank = i + 1.0 - var pUseful = gain(lst[i], maxG) - var disc = pUseful/rank + let ERR = 0.0 + let trust = 1.0 + for (i = 0; i < lst.length; i++) { + let rank = i + 1.0 + let pUseful = gain(lst[i], maxG) + let disc = pUseful/rank ERR = ERR + trust * disc trust = trust * (1.0 - pUseful) } From 39f8b0f316606f0da9427aca18962c5b693de009 Mon Sep 17 00:00:00 2001 From: David Fisher Date: Fri, 3 Jan 2025 11:48:36 -0500 Subject: [PATCH 10/11] Address javascript deepsource warnings. --- db/scorers/err@10.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/scorers/err@10.js b/db/scorers/err@10.js index f241d39f1..2bf2dd5d5 100644 --- a/db/scorers/err@10.js +++ b/db/scorers/err@10.js @@ -19,9 +19,9 @@ function err(lst, maxG=3.0) { let ERR = 0.0 let trust = 1.0 for (i = 0; i < lst.length; i++) { - let rank = i + 1.0 - let pUseful = gain(lst[i], maxG) - let disc = pUseful/rank + const rank = i + 1.0 + const pUseful = gain(lst[i], maxG) + const disc = pUseful/rank ERR = ERR + trust * disc trust = trust * (1.0 - pUseful) } From 66b7c4d738971bd09b1f47a86e930b3777625cc4 Mon Sep 17 00:00:00 2001 From: David Fisher Date: Tue, 7 Jan 2025 15:02:45 -0500 Subject: [PATCH 11/11] Refactor for tidyness. Add the migration. Add to seeds --- db/migrate/20250107195202_add_err.rb | 12 ++++++++++++ db/schema.rb | 2 +- db/scorers/err@10.js | 7 ++++--- db/seeds.rb | 9 +++++++++ 4 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 db/migrate/20250107195202_add_err.rb diff --git a/db/migrate/20250107195202_add_err.rb b/db/migrate/20250107195202_add_err.rb new file mode 100644 index 000000000..546710309 --- /dev/null +++ b/db/migrate/20250107195202_add_err.rb @@ -0,0 +1,12 @@ +class AddErr < ActiveRecord::Migration[8.0] + def change + Scorer.where(name: 'ERR@10').first_or_create( + scale: (0..3).to_a, + scale_with_labels: {"0":"Poor","1":"Fair","2":"Good","3":"Perfect"}, + show_scale_labels: true, + code: File.readlines('./db/scorers/err@10.js','\n').join('\n'), + name: 'ERR@10', + communal: true + ) + end +end diff --git a/db/schema.rb b/db/schema.rb index 6e585e0b7..12533d079 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2024_12_18_173819) do +ActiveRecord::Schema[8.0].define(version: 2025_01_07_195202) do create_table "active_storage_attachments", charset: "utf8mb4", collation: "utf8mb4_bin", force: :cascade do |t| t.string "name", null: false t.string "record_type", null: false diff --git a/db/scorers/err@10.js b/db/scorers/err@10.js index 2bf2dd5d5..5d72a5cdc 100644 --- a/db/scorers/err@10.js +++ b/db/scorers/err@10.js @@ -12,15 +12,16 @@ for (i = 0; i < k; i++) { } } -function gain(grade, maxGrade=3.0) { +function gain(grade, maxGrade) { return (Math.pow(2,grade) - 1.0)/Math.pow(2,maxGrade) } -function err(lst, maxG=3.0) { +function err(lst) { let ERR = 0.0 let trust = 1.0 for (i = 0; i < lst.length; i++) { const rank = i + 1.0 - const pUseful = gain(lst[i], maxG) + // max is the maximum possible rating for the scorer. + const pUseful = gain(lst[i], max) const disc = pUseful/rank ERR = ERR + trust * disc trust = trust * (1.0 - pUseful) diff --git a/db/seeds.rb b/db/seeds.rb index 4010d1f3c..9e40ac120 100755 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -60,6 +60,15 @@ communal: true ) +Scorer.where(name: 'ERR@10').first_or_create( + scale: (0..3).to_a, + scale_with_labels: {"0":"Poor","1":"Fair","2":"Good","3":"Perfect"}, + show_scale_labels: true, + code: File.readlines('./db/scorers/err@10.js','\n').join('\n'), + name: 'ERR@10', + communal: true +) + SelectionStrategy.where(name: 'Single Rater').first_or_create( name: 'Single Rater', description: 'A single rating for each query/doc pair is all that is required. The fastest way to get a lot of ratings, with lower quality.'