OpenAPI query limits being hit (#1194)

* Migrate to using Faraday

* Properly store the book_id, and provide a link back.

* Embrace retry

Gemfile

@@ -21,8 +21,7 @@ gem 'd3-rails', '~> 3.5.5' # For cal heatmap
 gem 'devise', '>= 4.6.2'
 gem 'devise_invitable', '~> 2.0'
 
-# Using this as it wires in via Sprockets and I can't get npm version to work with the main app.
-# Had no luck with js/svg approach ;-(
+gem 'faraday-retry'
 gem 'foreman'
 gem 'importmap-rails', '~> 2.1'
 gem 'intercom-rails'

Gemfile.lock

@@ -195,6 +195,8 @@ GEM
       faraday (>= 1, < 3)
     faraday-net_http (3.4.0)
       net-http (>= 0.5.0)
+    faraday-retry (2.2.1)
+      faraday (~> 2.0)
     ffi (1.17.1-x86_64-darwin)
     ffi (1.17.1-x86_64-linux-gnu)
     foreman (0.88.1)
@@ -563,6 +565,7 @@ DEPENDENCIES
   derailed_benchmarks
   devise (>= 4.6.2)
   devise_invitable (~> 2.0)
+  faraday-retry
   foreman
   importmap-rails (~> 2.1)
   intercom-rails

app/controllers/ai_judges/prompts_controller.rb

@@ -2,20 +2,20 @@
 
 module AiJudges
   class PromptsController < ApplicationController
+    before_action :set_book
     def edit
       @ai_judge = User.find(params[:ai_judge_id])
 
-      if params[:book_id]
-        @book = Book.find(params[:book_id])
-        @query_doc_pair = @book.query_doc_pairs.sample
-      else
-        # grab any query_doc_pair that the judge has access to
-        @query_doc_pair = QueryDocPair
-          .joins(book: { teams: :members })
-          .where(teams: { teams_members: { member_id: @ai_judge.id } })
-          .order('RAND()')
-          .first
-      end
+      @query_doc_pair = if @book
+                          @book.query_doc_pairs.sample
+                        else
+                          # grab any query_doc_pair that the judge has access to
+                          QueryDocPair
+                            .joins(book: { teams: :members })
+                            .where(teams: { teams_members: { member_id: @ai_judge.id } })
+                            .order('RAND()')
+                            .first
+                        end
 
       @query_doc_pair = QueryDocPair.new if @query_doc_pair.nil?
     end
@@ -27,14 +27,18 @@ def update
       @query_doc_pair = QueryDocPair.new(query_doc_pair_params)
 
       llm_service = LlmService.new(@ai_judge.openai_key)
-      @judgement = llm_service.make_judgement @ai_judge, @query_doc_pair
-      pp @judgement
+      @judgement = Judgement.new(query_doc_pair: @query_doc_pair, user: @ai_judge)
+      llm_service.perform_safe_judgement @judgement
 
       render :edit
     end
 
     private
 
+    def set_book
+      @book = current_user.books_involved_with.where(id: params[:book_id]).first
+    end
+
     # Only allow a list of trusted parameters through.
     def ai_judge_params
       params.expect(user: [ :openai_key, :system_prompt ])

app/jobs/run_judge_judy_job.rb

@@ -25,17 +25,8 @@ def perform book, judge, number_of_pairs
       break if query_doc_pair.nil?
 
       judgement = Judgement.new(query_doc_pair: query_doc_pair, user: judge)
-      begin
-        judgement = llm_service.perform_judgement(judgement)
-      rescue RuntimeError => e
-        case e.message
-        when /401/
-          raise # we can't do anything about this, so pass it up
-        else
-          judgement.explanation = "BOOM: #{e}"
-          judgement.unrateable = true
-        end
-      end
+
+      llm_service.perform_safe_judgement(judgement)
 
       judgement.save!
       counter += 1

app/services/llm_service.rb

@@ -1,13 +1,26 @@
 # frozen_string_literal: true
 
-require 'net/http'
+require 'faraday'
+require 'faraday/retry'
 require 'json'
 
 class LlmService
   def initialize openai_key, _opts = {}
     @openai_key = openai_key
   end
 
+  def perform_safe_judgement judgement
+    perform_judgement(judgement)
+  rescue RuntimeError => e
+    case e.message
+    when /401/
+      raise # we can't do anything about this, so pass it up
+    else
+      judgement.explanation = "BOOM: #{e}"
+      judgement.unrateable = true
+    end
+  end
+
   def perform_judgement judgement
     user_prompt = make_user_prompt judgement.query_doc_pair
     results = get_llm_response user_prompt, judgement.user.system_prompt
@@ -19,7 +32,8 @@ def perform_judgement judgement
   end
 
   def make_user_prompt query_doc_pair
-    fields = JSON.parse(query_doc_pair.document_fields).to_yaml
+    document_fields = query_doc_pair.document_fields
+    fields = document_fields.blank? ? '' : JSON.parse(document_fields).to_yaml
 
     user_prompt = <<~TEXT
       Query: #{query_doc_pair.query_text}
@@ -32,44 +46,59 @@ def make_user_prompt query_doc_pair
   end
 
   # rubocop:disable Metrics/MethodLength
+  # rubocop:disable Metrics/AbcSize
   def get_llm_response user_prompt, system_prompt
-    uri = URI('https://api.openai.com/v1/chat/completions')
+    conn = Faraday.new(url: 'https://api.openai.com') do |f|
+      f.request :json
+      f.response :json
+      f.adapter Faraday.default_adapter
+      f.request :retry, {
+        max:                 3,
+        interval:            2,
+        interval_randomness: 0.5,
+        backoff_factor:      2,
+        # exceptions: [
+        #  Faraday::ConnectionFailed,
+        ##  Faraday::TimeoutError,
+        #  'Timeout::Error',
+        #  'Error::TooManyRequests'
+        # ],
+        retry_statuses:      [ 429 ],
+      }
+    end
 
-    headers = {
-      'Content-Type'  => 'application/json',
-      'Authorization' => "Bearer #{@openai_key}",
-    }
     body = {
       model:    'gpt-4',
       messages: [
         { role: 'system', content: system_prompt },
         { role: 'user', content: user_prompt }
       ],
     }
-    response = Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
-      request = Net::HTTP::Post.new(uri, headers)
-      request.body = body.to_json
-      http.request(request)
+
+    response = conn.post('/v1/chat/completions') do |req|
+      req.headers['Authorization'] = "Bearer #{@openai_key}"
+      req.headers['Content-Type'] = 'application/json'
+      req.body = body
     end
-    if response.is_a?(Net::HTTPSuccess)
-      json_response = JSON.parse(response.body)
-      content = json_response['choices']&.first&.dig('message', 'content')
-      parsed_content = begin
-        JSON.parse(content)
-      rescue StandardError
-        {}
-      end
 
-      parsed_content = parsed_content['response'] if parsed_content['response']
-      # puts "here is parsed"
-      # puts parsed_content
-      {
-        explanation: parsed_content['explanation'],
-        judgment:    parsed_content['judgment'],
-      }
+    if response.success?
+      begin
+        json_response = JSON.parse(response.env.response_body)
+        content = json_response['choices']&.first&.dig('message', 'content')
+
+        parsed_content = JSON.parse(content)
+        {
+          explanation: parsed_content['explanation'],
+          judgment:    parsed_content['judgment'],
+        }
+      rescue RuntimeError => e
+        puts e
+        raise "Error: Could not parse response from OpenAI: #{e} - #{response.env.response_body}"
+      end
     else
-      raise "Error: #{response.code} - #{response.message}"
+      raise "Error: #{response.status} - #{response.body}"
     end
   end
+  # rubocop:enable Metrics/AbcSize
   # rubocop:enable Metrics/MethodLength
 end

app/views/ai_judges/prompts/_form.html.erb

@@ -11,7 +11,7 @@
     </div>
   <% end %>
 
-  <%= form.hidden_field :book_id, value: @book.id if @book %>
+  <%= hidden_field_tag :book_id, @book.id if @book%>
 
   <div class="row gx-5">
     <div class="col-md-6">
@@ -28,7 +28,9 @@
         </div>
 
         <div>
-          <%= form.submit 'Refine Prompt' %>
+          <%= form.submit 'Refine Prompt', class: 'btn btn-default btn-primary' %>
+
+          <%= link_to 'Back to Book', book_path(@book), method: :get, class: 'btn btn-block btn-light' if @book %>
         </div>
       </div>
       <% if @judgement %>

test/services/llm_service_test.rb

@@ -43,7 +43,7 @@ class LlmServiceTest < ActiveSupport::TestCase
 
     test 'creating a judgement' do
       judgement = Judgement.new(query_doc_pair: query_doc_pair, user: judge)
-      judgement = service.perform_judgement judgement
+      service.perform_judgement judgement
 
       assert_instance_of Float, judgement.rating
       assert_not_nil judgement.explanation
@@ -52,20 +52,26 @@ class LlmServiceTest < ActiveSupport::TestCase
 
   describe 'error conditions' do
     test 'using a bad API key' do
-      # WebMock.disable!
       service = LlmService.new 'BAD_OPENAI_KEY'
       user_prompt = DEFAULT_USER_PROMPT
       system_prompt = AiJudgesController::DEFAULT_SYSTEM_PROMPT
 
-      assert_raises(RuntimeError, '401 - Unauthorized') do
+      error = assert_raises(RuntimeError) do
         service.get_llm_response(user_prompt, system_prompt)
       end
-
-      # WebMock.enable!
+      assert_equal 'Error: 401 - Unauthorized', error.message
     end
 
-    test 'it all blows up' do
-      assert true
+    test 'handle and back off a 429 error' do
+      # the Faraday Retry may mean we don't need this
+      service = LlmService.new 'OPENAI_429_ERROR'
+      user_prompt = DEFAULT_USER_PROMPT
+      system_prompt = AiJudgesController::DEFAULT_SYSTEM_PROMPT
+
+      error = assert_raises(RuntimeError) do
+        service.get_llm_response(user_prompt, system_prompt)
+      end
+      assert_equal 'Error: 429 - Too Many Requests', error.message
     end
   end
 end

test/support/webmock.rb

@@ -298,7 +298,10 @@ def setup
 
       stub_request(:post, 'https://api.openai.com/v1/chat/completions')
         .with(headers: { 'Authorization' => 'Bearer BAD_OPENAI_KEY' })
-        .to_return(status: 401)
+        .to_return(status: 401, body: 'Unauthorized')
+      stub_request(:post, 'https://api.openai.com/v1/chat/completions')
+        .with(headers: { 'Authorization' => 'Bearer OPENAI_429_ERROR' })
+        .to_return(status: 429, body: 'Too Many Requests')
     end
 
     # rubocop:enable Metrics/MethodLength