Finished with uri-list

README.md

@@ -21,23 +21,25 @@ A group of commands that will allow you to:
 
 ## Structures
 
-## URI List
-- Just a list of URIs, one per line.
+### URI List
+- One URI per line. Blank lines, or lines that begin with '#', are ignored.
 
 ### Session List
-- At its simplest form, a list of URLs which can be used for a snapshot.
-- More precisely, a list of requests, one per line: 
-in addition to the URL, each request may specify POST or GET, HTTP headers, and parameters. 
-- Each line in the file can actually constitute a session, with an optional login/logout, and multiple requests.
+- One request session per line. Blank lines, or lines that begin with '#', are ignored.
+- A request may be a single URL.
+- A request may also specify POST or GET, HTTP headers, and parameters. 
+- A session may contain multiple requests, with an optional login/logout specification.
 
 ## Commands
 
 ```
-vivosnap.rb prepare uri-list [classlist_file] [VIVO_homepage_URL] [uri_list_file]
+vivosnap.rb prepare uri-list [class_list_file] [VIVO_homepage_URL] {uri_list_file {REPLACE}}
 ```
 **NOT IMPLEMENTED**  
 Create a list of URIs. You provide a file with class URIs, and a URL for VIVO. 
-The tool will make requests of VIVOs ListRDF API, and write the results to the URI list file.
+The tool will make requests of VIVOs ListRDF API, and write the results. 
+If uri_list_file already exists, you must specify REPLACE. If uri_list_file is not provided,
+the URI list goes to stdout, along with the summary info. 
 
 ```
 vivosnap.rb prepare session-list [uri_list_file] [account_email] [account_password] [session_list_file]
@@ -98,6 +100,10 @@ does not allow for expected changes.
 
 ## Data structure details
 
+### Class list
+- One URI per line.
+- Blank lines and lines that begin with '#' will be treated as comments.
+
 ### Session list
 Here is a pseudo-syntax. URLs are all relative to the VIVO home page.
 ```

cmd_prepare_uri_list/cmd_prepare_uri_list.rb

@@ -1,9 +1,103 @@
+=begin
+--------------------------------------------------------------------------------
+
+execute LISTRDF requests, like this:
+http://localhost:8080/vivo/listrdf?vclass=http%3A%2F%2Fvivoweb.org%2Fontology%2Fcore%23FacultyMember
+
+write the results to the file, one URI per line.
+
+--------------------------------------------------------------------------------
+=end
+
 class CmdPrepareUriList
+  USAGE = 'prepare uri-list [class_list_file] [VIVO_homepage_URL] {uri_list_file {REPLACE}}'
   def initialize(args)
-    bogus "CmdPrepareUriList.initialize NOT IMPLEMENTED"
+    @args = args
+    complain("usage: #{USAGE}") unless (2..4).include? args.size
+
+    @class_list_file = args[0]
+    complain("'#{@class_list_file}' does not exist.") unless File.exist?(@class_list_file)
+
+    @vivo_home_url = args[1]
+    begin
+      HttpRequest.new(@vivo_home_url).exec do |response|
+        complain("Response from '#{@vivo_home_url}' does not look like VIVO's home page.") unless response.body =~ /body class="home"/
+      end
+    rescue
+      puts "#{$!}\n#{$!.backtrace.join("\n")}"
+      complain("Can't contact VIVO at '#{@vivo_home_url}': #{$!}")
+    end
+
+    @uri_list_file = args[2]
+
+    if @uri_list_file
+      complain("'#{@uri_list_file}' already exists. Specify REPLACE to replace it.") if File.exist?(@uri_list_file) unless 'REPLACE' == args[3]
+      @output = File.open(@uri_list_file, 'w')
+    else
+      @output = $stdout
+    end
   end
-  
+
   def run()
-    bogus "CmdPrepareUriList.run NOT IMPLEMENTED"
+    write_heading
+    @stats = []
+
+    File.open(@class_list_file) do |f|
+      f.each_line do |line|
+        line.strip!
+        perform_listrdf(line) unless line.empty? || line.start_with?('#')
+      end
+    end
+
+    write_report
+  end
+
+  def perform_listrdf(class_uri)
+    HttpRequest.new(listrdf_url).parameters('vclass' => class_uri).headers('Accept' => 'text/plain').exec do |response|
+      process_response(class_uri, response.body.lines)
+    end
+  end
+
+  def process_response(class_uri, lines)
+    @stats << [class_uri, lines.size]
+    @output.puts
+    @output.puts "# class = #{class_uri}"
+    @output.puts
+
+    lines.each do |line|
+      if line =~ /<([^>]+)>/
+        @output.puts $1
+      else
+        warning("Unexpected line in response '#{line}'")
+      end
+    end
+  end
+
+  def listrdf_url()
+    if @vivo_home_url.end_with?('/')
+      @vivo_home_url + 'listrdf'
+    else
+      @vivo_home_url + '/listrdf'
+    end
+  end
+
+  def write_heading()
+    @output.puts "#"
+    @output.puts "# prepare uri-list #{@args.join(' ')}"
+    @output.puts "# #{Time.now.strftime('%Y-%m-%d %H:%M:%S')}"
+    @output.puts "#"
+  end
+
+  def write_report()
+    width = @stats.map { |s| s[0].size }.max
+
+    report = "\n"
+    @stats.each do |s|
+      report += "#   %-#{width}s  %5d \n" % s
+    end
+    report += "#   %-#{width}s  %5d \n" % ['TOTAL', @stats.map{|s| s[1]}.inject(:+)]
+
+    @output.print(report)
+    print(report) if @output != $stdout
   end
 end

common.rb

@@ -11,10 +11,12 @@ def warning(message)
   puts("WARNING: #{message}")
 end
 
-module Kernel
-  def bogus(message)
-    puts(">>>>>>>>>>>>>BOGUS #{message}")
-  end
+def bogus(message)
+  puts(">>>>>>>>>>>>>BOGUS #{message}")
+end
+
+def complain(message)
+  raise UserInputError.new(message)
 end
 
 require_relative 'cmd_prepare_uri_list/cmd_prepare_uri_list'
@@ -25,3 +27,4 @@ def bogus(message)
 require_relative 'cmd_compare/cmd_compare'
 require_relative 'cmd_compare_again/cmd_compare_again'
 require_relative 'cmd_display/cmd_display'
+require_relative 'utils/http_request'

utils/http_request.rb

@@ -0,0 +1,69 @@
+require 'net/http'
+require 'uri'
+require 'cgi'
+
+class HttpRequest
+  def initialize(url, method = :GET)
+    @url = url
+    @method = method
+    @headers = {}
+    @parameters = {}
+  end
+
+  def headers(heads)
+    @headers = heads
+    self
+  end
+
+  def parameters(params)
+    @parameters = params
+    self
+  end
+
+  def exec()
+    uri = URI.parse(@url)
+
+    Net::HTTP.start(uri.host, uri.port) do |http|
+      if @method == :GET
+        full_url = add_parameters(uri.request_uri)
+        request = Net::HTTP::Get.new(full_url)
+      elsif @method == :POST
+        request = Net::HTTP::Post.new(uri.request_uri)
+        request.set_form_data(@parameters)
+      else
+        raise 'Method must be :GET or :POST'
+      end
+
+      @headers.each_pair { |k, v| request[k] = v }
+
+      begin
+        http.request(request) do |response|
+          response.value
+          yield response if block_given?
+        end
+      rescue Net::HTTPServerException => e
+        raise e.exception(e.message << "\nProblem request: \n#{inspect_request(request, @url)}")
+      rescue IOError => e
+        raise e.exception(e.message << "\nProblem request: \n#{inspect_request(request, @url)}")
+      end
+    end
+  end
+
+  def add_parameters(request_uri)
+    if @parameters.empty?
+      request_uri
+    elsif request_uri.include?("?")
+      request_uri + "&" + URI.encode_www_form(@parameters)
+    else
+      request_uri + "?" + URI.encode_www_form(@parameters)
+    end
+
+  end
+
+  def inspect_request(r, url)
+    headers = r.to_hash.to_a.map{|h| "   #{h[0]} ==> #{h[1]}"}.join("\n")
+    body = body ? CGI.unescape(r.body) : 'NO BODY'
+    "#{r.method} #{url}\n#{headers}\n#{body}"
+  end
+
+end

vivosnap.rb

@@ -24,7 +24,7 @@ def initialize(args)
         return
       end
     end
-    raise UserInputError.new("Arguments not valid: #{args.join(' ')}\nValid choices are #{format_cmds}")
+    complain("Arguments not valid: #{args.join(' ')}\nValid choices are #{format_cmds}")
   end
 
   def format_cmds()