(maint) integration: use puppetserver uberjar to install gems

Currently, puppetserver has transitioned to jetty10 and puppetdb is
still using jetty9 creating dependency conflicts when integration tests
are run. Once we transition to jetty10 as well, there will still be
dependency conflicts when we run integration tests against puppetserver
7.x, which is still on jetty9. Remove puppetserver from the project file
to avoid dependency conflicts.

d7e619d moved puppetserver to run in
its own JVM for integration tests, so it is no longer necessary to have
puppetserver in our project.clj file, but we still had it there to
facilitate installing the vendored gems that puppetserver needs. This
moves gem installation for puppetserver out of our project file and
clojure code, and instead uses the test setup scripts and the
puppetserver uberjar to create the vendored gem directory.

The jetty service name changed from jetty9-service to jetty10-service.
Adds an additional bootstrap file, used only for 7.x integration
testing, so each puppetserver version loads the correct tk services.

ext/bin/config-puppetserver-test-ref

@@ -78,6 +78,23 @@ fi
  dep_ver="$(lein-pprint :version)"
  echo "$dep_ver" > "$top/ext/test-conf/puppetserver-dep")
 
-# Install Puppet Server Ruby dependencies using JRuby
-# Procedure defined in src-gems/puppetlabs/puppetdb/integration/install_gems.clj
-lein install-gems
+gem_list=()
+while read LINE
+do
+  gem_name=$(echo $LINE |awk '{print $1}')
+  gem_version=$(echo $LINE |awk '{print $2}')
+  gem_list+=("$gem_name:$gem_version")
+done < ./puppetserver/resources/ext/build-scripts/jruby-gem-list.txt
+
+function gem-install() {
+java -cp puppetserver/target/puppet-server-release.jar clojure.main \
+  -m puppetlabs.puppetserver.cli.gem \
+  --config ./test-resources/puppetserver/puppetserver.conf -- install --no-document "$@"
+}
+
+echo "Installing gems"
+gem-install facter
+gem-install hiera
+gem-install puppet-pson
+gem-install "${gem_list[@]}"
+echo "Finished install gems"

project.clj

@@ -42,31 +42,6 @@
     [["releases" "https://artifactory.delivery.puppetlabs.net/artifactory/list/clojure-releases__local/"]
      ["snapshots" "https://artifactory.delivery.puppetlabs.net/artifactory/list/clojure-snapshots__local/"]]))
 
-;; See the integration tests section in documentation/CONTRIBUTING.md.
-(def puppetserver-test-dep-ver
-  (some-> (try
-            (slurp (str "ext/test-conf/puppetserver-dep"))
-            (catch java.io.FileNotFoundException ex
-              (binding [*out* *err*]
-                (println "puppetserver test dependency unconfigured (ignoring)"))
-              nil))
-          clojure.string/trim))
-
-(def puppetserver-test-dep-gem-list
-  (when puppetserver-test-dep-ver
-    (let [[major minor] (->> (re-matches #"^([0-9]+)\.([0-9]+)\..*" puppetserver-test-dep-ver)
-                             next
-                             (take 2)
-                             (map #(Integer/parseInt %)))]
-      (if (neg? (compare [major minor] [5 3]))
-        "gem-list.txt"
-        "jruby-gem-list.txt"))))
-
-(def puppetserver-test-deps
-  (when puppetserver-test-dep-ver
-    `[[puppetlabs/puppetserver ~puppetserver-test-dep-ver]
-      [puppetlabs/puppetserver ~puppetserver-test-dep-ver :classifier "test"]]))
-
 (def pdb-dev-deps
   (concat
    '[[ring/ring-mock]
@@ -82,8 +57,7 @@
      [org.yaml/snakeyaml]
 
      ;; Only needed for :integration tests
-     [puppetlabs/trapperkeeper-filesystem-watcher nil]]
-   puppetserver-test-deps))
+     [puppetlabs/trapperkeeper-filesystem-watcher nil]]))
 
 ;; Don't use lein :clean-targets so that we don't have to repeat
 ;; ourselves, given that we need to remove some protected files, and
@@ -319,9 +293,6 @@
                          ;; compile test files, and crashes because
                          ;; "src" namespaces aren't available.
                          :aot ^:replace []}
-             :install-gems {:source-paths ^:replace ["src-gems"]
-                            :target-path "target-gems"
-                            :dependencies ~puppetserver-test-deps}
              :ci {:plugins [[lein-pprint "1.1.1"]
                             [lein-exec "0.3.7"]]}
              ; We only want to include bouncycastle in the FOSS uberjar.
@@ -356,14 +327,7 @@
              ;; yet, disable it for now.
              :exclude-linters [:local-shadows-var]}
 
-  :aliases {"gem" ["with-profile" "install-gems,dev"
-                   "trampoline" "run" "-m" "puppetlabs.puppetserver.cli.gem"
-                   "--config" "./test-resources/puppetserver/puppetserver.conf"]
-            "install-gems" ["with-profile" "install-gems,dev"
-                            "trampoline" "run" "-m" "puppetlabs.puppetdb.integration.install-gems"
-                            ~puppetserver-test-dep-gem-list
-                            "--config" "./test-resources/puppetserver/puppetserver.conf"]
-            "kondo" ["with-profile" "+kondo" "run" "-m" "clj-kondo.main"]
+  :aliases {"kondo" ["with-profile" "+kondo" "run" "-m" "clj-kondo.main"]
             "clean" ~(pdb-run-clean pdb-clean-paths)
             "distclean" ~(pdb-run-clean pdb-distclean-paths)
             "time-shift-export" ^{:doc (clojure.string/join "" ["Shifts all timestamps from a PuppetDB archive with"

test-resources/puppetserver/bootstrap-7.x.cfg

@@ -0,0 +1,21 @@
+puppetlabs.services.request-handler.request-handler-service/request-handler-service
+puppetlabs.services.jruby.jruby-puppet-service/jruby-puppet-pooled-service
+puppetlabs.services.jruby-pool-manager.jruby-pool-manager-service/jruby-pool-manager-service
+puppetlabs.services.puppet-profiler.puppet-profiler-service/puppet-profiler-service
+puppetlabs.trapperkeeper.services.webserver.jetty9-service/jetty9-service
+puppetlabs.trapperkeeper.services.webrouting.webrouting-service/webrouting-service
+puppetlabs.services.config.puppet-server-config-service/puppet-server-config-service
+puppetlabs.services.master.master-service/master-service
+puppetlabs.services.legacy-routes.legacy-routes-service/legacy-routes-service
+puppetlabs.services.puppet-admin.puppet-admin-service/puppet-admin-service
+puppetlabs.trapperkeeper.services.authorization.authorization-service/authorization-service
+puppetlabs.services.versioned-code-service.versioned-code-service/versioned-code-service
+puppetlabs.trapperkeeper.services.scheduler.scheduler-service/scheduler-service
+puppetlabs.trapperkeeper.services.status.status-service/status-service
+puppetlabs.trapperkeeper.services.metrics.metrics-service/metrics-service
+puppetlabs.services.jruby.jruby-metrics-service/jruby-metrics-service
+puppetlabs.trapperkeeper.services.watcher.filesystem-watch-service/filesystem-watch-service
+# To enable the CA service, leave the following line uncommented
+puppetlabs.services.ca.certificate-authority-service/certificate-authority-service
+# To disable the CA service, comment out the above line and uncomment the line below
+#puppetlabs.services.ca.certificate-authority-disabled-service/certificate-authority-disabled-service

test-resources/puppetserver/bootstrap.cfg

@@ -2,7 +2,7 @@ puppetlabs.services.request-handler.request-handler-service/request-handler-serv
 puppetlabs.services.jruby.jruby-puppet-service/jruby-puppet-pooled-service
 puppetlabs.services.jruby-pool-manager.jruby-pool-manager-service/jruby-pool-manager-service
 puppetlabs.services.puppet-profiler.puppet-profiler-service/puppet-profiler-service
-puppetlabs.trapperkeeper.services.webserver.jetty9-service/jetty9-service
+puppetlabs.trapperkeeper.services.webserver.jetty10-service/jetty10-service
 puppetlabs.trapperkeeper.services.webrouting.webrouting-service/webrouting-service
 puppetlabs.services.config.puppet-server-config-service/puppet-server-config-service
 puppetlabs.services.master.master-service/master-service

test/puppetlabs/puppetdb/integration/fixtures.clj

@@ -164,8 +164,14 @@
     (doto process .destroy .waitFor) ;; Assumes destroy sends a SIGTERM
     (doseq [f files-to-cleanup] (fs/delete f))))
 
+(def pup-srv-version
+  (string/trim (slurp "./ext/test-conf/puppetserver-dep")))
+
 (def dev-config-file "./test-resources/puppetserver/puppetserver.conf")
-(def dev-bootstrap-file "./test-resources/puppetserver/bootstrap.cfg")
+(def dev-bootstrap-file 
+  (if (string/starts-with? pup-srv-version "7.")
+    "./test-resources/puppetserver/bootstrap-7.x.cfg"
+    "./test-resources/puppetserver/bootstrap.cfg"))
 
 (defn mri-agent-dir []
   (-> (sh "bundle" "show" "puppet") :out clojure.string/trim))