From ffabbc1cfb6f4c9f5cf0069c868e690769fcbf0c Mon Sep 17 00:00:00 2001 From: Austin Blatt Date: Mon, 4 Dec 2023 10:50:55 -0800 Subject: [PATCH] (maint) integration: use puppetserver uberjar to install gems Currently, puppetserver has transitioned to jetty10 and puppetdb is still using jetty9 creating dependency conflicts when integration tests are run. Once we transition to jetty10 as well, there will still be dependency conflicts when we run integration tests against puppetserver 7.x, which is still on jetty9. Remove puppetserver from the project file to avoid dependency conflicts. d7e619df148f52f3d9661cfd8c2c2f7e796d6ca9 moved puppetserver to run in its own JVM for integration tests, so it is no longer necessary to have puppetserver in our project.clj file, but we still had it there to facilitate installing the vendored gems that puppetserver needs. This moves gem installation for puppetserver out of our project file and clojure code, and instead uses the test setup scripts and the puppetserver uberjar to create the vendored gem directory. The jetty service name changed from jetty9-service to jetty10-service. Adds an additional bootstrap file, used only for 7.x integration testing, so each puppetserver version loads the correct tk services. --- ext/bin/config-puppetserver-test-ref | 23 +++++++++-- project.clj | 40 +------------------ .../puppetdb/integration/install_gems.clj | 30 -------------- test-resources/puppetserver/bootstrap-7.x.cfg | 21 ++++++++++ test-resources/puppetserver/bootstrap.cfg | 2 +- .../puppetdb/integration/fixtures.clj | 8 +++- 6 files changed, 51 insertions(+), 73 deletions(-) delete mode 100644 src-gems/puppetlabs/puppetdb/integration/install_gems.clj create mode 100644 test-resources/puppetserver/bootstrap-7.x.cfg diff --git a/ext/bin/config-puppetserver-test-ref b/ext/bin/config-puppetserver-test-ref index dd41b491d5..2062104244 100755 --- a/ext/bin/config-puppetserver-test-ref +++ b/ext/bin/config-puppetserver-test-ref @@ -78,6 +78,23 @@ fi dep_ver="$(lein-pprint :version)" echo "$dep_ver" > "$top/ext/test-conf/puppetserver-dep") -# Install Puppet Server Ruby dependencies using JRuby -# Procedure defined in src-gems/puppetlabs/puppetdb/integration/install_gems.clj -lein install-gems +gem_list=() +while read LINE +do + gem_name=$(echo $LINE |awk '{print $1}') + gem_version=$(echo $LINE |awk '{print $2}') + gem_list+=("$gem_name:$gem_version") +done < ./puppetserver/resources/ext/build-scripts/jruby-gem-list.txt + +function gem-install() { +java -cp puppetserver/target/puppet-server-release.jar clojure.main \ + -m puppetlabs.puppetserver.cli.gem \ + --config ./test-resources/puppetserver/puppetserver.conf -- install --no-document "$@" +} + +echo "Installing gems" +gem-install facter +gem-install hiera +gem-install puppet-pson +gem-install "${gem_list[@]}" +echo "Finished install gems" diff --git a/project.clj b/project.clj index 2a774eec01..b11a8ec84c 100644 --- a/project.clj +++ b/project.clj @@ -42,31 +42,6 @@ [["releases" "https://artifactory.delivery.puppetlabs.net/artifactory/list/clojure-releases__local/"] ["snapshots" "https://artifactory.delivery.puppetlabs.net/artifactory/list/clojure-snapshots__local/"]])) -;; See the integration tests section in documentation/CONTRIBUTING.md. -(def puppetserver-test-dep-ver - (some-> (try - (slurp (str "ext/test-conf/puppetserver-dep")) - (catch java.io.FileNotFoundException ex - (binding [*out* *err*] - (println "puppetserver test dependency unconfigured (ignoring)")) - nil)) - clojure.string/trim)) - -(def puppetserver-test-dep-gem-list - (when puppetserver-test-dep-ver - (let [[major minor] (->> (re-matches #"^([0-9]+)\.([0-9]+)\..*" puppetserver-test-dep-ver) - next - (take 2) - (map #(Integer/parseInt %)))] - (if (neg? (compare [major minor] [5 3])) - "gem-list.txt" - "jruby-gem-list.txt")))) - -(def puppetserver-test-deps - (when puppetserver-test-dep-ver - `[[puppetlabs/puppetserver ~puppetserver-test-dep-ver] - [puppetlabs/puppetserver ~puppetserver-test-dep-ver :classifier "test"]])) - (def pdb-dev-deps (concat '[[ring/ring-mock] @@ -82,8 +57,7 @@ [org.yaml/snakeyaml] ;; Only needed for :integration tests - [puppetlabs/trapperkeeper-filesystem-watcher nil]] - puppetserver-test-deps)) + [puppetlabs/trapperkeeper-filesystem-watcher nil]])) ;; Don't use lein :clean-targets so that we don't have to repeat ;; ourselves, given that we need to remove some protected files, and @@ -319,9 +293,6 @@ ;; compile test files, and crashes because ;; "src" namespaces aren't available. :aot ^:replace []} - :install-gems {:source-paths ^:replace ["src-gems"] - :target-path "target-gems" - :dependencies ~puppetserver-test-deps} :ci {:plugins [[lein-pprint "1.1.1"] [lein-exec "0.3.7"]]} ; We only want to include bouncycastle in the FOSS uberjar. @@ -356,14 +327,7 @@ ;; yet, disable it for now. :exclude-linters [:local-shadows-var]} - :aliases {"gem" ["with-profile" "install-gems,dev" - "trampoline" "run" "-m" "puppetlabs.puppetserver.cli.gem" - "--config" "./test-resources/puppetserver/puppetserver.conf"] - "install-gems" ["with-profile" "install-gems,dev" - "trampoline" "run" "-m" "puppetlabs.puppetdb.integration.install-gems" - ~puppetserver-test-dep-gem-list - "--config" "./test-resources/puppetserver/puppetserver.conf"] - "kondo" ["with-profile" "+kondo" "run" "-m" "clj-kondo.main"] + :aliases {"kondo" ["with-profile" "+kondo" "run" "-m" "clj-kondo.main"] "clean" ~(pdb-run-clean pdb-clean-paths) "distclean" ~(pdb-run-clean pdb-distclean-paths) "time-shift-export" ^{:doc (clojure.string/join "" ["Shifts all timestamps from a PuppetDB archive with" diff --git a/src-gems/puppetlabs/puppetdb/integration/install_gems.clj b/src-gems/puppetlabs/puppetdb/integration/install_gems.clj deleted file mode 100644 index 8a3c2b24f7..0000000000 --- a/src-gems/puppetlabs/puppetdb/integration/install_gems.clj +++ /dev/null @@ -1,30 +0,0 @@ -(ns puppetlabs.puppetdb.integration.install-gems - (:require [clojure.java.io :as io] - [puppetlabs.puppetserver.cli.subcommand :as cli] - [puppetlabs.services.jruby.jruby-puppet-core :as jruby-puppet-core] - [puppetlabs.services.jruby-pool-manager.jruby-core :as jruby-core] - [clojure.string :as string])) - -(defn gem-run! - [config & args] - (let [jruby-config (jruby-puppet-core/initialize-and-create-jruby-config config)] - (jruby-core/cli-run! jruby-config "gem" args))) - -(defn install-gems [gem-list-name config _] - (gem-run! config "install" "facter" "--no-document") - (gem-run! config "install" "hiera" "--no-document") - - ;; Used by the test file-with-binary-template to ensure we handle PSON - ;; serialized catalogs properly. - (gem-run! config "install" "puppet-pson" "--no-document") - - ;; Install the puppetserver vendored gems listed inside its jar; this is where - ;; ezbake gets them - (let [gem-list (string/split (slurp (io/resource (str "ext/build-scripts/" gem-list-name))) #"\n")] - (doseq [[gem version] (map #(string/split % #"\s") gem-list)] - (gem-run! config "install" gem "--no-document" "--version" version)))) - -(defn -main - [& args] - (let [[gem-list-name & others] args] - (cli/run (partial install-gems gem-list-name) others))) diff --git a/test-resources/puppetserver/bootstrap-7.x.cfg b/test-resources/puppetserver/bootstrap-7.x.cfg new file mode 100644 index 0000000000..62d970f887 --- /dev/null +++ b/test-resources/puppetserver/bootstrap-7.x.cfg @@ -0,0 +1,21 @@ +puppetlabs.services.request-handler.request-handler-service/request-handler-service +puppetlabs.services.jruby.jruby-puppet-service/jruby-puppet-pooled-service +puppetlabs.services.jruby-pool-manager.jruby-pool-manager-service/jruby-pool-manager-service +puppetlabs.services.puppet-profiler.puppet-profiler-service/puppet-profiler-service +puppetlabs.trapperkeeper.services.webserver.jetty9-service/jetty9-service +puppetlabs.trapperkeeper.services.webrouting.webrouting-service/webrouting-service +puppetlabs.services.config.puppet-server-config-service/puppet-server-config-service +puppetlabs.services.master.master-service/master-service +puppetlabs.services.legacy-routes.legacy-routes-service/legacy-routes-service +puppetlabs.services.puppet-admin.puppet-admin-service/puppet-admin-service +puppetlabs.trapperkeeper.services.authorization.authorization-service/authorization-service +puppetlabs.services.versioned-code-service.versioned-code-service/versioned-code-service +puppetlabs.trapperkeeper.services.scheduler.scheduler-service/scheduler-service +puppetlabs.trapperkeeper.services.status.status-service/status-service +puppetlabs.trapperkeeper.services.metrics.metrics-service/metrics-service +puppetlabs.services.jruby.jruby-metrics-service/jruby-metrics-service +puppetlabs.trapperkeeper.services.watcher.filesystem-watch-service/filesystem-watch-service +# To enable the CA service, leave the following line uncommented +puppetlabs.services.ca.certificate-authority-service/certificate-authority-service +# To disable the CA service, comment out the above line and uncomment the line below +#puppetlabs.services.ca.certificate-authority-disabled-service/certificate-authority-disabled-service diff --git a/test-resources/puppetserver/bootstrap.cfg b/test-resources/puppetserver/bootstrap.cfg index 62d970f887..1a4881053a 100644 --- a/test-resources/puppetserver/bootstrap.cfg +++ b/test-resources/puppetserver/bootstrap.cfg @@ -2,7 +2,7 @@ puppetlabs.services.request-handler.request-handler-service/request-handler-serv puppetlabs.services.jruby.jruby-puppet-service/jruby-puppet-pooled-service puppetlabs.services.jruby-pool-manager.jruby-pool-manager-service/jruby-pool-manager-service puppetlabs.services.puppet-profiler.puppet-profiler-service/puppet-profiler-service -puppetlabs.trapperkeeper.services.webserver.jetty9-service/jetty9-service +puppetlabs.trapperkeeper.services.webserver.jetty10-service/jetty10-service puppetlabs.trapperkeeper.services.webrouting.webrouting-service/webrouting-service puppetlabs.services.config.puppet-server-config-service/puppet-server-config-service puppetlabs.services.master.master-service/master-service diff --git a/test/puppetlabs/puppetdb/integration/fixtures.clj b/test/puppetlabs/puppetdb/integration/fixtures.clj index c4affe51b4..01f13ab29c 100644 --- a/test/puppetlabs/puppetdb/integration/fixtures.clj +++ b/test/puppetlabs/puppetdb/integration/fixtures.clj @@ -164,8 +164,14 @@ (doto process .destroy .waitFor) ;; Assumes destroy sends a SIGTERM (doseq [f files-to-cleanup] (fs/delete f)))) +(def pup-srv-version + (string/trim (slurp "./ext/test-conf/puppetserver-dep"))) + (def dev-config-file "./test-resources/puppetserver/puppetserver.conf") -(def dev-bootstrap-file "./test-resources/puppetserver/bootstrap.cfg") +(def dev-bootstrap-file + (if (string/starts-with? pup-srv-version "7.") + "./test-resources/puppetserver/bootstrap-7.x.cfg" + "./test-resources/puppetserver/bootstrap.cfg")) (defn mri-agent-dir [] (-> (sh "bundle" "show" "puppet") :out clojure.string/trim))