diff --git a/scripts/setup.sh b/scripts/setup.sh index 5fde2ed..f5d0175 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -322,11 +322,25 @@ init_tdp_ui() { } download_tdp_binaries() { - while IFS=";" read -r uri_file_name file_name - do - local target_file=${file_name:-$(basename ${uri_file_name})} - wget --continue --output-document="files/${target_file}" ${uri_file_name} - done < scripts/tdp-release-uris.txt + + local collection=$1 + + downloads_file="scripts/tdp-release-uris.json" + + collection_query=".\"$collection\" | keys[]" + archives=$(jq -r "$collection_query" "$downloads_file") + + for archive in $archives; do + + download_query=".\"$collection\".\"$archive\".link" + download_url=$(jq -r "$download_query" "$downloads_file") + + file_query=".\"$collection\".\"$archive\".name" + dest_file=$(jq -r "$file_query" "$downloads_file") + + echo "downloading : $dest_file" + wget --continue --output-document="files/${dest_file}" "$download_url" + done } main() { @@ -360,7 +374,15 @@ main() { esac done - download_tdp_binaries + download_tdp_binaries tdp-collection + + for feature in "${FEATURES[@]}"; do + case "$feature" in + extras) download_tdp_binaries tdp-extras;; + observability) download_tdp_binaries tdp-observability;; + esac + done + } main "$@" diff --git a/scripts/tdp-release-uris.json b/scripts/tdp-release-uris.json new file mode 100644 index 0000000..c9e5861 --- /dev/null +++ b/scripts/tdp-release-uris.json @@ -0,0 +1,152 @@ +{ + "tdp-collection": { + "hbase": { + "name": "hbase-2.1.10-1.0-bin.tar.gz", + "link": "https://github.com/TOSIT-IO/hbase/releases/download/2.1.10-1.0/hbase-2.1.10-1.0-bin.tar.gz" + }, + "phoenix_queryserver_daemon": { + "name": "phoenix-queryserver-6.0.0-0.0-bin.tar.gz", + "link": "https://github.com/TOSIT-IO/phoenix-queryserver/releases/download/6.0.0-0.0/phoenix-queryserver-6.0.0-0.0-bin.tar.gz" + }, + "phoenix": { + "name": "phoenix-hbase-2.1-5.1.3-1.0-bin.tar.gz", + "link": "https://github.com/TOSIT-IO/phoenix/releases/download/5.1.3-1.0/phoenix-hbase-2.1-5.1.3-1.0-bin.tar.gz" + }, + "hbase-spark": { + "name": "hbase-spark-2.3.4-1.0.0-0.0.jar", + "link": "https://github.com/TOSIT-IO/hbase-connectors/releases/download/2.3.4-1.0.0-0.0/hbase-spark-2.3.4-1.0.0-0.0.jar" + }, + "hbck2": { + "name": "hbase-hbck2-1.1.0-0.0.jar", + "link": "https://github.com/TOSIT-IO/hbase-operator-tools/releases/download/1.1.0-0.0/hbase-hbck2-1.1.0-0.0.jar" + }, + "hadoop": { + "name": "hadoop-3.1.1-0.0.tar.gz.1", + "link": "https://github.com/TOSIT-IO/hadoop/releases/download/3.1.1-0.0/hadoop-3.1.1-0.0.tar.gz" + }, + "hive": { + "name": "apache-hive-3.1.3-1.0-bin.tar.gz", + "link": "https://github.com/TOSIT-IO/hive/releases/download/3.1.3-1.0/apache-hive-3.1.3-1.0-bin.tar.gz" + }, + "tez": { + "name": "tez-0.9.1-1.0.tar.gz", + "link": "https://github.com/TOSIT-IO/tez/releases/download/0.9.1-1.0/tez-0.9.1-1.0.tar.gz" + }, + "knox": { + "name": "knox-1.6.1-0.0.tar.gz", + "link": "https://github.com/TOSIT-IO/knox/releases/download/1.6.1-0.0/knox-1.6.1-0.0.tar.gz" + }, + "knox-shell": { + "name": "knoxshell-1.6.1-0.0.tar.gz", + "link": "https://github.com/TOSIT-IO/knox/releases/download/1.6.1-0.0/knoxshell-1.6.1-0.0.tar.gz" + }, + "ranger-admin": { + "name": "ranger-2.0.0-1.0-admin.tar.gz", + "link": "https://github.com/TOSIT-IO/ranger/releases/download/2.0.0-1.0/ranger-2.0.0-1.0-admin.tar.gz" + }, + "ranger-hdfs": { + "name": "ranger-2.0.0-1.0-hdfs-plugin.tar.gz", + "link": "https://github.com/TOSIT-IO/ranger/releases/download/2.0.0-1.0/ranger-2.0.0-1.0-hdfs-plugin.tar.gz" + }, + "ranger-hive": { + "name": "ranger-2.0.0-1.0-hive-plugin.tar.gz", + "link": "https://github.com/TOSIT-IO/ranger/releases/download/2.0.0-1.0/ranger-2.0.0-1.0-hive-plugin.tar.gz" + }, + "ranger-knox": { + "name": "ranger-2.0.0-1.0-knox-plugin.tar.gz", + "link": "https://github.com/TOSIT-IO/ranger/releases/download/2.0.0-1.0/ranger-2.0.0-1.0-knox-plugin.tar.gz" + }, + "ranger-yarn": { + "name": "ranger-2.0.0-1.0-yarn-plugin.tar.gz", + "link": "https://github.com/TOSIT-IO/ranger/releases/download/2.0.0-1.0/ranger-2.0.0-1.0-yarn-plugin.tar.gz" + }, + "ranger-hbase": { + "name": "ranger-2.0.0-1.0-hbase-plugin.tar.gz", + "link": "https://github.com/TOSIT-IO/ranger/releases/download/2.0.0-1.0/ranger-2.0.0-1.0-hbase-plugin.tar.gz" + }, + "ranger-ugsync": { + "name": "ranger-2.0.0-1.0-usersync.tar.gz", + "link": "https://github.com/TOSIT-IO/ranger/releases/download/2.0.0-1.0/ranger-2.0.0-1.0-usersync.tar.gz" + }, + "ranger-kms": { + "name": "ranger-2.0.0-1.0-kms.tar.gz", + "link": "https://github.com/TOSIT-IO/ranger/releases/download/2.0.0-1.0/ranger-2.0.0-1.0-kms.tar.gz" + }, + "solr": { + "name": "solr-7.7.3.tgz", + "link": "https://archive.apache.org/dist/lucene/solr/7.7.3/solr-7.7.3.tgz" + }, + "spark2": { + "name": "spark-2.3.4-1.0-bin-tdp.tgz", + "link": "https://github.com/TOSIT-IO/spark/releases/download/2.3.4-1.0/spark-2.3.4-1.0-bin-tdp.tgz" + }, + "spark3": { + "name": "spark-3.2.2-0.0-bin-tdp.tgz", + "link": "https://github.com/TOSIT-IO/spark/releases/download/3.2.2-0.0/spark-3.2.2-0.0-bin-tdp.tgz" + }, + "yarn": { + "name": "spark-2.3.4-1.0-bin-tdp.tgz", + "link": "https://github.com/TOSIT-IO/spark/releases/download/2.3.4-1.0/spark-2.3.4-1.0-bin-tdp.tgz" + }, + "zookeeper-3.4": { + "name": "zookeeper-3.4.6.tar.gz", + "link": "https://archive.apache.org/dist/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz" + }, + "zookeeper-3.5": { + "name": "apache-zookeeper-3.5.9-bin.tar.gz", + "link": "https://archive.apache.org/dist/zookeeper/zookeeper-3.5.9/apache-zookeeper-3.5.9-bin.tar.gz" + } + }, + "tdp-observability": { + "node-exporter": { + "name": "node_exporter-1.3.1.linux-amd64.tar.gz", + "link": "https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz" + }, + "postgres-exporter": { + "name": "postgres_exporter-0.12.0.linux-amd64.tar.gz", + "link": "https://github.com/prometheus-community/postgres_exporter/releases/download/v0.12.0/postgres_exporter-0.12.0.linux-amd64.tar.gz" + }, + "jmx": { + "name": "jmx_prometheus_javaagent-0.16.1.jar", + "link": "https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar" + }, + "prometheus": { + "name": "prometheus-2.37.6.linux-amd64.tar.gz", + "link": "https://github.com/prometheus/prometheus/releases/download/v2.37.6/prometheus-2.37.6.linux-amd64.tar.gz" + }, + "grafana": { + "name": "grafana-9.2.0.linux-amd64.tar.gz", + "link": "https://dl.grafana.com/oss/release/grafana-9.2.0.linux-amd64.tar.gz" + }, + "loki": { + "name": "loki-linux-amd64.zip;loki-2.7.2-linux-amd64.zip", + "link": "https://github.com/grafana/loki/releases/download/v2.7.2/loki-linux-amd64.zip;loki-2.7.2-linux-amd64.zip" + }, + "promtail": { + "name": "promtail-linux-amd64.zip;promtail-2.7.2-linux-amd64.zip", + "link": "https://github.com/grafana/loki/releases/download/v2.7.2/promtail-linux-amd64.zip;promtail-2.7.2-linux-amd64.zip" + } + }, + "tdp-extras": { + "kafka": { + "name": "kafka_2.13-2.8.2-TDP-0.1.0-SNAPSHOT.tgz", + "link": "https://github.com/TOSIT-IO/kafka/releases/download/2.8.2-TDP-0.1.0/kafka_2.13-2.8.2-TDP-0.1.0-SNAPSHOT.tgz" + }, + "ranger-kafka": { + "name": "ranger-2.0.0-1.0-kafka-plugin.tar.gz", + "link": "https://github.com/TOSIT-IO/ranger/releases/download/2.0.0-1.0/ranger-2.0.0-1.0-kafka-plugin.tar.gz" + }, + "jupyter-hub": { + "name": "jupyterhub-2.3.1-TDP-0.1.0.tar.gz", + "link": "https://github.com/TOSIT-IO/jupyterhub-pkg/releases/download/v2.3.1-TDP-0.1.0/jupyterhub-2.3.1-TDP-0.1.0.tar.gz" + }, + "livy": { + "name": "apache-livy-0.8.0-incubating-TDP-0.1.0-SNAPSHOT-bin.zip", + "link": "https://github.com/TOSIT-IO/incubator-livy/releases/download/v0.8.0-incubating-TDP-0.1.0/apache-livy-0.8.0-incubating-TDP-0.1.0-SNAPSHOT-bin.zip" + }, + "hue": { + "name": "hue-release-4.11.0-TDP-0.1.0-python38.tar.gz", + "link": "https://github.com/TOSIT-IO/hue/releases/download/hue-release-4.11.0-TDP-0.1.0-python38/hue-release-4.11.0-TDP-0.1.0-python38.tar.gz" + } + } +}