Supported new model lifecycle

legion-platform · May 13, 2019 · f24e3ae · f24e3ae
1 parent 36dde9f
commit f24e3ae
Show file tree

Hide file tree

Showing 31 changed files with 469 additions and 1,119 deletions.
diff --git a/.gitignore b/.gitignore
@@ -142,8 +142,8 @@ ENV/
 workspace/
 keys/
 
-tests/robot/*.html
-tests/robot/*.xml
+tests/*.html
+tests/*.xml
 tests/python/cover/*
 
 legion/tests/__init__\.py

diff --git a/Makefile b/Makefile
@@ -0,0 +1,67 @@
+SHELL := /bin/bash
+
+PROJECTNAME := $(shell basename "$(PWD)")
+CREDENTIAL_SECRETS=.secrets.yaml
+ROBOT_FILES=**/*.robot
+CLUSTER_NAME=
+PATH_TO_PROFILES_DIR=profiles
+E2E_PYTHON_TAGS=
+COMMIT_ID=
+TEMP_DIRECTORY=
+TAG=
+# Example of DOCKER_REGISTRY: nexus.domain.com:443/
+DOCKER_REGISTRY=
+HELM_ADDITIONAL_PARAMS=
+
+-include .env
+
+.EXPORT_ALL_VARIABLES:
+
+.PHONY: install-all install-cli install-services install-sdk
+
+check-tag:
+	@if [ "${TAG}" == "" ]; then \
+	    echo "TAG not defined, please define TAG variable" ; exit 1 ;\
+	fi
+	@if [ "${DOCKER_REGISTRY}" == "" ]; then \
+	    echo "DOCKER_REGISTRY not defined, please define DOCKER_REGISTRY variable" ; exit 1 ;\
+	fi
+
+## docker-build-jenkins: Build jenkins docker image
+docker-build-jenkins:
+	docker build -t legion/k8s-jenkins:latest -f containers/jenkins/Dockerfile .
+
+## docker-build-ansible: Build ansible docker image
+docker-build-ansible:
+	docker build -t legion/k8s-jenkins-ansible:latest -f containers/toolchains/python/Dockerfile .
+
+## docker-build-agent: Build agent docker image
+docker-build-agent:
+	docker build -t legion/jenkins-pipeline-agent:latest -f containers/agent/Dockerfile .
+
+## docker-push-jenkins: Push jenkins docker image
+docker-push-jenkins:
+	docker tag legion/k8s-jenkins:latest ${DOCKER_REGISTRY}/legion/k8s-jenkins:${TAG}
+	docker push ${DOCKER_REGISTRY}/legion/k8s-jenkins:${TAG}
+
+## docker-push-ansible: Push ansible docker image
+docker-push-ansible:  check-tag
+	docker tag legion/k8s-jenkins-ansible:latest ${DOCKER_REGISTRY}/legion/k8s-jenkins-ansible:${TAG}
+	docker push ${DOCKER_REGISTRY}/legion/k8s-jenkins-ansible:${TAG}
+
+## docker-push-agent: Push agent docker image
+docker-push-agent:  check-tag
+	docker tag legion/jenkins-pipeline-agent:latest ${DOCKER_REGISTRY}/legion/jenkins-pipeline-agent:${TAG}
+	docker push ${DOCKER_REGISTRY}/legion/jenkins-pipeline-agent:${TAG}
+
+## e2e-robot: Run e2e robot tests
+e2e-robot:
+	pabot --verbose --processes 6 \
+	      -v PATH_TO_PROFILES_DIR:profiles \
+	      --listener legion.robot.process_reporter \
+	      --outputdir target legion/tests/e2e/robot/tests/${ROBOT_FILES}
+
+help: Makefile
+	@echo "Choose a command run in "$(PROJECTNAME)":"
+	@sed -n 's/^##//p' $< | column -t -s ':' |  sed -e 's/^/ /'
+	@echo
diff --git a/containers/agent/Dockerfile b/containers/agent/Dockerfile
@@ -42,7 +42,7 @@ ENV LC_ALL="en_US.UTF-8" \
     LANGUAGE="en_US.UTF-8"
 
 # Install legion_jenkins_test package
-COPY tests/robot/libraries/legion_jenkins_test /opt
+COPY tests/libraries/legion_jenkins_test /opt
 RUN cd /opt/ && python3 setup.py install && rm -rf /opt/legion_jenkins_test
 
 WORKDIR /opt
diff --git a/containers/jenkins/Dockerfile b/containers/jenkins/Dockerfile
@@ -17,15 +17,18 @@
 # Build Legion Jenkins plugin
 FROM maven:3.5.3-jdk-8 as builder
 
-ENV JENKINS_PLUGIN_VERSION=$legion_plugin_version
-
-# ENV JAVA_HOME="$(readlink -f /usr/bin/java | sed 's:bin/java::')"
 ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/jre/"
+WORKDIR "/src/legion-jenkins-plugin"
 
-COPY legion-jenkins-plugin /src/legion-jenkins-plugin
+COPY legion-jenkins-plugin/pom.xml ./
+# Download dependencies
+RUN mvn verify --fail-never
 
-RUN mvn -f /src/legion-jenkins-plugin/pom.xml versions:set -DnewVersion=${JENKINS_PLUGIN_VERSION} -Dmaven.repo.local=///tmp/.m2/repository
-RUN mvn -f /src/legion-jenkins-plugin/pom.xml install -Dmaven.repo.local=/tmp/.m2/repository
+COPY legion-jenkins-plugin ./
+
+ENV JENKINS_PLUGIN_VERSION=$legion_plugin_version
+RUN mvn versions:set -DnewVersion=${JENKINS_PLUGIN_VERSION} -Dmaven.repo.local=///tmp/.m2/repository
+RUN mvn install -Dmaven.repo.local=/tmp/.m2/repository
 
 # Build Jenkins image
 FROM jenkins/jenkins:2.121.3

diff --git a/legion-jenkins-plugin/src/main/resources/legion.groovy b/legion-jenkins-plugin/src/main/resources/legion.groovy
@@ -76,8 +76,8 @@ def pod(Map podParams=null, Closure body) {
 
     model_image = podParams.get('image', getDefaultImageName())
     builder_image = "${System.getenv('LEGION_EDI_IMAGE')}"
-    cpu = podParams.get('cpu', '330m')
-    ram = podParams.get('ram', '4Gi')
+    cpu = podParams.get('cpu', '250m')
+    ram = podParams.get('ram', '256Mi')
     iamRole = podParams.get('iamRole', "${System.getenv('CLUSTER_NAME')}-${params.Enclave}-jslave-role")
     annotations << podAnnotation(key: 'iam.amazonaws.com/role', value: iamRole)
 
@@ -97,56 +97,21 @@ def pod(Map podParams=null, Closure body) {
 
     label = "jenkins-build-${UUID.randomUUID().toString()}"
 
-    def tolerations = """
-spec:
-  containers:
-  - name: builder
-    env:
-    - name: POD_NAME
-      valueFrom:
-        fieldRef:
-          fieldPath: metadata.name
-    volumeMounts:
-    - mountPath: "/var/run/docker.sock"
-      name: docker-socket
-  volumes:
-  - name: docker-socket
-    hostPath:
-      path: "/var/run/docker.sock"
-  tolerations:
-  - key: "dedicated"
-    operator: "Equal"
-    value: "jenkins-slave"
-    effect: "NoSchedule"
-"""
-
     podTemplate(
-            label: label, yaml: tolerations,
+            label: label,
             namespace: "${params.Enclave}",
             annotations: annotations,
             containers: [
                     containerTemplate(
-                            name: 'model',
+                            name: 'slave',
                             image: model_image,
                             resourceLimitMemory: ram,
                             resourceLimitCpu: cpu,
                             ttyEnabled: true,
                             command: 'cat',
                             envVars: envVars
-                    ),
-                    containerTemplate(
-                            name: 'builder',
-                            image: builder_image,
-                            resourceLimitMemory: '100Mi',
-                            resourceLimitCpu: '100m',
-                            ttyEnabled: true,
-                            command: '/usr/local/bin/uwsgi',
-                            serviceAccount: 'jenkins',
-                            args: '--strict --ini /etc/uwsgi/model_builder_uwsgi.ini',
-                            envVars: envVars
                     )
-            ],
-            serviceAccount: "model-builder"
+            ]
     ) {
         node(label){
             container('model', body)

diff --git a/pipelines/Pipeline.groovy b/pipelines/Pipeline.groovy
@@ -59,14 +59,14 @@ def runRobotTests() {
                         dir("${WORKSPACE}"){
                             def nose_report = 0
                             def robot_report = 0
-                            sh "./tests/robot/run_robot_tests.sh ${env.param_profile} ${env.param_legion_jenkins_version}"
+                            sh "./tests/run_robot_tests.sh ${env.param_profile} ${env.param_legion_jenkins_version}"
 
-                            robot_report = sh(script: 'find tests/robot/ -name "*.xml" | wc -l', returnStdout: true)
+                            robot_report = sh(script: 'find tests/ -name "*.xml" | wc -l', returnStdout: true)
 
                             if (robot_report.toInteger() > 0) {
                                 step([
                                     $class : 'RobotPublisher',
-                                    outputPath : 'tests/robot/',
+                                    outputPath : 'tests/',
                                     outputFileName : "*.xml",
                                     disableArchiveOutput : false,
                                     passThreshold : 100,

diff --git a/...libraries/legion_jenkins_test/MANIFEST.in → ...libraries/legion_jenkins_test/MANIFEST.in b/...libraries/legion_jenkins_test/MANIFEST.in → ...libraries/legion_jenkins_test/MANIFEST.in
diff --git a/tests/libraries/legion_jenkins_test/Pipfile b/tests/libraries/legion_jenkins_test/Pipfile
@@ -0,0 +1,20 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+six = "==1.12.0"
+urllib3 = "==1.24.2"
+requests = "==2.21.0"
+kubernetes = "==8.0.1"
+Jinja2 = "==2.10.1"
+boto3 = "==1.9.75"
+botocore = "==1.12.75"
+python-jenkins = "==0.4.15"
+robotframework = "==3.0.2"
+robotframework-pabot = "==0.44"
+PyYAML = "==5.1"
+
+[requires]
+python_version = "3.6"