Render auth and env json to single yml file (#4906)

* Render lms and studio config as yml
openedx-unsupported · Jan 3, 2019 · ed8f34e · ed8f34e
1 parent 073244b
commit ed8f34e
Show file tree

Hide file tree

Showing 27 changed files with 133 additions and 145 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,6 +1,7 @@
 *
 !playbooks/
 !docker/build/*
+!docker/devstack_common_ansible_overrides.yml
 !docker/build/*/*.yml
 docker/build/*/Dockerfile
 !docker/plays/

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,6 @@
 - Role: edxapp
   - Renamed proctoring backend setting to work with edx-proctoring 1.5.0
+  - Render auth and env config to a single yml file
 
 - Role: edxapp
   - Remove low priority queue, use default instead.

diff --git a/docker/build/credentials/Dockerfile b/docker/build/credentials/Dockerfile
@@ -17,7 +17,7 @@ ADD . /edx/app/edx_ansible/edx_ansible
 WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
 
 COPY docker/build/credentials/ansible_overrides.yml /
-COPY docker/build/devstack/ansible_overrides.yml /devstack/ansible_overrides.yml
+COPY docker/devstack_common_ansible_overrides.yml /devstack/ansible_overrides.yml
 
 ARG OPENEDX_RELEASE=master
 ENV OPENEDX_RELEASE=${OPENEDX_RELEASE}

diff --git a/docker/build/devpi/Dockerfile b/docker/build/devpi/Dockerfile
@@ -21,7 +21,7 @@ WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
 
 RUN apt-get update
 
-COPY docker/build/devstack/ansible_overrides.yml /devstack/ansible_overrides.yml
+COPY docker/devstack_common_ansible_overrides.yml /devstack/ansible_overrides.yml
 
 RUN sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook devpi.yml \
     -c local -i '127.0.0.1,' \

diff --git a/docker/build/discovery/Dockerfile b/docker/build/discovery/Dockerfile
@@ -17,7 +17,7 @@ ADD . /edx/app/edx_ansible/edx_ansible
 WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
 
 COPY docker/build/discovery/ansible_overrides.yml /
-COPY docker/build/devstack/ansible_overrides.yml /devstack/ansible_overrides.yml
+COPY docker/devstack_common_ansible_overrides.yml /devstack/ansible_overrides.yml
 
 ARG OPENEDX_RELEASE=master
 ENV OPENEDX_RELEASE=${OPENEDX_RELEASE}

diff --git a/docker/build/ecommerce/Dockerfile b/docker/build/ecommerce/Dockerfile
@@ -17,7 +17,7 @@ ADD . /edx/app/edx_ansible/edx_ansible
 WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
 
 COPY docker/build/ecommerce/ansible_overrides.yml /
-COPY docker/build/devstack/ansible_overrides.yml /devstack/ansible_overrides.yml
+COPY docker/devstack_common_ansible_overrides.yml /devstack/ansible_overrides.yml
 
 ARG OPENEDX_RELEASE=master
 ENV OPENEDX_RELEASE=${OPENEDX_RELEASE}

diff --git a/docker/build/edxapp/Dockerfile b/docker/build/edxapp/Dockerfile
@@ -18,7 +18,7 @@ WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
 
 COPY docker/build/edxapp/ansible_overrides.yml /
 COPY docker/build/edxapp/devstack.yml /
-COPY docker/build/devstack/ansible_overrides.yml /devstack/ansible_overrides.yml
+COPY docker/devstack_common_ansible_overrides.yml /devstack/ansible_overrides.yml
 
 ARG OPENEDX_RELEASE=master
 ENV OPENEDX_RELEASE=${OPENEDX_RELEASE}

diff --git a/docker/build/edxapp/ansible_overrides.yml b/docker/build/edxapp/ansible_overrides.yml
@@ -14,7 +14,6 @@ EDXAPP_LMS_GUNICORN_EXTRA_CONF: 'reload = True'
 
 EDXAPP_NO_PREREQ_INSTALL: 0
 COMMON_SSH_PASSWORD_AUTH: "yes"
-EDXAPP_LMS_BASE: "edx.devstack.lms:18000"
 EDXAPP_CMS_BASE: "edx.devstack.studio:18010"
 EDXAPP_OAUTH_ENFORCE_SECURE: false
 EDXAPP_LMS_BASE_SCHEME: http

diff --git a/docker/build/jenkins_worker/Dockerfile b/docker/build/jenkins_worker/Dockerfile
@@ -17,7 +17,7 @@ ADD . /edx/app/edx_ansible/edx_ansible
 WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
 
 COPY docker/build/jenkins_worker/ansible_overrides.yml /jenkins_worker/ansible_overrides.yml
-COPY docker/build/devstack/ansible_overrides.yml /devstack/ansible_overrides.yml
+COPY docker/devstack_common_ansible_overrides.yml /devstack/ansible_overrides.yml
 
 ARG OPENEDX_RELEASE=master
 ENV OPENEDX_RELEASE=${OPENEDX_RELEASE}

diff --git a/docker/build/devstack/ansible_overrides.yml → docker/devstack_common_ansible_overrides.yml b/docker/build/devstack/ansible_overrides.yml → docker/devstack_common_ansible_overrides.yml
@@ -1,3 +1,6 @@
+
+# These variables are loaded into most devstack images via their Dockerfile
+
 EDXAPP_LMS_BASE: 'edx.devstack.lms:18000'
 EDXAPP_LMS_ROOT_URL: 'http://{{ EDXAPP_LMS_BASE }}'
 EDXAPP_LMS_PUBLIC_ROOT_URL: 'http://localhost:18000'

diff --git a/playbooks/roles/common/tasks/main.yml b/playbooks/roles/common/tasks/main.yml
@@ -118,6 +118,10 @@
     group: "{{ item.group | default('root') }}"
     mode: "0755"
   with_items: "{{ common_directories }}"
+  tags:
+    - install
+    - install:configuration
+    - common_directories
 
 - name: upload sudo config for key forwarding as root
   copy:

diff --git a/playbooks/roles/container/defaults/main.yml b/playbooks/roles/container/defaults/main.yml
diff --git a/playbooks/roles/container/meta/main.yml b/playbooks/roles/container/meta/main.yml
diff --git a/playbooks/roles/container/tasks/main.yml b/playbooks/roles/container/tasks/main.yml
diff --git a/playbooks/roles/docker-tools/meta/main.yml b/playbooks/roles/docker-tools/meta/main.yml
@@ -0,0 +1,4 @@
+---
+
+dependencies:
+  - common
diff --git a/playbooks/roles/docker-tools/tasks/main.yml b/playbooks/roles/docker-tools/tasks/main.yml
@@ -50,11 +50,20 @@
     - install
     - install:system-requirements
 
+- name: Are we in a Docker container
+  shell: echo $(egrep -q 'docker' /proc/self/cgroup && echo 'yes' || echo 'no')
+  ignore_errors: yes
+  register: docker_container
+  tags:
+    - install
+    - install:base
+
 - name: start docker service
   service:
     name: docker
     enabled: yes
     state: started
+  when: docker_container.stdout != 'yes'
   tags:
     - install
     - install:configuration

diff --git a/playbooks/roles/docker/defaults/main.yml b/playbooks/roles/docker/defaults/main.yml
diff --git a/playbooks/roles/docker/meta/main.yml b/playbooks/roles/docker/meta/main.yml
diff --git a/playbooks/roles/docker/tasks/main.yml b/playbooks/roles/docker/tasks/main.yml
diff --git a/playbooks/roles/edxapp/tasks/service_variant_config.yml b/playbooks/roles/edxapp/tasks/service_variant_config.yml
@@ -16,6 +16,58 @@
     - install:app-configuration
     - edxapp_cfg # Old deprecated tag, will remove when possible
 
+- name: combined lms auth env for yml
+  set_fact:
+    lms_combined_config: '{{lms_env_config|combine(lms_auth_config)}}'
+  when: '"lms" in service_variants_enabled'
+  tags:
+    - install
+    - install:configuration
+    - install:app-configuration
+    - edxapp_cfg # Old deprecated tag, will remove when possible
+
+- name: render lms yml config # meant to replace existing json config eventually
+  template:
+    src: "lms.yml.j2"
+    dest: "{{ COMMON_CFG_DIR }}/lms.yml"
+    owner: "{{ edxapp_user }}"
+    group: "{{ common_web_group }}"
+    mode: 0640
+  become: true
+  with_items: "{{ service_variants_enabled }}"
+  when: '"lms" in service_variants_enabled'
+  tags:
+    - install
+    - install:configuration
+    - install:app-configuration
+    - edxapp_cfg # Old deprecated tag, will remove when possible
+
+- name: combined cms auth env for yml
+  set_fact:
+    cms_combined_config: '{{cms_env_config|combine(cms_auth_config)}}'
+  when: '"cms" in service_variants_enabled'
+  tags:
+    - install
+    - install:configuration
+    - install:app-configuration
+    - edxapp_cfg # Old deprecated tag, will remove when possible
+
+- name: render studio yml config # meant to replace existing json config eventually
+  template:
+    src: "studio.yml.j2"
+    dest: "{{ COMMON_CFG_DIR }}/studio.yml"
+    owner: "{{ edxapp_user }}"
+    group: "{{ common_web_group }}"
+    mode: 0640
+  become: true
+  with_items: "{{ service_variants_enabled }}"
+  when: '"cms" in service_variants_enabled'
+  tags:
+    - install
+    - install:configuration
+    - install:app-configuration
+    - edxapp_cfg # Old deprecated tag, will remove when possible
+
 # write the supervisor scripts for the service variants
 - name: "writing {{ item }} supervisor script"
   template:

diff --git a/playbooks/roles/edxapp/templates/lms.yml.j2 b/playbooks/roles/edxapp/templates/lms.yml.j2
@@ -0,0 +1,3 @@
+{% if lms_combined_config %}
+{{ lms_combined_config | to_nice_yaml }}
+{% endif %}
diff --git a/playbooks/roles/edxapp/templates/studio.yml.j2 b/playbooks/roles/edxapp/templates/studio.yml.j2
@@ -0,0 +1,3 @@
+{% if cms_combined_config %}
+{{ cms_combined_config | to_nice_yaml }}
+{% endif %}
diff --git a/playbooks/roles/journals/meta/main.yml b/playbooks/roles/journals/meta/main.yml
@@ -19,7 +19,8 @@
 #   my_role_var1: 'bar'
 # }
 dependencies:
-  - role: docker
+  # docker-tools is used by devstacks to launch elasticsearch, it likely does not require the same docker version
+  # as is used in docker-tools and should be later changed to an up to date version
   - role: docker-tools
   - role: edx_django_service
     edx_django_service_version: '{{ JOURNALS_VERSION }}'

diff --git a/tests/test_playbooks.sh b/tests/test_playbooks.sh
@@ -16,7 +16,7 @@ ansible-playbook -i localhost, --syntax-check travis-test.yml
 
 output_dir="$PWD/test_output/env-dep"
 mkdir -p $output_dir
-ansible-playbook -i localhost, -c local --tags edxapp_cfg edxapp.yml -e edxapp_user=`whoami` -e edxapp_app_dir=$output_dir -e edxapp_code_dir=$output_dir
+ansible-playbook -i localhost, -c local --tags "common_directories,edxapp_cfg" edxapp.yml -e edxapp_user=`whoami` -e edxapp_app_dir=$output_dir -e edxapp_code_dir=$output_dir
 
 root_dir=$output_dir
 environment_deployments="."

diff --git a/util/check_for_key_collisions/README.md b/util/check_for_key_collisions/README.md
@@ -0,0 +1,4 @@
+Finds if there are colliding keys in a set of yaml/json files that might collide when ansible merges happen
+
+USAGE:
+python check_for_yaml_key_collisions/check_for_yaml_key_collisions.py --files file1.yml --files file2.json
diff --git a/util/check_for_key_collisions/check_for_key_collisions.py b/util/check_for_key_collisions/check_for_key_collisions.py
@@ -0,0 +1,38 @@
+import click
+import yaml
+import json
+from collections import defaultdict
+
+@click.command()
+@click.option('--files', '-m', multiple=True)
+def check_for_yaml_key_collisions(files):
+    values_for_keys = defaultdict(lambda: [])
+    for file_path in files:
+        content = None
+        if file_path.endswith(".yml") or file_path.endswith(".yaml"):
+            stream = file(file_path, 'r')
+            content = yaml.load(stream)
+        elif file_path.endswith(".json"):
+            with open(file_path, "r") as read_file:
+                content = json.load(read_file)
+        for key, value in content.iteritems():
+            values_for_keys[key].append(value)
+
+    collisions = {}
+
+    for key,value in values_for_keys.iteritems():
+        if len(value) > 1:
+            collisions[key] = value
+
+
+    if len(collisions.keys()) > 0:
+        print(str.format("Found key collisions: {}", len(collisions)))
+        for key,value in collisions.iteritems():
+            print(str.format("{} {}", key, value))
+        exit(1)
+    else:
+        print("No collisions found")
+        exit(0)
+
+if __name__ == '__main__':
+    check_for_yaml_key_collisions()
diff --git a/util/check_for_key_collisions/requirements.txt b/util/check_for_key_collisions/requirements.txt
@@ -0,0 +1,2 @@
+Click==7.0
+PyYAML==3.13