From 9ddb1ba4a29690747ce3f33e937b31f73352f218 Mon Sep 17 00:00:00 2001 From: Artyom Bakhtin Date: Mon, 22 Apr 2019 10:52:23 +0300 Subject: [PATCH 1/2] refactor Ansible role Signed-off-by: Artyom Bakhtin --- deploy/ansible/roles/iroha-docker/README.md | 18 +-- .../roles/iroha-docker/defaults/main.yml | 53 ++++++--- .../roles/iroha-docker/handlers/main.yml | 8 ++ .../roles/iroha-docker/tasks/config-gen.yml | 80 +++++-------- .../roles/iroha-docker/tasks/deploy.yml | 108 ++++++------------ .../roles/iroha-docker/tasks/init-vars.yml | 44 +++++++ .../ansible/roles/iroha-docker/tasks/main.yml | 7 +- .../templates/docker-compose.yml.j2 | 8 +- 8 files changed, 178 insertions(+), 148 deletions(-) create mode 100644 deploy/ansible/roles/iroha-docker/handlers/main.yml create mode 100644 deploy/ansible/roles/iroha-docker/tasks/init-vars.yml diff --git a/deploy/ansible/roles/iroha-docker/README.md b/deploy/ansible/roles/iroha-docker/README.md index 7031632d8a6..df639d8c6e4 100644 --- a/deploy/ansible/roles/iroha-docker/README.md +++ b/deploy/ansible/roles/iroha-docker/README.md @@ -1,4 +1,4 @@ -##### Description +# Description [This role](https://github.com/hyperledger/iroha/tree/master/deploy/ansible/roles/iroha-docker) deploys multiple replicas of Iroha containers (one Iroha peer per container) on remote hosts. Each Iroha peer can communicate with others in two ways: - using public IP addresses or hostnames set in inventory list OR - using private IP addresses of the Docker overlay network @@ -7,11 +7,11 @@ The first one is easier to implement since it does not require preliminary confi This option is enabled by default. -The second one can be used when there exists an overlay network between the hosts. In short, overlay network allows for Docker containers to communicate using a single subnet. Such that each container would have a unique IP address in that subnet. Learn more in official Docker documentation (https://docs.docker.com/network/overlay). We recommend to use Calico for setting up Docker overlay network since it can be used as a network plugin (https://docs.projectcalico.org/v1.5/getting-started/docker/tutorials/basic). +The second one can be used when there exists an overlay network between the hosts. In short, overlay network allows for Docker containers to communicate using a single subnet. Such that each container would have a unique IP address in that subnet. Learn more in official Docker documentation (https://docs.docker.com/network/overlay). Overlay network can be created if your instance is part of a Swarm cluster. Another method does not involve creating a Swarm cluster but requires a distributed key-value storage. [There is a guide](https://docker-k8s-lab.readthedocs.io/en/latest/docker/docker-etcd.html) on how to create such overlay network. -The second way is also suitable for local-only deployments. +This method is also suitable for local-only deployments and does not require any key-value storage. -##### Requirements +# Requirements Tested on Ubuntu 16.04, 18.04 - Local: - python3, python3-dev @@ -26,7 +26,7 @@ The second way is also suitable for local-only deployments. **Note:** > `docker.io` package from Ubuntu repos will not work. Either use Ansible role or install Docker following official instructions for your OS flavor. -##### Quick Start +# Quick Start 1. Install Ansible ``` pip3 install ansible @@ -49,11 +49,11 @@ It will also install Docker along with required python modules. If you want to s **Note:** > This command escalates privileges on a remote host during the run. It is required to be able to spin up Docker containers. We recommend to run the playbook using a passwordless remote sudo user. -##### Initial configuration +# Initial configuration See `defaults/main.yml` file to get more details about available configuration options. -##### Examples +# Examples **Example 1** Deploying 6 Iroha peers on two remote hosts communicating using public IP addresses. With 2 and 4 replicas on each host respectively. @@ -77,12 +77,12 @@ Deploying 6 Iroha peers on two remote hosts communicating using public IP addres **192.168.122.109.yml** ``` - replicas: 2 + iroha_replicas: 2 ``` **192.168.122.30.yml** ``` - replicas: 4 + iroha_replicas: 4 ``` 4. Run the playbook diff --git a/deploy/ansible/roles/iroha-docker/defaults/main.yml b/deploy/ansible/roles/iroha-docker/defaults/main.yml index c301ca51428..3dc9d4d4952 100644 --- a/deploy/ansible/roles/iroha-docker/defaults/main.yml +++ b/deploy/ansible/roles/iroha-docker/defaults/main.yml @@ -1,8 +1,8 @@ # How many Iroha peers (containers) are deployed on each host. Can be set on a per-host level. -# Minimum total nodes is 6 (5f+1) in order for consensus to work properly. +# Minimum total nodes is 4 (3f+1) in order for consensus to work properly. # -# Default: 6 -replicas: 6 +# Default: 4 +iroha_replicas: 4 # Whether to use custom hostname for EACH container. # If set to `true`, Iroha peers will communicate using these hostnames. Hostnames should be set using @@ -11,39 +11,61 @@ replicas: 6 # variable. Container and service names in Docker Compose files will be auto-generated. # # Default: false -custom_hostnames: false +iroha_custom_hostnames: false # Affects how Iroha peers are communicated. If set to `true`, Docker overlay network with that -# name will be used. It must be created beforehand. The recommended way is to use Calico plugin -# for Docker (projectcalico.org). +# name will be used. It must be created beforehand. The recommended way is to use Swarm cluster +# or create a network manually using a KV storage. # Setting it to `true` without creating the network will only work for a single host deployment # (Iroha peers will only be able to communicate within that host). Suitable for local-only # deployments for testing purposes. # # Default: false -overlay_network: false +iroha_overlay_network: false ## Deployment configs -container_basename: iroha +iroha_container_basename: iroha # Path on a local machine for generated configs prior moving them on the remote -config_local_dir: /tmp/iroha-ansible-conf +iroha_config_local_dir: /tmp/iroha-ansible-conf # Path on a remote machine for generated configs -deploy_dir: /opt/iroha-deploy +iroha_deploy_dir: /opt/iroha-deploy # Overlay network name # If using overlay network plugin (like Calico) the network must be created prior running this role -# The network will be created automatically if deploying locally or with `overlay_network` +# The network will be created automatically if deploying locally or with `iroha_overlay_network` # set to `false` iroha_network_name: iroha-net # The role is incompatible with Iroha versions below RC2 since keys format has changed. # Apply the patch (files/old-keys-format.patch) if you need support for previous Iroha versions. -iroha_docker_tag: 1.0.0_rc2 -postgres_docker_tag: '9.5' +iroha_docker_tag: 1.0.0_rc5 +iroha_postgres_docker_tag: '9.5' + +## The block of variables that toggles certain tasks of the role +# Toggle tasks in `init-vars.yml` file. Will initialize variables for container names, +# Iroha hostnames, etc. No configs will be changed. +# Default: true +iroha_init_vars: true +# Toggle tasks in `config-gen.yml` file. Will generate new keys for Iroha with a new Genesis +# Block +# Default: true +iroha_generate_init_configs: true +# Toggle tasks in `deploy.yml` file. Enable if you just need to update Iroha configs and Docker +# Compose file +# Default: true +iroha_update_runtime_configs: true + +# Toggles usage of a custom Genesis Block. Useful if you have custom roles, users, permissions, +# etc. The role will fill your custom Genesis Block with Iroha peers while preserving everything +# else. +# Default: false +iroha_custom_genesis_block: false +# Path to the custom Genesis Block. Used only when `iroha_custom_genesis_block` set to `true` +iroha_custom_genesis_block_path: '' ## Iroha config -# This value will be used as a base peer port for Iroha container. E.g. setting `replicas` option to +# This value will be used as a base peer port for Iroha container. E.g. setting `iroha_replicas` option to # 6 will generate Docker Compose file with port numbers starting from 10001 counting up to 10006. -# If `custom_hostnames` set to `true` AND contains port numbers this option is ignored +# If `iroha_custom_hostnames` set to `true` AND contains port numbers this option is ignored iroha_peer_port: 10001 # As above but for base Torii port iroha_torii_port: 50051 @@ -66,3 +88,4 @@ iroha_blockstore_path: /tmp/block_store #iroha_mst_expiration_time: 1440 #iroha_max_rounds_delay: 3000 #iroha_stale_stream_max_rounds: 2 + diff --git a/deploy/ansible/roles/iroha-docker/handlers/main.yml b/deploy/ansible/roles/iroha-docker/handlers/main.yml new file mode 100644 index 00000000000..c234c96d7a0 --- /dev/null +++ b/deploy/ansible/roles/iroha-docker/handlers/main.yml @@ -0,0 +1,8 @@ +# handlers file for Iroha +--- +- name: restart Iroha + docker_service: + project_src: "{{ iroha_deploy_dir }}" + pull: yes + state: present + recreate: always diff --git a/deploy/ansible/roles/iroha-docker/tasks/config-gen.yml b/deploy/ansible/roles/iroha-docker/tasks/config-gen.yml index 14c5fd4fe68..cc05cd58bd7 100644 --- a/deploy/ansible/roles/iroha-docker/tasks/config-gen.yml +++ b/deploy/ansible/roles/iroha-docker/tasks/config-gen.yml @@ -1,81 +1,63 @@ - block: - - set_fact: - iroha_nodes: [] - iroha_all_nodes: [] - - - name: generate hostnames (no overlay) - set_fact: - iroha_nodes: "{{ iroha_nodes }} + [ '{{ inventory_hostname }}:{{ iroha_peer_port | int + item | int }}' ]" - loop: "{{ range(0, replicas |int) | list }}" - when: not custom_hostnames and not overlay_network - - - shell: "cat /dev/urandom | tr -cd [:xdigit:] | tr '[:upper:]' '[:lower:]' | head -c {{ 8 * replicas | int}}" - register: random_hex - when: not custom_hostnames and overlay_network - - - name: generate hostnames (overlay) - set_fact: - iroha_nodes: "{{ iroha_nodes }} + [ '{{ container_basename }}-{{ item }}-{{ random_hex.stdout[item|int*8:(item|int+1)*8-1] }}:{{ iroha_peer_port }}' ]" - loop: "{{ range(0, replicas) | list }}" - when: not custom_hostnames and overlay_network - - - name: generate hostnames (custom_hostnames) - set_fact: - iroha_nodes: "{{ iroha_nodes }} + [ '{% if item.split(':') | length < 2 %}{{ item }}:{{ iroha_peer_port | int + idx }}{% else %}{{ item }}{% endif %}' ]" - when: custom_hostnames and hostnames is defined - loop: "{{ hostnames }}" - loop_control: - index_var: idx - - - set_fact: - iroha_peers_map: "{{ iroha_peers_map | default([]) }} + [ {{ {'hostname': item, 'human_hostname': 'c_' + item | regex_replace('\\.', '_') | regex_replace(':', '_'), 'peer_port': item.split(':')[1] } }} ]" - loop: "{{ iroha_nodes }}" - - - debug: - var: iroha_peers_map - - - set_fact: - iroha_nodes: "{{ iroha_peers_map }}" - - - set_fact: - iroha_all_nodes: "{{ iroha_all_nodes }} + {{ hostvars[item]['iroha_nodes'] }}" - loop: "{{ ansible_play_hosts }}" + - name: copy custom genesis block + copy: + src: "{{ iroha_custom_genesis_block_path }}" + dest: "{{ role_path }}/files/genesis.block" run_once: yes + when: iroha_custom_genesis_block + failed_when: iroha_custom_genesis_block_path is not defined - name: generate Iroha configs - command: "./config_gen.sh -p{{ iroha_all_nodes | join(',', attribute='hostname') }} -o{{ config_local_dir }} -f" + command: "./config_gen.sh -p{{ iroha_all_nodes | join(',', attribute='hostname') }} -o{{ iroha_config_local_dir }} -f" run_once: yes args: chdir: "{{ role_path }}/files" - name: make config dirs file: - path: "{{ config_local_dir }}/{{ inventory_hostname }}/conf/{{ item.human_hostname }}" + path: "{{ iroha_config_local_dir }}/{{ inventory_hostname }}/conf/{{ item.human_hostname }}" state: directory loop: "{{ iroha_nodes }}" - name: move keys copy: - src: "{{ config_local_dir }}/{{ item[0].hostname }}{{ item[1] }}" - dest: "{{ config_local_dir }}/{{ inventory_hostname }}/conf/{{ item[0].human_hostname }}" + src: "{{ iroha_config_local_dir }}/{{ item[0].hostname }}{{ item[1] }}" + dest: "{{ iroha_config_local_dir }}/{{ inventory_hostname }}/conf/{{ item[0].human_hostname }}" loop: "{{ iroha_nodes | product(['.priv', '.pub']) | list }}" - name: move genesis.block copy: - src: "{{ config_local_dir }}/genesis.block" - dest: "{{ config_local_dir }}/{{ inventory_hostname }}/conf/{{ item.human_hostname }}" + src: "{{ iroha_config_local_dir }}/genesis.block" + dest: "{{ iroha_config_local_dir }}/{{ inventory_hostname }}/conf/{{ item.human_hostname }}" loop: "{{ iroha_nodes }}" - name: cleanup file: - path: "{{ config_local_dir }}/{{ item[0].hostname }}{{ item[1] }}" + path: "{{ iroha_config_local_dir }}/{{ item[0].hostname }}{{ item[1] }}" state: absent loop: "{{ iroha_nodes | product(['.priv', '.pub']) | list }}" - name: cleanup file: - path: "{{ config_local_dir }}/genesis.block" + path: "{{ iroha_config_local_dir }}/genesis.block" state: absent run_once: yes become: no delegate_to: localhost + when: iroha_generate_init_configs + tags: ["iroha-config-gen"] + +- name: create deploy dir + file: + state: directory + path: "{{ iroha_deploy_dir }}/conf" + tags: ["iroha-config-gen"] + +- name: copy config files + synchronize: + src: "{{ iroha_config_local_dir }}/{{ inventory_hostname }}/conf/" + dest: "{{ iroha_deploy_dir }}/conf/" + recursive: yes + checksum: yes + when: iroha_generate_init_configs + tags: ["iroha-config-gen"] diff --git a/deploy/ansible/roles/iroha-docker/tasks/deploy.yml b/deploy/ansible/roles/iroha-docker/tasks/deploy.yml index 855e86d8fd8..bb35bc7bf0a 100644 --- a/deploy/ansible/roles/iroha-docker/tasks/deploy.yml +++ b/deploy/ansible/roles/iroha-docker/tasks/deploy.yml @@ -1,71 +1,37 @@ -- name: create deploy dir - file: - state: directory - path: "{{ deploy_dir }}/conf" - -- stat: - path: "{{ deploy_dir }}/docker-compose.yml" - register: docker_compose_file - -- name: stop Iroha - docker_service: - project_src: "{{ deploy_dir }}" - project_name: ansible_iroha - remove_volumes: yes - state: absent - when: docker_compose_file.stat.exists - -- name: remove Iroha network (overlay) - docker_network: - name: "{{ iroha_network_name }}" - driver: overlay - state: absent - run_once: yes - when: docker_compose_file.stat.exists and overlay_network - -- name: remove Iroha network - docker_network: - name: "{{ iroha_network_name }}" - state: absent - when: docker_compose_file.stat.exists and not overlay_network - -- name: generate Docker compose - template: - src: docker-compose.yml.j2 - dest: "{{ deploy_dir }}/docker-compose.yml" - -- name: copy config files - synchronize: - src: "{{ config_local_dir }}/{{ inventory_hostname }}/conf/" - dest: "{{ deploy_dir }}/conf/" - recursive: yes - delete: yes - -- name: generate config files - template: - src: config.docker.j2 - dest: "{{ deploy_dir }}/conf/{{ item.human_hostname }}/config.docker" - loop: "{{ iroha_nodes }}" - -- name: create Docker network - docker_network: - name: "{{ iroha_network_name }}" - driver: overlay - run_once: yes - when: overlay_network - -- name: create Docker network - docker_network: - name: "{{ iroha_network_name }}" - when: not overlay_network - -- name: pull Iroha - command: docker-compose pull - args: - chdir: "{{ deploy_dir }}" - changed_when: False - -- name: run Iroha - docker_service: - project_src: "{{ deploy_dir }}" - project_name: ansible_iroha +- block: + - name: create deploy dir + file: + state: directory + path: "{{ iroha_deploy_dir }}/conf" + + # Attachable overlay network cannot be created with a `docker_network` module + - name: create Docker network + shell: "docker network create -d overlay --attachable {{ iroha_network_name }} || true" + when: iroha_overlay_network + run_once: yes + + - name: create Docker network + docker_network: + name: "{{ iroha_network_name }}" + when: not iroha_overlay_network + + - name: generate Docker Compose + template: + src: docker-compose.yml.j2 + dest: "{{ iroha_deploy_dir }}/docker-compose.yml" + notify: restart Iroha + tags: ["iroha-deploy"] + + - name: generate config files + template: + src: config.docker.j2 + dest: "{{ iroha_deploy_dir }}/conf/{{ item.human_hostname }}/config.docker" + notify: restart Iroha + loop: "{{ iroha_nodes }}" + tags: ["iroha-deploy"] + + - name: run Iroha + docker_service: + project_src: "{{ iroha_deploy_dir }}" + tags: ["iroha-deploy"] + when: iroha_update_runtime_configs diff --git a/deploy/ansible/roles/iroha-docker/tasks/init-vars.yml b/deploy/ansible/roles/iroha-docker/tasks/init-vars.yml new file mode 100644 index 00000000000..2196447ba86 --- /dev/null +++ b/deploy/ansible/roles/iroha-docker/tasks/init-vars.yml @@ -0,0 +1,44 @@ +- block: + - set_fact: + iroha_nodes: [] + iroha_peers_map: [] + + - name: generate hostnames (no overlay) + set_fact: + iroha_nodes: "{{ iroha_nodes }} + [ '{{ inventory_hostname }}:{{ iroha_peer_port | int + item | int }}' ]" + loop: "{{ range(0, iroha_replicas |int) | list }}" + when: not iroha_custom_hostnames and not iroha_overlay_network + + - name: generate hostnames (overlay) + set_fact: + iroha_nodes: "{{ iroha_nodes }} + [ '{{ iroha_container_basename }}-{{ item }}-{{ inventory_hostname | checksum | regex_replace('(^[a-fA-F0-9]{8}).+$', '\\1') }}:{{ iroha_peer_port }}' ]" + loop: "{{ range(0, iroha_replicas) | list }}" + when: not iroha_custom_hostnames and iroha_overlay_network + + - name: generate hostnames (iroha_custom_hostnames) + set_fact: + iroha_nodes: "{{ iroha_nodes }} + [ '{% if item.split(':') | length < 2 %}{{ item }}:{{ iroha_peer_port | int + idx }}{% else %}{{ item }}{% endif %}' ]" + when: iroha_custom_hostnames and hostnames is defined + loop: "{{ hostnames }}" + loop_control: + index_var: idx + + - set_fact: + iroha_peers_map: "{{ iroha_peers_map }} + [ {{ {'hostname': item, 'human_hostname': 'c_' + item | regex_replace('\\.', '_') | regex_replace(':', '_'), 'peer_port': item.split(':')[1] } }} ]" + loop: "{{ iroha_nodes }}" + + - debug: + var: iroha_peers_map + + - set_fact: + iroha_nodes: "{{ iroha_peers_map }}" + + - set_fact: + iroha_all_nodes: "{{ iroha_all_nodes | default([]) }} + {{ hostvars[item]['iroha_nodes'] }}" + loop: "{{ ansible_play_hosts }}" + run_once: yes + + become: no + delegate_to: localhost + when: iroha_init_vars + tags: ["iroha-init-vars"] diff --git a/deploy/ansible/roles/iroha-docker/tasks/main.yml b/deploy/ansible/roles/iroha-docker/tasks/main.yml index 4e357468f0c..6d3fd2443b7 100644 --- a/deploy/ansible/roles/iroha-docker/tasks/main.yml +++ b/deploy/ansible/roles/iroha-docker/tasks/main.yml @@ -1,3 +1,6 @@ +- name: init vars + include: init-vars.yml + - name: generate Iroha configs include: config-gen.yml @@ -5,4 +8,6 @@ include: deploy.yml - debug: - msg: "Stop Iroha containers with `docker-compose -f {{ deploy_dir }}/docker-compose.yml -p ansible_iroha down -v`" + msg: "Stop Iroha containers with `docker-compose -f {{ iroha_deploy_dir }}/docker-compose.yml down`" + when: iroha_update_runtime_configs + run_once: yes diff --git a/deploy/ansible/roles/iroha-docker/templates/docker-compose.yml.j2 b/deploy/ansible/roles/iroha-docker/templates/docker-compose.yml.j2 index 2d26816a11a..40936e4747d 100644 --- a/deploy/ansible/roles/iroha-docker/templates/docker-compose.yml.j2 +++ b/deploy/ansible/roles/iroha-docker/templates/docker-compose.yml.j2 @@ -4,12 +4,12 @@ services: {% for node in iroha_nodes %} {{ node.human_hostname }}: image: hyperledger/iroha:{{ iroha_docker_tag }} -{% if overlay_network %} +{% if iroha_overlay_network %} container_name: {{ node.hostname.split(':')[0] }} expose: - {{ iroha_peer_port }} ports: - - 127.0.0.1:{{ iroha_torii_port | int + loop.index - 1 }}:{{ iroha_torii_port }} + - {{ iroha_torii_port | int + loop.index - 1 }}:{{ iroha_torii_port }} {% else %} container_name: {{ node.human_hostname }} ports: @@ -27,9 +27,10 @@ services: networks: - {{ iroha_network_name }} - iroha-db-net + restart: always {{ node.human_hostname }}-postgres: - image: postgres:{{ postgres_docker_tag }} + image: postgres:{{ iroha_postgres_docker_tag }} container_name: {{ node.human_hostname }}-postgres environment: POSTGRES_PASSWORD: {{ iroha_postgres_password }} @@ -39,6 +40,7 @@ services: - psql_storage-{{ node.human_hostname }}:/var/lib/postgresql/data networks: - iroha-db-net + restart: always {% if iroha_postgres_max_prepared_transactions is defined %} command: -c max_prepared_transactions={{ iroha_postgres_max_prepared_transactions }} {% endif %} From 010941bfb59e9b029a3279d721d0776c3e84867e Mon Sep 17 00:00:00 2001 From: Artyom Bakhtin Date: Thu, 25 Apr 2019 09:13:12 +0300 Subject: [PATCH 2/2] add overlay network parameters Signed-off-by: Artyom Bakhtin --- deploy/ansible/roles/iroha-docker/defaults/main.yml | 2 ++ deploy/ansible/roles/iroha-docker/tasks/deploy.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/deploy/ansible/roles/iroha-docker/defaults/main.yml b/deploy/ansible/roles/iroha-docker/defaults/main.yml index 3dc9d4d4952..b8914949c71 100644 --- a/deploy/ansible/roles/iroha-docker/defaults/main.yml +++ b/deploy/ansible/roles/iroha-docker/defaults/main.yml @@ -34,6 +34,8 @@ iroha_deploy_dir: /opt/iroha-deploy # The network will be created automatically if deploying locally or with `iroha_overlay_network` # set to `false` iroha_network_name: iroha-net +iroha_network_address: '10.10.111.0/24' +iroha_network_gateway: '10.10.111.1' # The role is incompatible with Iroha versions below RC2 since keys format has changed. # Apply the patch (files/old-keys-format.patch) if you need support for previous Iroha versions. diff --git a/deploy/ansible/roles/iroha-docker/tasks/deploy.yml b/deploy/ansible/roles/iroha-docker/tasks/deploy.yml index bb35bc7bf0a..f50ac10a833 100644 --- a/deploy/ansible/roles/iroha-docker/tasks/deploy.yml +++ b/deploy/ansible/roles/iroha-docker/tasks/deploy.yml @@ -6,7 +6,7 @@ # Attachable overlay network cannot be created with a `docker_network` module - name: create Docker network - shell: "docker network create -d overlay --attachable {{ iroha_network_name }} || true" + shell: "docker network create -d overlay --attachable {{ iroha_network_name }} --gateway {{ iroha_network_gateway }} --subnet {{ iroha_network_address }} || true" when: iroha_overlay_network run_once: yes