From 4ba1a83c0fcf03325fba4903f508849846937cc7 Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Mon, 25 Mar 2019 22:23:40 +0100 Subject: [PATCH] Refactor CI pipelien and enable molecule testing (#1) --- .drone.jsonnet | 110 +++++++++++++++++ .drone.yml | 130 +++++++++++++++++--- .gitignore | 5 + defaults/main.yml | 9 +- molecule/gce-centos-7/create.yml | 73 +++++++++++ molecule/gce-centos-7/destroy.yml | 39 ++++++ molecule/gce-centos-7/molecule.yml | 28 +++++ molecule/gce-centos-7/playbook.yml | 8 ++ molecule/gce-centos-7/prepare.yml | 9 ++ molecule/gce-centos-7/tests/test_default.py | 30 +++++ molecule/pytest.ini | 3 + tasks/install.yml | 6 +- tasks/tls.yml | 17 +-- 13 files changed, 424 insertions(+), 43 deletions(-) create mode 100644 .drone.jsonnet create mode 100644 .gitignore create mode 100644 molecule/gce-centos-7/create.yml create mode 100644 molecule/gce-centos-7/destroy.yml create mode 100644 molecule/gce-centos-7/molecule.yml create mode 100644 molecule/gce-centos-7/playbook.yml create mode 100644 molecule/gce-centos-7/prepare.yml create mode 100644 molecule/gce-centos-7/tests/test_default.py create mode 100644 molecule/pytest.ini diff --git a/.drone.jsonnet b/.drone.jsonnet new file mode 100644 index 0000000..87e0641 --- /dev/null +++ b/.drone.jsonnet @@ -0,0 +1,110 @@ +local AnsibleVersions(version="latest", package="ansible") = { + name: "ansible-" + version, + image: "python:3.7", + pull: "always", + environment: { + PY_COLORS: 1 + }, + commands: [ + "pip install " + package + " ansible-later -qq", + "git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy", + "git ls-files *[^LICENSE,.md,molecule,.ini] | xargs ansible-later -c ~/policy/config.ini" + ], + depends_on: [ + "clone", + ], +}; + +local PipelineLinting = { + kind: "pipeline", + name: "linting", + platform: { + os: "linux", + arch: "amd64", + }, + steps: [ + AnsibleVersions(version="latest", package="ansible"), + AnsibleVersions(version="master", package="git+https://github.com/ansible/ansible.git@devel"), + ], + trigger: { + ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"], + }, +}; + +local PipelineDeployment = { + kind: "pipeline", + name: "deployment", + platform: { + os: "linux", + arch: "amd64", + }, + workspace: { + base: "/drone/src", + path: "xoxys.nginx" + }, + steps: [ + { + name: "molecule", + image: "xoxys/molecule:gce-linux-amd64", + pull: "always", + environment: { + GCE_SSH_KEY: { "from_secret": "gce_ssh_key" }, + GCE_SERVICE_ACCOUNT_EMAIL: { "from_secret": "gce_service_account_email" }, + GCE_PROJECT_ID: { "from_secret": "gce_project_id" }, + GCE_CREDENTIALS_JSON: { "from_secret": "gce_credentials_json" }, + GCE_SSH_USER: { "from_secret": "gce_ssh_user" }, + GCE_CREDENTIALS_FILE: "/root/ansible-testing.json", + MOLECULE_CUSTOM_MODULES_REPO: "https://gitea.rknet.org/ansible/custom_modules", + PY_COLORS: 1 + }, + commands: [ + "/bin/bash /docker-entrypoint.sh", + "molecule create --scenario-name gce-centos-7", + "molecule converge --scenario-name gce-centos-7", + "molecule verify --scenario-name gce-centos-7", + "molecule destroy --scenario-name gce-centos-7", + ], + }, + ], + depends_on: [ + "linting", + ], + trigger: { + ref: ["refs/heads/master", "refs/tags/**"], + }, +}; + +local PipelineNotifications = { + kind: "pipeline", + name: "notifications", + platform: { + os: "linux", + arch: "amd64", + }, + steps: [ + { + name: "matrix", + image: "plugins/matrix", + settings: { + homeserver: "https://matrix.rknet.org", + roomid: "MtidqQXWWAtQcByBhH:rknet.org", + template: "Status: **{{ build.status }}**
Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}
Message: {{ build.message }}", + username: { "from_secret": "matrix_username" }, + password: { "from_secret": "matrix_password" }, + }, + }, + ], + depends_on: [ + "deployment", + ], + trigger: { + status: [ "success", "failure" ], + ref: ["refs/heads/master", "refs/tags/**"], + }, +}; + +[ + PipelineLinting, + PipelineDeployment, + PipelineNotifications, +] diff --git a/.drone.yml b/.drone.yml index 289d224..097555f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,22 +1,116 @@ --- kind: pipeline -name: default +name: linting + +platform: + os: linux + arch: amd64 steps: - - name: ansible-latest - image: python:2.7 - pull: always - commands: - - pip install ansible ansible-later -q - - git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy - - git ls-files *[^LICENSE,.md] | xargs ansible-later -c ~/policy/config.ini - depends_on: [ clone ] - - - name: ansible-master - image: python:2.7 - pull: always - commands: - - pip install ansible ansible-later -q - - git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy - - git ls-files *[^LICENSE,.md] | xargs ansible-later -c ~/policy/config.ini - depends_on: [ clone ] +- name: ansible-latest + pull: always + image: python:3.7 + commands: + - pip install ansible ansible-later -qq + - git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy + - "git ls-files *[^LICENSE,.md,molecule,.ini] | xargs ansible-later -c ~/policy/config.ini" + environment: + PY_COLORS: 1 + depends_on: + - clone + +- name: ansible-master + pull: always + image: python:3.7 + commands: + - "pip install git+https://github.com/ansible/ansible.git@devel ansible-later -qq" + - git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy + - "git ls-files *[^LICENSE,.md,molecule,.ini] | xargs ansible-later -c ~/policy/config.ini" + environment: + PY_COLORS: 1 + depends_on: + - clone + +trigger: + ref: + - refs/heads/master + - "refs/tags/**" + - "refs/pull/**" + +--- +kind: pipeline +name: deployment + +platform: + os: linux + arch: amd64 + +workspace: + base: /drone/src + path: xoxys.nginx + +steps: +- name: molecule + pull: always + image: xoxys/molecule:gce-linux-amd64 + commands: + - /bin/bash /docker-entrypoint.sh + - molecule create --scenario-name gce-centos-7 + - molecule converge --scenario-name gce-centos-7 + - molecule verify --scenario-name gce-centos-7 + - molecule destroy --scenario-name gce-centos-7 + environment: + GCE_CREDENTIALS_FILE: /root/ansible-testing.json + GCE_CREDENTIALS_JSON: + from_secret: gce_credentials_json + GCE_PROJECT_ID: + from_secret: gce_project_id + GCE_SERVICE_ACCOUNT_EMAIL: + from_secret: gce_service_account_email + GCE_SSH_KEY: + from_secret: gce_ssh_key + GCE_SSH_USER: + from_secret: gce_ssh_user + MOLECULE_CUSTOM_MODULES_REPO: https://gitea.rknet.org/ansible/custom_modules + PY_COLORS: 1 + +trigger: + ref: + - refs/heads/master + - "refs/tags/**" + +depends_on: +- linting + +--- +kind: pipeline +name: notifications + +platform: + os: linux + arch: amd64 + +steps: +- name: matrix + image: plugins/matrix + settings: + homeserver: https://matrix.rknet.org + password: + from_secret: matrix_password + roomid: MtidqQXWWAtQcByBhH:rknet.org + template: "Status: **{{ build.status }}**
Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}
Message: {{ build.message }}" + username: + from_secret: matrix_username + +trigger: + ref: + - refs/heads/master + - "refs/tags/**" + status: + - success + - failure + +depends_on: +- deployment + +... diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..69810f1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +# ---> Python +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class diff --git a/defaults/main.yml b/defaults/main.yml index 31dbce4..17f11ac 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -48,15 +48,10 @@ nginx_open_ports: - 443 nginx_tls_enabled: False -# You can deploy your certificates from a file or from content. -# If you enable nginx_tls_source_use_content you have to put the content of your cert files into -# nginx_tls_cert_file and nginx_tls_cert_file. -nginx_tls_source_use_content: False -# If you enable nginx_tls_source_use_files theses variables have to contain the path to your -# certificate files located on the ansible "master" host -nginx_tls_source_use_files: True +# Source has to be a file nginx_tls_cert_source: mycert.pem nginx_tls_key_source: mykey.pem +# Set the destination filename nginx_tls_cert_file: mycert.pem nginx_tls_key_file: mykey.pem # nginx_tls_dhparam_file: # defaults to not set diff --git a/molecule/gce-centos-7/create.yml b/molecule/gce-centos-7/create.yml new file mode 100644 index 0000000..6fbb72e --- /dev/null +++ b/molecule/gce-centos-7/create.yml @@ -0,0 +1,73 @@ +--- +- name: Create + hosts: localhost + connection: local + gather_facts: false + #no_log: "{{ not (lookup('env', 'MOLECULE_DEBUG') | bool or molecule_yml.provisioner.log|default(false) | bool) }}" + vars: + ssh_port: 22 + ssh_user: "{{ lookup('env', 'GCE_SSH_USER') }}" + ssh_identity_file: "{{ lookup('env', 'HOME') }}/.ssh/google_compute_engine" + tasks: + - name: Create molecule instance(s) + gce: + instance_names: "{{ item.name }}" + zone: "{{ item.zone }}" + machine_type: "{{ item.machine_type }}" + image: "{{ item.image }}" + service_account_email: "{{ lookup('env', 'GCE_SERVICE_ACCOUNT_EMAIL') }}" + credentials_file: "{{ lookup('env', 'GCE_CREDENTIALS_FILE') }}" + project_id: "{{ lookup('env', 'GCE_PROJECT_ID') }}" + register: server + with_items: "{{ molecule_yml.platforms }}" + loop_control: + label: "{{ item.name }}:{{ item.machine_type }}" + async: 7200 + poll: 0 + + - name: Wait for instance(s) creation to complete + async_status: + jid: "{{ item.ansible_job_id }}" + register: gce_jobs + until: gce_jobs.finished + retries: 300 + with_items: "{{ server.results }}" + loop_control: + label: "{{ item.item.name }}:{{ item.item.machine_type }}" + + # Mandatory configuration for Molecule to function. + + - name: Populate instance config dict + set_fact: + instance_conf_dict: { + 'instance': "{{ item.instance_data[0].name }}", + 'address': "{{ item.instance_data[0].public_ip }}", + 'user': "{{ ssh_user }}", + 'port': "{{ ssh_port }}", + 'identity_file': "{{ ssh_identity_file }}", } + with_items: "{{ gce_jobs.results }}" + loop_control: + label: "{{ item.instance_data[0].name }}:{{ item.instance_data[0].machine_type }}" + register: instance_config_dict + when: server.changed | bool + + - name: Convert instance config dict to a list + set_fact: + instance_conf: "{{ instance_config_dict.results | map(attribute='ansible_facts.instance_conf_dict') | list }}" + when: server.changed | bool + + - name: Dump instance config + copy: + content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}" + dest: "{{ molecule_instance_config }}" + when: server.changed | bool + + - name: Wait for SSH + wait_for: + port: "{{ ssh_port }}" + host: "{{ item.address }}" + search_regex: SSH + delay: 10 + with_items: "{{ lookup('file', molecule_instance_config) | molecule_from_yaml }}" + loop_control: + label: "{{ item.instance }}" diff --git a/molecule/gce-centos-7/destroy.yml b/molecule/gce-centos-7/destroy.yml new file mode 100644 index 0000000..abe9cf0 --- /dev/null +++ b/molecule/gce-centos-7/destroy.yml @@ -0,0 +1,39 @@ +--- +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ not (lookup('env', 'MOLECULE_DEBUG') | bool or molecule_yml.provisioner.log|default(false) | bool) }}" + tasks: + - name: Destroy molecule instance(s) + gce: + instance_names: "{{ item.name }}" + state: absent + zone: "{{ item.zone }}" + service_account_email: "{{ lookup('env', 'GCE_SERVICE_ACCOUNT_EMAIL') }}" + credentials_file: "{{ lookup('env', 'GCE_CREDENTIALS_FILE') }}" + project_id: "{{ lookup('env', 'GCE_PROJECT_ID') }}" + register: server + with_items: "{{ molecule_yml.platforms }}" + async: 7200 + poll: 0 + + - name: Wait for instance(s) deletion to complete + async_status: + jid: "{{ item.ansible_job_id }}" + register: gce_jobs + until: gce_jobs.finished + retries: 300 + with_items: "{{ server.results }}" + + # Mandatory configuration for Molecule to function. + + - name: Populate instance config + set_fact: + instance_conf: {} + + - name: Dump instance config + copy: + content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}" + dest: "{{ molecule_instance_config }}" + when: server.changed | bool diff --git a/molecule/gce-centos-7/molecule.yml b/molecule/gce-centos-7/molecule.yml new file mode 100644 index 0000000..c7b5d82 --- /dev/null +++ b/molecule/gce-centos-7/molecule.yml @@ -0,0 +1,28 @@ +--- +dependency: + name: galaxy +driver: + name: gce +lint: + name: yamllint + enabled: False +platforms: + - name: gce-centos-7 + zone: europe-north1-a + machine_type: f1-micro + image: centos-7 +provisioner: + name: ansible + lint: + name: ansible-lint +scenario: + name: gce-centos-7 +verifier: + name: testinfra + lint: + name: flake8 +ansible: + ansiblecfg_defaults: + force_color: true + roles_path: /drone/src + timeout: 60 diff --git a/molecule/gce-centos-7/playbook.yml b/molecule/gce-centos-7/playbook.yml new file mode 100644 index 0000000..71caa98 --- /dev/null +++ b/molecule/gce-centos-7/playbook.yml @@ -0,0 +1,8 @@ +--- +- name: Converge + hosts: all + vars: + nginx_default_page_enabled: True + + roles: + - role: xoxys.nginx diff --git a/molecule/gce-centos-7/prepare.yml b/molecule/gce-centos-7/prepare.yml new file mode 100644 index 0000000..ddb01fb --- /dev/null +++ b/molecule/gce-centos-7/prepare.yml @@ -0,0 +1,9 @@ +--- +- name: Prepare + hosts: all + gather_facts: false + tasks: + - name: Install python for Ansible + raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal python-zipstream) + become: true + changed_when: false diff --git a/molecule/gce-centos-7/tests/test_default.py b/molecule/gce-centos-7/tests/test_default.py new file mode 100644 index 0000000..fcf5946 --- /dev/null +++ b/molecule/gce-centos-7/tests/test_default.py @@ -0,0 +1,30 @@ +import os + +import testinfra.utils.ansible_runner + +import warnings +warnings.filterwarnings("ignore", category=DeprecationWarning) + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_nginx_is_installed(host): + nginx = host.package("nginx") + assert nginx.is_installed + +def test_nginx_running_and_enabled(host): + nginx = host.service("nginx") + assert nginx.is_running + assert nginx.is_enabled + +def test_nginx_process(host): + # Verify worker procs are running + master = host.process.get(user="root", comm="nginx") + workers = host.process.filter(ppid=master.pid) + assert len(workers) > 0 + +def test_nginx_socket(host): + # Verify the socket is listening for HTTP traffic + assert host.socket("tcp://0.0.0.0:80").is_listening + diff --git a/molecule/pytest.ini b/molecule/pytest.ini new file mode 100644 index 0000000..c24fe5b --- /dev/null +++ b/molecule/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +filterwarnings = + ignore::DeprecationWarning diff --git a/tasks/install.yml b/tasks/install.yml index 280344d..d385b8a 100644 --- a/tasks/install.yml +++ b/tasks/install.yml @@ -36,7 +36,7 @@ owner: "{{ nginx_user }}" group: "{{ nginx_group }}" mode: 0750 - with_items: + loop: - "{{ nginx_vhosts_dir }}" - "{{ nginx_vhosts_dir }}/default" @@ -47,7 +47,7 @@ owner: root group: root mode: 0640 - with_items: + loop: - /etc/nginx/sites-available - /etc/nginx/sites-enabled @@ -88,7 +88,7 @@ name: "{{ item.name }}" state: "{{ item.state }}" persistent: "{{ item.persistent }}" - with_items: "{{ nginx_set_sebooleans }}" + loop: "{{ nginx_set_sebooleans }}" when: nginx_set_sebooleans is defined - name: Fix selinux file context mappaing for pid file diff --git a/tasks/tls.yml b/tasks/tls.yml index 6b8ec18..7b20bee 100644 --- a/tasks/tls.yml +++ b/tasks/tls.yml @@ -1,24 +1,11 @@ --- - block: - - name: Copy certs and private key (content) - copy: - content: "{{ item.src }}" - dest: "{{ item.dest }}" - mode: "{{ item.mode }}" - with_items: - - { src: "{{ nginx_tls_key_source }}", dest: '/etc/pki/tls/private/{{ nginx_tls_key_file }}', mode: '0600' } - - { src: "{{ nginx_tls_cert_source }}", dest: '/etc/pki/tls/certs/{{ nginx_tls_cert_file }}', mode: '0750' } - loop_control: - label: "{{ item.dest }}" - notify: __nginx_reload - when: nginx_tls_source_use_content - - - name: Copy certs and private key (files) + - name: Copy certs and private key copy: src: "{{ item.src }}" dest: "{{ item.dest }}" mode: "{{ item.mode }}" - with_items: + loop: - { src: "{{ nginx_tls_key_source }}", dest: '/etc/pki/tls/private/{{ nginx_tls_key_file }}', mode: '0600' } - { src: "{{ nginx_tls_cert_source }}", dest: '/etc/pki/tls/certs/{{ nginx_tls_cert_file }}', mode: '0750' } loop_control: