Refactor CI pipelien and enable molecule testing (#1)

.drone.jsonnet

@@ -0,0 +1,110 @@
+local AnsibleVersions(version="latest", package="ansible") = {
+    name: "ansible-" + version,
+    image: "python:3.7",
+    pull: "always",
+    environment: {
+      PY_COLORS: 1
+    },
+    commands: [
+      "pip install " + package + " ansible-later -qq",
+      "git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy",
+      "git ls-files *[^LICENSE,.md,molecule,.ini] | xargs ansible-later -c ~/policy/config.ini"
+    ],
+    depends_on: [
+      "clone",
+    ],
+};
+
+local PipelineLinting = {
+  kind: "pipeline",
+  name: "linting",
+  platform: {
+    os: "linux",
+    arch: "amd64",
+  },
+  steps: [
+    AnsibleVersions(version="latest", package="ansible"),
+    AnsibleVersions(version="master", package="git+https://github.com/ansible/ansible.git@devel"),
+  ],
+  trigger: {
+    ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"],
+  },
+};
+
+local PipelineDeployment = {
+  kind: "pipeline",
+  name: "deployment",
+  platform: {
+    os: "linux",
+    arch: "amd64",
+  },
+  workspace: {
+    base: "/drone/src",
+    path: "xoxys.nginx"
+  },
+  steps: [
+    {
+      name: "molecule",
+      image: "xoxys/molecule:gce-linux-amd64",
+      pull: "always",
+      environment: {
+        GCE_SSH_KEY: { "from_secret": "gce_ssh_key" },
+        GCE_SERVICE_ACCOUNT_EMAIL: { "from_secret": "gce_service_account_email" },
+        GCE_PROJECT_ID: { "from_secret": "gce_project_id" },
+        GCE_CREDENTIALS_JSON: { "from_secret": "gce_credentials_json" },
+        GCE_SSH_USER: { "from_secret": "gce_ssh_user" },
+        GCE_CREDENTIALS_FILE: "/root/ansible-testing.json",
+        MOLECULE_CUSTOM_MODULES_REPO: "https://gitea.rknet.org/ansible/custom_modules",
+        PY_COLORS: 1
+      },
+      commands: [
+        "/bin/bash /docker-entrypoint.sh",
+        "molecule create --scenario-name gce-centos-7",
+        "molecule converge --scenario-name gce-centos-7",
+        "molecule verify --scenario-name gce-centos-7",
+        "molecule destroy --scenario-name gce-centos-7",
+      ],
+    },
+  ],
+  depends_on: [
+    "linting",
+  ],
+  trigger: {
+    ref: ["refs/heads/master", "refs/tags/**"],
+  },
+};
+
+local PipelineNotifications = {
+  kind: "pipeline",
+  name: "notifications",
+  platform: {
+    os: "linux",
+    arch: "amd64",
+  },
+  steps: [
+    {
+      name: "matrix",
+      image: "plugins/matrix",
+      settings: {
+        homeserver: "https://matrix.rknet.org",
+        roomid: "MtidqQXWWAtQcByBhH:rknet.org",
+        template: "Status: **{{ build.status }}**<br/> Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}<br/> Message: {{ build.message }}",
+        username: { "from_secret": "matrix_username" },
+        password: { "from_secret": "matrix_password" },
+      },
+    },
+  ],
+  depends_on: [
+    "deployment",
+  ],
+  trigger: {
+    status: [ "success", "failure" ],
+    ref: ["refs/heads/master", "refs/tags/**"],
+  },
+};
+
+[
+  PipelineLinting,
+  PipelineDeployment,
+  PipelineNotifications,
+]

.drone.yml

@@ -1,22 +1,116 @@
 ---
 kind: pipeline
-name: default
+name: linting
+
+platform:
+  os: linux
+  arch: amd64
 
 steps:
-  - name: ansible-latest
-    image: python:2.7
+- name: ansible-latest
   pull: always
+  image: python:3.7
   commands:
-      - pip install ansible ansible-later -q
+  - pip install ansible ansible-later -qq
   - git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy
-      - git ls-files *[^LICENSE,.md] | xargs ansible-later -c ~/policy/config.ini
-    depends_on: [ clone ]
+  - "git ls-files *[^LICENSE,.md,molecule,.ini] | xargs ansible-later -c ~/policy/config.ini"
+  environment:
+    PY_COLORS: 1
+  depends_on:
+  - clone
 
-  - name: ansible-master
-    image: python:2.7
+- name: ansible-master
   pull: always
+  image: python:3.7
   commands:
-      - pip install ansible ansible-later -q
+  - "pip install git+https://github.com/ansible/ansible.git@devel ansible-later -qq"
   - git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy
-      - git ls-files *[^LICENSE,.md] | xargs ansible-later -c ~/policy/config.ini
-    depends_on: [ clone ]
+  - "git ls-files *[^LICENSE,.md,molecule,.ini] | xargs ansible-later -c ~/policy/config.ini"
+  environment:
+    PY_COLORS: 1
+  depends_on:
+  - clone
+
+trigger:
+  ref:
+  - refs/heads/master
+  - "refs/tags/**"
+  - "refs/pull/**"
+
+---
+kind: pipeline
+name: deployment
+
+platform:
+  os: linux
+  arch: amd64
+
+workspace:
+  base: /drone/src
+  path: xoxys.nginx
+
+steps:
+- name: molecule
+  pull: always
+  image: xoxys/molecule:gce-linux-amd64
+  commands:
+  - /bin/bash /docker-entrypoint.sh
+  - molecule create --scenario-name gce-centos-7
+  - molecule converge --scenario-name gce-centos-7
+  - molecule verify --scenario-name gce-centos-7
+  - molecule destroy --scenario-name gce-centos-7
+  environment:
+    GCE_CREDENTIALS_FILE: /root/ansible-testing.json
+    GCE_CREDENTIALS_JSON:
+      from_secret: gce_credentials_json
+    GCE_PROJECT_ID:
+      from_secret: gce_project_id
+    GCE_SERVICE_ACCOUNT_EMAIL:
+      from_secret: gce_service_account_email
+    GCE_SSH_KEY:
+      from_secret: gce_ssh_key
+    GCE_SSH_USER:
+      from_secret: gce_ssh_user
+    MOLECULE_CUSTOM_MODULES_REPO: https://gitea.rknet.org/ansible/custom_modules
+    PY_COLORS: 1
+
+trigger:
+  ref:
+  - refs/heads/master
+  - "refs/tags/**"
+
+depends_on:
+- linting
+
+---
+kind: pipeline
+name: notifications
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: matrix
+  image: plugins/matrix
+  settings:
+    homeserver: https://matrix.rknet.org
+    password:
+      from_secret: matrix_password
+    roomid: MtidqQXWWAtQcByBhH:rknet.org
+    template: "Status: **{{ build.status }}**<br/> Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}<br/> Message: {{ build.message }}"
+    username:
+      from_secret: matrix_username
+
+trigger:
+  ref:
+  - refs/heads/master
+  - "refs/tags/**"
+  status:
+  - success
+  - failure
+
+depends_on:
+- deployment
+
+...

.gitignore

@@ -0,0 +1,5 @@
+# ---> Python
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class

defaults/main.yml

@@ -48,15 +48,10 @@ nginx_open_ports:
   - 443
 
 nginx_tls_enabled: False
-# You can deploy your certificates from a file or from content.
-# If you enable nginx_tls_source_use_content you have to put the content of your cert files into
-# nginx_tls_cert_file and nginx_tls_cert_file.
-nginx_tls_source_use_content: False
-# If you enable nginx_tls_source_use_files theses variables have to contain the path to your
-# certificate files located on the ansible "master" host
-nginx_tls_source_use_files: True
+# Source has to be a file
 nginx_tls_cert_source: mycert.pem
 nginx_tls_key_source: mykey.pem
+# Set the destination filename
 nginx_tls_cert_file: mycert.pem
 nginx_tls_key_file: mykey.pem
 # nginx_tls_dhparam_file: # defaults to not set

molecule/gce-centos-7/create.yml

@@ -0,0 +1,73 @@
+---
+- name: Create
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  #no_log: "{{ not (lookup('env', 'MOLECULE_DEBUG') | bool or molecule_yml.provisioner.log|default(false) | bool) }}"
+  vars:
+    ssh_port: 22
+    ssh_user: "{{ lookup('env', 'GCE_SSH_USER') }}"
+    ssh_identity_file: "{{ lookup('env', 'HOME') }}/.ssh/google_compute_engine"
+  tasks:
+    - name: Create molecule instance(s)
+      gce:
+        instance_names: "{{ item.name }}"
+        zone: "{{ item.zone }}"
+        machine_type: "{{ item.machine_type }}"
+        image: "{{ item.image }}"
+        service_account_email: "{{ lookup('env', 'GCE_SERVICE_ACCOUNT_EMAIL') }}"
+        credentials_file: "{{ lookup('env', 'GCE_CREDENTIALS_FILE') }}"
+        project_id: "{{ lookup('env', 'GCE_PROJECT_ID') }}"
+      register: server
+      with_items: "{{ molecule_yml.platforms }}"
+      loop_control:
+        label: "{{ item.name }}:{{ item.machine_type }}"
+      async: 7200
+      poll: 0
+
+    - name: Wait for instance(s) creation to complete
+      async_status:
+        jid: "{{ item.ansible_job_id }}"
+      register: gce_jobs
+      until: gce_jobs.finished
+      retries: 300
+      with_items: "{{ server.results }}"
+      loop_control:
+        label: "{{ item.item.name }}:{{ item.item.machine_type }}"
+
+    # Mandatory configuration for Molecule to function.
+
+    - name: Populate instance config dict
+      set_fact:
+        instance_conf_dict: {
+          'instance': "{{ item.instance_data[0].name }}",
+          'address': "{{ item.instance_data[0].public_ip }}",
+          'user': "{{ ssh_user }}",
+          'port': "{{ ssh_port }}",
+          'identity_file': "{{ ssh_identity_file }}", }
+      with_items: "{{ gce_jobs.results }}"
+      loop_control:
+        label: "{{ item.instance_data[0].name }}:{{ item.instance_data[0].machine_type }}"
+      register: instance_config_dict
+      when: server.changed | bool
+
+    - name: Convert instance config dict to a list
+      set_fact:
+        instance_conf: "{{ instance_config_dict.results | map(attribute='ansible_facts.instance_conf_dict') | list }}"
+      when: server.changed | bool
+
+    - name: Dump instance config
+      copy:
+        content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}"
+        dest: "{{ molecule_instance_config }}"
+      when: server.changed | bool
+
+    - name: Wait for SSH
+      wait_for:
+        port: "{{ ssh_port }}"
+        host: "{{ item.address }}"
+        search_regex: SSH
+        delay: 10
+      with_items: "{{ lookup('file', molecule_instance_config) | molecule_from_yaml }}"
+      loop_control:
+        label: "{{ item.instance }}"

molecule/gce-centos-7/destroy.yml

@@ -0,0 +1,39 @@
+---
+- name: Destroy
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  no_log: "{{ not (lookup('env', 'MOLECULE_DEBUG') | bool or molecule_yml.provisioner.log|default(false) | bool) }}"
+  tasks:
+    - name: Destroy molecule instance(s)
+      gce:
+        instance_names: "{{ item.name }}"
+        state: absent
+        zone: "{{ item.zone }}"
+        service_account_email: "{{ lookup('env', 'GCE_SERVICE_ACCOUNT_EMAIL') }}"
+        credentials_file: "{{ lookup('env', 'GCE_CREDENTIALS_FILE') }}"
+        project_id: "{{ lookup('env', 'GCE_PROJECT_ID') }}"
+      register: server
+      with_items: "{{ molecule_yml.platforms }}"
+      async: 7200
+      poll: 0
+
+    - name: Wait for instance(s) deletion to complete
+      async_status:
+        jid: "{{ item.ansible_job_id }}"
+      register: gce_jobs
+      until: gce_jobs.finished
+      retries: 300
+      with_items: "{{ server.results }}"
+
+    # Mandatory configuration for Molecule to function.
+
+    - name: Populate instance config
+      set_fact:
+        instance_conf: {}
+
+    - name: Dump instance config
+      copy:
+        content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}"
+        dest: "{{ molecule_instance_config }}"
+      when: server.changed | bool

molecule/gce-centos-7/molecule.yml

@@ -0,0 +1,28 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: gce
+lint:
+  name: yamllint
+  enabled: False
+platforms:
+  - name: gce-centos-7
+    zone: europe-north1-a
+    machine_type: f1-micro
+    image: centos-7
+provisioner:
+  name: ansible
+  lint:
+    name: ansible-lint
+scenario:
+  name: gce-centos-7
+verifier:
+  name: testinfra
+  lint:
+    name: flake8
+ansible:
+  ansiblecfg_defaults:
+    force_color: true
+    roles_path: /drone/src
+    timeout: 60

molecule/gce-centos-7/playbook.yml

@@ -0,0 +1,8 @@
+---
+- name: Converge
+  hosts: all
+  vars:
+    nginx_default_page_enabled: True
+
+  roles:
+    - role: xoxys.nginx

molecule/gce-centos-7/prepare.yml

@@ -0,0 +1,9 @@
+---
+- name: Prepare
+  hosts: all
+  gather_facts: false
+  tasks:
+    - name: Install python for Ansible
+      raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal python-zipstream)
+      become: true
+      changed_when: false

molecule/gce-centos-7/tests/test_default.py

@@ -0,0 +1,30 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+import warnings
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_nginx_is_installed(host):
+    nginx = host.package("nginx")
+    assert nginx.is_installed
+
+def test_nginx_running_and_enabled(host):
+    nginx = host.service("nginx")
+    assert nginx.is_running
+    assert nginx.is_enabled
+
+def test_nginx_process(host):
+    # Verify worker procs are running
+    master = host.process.get(user="root", comm="nginx")
+    workers = host.process.filter(ppid=master.pid)
+    assert len(workers) > 0
+
+def test_nginx_socket(host):
+    # Verify the socket is listening for HTTP traffic
+    assert host.socket("tcp://0.0.0.0:80").is_listening
+

molecule/pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+filterwarnings =
+    ignore::DeprecationWarning

tasks/install.yml

@@ -36,7 +36,7 @@
         owner: "{{ nginx_user }}"
         group: "{{ nginx_group }}"
         mode: 0750
-      with_items:
+      loop:
         - "{{ nginx_vhosts_dir }}"
         - "{{ nginx_vhosts_dir }}/default"
 
@@ -47,7 +47,7 @@
         owner: root
         group: root
         mode: 0640
-      with_items:
+      loop:
         - /etc/nginx/sites-available
         - /etc/nginx/sites-enabled
 
@@ -88,7 +88,7 @@
         name: "{{ item.name }}"
         state: "{{ item.state }}"
         persistent: "{{ item.persistent }}"
-      with_items: "{{ nginx_set_sebooleans }}"
+      loop: "{{ nginx_set_sebooleans }}"
       when: nginx_set_sebooleans is defined
 
     - name: Fix selinux file context mappaing for pid file

tasks/tls.yml

@@ -1,24 +1,11 @@
 ---
 - block:
-    - name: Copy certs and private key (content)
-      copy:
-        content: "{{ item.src }}"
-        dest: "{{ item.dest }}"
-        mode: "{{ item.mode }}"
-      with_items:
-        - { src: "{{ nginx_tls_key_source }}", dest: '/etc/pki/tls/private/{{ nginx_tls_key_file }}', mode: '0600' }
-        - { src: "{{ nginx_tls_cert_source }}", dest: '/etc/pki/tls/certs/{{ nginx_tls_cert_file }}', mode: '0750' }
-      loop_control:
-        label: "{{ item.dest }}"
-      notify: __nginx_reload
-      when: nginx_tls_source_use_content
-
-    - name: Copy certs and private key (files)
+    - name: Copy certs and private key
       copy:
         src: "{{ item.src }}"
         dest: "{{ item.dest }}"
         mode: "{{ item.mode }}"
-      with_items:
+      loop:
         - { src: "{{ nginx_tls_key_source }}", dest: '/etc/pki/tls/private/{{ nginx_tls_key_file }}', mode: '0600' }
         - { src: "{{ nginx_tls_cert_source }}", dest: '/etc/pki/tls/certs/{{ nginx_tls_cert_file }}', mode: '0750' }
       loop_control: