refactor ci pipeline

.drone.jsonnet

@@ -1,109 +1,148 @@
-local AnsibleVersions(version="latest", package="ansible") = {
-    name: "ansible-" + version,
-    image: "python:3.7",
-    pull: "always",
-    environment: {
-      PY_COLORS: 1
-    },
-    commands: [
-      "pip install " + package + " ansible-later~=0.2.0 -qq",
-      "git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy",
-      "ansible-later -c ~/policy/config.yml"
-    ],
-    depends_on: [
-      "clone",
-    ],
-};
-
 local PipelineLinting = {
-  kind: "pipeline",
+    kind: "pipeline",
-  name: "linting",
+    name: "linting",
-  platform: {
+    platform: {
-    os: "linux",
+        os: "linux",
-    arch: "amd64",
+        arch: "amd64",
-  },
+    },
-  steps: [
+    steps: [
-    AnsibleVersions(version="latest", package="ansible"),
+        {
-    AnsibleVersions(version="master", package="git+https://github.com/ansible/ansible.git@devel"),
+            name: "ansible-later",
-  ],
+            image: "xoxys/ansible-later:latest",
-  trigger: {
+            environment: {
-    ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"],
+                PY_COLORS: 1
-  },
+            },
+            commands: [
+                "git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy",
+                "ansible-later -c ~/policy/config.yml"
+            ],
+        },
+    ],
+    trigger: {
+        ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"],
+    },
 };
 
 local PipelineDeployment = {
-  kind: "pipeline",
+    kind: "pipeline",
-  name: "deployment",
+    name: "deployment",
-  platform: {
+    platform: {
-    os: "linux",
+        os: "linux",
-    arch: "amd64",
+        arch: "amd64",
-  },
+    },
-  workspace: {
+    concurrency: {
-    base: "/drone/src",
+        limit: 1
-    path: "xoxys.certbot"
+    },
-  },
+    workspace: {
-  steps: [
+        base: "/drone/src",
-    {
+        path: "xoxys.certbot"
-      name: "molecule",
+    },
-      image: "xoxys/molecule:ec2-linux-amd64",
+    steps: [
-      pull: "always",
+        {
-      environment: {
+            name: "ansible-molecule",
-        ANSIBLE_ROLES_PATH: "/drone/src",
+            image: "xoxys/molecule:do-linux-amd64",
-        AWS_ACCESS_KEY_ID: { "from_secret": "aws_access_key_id" },
+            environment: {
-        AWS_SECRET_ACCESS_KEY: { "from_secret": "aws_secret_access_key" },
+                DO_API_KEY: { "from_secret": "do_api_key" },
-        AWS_REGION: "eu-central-1",
+                USER: "root",
-        MOLECULE_CUSTOM_MODULES_REPO: "https://gitea.rknet.org/ansible/custom_modules",
+                MOLECULE_CUSTOM_MODULES_REPO: "https://gitea.rknet.org/ansible/custom_modules",
-        MOLECULE_CUSTOM_FILTERS_REPO: "https://gitea.rknet.org/ansible/custom_filters",
+                MOLECULE_CUSTOM_FILTERS_REPO: "https://gitea.rknet.org/ansible/custom_filters",
-        PY_COLORS: 1
+                PY_COLORS: 1
-      },
+            },
-      commands: [
+            commands: [
-        "/bin/bash /docker-entrypoint.sh",
+                "/bin/bash /docker-entrypoint.sh",
-        "molecule test --scenario-name ec2-centos-7",
+                "molecule test -s default",
-      ],
+            ],
+        },
+    ],
+    depends_on: [
+        "linting",
+    ],
+    trigger: {
+        ref: ["refs/heads/master", "refs/tags/**"],
     },
-  ],
-  depends_on: [
-    "linting",
-  ],
-  trigger: {
-    ref: ["refs/heads/master", "refs/tags/**"],
-  },
 };
 
-local PipelineNotifications = {
+local PipelineDocumentation = {
-  kind: "pipeline",
+    kind: "pipeline",
-  name: "notifications",
+    name: "deployment",
-  platform: {
+    platform: {
-    os: "linux",
+        os: "linux",
-    arch: "amd64",
+        arch: "amd64",
-  },
+    },
-  clone: {
+    steps: [
-    disable: true,
+        {
-  },
+            name: "ansible-doctor",
-  steps: [
+            image: "xoxys/ansible-doctor:latest",
-    {
+            environment: {
-      name: "matrix",
+                ANSIBLE_DOCTOR_LOG_LEVEL: "INFO",
-      image: "plugins/matrix",
+                ANSIBLE_DOCTOR_FORCE_OVERWRITE: true,
-      settings: {
+                ANSIBLE_DOCTOR_EXCLUDE_FILES: "molecule/",
-        homeserver: "https://matrix.rknet.org",
+                ANSIBLE_DOCTOR_CUSTOM_HEADER: "HEADER.md",
-        roomid: "MtidqQXWWAtQcByBhH:rknet.org",
+                PY_COLORS: 1
-        template: "Status: **{{ build.status }}**<br/> Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}<br/> Message: {{ build.message }}",
+            },
-        username: { "from_secret": "matrix_username" },
+        },
-        password: { "from_secret": "matrix_password" },
+        {
-      },
+            name: "push-to-repo",
+            image: "plugins/git-action:latest",
+            settings: {
+                actions: ["commit", "push"],
+                author_email: "shipper@rknet.org",
+                author_name: "DroneShipper",
+                branch: "master",
+                message: "[SKIP CI] update readme",
+                remote: "https://gitea.rknet.org/ansible/xoxys.certbot",
+                netrc_machine: "gitea.rknet.org",
+                netrc_username: {"from_secret": "gitea_username"},
+                netrc_password: {"from_secret": "gitea_token"},
+            },
+            when: {
+                ref: ["refs/heads/master"],
+            },
+        },
+    ],
+    depends_on: [
+        "deployment",
+    ],
+    trigger: {
+        ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"],
+    },
+};
+
+local PipelineNotification= {
+    kind: "pipeline",
+    name: "notification",
+    platform: {
+        os: "linux",
+        arch: "amd64",
+    },
+    clone: {
+        disable: true,
+    },
+    steps: [
+        {
+            name: "matrix",
+            image: "plugins/matrix",
+            settings: {
+                homeserver: { "from_secret": "matrix_homeserver" },
+                roomid: { "from_secret": "matrix_roomid" },
+                template: "Status: **{{ build.status }}**<br/> Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}<br/> Message: {{ build.message }}",
+                username: { "from_secret": "matrix_username" },
+                password: { "from_secret": "matrix_password" },
+            },
+        },
+    ],
+    depends_on: [
+        "documentation",
+    ],
+    trigger: {
+        status: [ "success", "failure" ],
+        ref: ["refs/heads/master", "refs/tags/**"],
     },
-  ],
-  depends_on: [
-    "deployment",
-  ],
-  trigger: {
-    status: [ "success", "failure" ],
-    ref: ["refs/heads/master", "refs/tags/**"],
-  },
 };
 
 [
-  PipelineLinting,
+    PipelineLinting,
-  PipelineDeployment,
+    PipelineDeployment,
-  PipelineNotifications,
+    PipelineDocumentation,
+    PipelineNotification,
 ]

.drone.yml

@@ -7,29 +7,13 @@ platform:
   arch: amd64
 
 steps:
-- name: ansible-latest
+- name: ansible-later
-  pull: always
+  image: xoxys/ansible-later:latest
-  image: python:3.7
   commands:
-  - pip install ansible ansible-later~=0.2.0 -qq
   - git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy
   - ansible-later -c ~/policy/config.yml
   environment:
     PY_COLORS: 1
-  depends_on:
-  - clone
-
-- name: ansible-master
-  pull: always
-  image: python:3.7
-  commands:
-  - "pip install git+https://github.com/ansible/ansible.git@devel ansible-later~=0.2.0 -qq"
-  - git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy
-  - ansible-later -c ~/policy/config.yml
-  environment:
-    PY_COLORS: 1
-  depends_on:
-  - clone
 
 trigger:
   ref:
@@ -45,27 +29,26 @@ platform:
   os: linux
   arch: amd64
 
+concurrency:
+  limit: 1
+
 workspace:
   base: /drone/src
   path: xoxys.certbot
 
 steps:
-- name: molecule
+- name: ansible-molecule
-  pull: always
+  image: xoxys/molecule:do-linux-amd64
-  image: xoxys/molecule:ec2-linux-amd64
   commands:
   - /bin/bash /docker-entrypoint.sh
-  - molecule test --scenario-name ec2-centos-7
+  - molecule test -s default
   environment:
-    ANSIBLE_ROLES_PATH: /drone/src
+    DO_API_KEY:
-    AWS_ACCESS_KEY_ID:
+      from_secret: do_api_key
-      from_secret: aws_access_key_id
-    AWS_REGION: eu-central-1
-    AWS_SECRET_ACCESS_KEY:
-      from_secret: aws_secret_access_key
     MOLECULE_CUSTOM_FILTERS_REPO: https://gitea.rknet.org/ansible/custom_filters
     MOLECULE_CUSTOM_MODULES_REPO: https://gitea.rknet.org/ansible/custom_modules
     PY_COLORS: 1
+    USER: root
 
 trigger:
   ref:
@@ -77,7 +60,54 @@ depends_on:
 
 ---
 kind: pipeline
-name: notifications
+name: deployment
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: ansible-doctor
+  image: xoxys/ansible-doctor:latest
+  environment:
+    ANSIBLE_DOCTOR_CUSTOM_HEADER: HEADER.md
+    ANSIBLE_DOCTOR_EXCLUDE_FILES: molecule/
+    ANSIBLE_DOCTOR_FORCE_OVERWRITE: true
+    ANSIBLE_DOCTOR_LOG_LEVEL: INFO
+    PY_COLORS: 1
+
+- name: push-to-repo
+  image: plugins/git-action:latest
+  settings:
+    actions:
+    - commit
+    - push
+    author_email: "shipper@rknet.org"
+    author_name: DroneShipper
+    branch: master
+    message: "[SKIP CI] update readme"
+    netrc_machine: gitea.rknet.org
+    netrc_password:
+      from_secret: gitea_token
+    netrc_username:
+      from_secret: gitea_username
+    remote: https://gitea.rknet.org/ansible/xoxys.certbot
+  when:
+    ref:
+    - refs/heads/master
+
+trigger:
+  ref:
+  - refs/heads/master
+  - "refs/tags/**"
+  - "refs/pull/**"
+
+depends_on:
+- deployment
+
+---
+kind: pipeline
+name: notification
 
 platform:
   os: linux
@@ -90,10 +120,12 @@ steps:
 - name: matrix
   image: plugins/matrix
   settings:
-    homeserver: https://matrix.rknet.org
+    homeserver:
+      from_secret: matrix_homeserver
     password:
       from_secret: matrix_password
-    roomid: MtidqQXWWAtQcByBhH:rknet.org
+    roomid:
+      from_secret: matrix_roomid
     template: "Status: **{{ build.status }}**<br/> Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}<br/> Message: {{ build.message }}"
     username:
       from_secret: matrix_username
@@ -107,10 +139,10 @@ trigger:
   - failure
 
 depends_on:
-- deployment
+- documentation
 
 ---
 kind: signature
-hmac: c7037b785902327e7b303293aa1d31291f60e67d8c5f59de19b7f61f28fafbfd
+hmac: cea6f69f523634c4df5c480608e4ff72f9434c93ff9808f7a6fccc68c356bb73
 
 ...

.gitignore

@@ -1,3 +1,11 @@
 # ---> Ansible
 *.retry
+filter/plugins/
+library
+
+# ---> Python
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
 

molecule/default/create.yml

@@ -3,100 +3,66 @@
   hosts: localhost
   connection: local
   gather_facts: false
-  no_log: "{{ not (lookup('env', 'MOLECULE_DEBUG') | bool or molecule_yml.provisioner.log|default(false) | bool) }}"
+  no_log: "{{ molecule_no_log }}"
   vars:
-    ssh_user: ubuntu
+    ssh_user: root
     ssh_port: 22
 
-    security_group_name: molecule
-    security_group_description: Security group for testing Molecule
-    security_group_rules:
-      - proto: tcp
-        from_port: "{{ ssh_port }}"
-        to_port: "{{ ssh_port }}"
-        cidr_ip: '0.0.0.0/0'
-      - proto: icmp
-        from_port: 8
-        to_port: -1
-        cidr_ip: '0.0.0.0/0'
-    security_group_rules_egress:
-      - proto: -1
-        from_port: 0
-        to_port: 0
-        cidr_ip: '0.0.0.0/0'
-
     keypair_name: molecule_key
     keypair_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/ssh_key"
   tasks:
-    - name: Create security group
+    - name: Create local keypair
-      ec2_group:
+      user:
-        name: "{{ security_group_name }}"
+        name: "{{ lookup('env', 'USER') }}"
-        description: "{{ security_group_name }}"
+        generate_ssh_key: true
-        rules: "{{ security_group_rules }}"
+        ssh_key_file: "{{ keypair_path }}"
-        rules_egress: "{{ security_group_rules_egress }}"
+      register: local_keypair
 
-    - name: Test for presence of local keypair
+    - name: Create remote keypair
-      stat:
+      digital_ocean_sshkey:
-        path: "{{ keypair_path }}"
-      register: keypair_local
-
-    - name: Delete remote keypair
-      ec2_key:
         name: "{{ keypair_name }}"
-        state: absent
+        ssh_pub_key: "{{ local_keypair.ssh_public_key }}"
-      when: not keypair_local.stat.exists
+        state: present
-
+      register: remote_keypair
-    - name: Create keypair
-      ec2_key:
-        name: "{{ keypair_name }}"
-      register: keypair
-
-    - name: Persist the keypair
-      copy:
-        dest: "{{ keypair_path }}"
-        content: "{{ keypair.key.private_key }}"
-        mode: 0600
-      when: keypair.changed
 
     - name: Create molecule instance(s)
-      ec2:
+      digital_ocean_droplet:
-        key_name: "{{ keypair_name }}"
+        name: "{{ item.name }}"
-        image: "{{ item.image }}"
+        unique_name: true
-        instance_type: "{{ item.instance_type }}"
+        region: "{{ item.region_id }}"
-        vpc_subnet_id: "{{ item.vpc_subnet_id }}"
+        image: "{{ item.image_id }}"
-        group: "{{ security_group_name }}"
+        size: "{{ item.size_id }}"
-        instance_tags:
+        ssh_keys: "{{ remote_keypair.data.ssh_key.id }}"
-          instance: "{{ item.name }}"
         wait: true
-        assign_public_ip: true
+        wait_timeout: 300
-        exact_count: 1
+        state: present
-        count_tag:
-          instance: "{{ item.name }}"
       register: server
-      with_items: "{{ molecule_yml.platforms }}"
+      loop: "{{ molecule_yml.platforms }}"
       async: 7200
       poll: 0
 
     - name: Wait for instance(s) creation to complete
       async_status:
         jid: "{{ item.ansible_job_id }}"
-      register: ec2_jobs
+      register: digitalocean_jobs
-      until: ec2_jobs.finished
+      until: digitalocean_jobs.finished
       retries: 300
-      with_items: "{{ server.results }}"
+      loop: "{{ server.results }}"
 
     # Mandatory configuration for Molecule to function.
 
     - name: Populate instance config dict
       set_fact:
         instance_conf_dict: {
-          'instance': "{{ item.instances[0].tags.instance }}",
+          'instance': "{{ item.data.droplet.name }}",
-          'address': "{{ item.instances[0].public_ip }}",
+          'address': "{{ item.data.ip_address }}",
           'user': "{{ ssh_user }}",
           'port': "{{ ssh_port }}",
           'identity_file': "{{ keypair_path }}",
-          'instance_ids': "{{ item.instance_ids }}", }
+          'droplet_id': "{{ item.data.droplet.id }}",
-      with_items: "{{ ec2_jobs.results }}"
+          'ssh_key_id': "{{ remote_keypair.data.ssh_key.id }}",
+        }
+      loop: "{{ digitalocean_jobs.results }}"
       register: instance_config_dict
       when: server.changed | bool
 
@@ -118,8 +84,4 @@
         search_regex: SSH
         delay: 10
         timeout: 320
-      with_items: "{{ lookup('file', molecule_instance_config) | molecule_from_yaml }}"
+      loop: "{{ lookup('file', molecule_instance_config) | molecule_from_yaml }}"
-
-    - name: Wait for boot process to finish
-      pause:
-        minutes: 2

molecule/default/destroy.yml

@@ -3,7 +3,7 @@
   hosts: localhost
   connection: local
   gather_facts: false
-  no_log: "{{ not (lookup('env', 'MOLECULE_DEBUG') | bool or molecule_yml.provisioner.log|default(false) | bool) }}"
+  no_log: "{{ molecule_no_log }}"
   tasks:
     - block:
         - name: Populate instance config
@@ -17,11 +17,12 @@
             skip_instances: true
 
     - name: Destroy molecule instance(s)
-      ec2:
+      digital_ocean_droplet:
+        name: "{{ item.instance }}"
+        id: "{{ item.droplet_id }}"
         state: absent
-        instance_ids: "{{ item.instance_ids }}"
       register: server
-      with_items: "{{ instance_conf }}"
+      loop: "{{ instance_conf | flatten(levels=1) }}"
       when: not skip_instances
       async: 7200
       poll: 0
@@ -29,10 +30,16 @@
     - name: Wait for instance(s) deletion to complete
       async_status:
         jid: "{{ item.ansible_job_id }}"
-      register: ec2_jobs
+      register: digitalocean_jobs
-      until: ec2_jobs.finished
+      until: digitalocean_jobs.finished
       retries: 300
-      with_items: "{{ server.results }}"
+      loop: "{{ server.results }}"
+
+    - name: Delete remote keypair
+      digital_ocean_sshkey:
+        fingerprint: "{{ item.ssh_key_id }}"
+        state: absent
+      loop: "{{ instance_conf | flatten(levels=1) }}"
 
     # Mandatory configuration for Molecule to function.
 
@@ -42,6 +49,6 @@
 
     - name: Dump instance config
       copy:
-        content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}"
+        content: "{{ instance_conf | molecule_to_yaml | molecule_header }}"
         dest: "{{ molecule_instance_config }}"
       when: server.changed | bool

molecule/default/molecule.yml

@@ -2,18 +2,20 @@
 dependency:
   name: galaxy
 driver:
-  name: ec2
+  name: digitalocean
+platforms:
+  - name: centos7-certbot
+    region_id: fra1
+    image_id: centos-7-x64
+    size_id: s-1vcpu-1gb
 lint:
   name: yamllint
-platforms:
+  enabled: False
-  - name: instance
-    image: ami-a5b196c0
-    instance_type: t2.micro
-    vpc_subnet_id: subnet-6456fd1f
 provisioner:
   name: ansible
   lint:
     name: ansible-lint
+    enabled: False
 verifier:
   name: testinfra
   lint:

molecule/default/playbook.yml

@@ -1,5 +1,9 @@
 ---
 - name: Converge
   hosts: all
+  vars:
+    certbot_packages_extra:
+      - epel-release
+
   roles:
     - role: xoxys.certbot

molecule/default/prepare.yml

@@ -4,6 +4,6 @@
   gather_facts: false
   tasks:
     - name: Install python for Ansible
-      raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal python-zipstream)
+      raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal)
       become: true
       changed_when: false

molecule/default/tests/test_default.py

@@ -2,13 +2,13 @@ import os
 
 import testinfra.utils.ansible_runner
 
+import warnings
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+
 testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
     os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
 
 
-def test_hosts_file(host):
+def test_certbot_is_installed(host):
-    f = host.file('/etc/hosts')
+    certbot = host.package("certbot")
-
+    assert certbot.is_installed
-    assert f.exists
-    assert f.user == 'root'
-    assert f.group == 'root'