refactor ci pipeline

2019-10-15 23:23:56 +02:00 · 2019-10-15 23:23:56 +02:00 · 3411598c1c
commit 3411598c1c
parent 3488c92b54
9 changed files with 276 additions and 222 deletions
--- a/.drone.jsonnet
+++ b/.drone.jsonnet
@ -1,109 +1,148 @@
-local AnsibleVersions(version="latest", package="ansible") = {
-    name: "ansible-" + version,
-    image: "python:3.7",
-    pull: "always",
-    environment: {
-      PY_COLORS: 1
-    },
-    commands: [
-      "pip install " + package + " ansible-later~=0.2.0 -qq",
-      "git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy",
-      "ansible-later -c ~/policy/config.yml"
-    ],
-    depends_on: [
-      "clone",
-    ],
-};
-
 local PipelineLinting = {
-  kind: "pipeline",
-  name: "linting",
-  platform: {
-    os: "linux",
-    arch: "amd64",
-  },
-  steps: [
-    AnsibleVersions(version="latest", package="ansible"),
-    AnsibleVersions(version="master", package="git+https://github.com/ansible/ansible.git@devel"),
-  ],
-  trigger: {
-    ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"],
-  },
+    kind: "pipeline",
+    name: "linting",
+    platform: {
+        os: "linux",
+        arch: "amd64",
+    },
+    steps: [
+        {
+            name: "ansible-later",
+            image: "xoxys/ansible-later:latest",
+            environment: {
+                PY_COLORS: 1
+            },
+            commands: [
+                "git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy",
+                "ansible-later -c ~/policy/config.yml"
+            ],
+        },
+    ],
+    trigger: {
+        ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"],
+    },
 };

 local PipelineDeployment = {
-  kind: "pipeline",
-  name: "deployment",
-  platform: {
-    os: "linux",
-    arch: "amd64",
-  },
-  workspace: {
-    base: "/drone/src",
-    path: "xoxys.certbot"
-  },
-  steps: [
-    {
-      name: "molecule",
-      image: "xoxys/molecule:ec2-linux-amd64",
-      pull: "always",
-      environment: {
-        ANSIBLE_ROLES_PATH: "/drone/src",
-        AWS_ACCESS_KEY_ID: { "from_secret": "aws_access_key_id" },
-        AWS_SECRET_ACCESS_KEY: { "from_secret": "aws_secret_access_key" },
-        AWS_REGION: "eu-central-1",
-        MOLECULE_CUSTOM_MODULES_REPO: "https://gitea.rknet.org/ansible/custom_modules",
-        MOLECULE_CUSTOM_FILTERS_REPO: "https://gitea.rknet.org/ansible/custom_filters",
-        PY_COLORS: 1
-      },
-      commands: [
-        "/bin/bash /docker-entrypoint.sh",
-        "molecule test --scenario-name ec2-centos-7",
-      ],
+    kind: "pipeline",
+    name: "deployment",
+    platform: {
+        os: "linux",
+        arch: "amd64",
+    },
+    concurrency: {
+        limit: 1
+    },
+    workspace: {
+        base: "/drone/src",
+        path: "xoxys.certbot"
+    },
+    steps: [
+        {
+            name: "ansible-molecule",
+            image: "xoxys/molecule:do-linux-amd64",
+            environment: {
+                DO_API_KEY: { "from_secret": "do_api_key" },
+                USER: "root",
+                MOLECULE_CUSTOM_MODULES_REPO: "https://gitea.rknet.org/ansible/custom_modules",
+                MOLECULE_CUSTOM_FILTERS_REPO: "https://gitea.rknet.org/ansible/custom_filters",
+                PY_COLORS: 1
+            },
+            commands: [
+                "/bin/bash /docker-entrypoint.sh",
+                "molecule test -s default",
+            ],
+        },
+    ],
+    depends_on: [
+        "linting",
+    ],
+    trigger: {
+        ref: ["refs/heads/master", "refs/tags/**"],
    },
-  ],
-  depends_on: [
-    "linting",
-  ],
-  trigger: {
-    ref: ["refs/heads/master", "refs/tags/**"],
-  },
 };

-local PipelineNotifications = {
-  kind: "pipeline",
-  name: "notifications",
-  platform: {
-    os: "linux",
-    arch: "amd64",
-  },
-  clone: {
-    disable: true,
-  },
-  steps: [
-    {
-      name: "matrix",
-      image: "plugins/matrix",
-      settings: {
-        homeserver: "https://matrix.rknet.org",
-        roomid: "MtidqQXWWAtQcByBhH:rknet.org",
-        template: "Status: **{{ build.status }}**<br/> Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}<br/> Message: {{ build.message }}",
-        username: { "from_secret": "matrix_username" },
-        password: { "from_secret": "matrix_password" },
-      },
+local PipelineDocumentation = {
+    kind: "pipeline",
+    name: "deployment",
+    platform: {
+        os: "linux",
+        arch: "amd64",
+    },
+    steps: [
+        {
+            name: "ansible-doctor",
+            image: "xoxys/ansible-doctor:latest",
+            environment: {
+                ANSIBLE_DOCTOR_LOG_LEVEL: "INFO",
+                ANSIBLE_DOCTOR_FORCE_OVERWRITE: true,
+                ANSIBLE_DOCTOR_EXCLUDE_FILES: "molecule/",
+                ANSIBLE_DOCTOR_CUSTOM_HEADER: "HEADER.md",
+                PY_COLORS: 1
+            },
+        },
+        {
+            name: "push-to-repo",
+            image: "plugins/git-action:latest",
+            settings: {
+                actions: ["commit", "push"],
+                author_email: "shipper@rknet.org",
+                author_name: "DroneShipper",
+                branch: "master",
+                message: "[SKIP CI] update readme",
+                remote: "https://gitea.rknet.org/ansible/xoxys.certbot",
+                netrc_machine: "gitea.rknet.org",
+                netrc_username: {"from_secret": "gitea_username"},
+                netrc_password: {"from_secret": "gitea_token"},
+            },
+            when: {
+                ref: ["refs/heads/master"],
+            },
+        },
+    ],
+    depends_on: [
+        "deployment",
+    ],
+    trigger: {
+        ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"],
+    },
+};
+
+local PipelineNotification= {
+    kind: "pipeline",
+    name: "notification",
+    platform: {
+        os: "linux",
+        arch: "amd64",
+    },
+    clone: {
+        disable: true,
+    },
+    steps: [
+        {
+            name: "matrix",
+            image: "plugins/matrix",
+            settings: {
+                homeserver: { "from_secret": "matrix_homeserver" },
+                roomid: { "from_secret": "matrix_roomid" },
+                template: "Status: **{{ build.status }}**<br/> Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}<br/> Message: {{ build.message }}",
+                username: { "from_secret": "matrix_username" },
+                password: { "from_secret": "matrix_password" },
+            },
+        },
+    ],
+    depends_on: [
+        "documentation",
+    ],
+    trigger: {
+        status: [ "success", "failure" ],
+        ref: ["refs/heads/master", "refs/tags/**"],
    },
-  ],
-  depends_on: [
-    "deployment",
-  ],
-  trigger: {
-    status: [ "success", "failure" ],
-    ref: ["refs/heads/master", "refs/tags/**"],
-  },
 };

 [
-  PipelineLinting,
-  PipelineDeployment,
-  PipelineNotifications,
+    PipelineLinting,
+    PipelineDeployment,
+    PipelineDocumentation,
+    PipelineNotification,
 ]
--- a/.drone.yml
+++ b/.drone.yml
@ -7,29 +7,13 @@ platform:
  arch: amd64

 steps:
- name: ansible-latest
-  pull: always
-  image: python:3.7
+- name: ansible-later
+  image: xoxys/ansible-later:latest
  commands:
-  - pip install ansible ansible-later~=0.2.0 -qq
  - git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy
  - ansible-later -c ~/policy/config.yml
  environment:
    PY_COLORS: 1
-  depends_on:
-  - clone
-
- name: ansible-master
-  pull: always
-  image: python:3.7
-  commands:
-  - "pip install git+https://github.com/ansible/ansible.git@devel ansible-later~=0.2.0 -qq"
-  - git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy
-  - ansible-later -c ~/policy/config.yml
-  environment:
-    PY_COLORS: 1
-  depends_on:
-  - clone

 trigger:
  ref:
@ -45,27 +29,26 @@ platform:
  os: linux
  arch: amd64

+concurrency:
+  limit: 1
+
 workspace:
  base: /drone/src
  path: xoxys.certbot

 steps:
- name: molecule
-  pull: always
-  image: xoxys/molecule:ec2-linux-amd64
+- name: ansible-molecule
+  image: xoxys/molecule:do-linux-amd64
  commands:
  - /bin/bash /docker-entrypoint.sh
-  - molecule test --scenario-name ec2-centos-7
+  - molecule test -s default
  environment:
-    ANSIBLE_ROLES_PATH: /drone/src
-    AWS_ACCESS_KEY_ID:
-      from_secret: aws_access_key_id
-    AWS_REGION: eu-central-1
-    AWS_SECRET_ACCESS_KEY:
-      from_secret: aws_secret_access_key
+    DO_API_KEY:
+      from_secret: do_api_key
    MOLECULE_CUSTOM_FILTERS_REPO: https://gitea.rknet.org/ansible/custom_filters
    MOLECULE_CUSTOM_MODULES_REPO: https://gitea.rknet.org/ansible/custom_modules
    PY_COLORS: 1
+    USER: root

 trigger:
  ref:
@ -77,7 +60,54 @@ depends_on:

 ---
 kind: pipeline
-name: notifications
+name: deployment
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: ansible-doctor
+  image: xoxys/ansible-doctor:latest
+  environment:
+    ANSIBLE_DOCTOR_CUSTOM_HEADER: HEADER.md
+    ANSIBLE_DOCTOR_EXCLUDE_FILES: molecule/
+    ANSIBLE_DOCTOR_FORCE_OVERWRITE: true
+    ANSIBLE_DOCTOR_LOG_LEVEL: INFO
+    PY_COLORS: 1
+
+- name: push-to-repo
+  image: plugins/git-action:latest
+  settings:
+    actions:
+    - commit
+    - push
+    author_email: "shipper@rknet.org"
+    author_name: DroneShipper
+    branch: master
+    message: "[SKIP CI] update readme"
+    netrc_machine: gitea.rknet.org
+    netrc_password:
+      from_secret: gitea_token
+    netrc_username:
+      from_secret: gitea_username
+    remote: https://gitea.rknet.org/ansible/xoxys.certbot
+  when:
+    ref:
+    - refs/heads/master
+
+trigger:
+  ref:
+  - refs/heads/master
+  - "refs/tags/**"
+  - "refs/pull/**"
+
+depends_on:
+- deployment
+
+---
+kind: pipeline
+name: notification

 platform:
  os: linux
@ -90,10 +120,12 @@ steps:
 - name: matrix
  image: plugins/matrix
  settings:
-    homeserver: https://matrix.rknet.org
+    homeserver:
+      from_secret: matrix_homeserver
    password:
      from_secret: matrix_password
-    roomid: MtidqQXWWAtQcByBhH:rknet.org
+    roomid:
+      from_secret: matrix_roomid
    template: "Status: **{{ build.status }}**<br/> Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}<br/> Message: {{ build.message }}"
    username:
      from_secret: matrix_username
@ -107,10 +139,10 @@ trigger:
  - failure

 depends_on:
- deployment
+- documentation

 ---
 kind: signature
-hmac: c7037b785902327e7b303293aa1d31291f60e67d8c5f59de19b7f61f28fafbfd
+hmac: cea6f69f523634c4df5c480608e4ff72f9434c93ff9808f7a6fccc68c356bb73

 ...
--- a/.gitignore
+++ b/.gitignore
@ -1,3 +1,11 @@
 # ---> Ansible
 *.retry
+filter/plugins/
+library
+
+# ---> Python
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class

--- a/molecule/default/create.yml
+++ b/molecule/default/create.yml
@ -3,100 +3,66 @@
  hosts: localhost
  connection: local
  gather_facts: false
-  no_log: "{{ not (lookup('env', 'MOLECULE_DEBUG') | bool or molecule_yml.provisioner.log|default(false) | bool) }}"
+  no_log: "{{ molecule_no_log }}"
  vars:
-    ssh_user: ubuntu
+    ssh_user: root
    ssh_port: 22

-    security_group_name: molecule
-    security_group_description: Security group for testing Molecule
-    security_group_rules:
-      - proto: tcp
-        from_port: "{{ ssh_port }}"
-        to_port: "{{ ssh_port }}"
-        cidr_ip: '0.0.0.0/0'
-      - proto: icmp
-        from_port: 8
-        to_port: -1
-        cidr_ip: '0.0.0.0/0'
-    security_group_rules_egress:
-      - proto: -1
-        from_port: 0
-        to_port: 0
-        cidr_ip: '0.0.0.0/0'
-
    keypair_name: molecule_key
    keypair_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/ssh_key"
  tasks:
-    - name: Create security group
-      ec2_group:
-        name: "{{ security_group_name }}"
-        description: "{{ security_group_name }}"
-        rules: "{{ security_group_rules }}"
-        rules_egress: "{{ security_group_rules_egress }}"
+    - name: Create local keypair
+      user:
+        name: "{{ lookup('env', 'USER') }}"
+        generate_ssh_key: true
+        ssh_key_file: "{{ keypair_path }}"
+      register: local_keypair

-    - name: Test for presence of local keypair
-      stat:
-        path: "{{ keypair_path }}"
-      register: keypair_local
-
-    - name: Delete remote keypair
-      ec2_key:
+    - name: Create remote keypair
+      digital_ocean_sshkey:
        name: "{{ keypair_name }}"
-        state: absent
-      when: not keypair_local.stat.exists
-
-    - name: Create keypair
-      ec2_key:
-        name: "{{ keypair_name }}"
-      register: keypair
-
-    - name: Persist the keypair
-      copy:
-        dest: "{{ keypair_path }}"
-        content: "{{ keypair.key.private_key }}"
-        mode: 0600
-      when: keypair.changed
+        ssh_pub_key: "{{ local_keypair.ssh_public_key }}"
+        state: present
+      register: remote_keypair

    - name: Create molecule instance(s)
-      ec2:
-        key_name: "{{ keypair_name }}"
-        image: "{{ item.image }}"
-        instance_type: "{{ item.instance_type }}"
-        vpc_subnet_id: "{{ item.vpc_subnet_id }}"
-        group: "{{ security_group_name }}"
-        instance_tags:
-          instance: "{{ item.name }}"
+      digital_ocean_droplet:
+        name: "{{ item.name }}"
+        unique_name: true
+        region: "{{ item.region_id }}"
+        image: "{{ item.image_id }}"
+        size: "{{ item.size_id }}"
+        ssh_keys: "{{ remote_keypair.data.ssh_key.id }}"
        wait: true
-        assign_public_ip: true
-        exact_count: 1
-        count_tag:
-          instance: "{{ item.name }}"
+        wait_timeout: 300
+        state: present
      register: server
-      with_items: "{{ molecule_yml.platforms }}"
+      loop: "{{ molecule_yml.platforms }}"
      async: 7200
      poll: 0

    - name: Wait for instance(s) creation to complete
      async_status:
        jid: "{{ item.ansible_job_id }}"
-      register: ec2_jobs
-      until: ec2_jobs.finished
+      register: digitalocean_jobs
+      until: digitalocean_jobs.finished
      retries: 300
-      with_items: "{{ server.results }}"
+      loop: "{{ server.results }}"

    # Mandatory configuration for Molecule to function.

    - name: Populate instance config dict
      set_fact:
        instance_conf_dict: {
-          'instance': "{{ item.instances[0].tags.instance }}",
-          'address': "{{ item.instances[0].public_ip }}",
+          'instance': "{{ item.data.droplet.name }}",
+          'address': "{{ item.data.ip_address }}",
          'user': "{{ ssh_user }}",
          'port': "{{ ssh_port }}",
          'identity_file': "{{ keypair_path }}",
-          'instance_ids': "{{ item.instance_ids }}", }
-      with_items: "{{ ec2_jobs.results }}"
+          'droplet_id': "{{ item.data.droplet.id }}",
+          'ssh_key_id': "{{ remote_keypair.data.ssh_key.id }}",
+        }
+      loop: "{{ digitalocean_jobs.results }}"
      register: instance_config_dict
      when: server.changed | bool

@ -118,8 +84,4 @@
        search_regex: SSH
        delay: 10
        timeout: 320
-      with_items: "{{ lookup('file', molecule_instance_config) | molecule_from_yaml }}"
-
-    - name: Wait for boot process to finish
-      pause:
-        minutes: 2
+      loop: "{{ lookup('file', molecule_instance_config) | molecule_from_yaml }}"
--- a/molecule/default/destroy.yml
+++ b/molecule/default/destroy.yml
@ -3,7 +3,7 @@
  hosts: localhost
  connection: local
  gather_facts: false
-  no_log: "{{ not (lookup('env', 'MOLECULE_DEBUG') | bool or molecule_yml.provisioner.log|default(false) | bool) }}"
+  no_log: "{{ molecule_no_log }}"
  tasks:
    - block:
        - name: Populate instance config
@ -17,11 +17,12 @@
            skip_instances: true

    - name: Destroy molecule instance(s)
-      ec2:
+      digital_ocean_droplet:
+        name: "{{ item.instance }}"
+        id: "{{ item.droplet_id }}"
        state: absent
-        instance_ids: "{{ item.instance_ids }}"
      register: server
-      with_items: "{{ instance_conf }}"
+      loop: "{{ instance_conf | flatten(levels=1) }}"
      when: not skip_instances
      async: 7200
      poll: 0
@ -29,10 +30,16 @@
    - name: Wait for instance(s) deletion to complete
      async_status:
        jid: "{{ item.ansible_job_id }}"
-      register: ec2_jobs
-      until: ec2_jobs.finished
+      register: digitalocean_jobs
+      until: digitalocean_jobs.finished
      retries: 300
-      with_items: "{{ server.results }}"
+      loop: "{{ server.results }}"
+
+    - name: Delete remote keypair
+      digital_ocean_sshkey:
+        fingerprint: "{{ item.ssh_key_id }}"
+        state: absent
+      loop: "{{ instance_conf | flatten(levels=1) }}"

    # Mandatory configuration for Molecule to function.

@ -42,6 +49,6 @@

    - name: Dump instance config
      copy:
-        content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}"
+        content: "{{ instance_conf | molecule_to_yaml | molecule_header }}"
        dest: "{{ molecule_instance_config }}"
      when: server.changed | bool
--- a/molecule/default/molecule.yml
+++ b/molecule/default/molecule.yml
@ -2,18 +2,20 @@
 dependency:
  name: galaxy
 driver:
-  name: ec2
+  name: digitalocean
+platforms:
+  - name: centos7-certbot
+    region_id: fra1
+    image_id: centos-7-x64
+    size_id: s-1vcpu-1gb
 lint:
  name: yamllint
-platforms:
-  - name: instance
-    image: ami-a5b196c0
-    instance_type: t2.micro
-    vpc_subnet_id: subnet-6456fd1f
+  enabled: False
 provisioner:
  name: ansible
  lint:
    name: ansible-lint
+    enabled: False
 verifier:
  name: testinfra
  lint:
--- a/molecule/default/playbook.yml
+++ b/molecule/default/playbook.yml
@ -1,5 +1,9 @@
 ---
 - name: Converge
  hosts: all
+  vars:
+    certbot_packages_extra:
+      - epel-release
+
  roles:
    - role: xoxys.certbot
--- a/molecule/default/prepare.yml
+++ b/molecule/default/prepare.yml
@ -4,6 +4,6 @@
  gather_facts: false
  tasks:
    - name: Install python for Ansible
-      raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal python-zipstream)
+      raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal)
      become: true
      changed_when: false
--- a/molecule/default/tests/test_default.py
+++ b/molecule/default/tests/test_default.py
@ -2,13 +2,13 @@ import os

 import testinfra.utils.ansible_runner

+import warnings
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+
 testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')


-def test_hosts_file(host):
-    f = host.file('/etc/hosts')
-
-    assert f.exists
-    assert f.user == 'root'
-    assert f.group == 'root'
+def test_certbot_is_installed(host):
+    certbot = host.package("certbot")
+    assert certbot.is_installed