refactor ci pipeline

This commit is contained in:
Robert Kaussow 2019-10-15 23:23:56 +02:00
parent 3488c92b54
commit 3411598c1c
9 changed files with 276 additions and 222 deletions

View File

@ -1,109 +1,148 @@
local AnsibleVersions(version="latest", package="ansible") = {
name: "ansible-" + version,
image: "python:3.7",
pull: "always",
environment: {
PY_COLORS: 1
},
commands: [
"pip install " + package + " ansible-later~=0.2.0 -qq",
"git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy",
"ansible-later -c ~/policy/config.yml"
],
depends_on: [
"clone",
],
};
local PipelineLinting = { local PipelineLinting = {
kind: "pipeline", kind: "pipeline",
name: "linting", name: "linting",
platform: { platform: {
os: "linux", os: "linux",
arch: "amd64", arch: "amd64",
}, },
steps: [ steps: [
AnsibleVersions(version="latest", package="ansible"), {
AnsibleVersions(version="master", package="git+https://github.com/ansible/ansible.git@devel"), name: "ansible-later",
], image: "xoxys/ansible-later:latest",
trigger: { environment: {
ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"], PY_COLORS: 1
}, },
commands: [
"git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy",
"ansible-later -c ~/policy/config.yml"
],
},
],
trigger: {
ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"],
},
}; };
local PipelineDeployment = { local PipelineDeployment = {
kind: "pipeline", kind: "pipeline",
name: "deployment", name: "deployment",
platform: { platform: {
os: "linux", os: "linux",
arch: "amd64", arch: "amd64",
}, },
workspace: { concurrency: {
base: "/drone/src", limit: 1
path: "xoxys.certbot" },
}, workspace: {
steps: [ base: "/drone/src",
{ path: "xoxys.certbot"
name: "molecule", },
image: "xoxys/molecule:ec2-linux-amd64", steps: [
pull: "always", {
environment: { name: "ansible-molecule",
ANSIBLE_ROLES_PATH: "/drone/src", image: "xoxys/molecule:do-linux-amd64",
AWS_ACCESS_KEY_ID: { "from_secret": "aws_access_key_id" }, environment: {
AWS_SECRET_ACCESS_KEY: { "from_secret": "aws_secret_access_key" }, DO_API_KEY: { "from_secret": "do_api_key" },
AWS_REGION: "eu-central-1", USER: "root",
MOLECULE_CUSTOM_MODULES_REPO: "https://gitea.rknet.org/ansible/custom_modules", MOLECULE_CUSTOM_MODULES_REPO: "https://gitea.rknet.org/ansible/custom_modules",
MOLECULE_CUSTOM_FILTERS_REPO: "https://gitea.rknet.org/ansible/custom_filters", MOLECULE_CUSTOM_FILTERS_REPO: "https://gitea.rknet.org/ansible/custom_filters",
PY_COLORS: 1 PY_COLORS: 1
}, },
commands: [ commands: [
"/bin/bash /docker-entrypoint.sh", "/bin/bash /docker-entrypoint.sh",
"molecule test --scenario-name ec2-centos-7", "molecule test -s default",
], ],
},
],
depends_on: [
"linting",
],
trigger: {
ref: ["refs/heads/master", "refs/tags/**"],
}, },
],
depends_on: [
"linting",
],
trigger: {
ref: ["refs/heads/master", "refs/tags/**"],
},
}; };
local PipelineNotifications = { local PipelineDocumentation = {
kind: "pipeline", kind: "pipeline",
name: "notifications", name: "deployment",
platform: { platform: {
os: "linux", os: "linux",
arch: "amd64", arch: "amd64",
}, },
clone: { steps: [
disable: true, {
}, name: "ansible-doctor",
steps: [ image: "xoxys/ansible-doctor:latest",
{ environment: {
name: "matrix", ANSIBLE_DOCTOR_LOG_LEVEL: "INFO",
image: "plugins/matrix", ANSIBLE_DOCTOR_FORCE_OVERWRITE: true,
settings: { ANSIBLE_DOCTOR_EXCLUDE_FILES: "molecule/",
homeserver: "https://matrix.rknet.org", ANSIBLE_DOCTOR_CUSTOM_HEADER: "HEADER.md",
roomid: "MtidqQXWWAtQcByBhH:rknet.org", PY_COLORS: 1
template: "Status: **{{ build.status }}**<br/> Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}<br/> Message: {{ build.message }}", },
username: { "from_secret": "matrix_username" }, },
password: { "from_secret": "matrix_password" }, {
}, name: "push-to-repo",
image: "plugins/git-action:latest",
settings: {
actions: ["commit", "push"],
author_email: "shipper@rknet.org",
author_name: "DroneShipper",
branch: "master",
message: "[SKIP CI] update readme",
remote: "https://gitea.rknet.org/ansible/xoxys.certbot",
netrc_machine: "gitea.rknet.org",
netrc_username: {"from_secret": "gitea_username"},
netrc_password: {"from_secret": "gitea_token"},
},
when: {
ref: ["refs/heads/master"],
},
},
],
depends_on: [
"deployment",
],
trigger: {
ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"],
},
};
local PipelineNotification= {
kind: "pipeline",
name: "notification",
platform: {
os: "linux",
arch: "amd64",
},
clone: {
disable: true,
},
steps: [
{
name: "matrix",
image: "plugins/matrix",
settings: {
homeserver: { "from_secret": "matrix_homeserver" },
roomid: { "from_secret": "matrix_roomid" },
template: "Status: **{{ build.status }}**<br/> Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}<br/> Message: {{ build.message }}",
username: { "from_secret": "matrix_username" },
password: { "from_secret": "matrix_password" },
},
},
],
depends_on: [
"documentation",
],
trigger: {
status: [ "success", "failure" ],
ref: ["refs/heads/master", "refs/tags/**"],
}, },
],
depends_on: [
"deployment",
],
trigger: {
status: [ "success", "failure" ],
ref: ["refs/heads/master", "refs/tags/**"],
},
}; };
[ [
PipelineLinting, PipelineLinting,
PipelineDeployment, PipelineDeployment,
PipelineNotifications, PipelineDocumentation,
PipelineNotification,
] ]

View File

@ -7,29 +7,13 @@ platform:
arch: amd64 arch: amd64
steps: steps:
- name: ansible-latest - name: ansible-later
pull: always image: xoxys/ansible-later:latest
image: python:3.7
commands: commands:
- pip install ansible ansible-later~=0.2.0 -qq
- git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy - git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy
- ansible-later -c ~/policy/config.yml - ansible-later -c ~/policy/config.yml
environment: environment:
PY_COLORS: 1 PY_COLORS: 1
depends_on:
- clone
- name: ansible-master
pull: always
image: python:3.7
commands:
- "pip install git+https://github.com/ansible/ansible.git@devel ansible-later~=0.2.0 -qq"
- git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy
- ansible-later -c ~/policy/config.yml
environment:
PY_COLORS: 1
depends_on:
- clone
trigger: trigger:
ref: ref:
@ -45,27 +29,26 @@ platform:
os: linux os: linux
arch: amd64 arch: amd64
concurrency:
limit: 1
workspace: workspace:
base: /drone/src base: /drone/src
path: xoxys.certbot path: xoxys.certbot
steps: steps:
- name: molecule - name: ansible-molecule
pull: always image: xoxys/molecule:do-linux-amd64
image: xoxys/molecule:ec2-linux-amd64
commands: commands:
- /bin/bash /docker-entrypoint.sh - /bin/bash /docker-entrypoint.sh
- molecule test --scenario-name ec2-centos-7 - molecule test -s default
environment: environment:
ANSIBLE_ROLES_PATH: /drone/src DO_API_KEY:
AWS_ACCESS_KEY_ID: from_secret: do_api_key
from_secret: aws_access_key_id
AWS_REGION: eu-central-1
AWS_SECRET_ACCESS_KEY:
from_secret: aws_secret_access_key
MOLECULE_CUSTOM_FILTERS_REPO: https://gitea.rknet.org/ansible/custom_filters MOLECULE_CUSTOM_FILTERS_REPO: https://gitea.rknet.org/ansible/custom_filters
MOLECULE_CUSTOM_MODULES_REPO: https://gitea.rknet.org/ansible/custom_modules MOLECULE_CUSTOM_MODULES_REPO: https://gitea.rknet.org/ansible/custom_modules
PY_COLORS: 1 PY_COLORS: 1
USER: root
trigger: trigger:
ref: ref:
@ -77,7 +60,54 @@ depends_on:
--- ---
kind: pipeline kind: pipeline
name: notifications name: deployment
platform:
os: linux
arch: amd64
steps:
- name: ansible-doctor
image: xoxys/ansible-doctor:latest
environment:
ANSIBLE_DOCTOR_CUSTOM_HEADER: HEADER.md
ANSIBLE_DOCTOR_EXCLUDE_FILES: molecule/
ANSIBLE_DOCTOR_FORCE_OVERWRITE: true
ANSIBLE_DOCTOR_LOG_LEVEL: INFO
PY_COLORS: 1
- name: push-to-repo
image: plugins/git-action:latest
settings:
actions:
- commit
- push
author_email: "shipper@rknet.org"
author_name: DroneShipper
branch: master
message: "[SKIP CI] update readme"
netrc_machine: gitea.rknet.org
netrc_password:
from_secret: gitea_token
netrc_username:
from_secret: gitea_username
remote: https://gitea.rknet.org/ansible/xoxys.certbot
when:
ref:
- refs/heads/master
trigger:
ref:
- refs/heads/master
- "refs/tags/**"
- "refs/pull/**"
depends_on:
- deployment
---
kind: pipeline
name: notification
platform: platform:
os: linux os: linux
@ -90,10 +120,12 @@ steps:
- name: matrix - name: matrix
image: plugins/matrix image: plugins/matrix
settings: settings:
homeserver: https://matrix.rknet.org homeserver:
from_secret: matrix_homeserver
password: password:
from_secret: matrix_password from_secret: matrix_password
roomid: MtidqQXWWAtQcByBhH:rknet.org roomid:
from_secret: matrix_roomid
template: "Status: **{{ build.status }}**<br/> Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}<br/> Message: {{ build.message }}" template: "Status: **{{ build.status }}**<br/> Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}<br/> Message: {{ build.message }}"
username: username:
from_secret: matrix_username from_secret: matrix_username
@ -107,10 +139,10 @@ trigger:
- failure - failure
depends_on: depends_on:
- deployment - documentation
--- ---
kind: signature kind: signature
hmac: c7037b785902327e7b303293aa1d31291f60e67d8c5f59de19b7f61f28fafbfd hmac: cea6f69f523634c4df5c480608e4ff72f9434c93ff9808f7a6fccc68c356bb73
... ...

8
.gitignore vendored
View File

@ -1,3 +1,11 @@
# ---> Ansible # ---> Ansible
*.retry *.retry
filter/plugins/
library
# ---> Python
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class

View File

@ -3,100 +3,66 @@
hosts: localhost hosts: localhost
connection: local connection: local
gather_facts: false gather_facts: false
no_log: "{{ not (lookup('env', 'MOLECULE_DEBUG') | bool or molecule_yml.provisioner.log|default(false) | bool) }}" no_log: "{{ molecule_no_log }}"
vars: vars:
ssh_user: ubuntu ssh_user: root
ssh_port: 22 ssh_port: 22
security_group_name: molecule
security_group_description: Security group for testing Molecule
security_group_rules:
- proto: tcp
from_port: "{{ ssh_port }}"
to_port: "{{ ssh_port }}"
cidr_ip: '0.0.0.0/0'
- proto: icmp
from_port: 8
to_port: -1
cidr_ip: '0.0.0.0/0'
security_group_rules_egress:
- proto: -1
from_port: 0
to_port: 0
cidr_ip: '0.0.0.0/0'
keypair_name: molecule_key keypair_name: molecule_key
keypair_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/ssh_key" keypair_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/ssh_key"
tasks: tasks:
- name: Create security group - name: Create local keypair
ec2_group: user:
name: "{{ security_group_name }}" name: "{{ lookup('env', 'USER') }}"
description: "{{ security_group_name }}" generate_ssh_key: true
rules: "{{ security_group_rules }}" ssh_key_file: "{{ keypair_path }}"
rules_egress: "{{ security_group_rules_egress }}" register: local_keypair
- name: Test for presence of local keypair - name: Create remote keypair
stat: digital_ocean_sshkey:
path: "{{ keypair_path }}"
register: keypair_local
- name: Delete remote keypair
ec2_key:
name: "{{ keypair_name }}" name: "{{ keypair_name }}"
state: absent ssh_pub_key: "{{ local_keypair.ssh_public_key }}"
when: not keypair_local.stat.exists state: present
register: remote_keypair
- name: Create keypair
ec2_key:
name: "{{ keypair_name }}"
register: keypair
- name: Persist the keypair
copy:
dest: "{{ keypair_path }}"
content: "{{ keypair.key.private_key }}"
mode: 0600
when: keypair.changed
- name: Create molecule instance(s) - name: Create molecule instance(s)
ec2: digital_ocean_droplet:
key_name: "{{ keypair_name }}" name: "{{ item.name }}"
image: "{{ item.image }}" unique_name: true
instance_type: "{{ item.instance_type }}" region: "{{ item.region_id }}"
vpc_subnet_id: "{{ item.vpc_subnet_id }}" image: "{{ item.image_id }}"
group: "{{ security_group_name }}" size: "{{ item.size_id }}"
instance_tags: ssh_keys: "{{ remote_keypair.data.ssh_key.id }}"
instance: "{{ item.name }}"
wait: true wait: true
assign_public_ip: true wait_timeout: 300
exact_count: 1 state: present
count_tag:
instance: "{{ item.name }}"
register: server register: server
with_items: "{{ molecule_yml.platforms }}" loop: "{{ molecule_yml.platforms }}"
async: 7200 async: 7200
poll: 0 poll: 0
- name: Wait for instance(s) creation to complete - name: Wait for instance(s) creation to complete
async_status: async_status:
jid: "{{ item.ansible_job_id }}" jid: "{{ item.ansible_job_id }}"
register: ec2_jobs register: digitalocean_jobs
until: ec2_jobs.finished until: digitalocean_jobs.finished
retries: 300 retries: 300
with_items: "{{ server.results }}" loop: "{{ server.results }}"
# Mandatory configuration for Molecule to function. # Mandatory configuration for Molecule to function.
- name: Populate instance config dict - name: Populate instance config dict
set_fact: set_fact:
instance_conf_dict: { instance_conf_dict: {
'instance': "{{ item.instances[0].tags.instance }}", 'instance': "{{ item.data.droplet.name }}",
'address': "{{ item.instances[0].public_ip }}", 'address': "{{ item.data.ip_address }}",
'user': "{{ ssh_user }}", 'user': "{{ ssh_user }}",
'port': "{{ ssh_port }}", 'port': "{{ ssh_port }}",
'identity_file': "{{ keypair_path }}", 'identity_file': "{{ keypair_path }}",
'instance_ids': "{{ item.instance_ids }}", } 'droplet_id': "{{ item.data.droplet.id }}",
with_items: "{{ ec2_jobs.results }}" 'ssh_key_id': "{{ remote_keypair.data.ssh_key.id }}",
}
loop: "{{ digitalocean_jobs.results }}"
register: instance_config_dict register: instance_config_dict
when: server.changed | bool when: server.changed | bool
@ -118,8 +84,4 @@
search_regex: SSH search_regex: SSH
delay: 10 delay: 10
timeout: 320 timeout: 320
with_items: "{{ lookup('file', molecule_instance_config) | molecule_from_yaml }}" loop: "{{ lookup('file', molecule_instance_config) | molecule_from_yaml }}"
- name: Wait for boot process to finish
pause:
minutes: 2

View File

@ -3,7 +3,7 @@
hosts: localhost hosts: localhost
connection: local connection: local
gather_facts: false gather_facts: false
no_log: "{{ not (lookup('env', 'MOLECULE_DEBUG') | bool or molecule_yml.provisioner.log|default(false) | bool) }}" no_log: "{{ molecule_no_log }}"
tasks: tasks:
- block: - block:
- name: Populate instance config - name: Populate instance config
@ -17,11 +17,12 @@
skip_instances: true skip_instances: true
- name: Destroy molecule instance(s) - name: Destroy molecule instance(s)
ec2: digital_ocean_droplet:
name: "{{ item.instance }}"
id: "{{ item.droplet_id }}"
state: absent state: absent
instance_ids: "{{ item.instance_ids }}"
register: server register: server
with_items: "{{ instance_conf }}" loop: "{{ instance_conf | flatten(levels=1) }}"
when: not skip_instances when: not skip_instances
async: 7200 async: 7200
poll: 0 poll: 0
@ -29,10 +30,16 @@
- name: Wait for instance(s) deletion to complete - name: Wait for instance(s) deletion to complete
async_status: async_status:
jid: "{{ item.ansible_job_id }}" jid: "{{ item.ansible_job_id }}"
register: ec2_jobs register: digitalocean_jobs
until: ec2_jobs.finished until: digitalocean_jobs.finished
retries: 300 retries: 300
with_items: "{{ server.results }}" loop: "{{ server.results }}"
- name: Delete remote keypair
digital_ocean_sshkey:
fingerprint: "{{ item.ssh_key_id }}"
state: absent
loop: "{{ instance_conf | flatten(levels=1) }}"
# Mandatory configuration for Molecule to function. # Mandatory configuration for Molecule to function.
@ -42,6 +49,6 @@
- name: Dump instance config - name: Dump instance config
copy: copy:
content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}" content: "{{ instance_conf | molecule_to_yaml | molecule_header }}"
dest: "{{ molecule_instance_config }}" dest: "{{ molecule_instance_config }}"
when: server.changed | bool when: server.changed | bool

View File

@ -2,18 +2,20 @@
dependency: dependency:
name: galaxy name: galaxy
driver: driver:
name: ec2 name: digitalocean
platforms:
- name: centos7-certbot
region_id: fra1
image_id: centos-7-x64
size_id: s-1vcpu-1gb
lint: lint:
name: yamllint name: yamllint
platforms: enabled: False
- name: instance
image: ami-a5b196c0
instance_type: t2.micro
vpc_subnet_id: subnet-6456fd1f
provisioner: provisioner:
name: ansible name: ansible
lint: lint:
name: ansible-lint name: ansible-lint
enabled: False
verifier: verifier:
name: testinfra name: testinfra
lint: lint:

View File

@ -1,5 +1,9 @@
--- ---
- name: Converge - name: Converge
hosts: all hosts: all
vars:
certbot_packages_extra:
- epel-release
roles: roles:
- role: xoxys.certbot - role: xoxys.certbot

View File

@ -4,6 +4,6 @@
gather_facts: false gather_facts: false
tasks: tasks:
- name: Install python for Ansible - name: Install python for Ansible
raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal python-zipstream) raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal)
become: true become: true
changed_when: false changed_when: false

View File

@ -2,13 +2,13 @@ import os
import testinfra.utils.ansible_runner import testinfra.utils.ansible_runner
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
def test_hosts_file(host): def test_certbot_is_installed(host):
f = host.file('/etc/hosts') certbot = host.package("certbot")
assert certbot.is_installed
assert f.exists
assert f.user == 'root'
assert f.group == 'root'