refactor ci pipeline

This commit is contained in:
Robert Kaussow 2019-10-15 23:23:56 +02:00
parent 3488c92b54
commit 3411598c1c
9 changed files with 276 additions and 222 deletions

View File

@ -1,20 +1,3 @@
local AnsibleVersions(version="latest", package="ansible") = {
name: "ansible-" + version,
image: "python:3.7",
pull: "always",
environment: {
PY_COLORS: 1
},
commands: [
"pip install " + package + " ansible-later~=0.2.0 -qq",
"git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy",
"ansible-later -c ~/policy/config.yml"
],
depends_on: [
"clone",
],
};
local PipelineLinting = {
kind: "pipeline",
name: "linting",
@ -23,8 +6,17 @@ local PipelineLinting = {
arch: "amd64",
},
steps: [
AnsibleVersions(version="latest", package="ansible"),
AnsibleVersions(version="master", package="git+https://github.com/ansible/ansible.git@devel"),
{
name: "ansible-later",
image: "xoxys/ansible-later:latest",
environment: {
PY_COLORS: 1
},
commands: [
"git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy",
"ansible-later -c ~/policy/config.yml"
],
},
],
trigger: {
ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"],
@ -38,27 +30,27 @@ local PipelineDeployment = {
os: "linux",
arch: "amd64",
},
concurrency: {
limit: 1
},
workspace: {
base: "/drone/src",
path: "xoxys.certbot"
},
steps: [
{
name: "molecule",
image: "xoxys/molecule:ec2-linux-amd64",
pull: "always",
name: "ansible-molecule",
image: "xoxys/molecule:do-linux-amd64",
environment: {
ANSIBLE_ROLES_PATH: "/drone/src",
AWS_ACCESS_KEY_ID: { "from_secret": "aws_access_key_id" },
AWS_SECRET_ACCESS_KEY: { "from_secret": "aws_secret_access_key" },
AWS_REGION: "eu-central-1",
DO_API_KEY: { "from_secret": "do_api_key" },
USER: "root",
MOLECULE_CUSTOM_MODULES_REPO: "https://gitea.rknet.org/ansible/custom_modules",
MOLECULE_CUSTOM_FILTERS_REPO: "https://gitea.rknet.org/ansible/custom_filters",
PY_COLORS: 1
},
commands: [
"/bin/bash /docker-entrypoint.sh",
"molecule test --scenario-name ec2-centos-7",
"molecule test -s default",
],
},
],
@ -70,9 +62,55 @@ local PipelineDeployment = {
},
};
local PipelineNotifications = {
local PipelineDocumentation = {
kind: "pipeline",
name: "notifications",
name: "deployment",
platform: {
os: "linux",
arch: "amd64",
},
steps: [
{
name: "ansible-doctor",
image: "xoxys/ansible-doctor:latest",
environment: {
ANSIBLE_DOCTOR_LOG_LEVEL: "INFO",
ANSIBLE_DOCTOR_FORCE_OVERWRITE: true,
ANSIBLE_DOCTOR_EXCLUDE_FILES: "molecule/",
ANSIBLE_DOCTOR_CUSTOM_HEADER: "HEADER.md",
PY_COLORS: 1
},
},
{
name: "push-to-repo",
image: "plugins/git-action:latest",
settings: {
actions: ["commit", "push"],
author_email: "shipper@rknet.org",
author_name: "DroneShipper",
branch: "master",
message: "[SKIP CI] update readme",
remote: "https://gitea.rknet.org/ansible/xoxys.certbot",
netrc_machine: "gitea.rknet.org",
netrc_username: {"from_secret": "gitea_username"},
netrc_password: {"from_secret": "gitea_token"},
},
when: {
ref: ["refs/heads/master"],
},
},
],
depends_on: [
"deployment",
],
trigger: {
ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"],
},
};
local PipelineNotification= {
kind: "pipeline",
name: "notification",
platform: {
os: "linux",
arch: "amd64",
@ -85,8 +123,8 @@ local PipelineNotifications = {
name: "matrix",
image: "plugins/matrix",
settings: {
homeserver: "https://matrix.rknet.org",
roomid: "MtidqQXWWAtQcByBhH:rknet.org",
homeserver: { "from_secret": "matrix_homeserver" },
roomid: { "from_secret": "matrix_roomid" },
template: "Status: **{{ build.status }}**<br/> Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}<br/> Message: {{ build.message }}",
username: { "from_secret": "matrix_username" },
password: { "from_secret": "matrix_password" },
@ -94,7 +132,7 @@ local PipelineNotifications = {
},
],
depends_on: [
"deployment",
"documentation",
],
trigger: {
status: [ "success", "failure" ],
@ -105,5 +143,6 @@ local PipelineNotifications = {
[
PipelineLinting,
PipelineDeployment,
PipelineNotifications,
PipelineDocumentation,
PipelineNotification,
]

View File

@ -7,29 +7,13 @@ platform:
arch: amd64
steps:
- name: ansible-latest
pull: always
image: python:3.7
- name: ansible-later
image: xoxys/ansible-later:latest
commands:
- pip install ansible ansible-later~=0.2.0 -qq
- git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy
- ansible-later -c ~/policy/config.yml
environment:
PY_COLORS: 1
depends_on:
- clone
- name: ansible-master
pull: always
image: python:3.7
commands:
- "pip install git+https://github.com/ansible/ansible.git@devel ansible-later~=0.2.0 -qq"
- git clone https://gitea.rknet.org/ansible/ansible-later-policy.git ~/policy
- ansible-later -c ~/policy/config.yml
environment:
PY_COLORS: 1
depends_on:
- clone
trigger:
ref:
@ -45,27 +29,26 @@ platform:
os: linux
arch: amd64
concurrency:
limit: 1
workspace:
base: /drone/src
path: xoxys.certbot
steps:
- name: molecule
pull: always
image: xoxys/molecule:ec2-linux-amd64
- name: ansible-molecule
image: xoxys/molecule:do-linux-amd64
commands:
- /bin/bash /docker-entrypoint.sh
- molecule test --scenario-name ec2-centos-7
- molecule test -s default
environment:
ANSIBLE_ROLES_PATH: /drone/src
AWS_ACCESS_KEY_ID:
from_secret: aws_access_key_id
AWS_REGION: eu-central-1
AWS_SECRET_ACCESS_KEY:
from_secret: aws_secret_access_key
DO_API_KEY:
from_secret: do_api_key
MOLECULE_CUSTOM_FILTERS_REPO: https://gitea.rknet.org/ansible/custom_filters
MOLECULE_CUSTOM_MODULES_REPO: https://gitea.rknet.org/ansible/custom_modules
PY_COLORS: 1
USER: root
trigger:
ref:
@ -77,7 +60,54 @@ depends_on:
---
kind: pipeline
name: notifications
name: deployment
platform:
os: linux
arch: amd64
steps:
- name: ansible-doctor
image: xoxys/ansible-doctor:latest
environment:
ANSIBLE_DOCTOR_CUSTOM_HEADER: HEADER.md
ANSIBLE_DOCTOR_EXCLUDE_FILES: molecule/
ANSIBLE_DOCTOR_FORCE_OVERWRITE: true
ANSIBLE_DOCTOR_LOG_LEVEL: INFO
PY_COLORS: 1
- name: push-to-repo
image: plugins/git-action:latest
settings:
actions:
- commit
- push
author_email: "shipper@rknet.org"
author_name: DroneShipper
branch: master
message: "[SKIP CI] update readme"
netrc_machine: gitea.rknet.org
netrc_password:
from_secret: gitea_token
netrc_username:
from_secret: gitea_username
remote: https://gitea.rknet.org/ansible/xoxys.certbot
when:
ref:
- refs/heads/master
trigger:
ref:
- refs/heads/master
- "refs/tags/**"
- "refs/pull/**"
depends_on:
- deployment
---
kind: pipeline
name: notification
platform:
os: linux
@ -90,10 +120,12 @@ steps:
- name: matrix
image: plugins/matrix
settings:
homeserver: https://matrix.rknet.org
homeserver:
from_secret: matrix_homeserver
password:
from_secret: matrix_password
roomid: MtidqQXWWAtQcByBhH:rknet.org
roomid:
from_secret: matrix_roomid
template: "Status: **{{ build.status }}**<br/> Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}<br/> Message: {{ build.message }}"
username:
from_secret: matrix_username
@ -107,10 +139,10 @@ trigger:
- failure
depends_on:
- deployment
- documentation
---
kind: signature
hmac: c7037b785902327e7b303293aa1d31291f60e67d8c5f59de19b7f61f28fafbfd
hmac: cea6f69f523634c4df5c480608e4ff72f9434c93ff9808f7a6fccc68c356bb73
...

8
.gitignore vendored
View File

@ -1,3 +1,11 @@
# ---> Ansible
*.retry
filter/plugins/
library
# ---> Python
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class

View File

@ -3,100 +3,66 @@
hosts: localhost
connection: local
gather_facts: false
no_log: "{{ not (lookup('env', 'MOLECULE_DEBUG') | bool or molecule_yml.provisioner.log|default(false) | bool) }}"
no_log: "{{ molecule_no_log }}"
vars:
ssh_user: ubuntu
ssh_user: root
ssh_port: 22
security_group_name: molecule
security_group_description: Security group for testing Molecule
security_group_rules:
- proto: tcp
from_port: "{{ ssh_port }}"
to_port: "{{ ssh_port }}"
cidr_ip: '0.0.0.0/0'
- proto: icmp
from_port: 8
to_port: -1
cidr_ip: '0.0.0.0/0'
security_group_rules_egress:
- proto: -1
from_port: 0
to_port: 0
cidr_ip: '0.0.0.0/0'
keypair_name: molecule_key
keypair_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/ssh_key"
tasks:
- name: Create security group
ec2_group:
name: "{{ security_group_name }}"
description: "{{ security_group_name }}"
rules: "{{ security_group_rules }}"
rules_egress: "{{ security_group_rules_egress }}"
- name: Create local keypair
user:
name: "{{ lookup('env', 'USER') }}"
generate_ssh_key: true
ssh_key_file: "{{ keypair_path }}"
register: local_keypair
- name: Test for presence of local keypair
stat:
path: "{{ keypair_path }}"
register: keypair_local
- name: Delete remote keypair
ec2_key:
- name: Create remote keypair
digital_ocean_sshkey:
name: "{{ keypair_name }}"
state: absent
when: not keypair_local.stat.exists
- name: Create keypair
ec2_key:
name: "{{ keypair_name }}"
register: keypair
- name: Persist the keypair
copy:
dest: "{{ keypair_path }}"
content: "{{ keypair.key.private_key }}"
mode: 0600
when: keypair.changed
ssh_pub_key: "{{ local_keypair.ssh_public_key }}"
state: present
register: remote_keypair
- name: Create molecule instance(s)
ec2:
key_name: "{{ keypair_name }}"
image: "{{ item.image }}"
instance_type: "{{ item.instance_type }}"
vpc_subnet_id: "{{ item.vpc_subnet_id }}"
group: "{{ security_group_name }}"
instance_tags:
instance: "{{ item.name }}"
digital_ocean_droplet:
name: "{{ item.name }}"
unique_name: true
region: "{{ item.region_id }}"
image: "{{ item.image_id }}"
size: "{{ item.size_id }}"
ssh_keys: "{{ remote_keypair.data.ssh_key.id }}"
wait: true
assign_public_ip: true
exact_count: 1
count_tag:
instance: "{{ item.name }}"
wait_timeout: 300
state: present
register: server
with_items: "{{ molecule_yml.platforms }}"
loop: "{{ molecule_yml.platforms }}"
async: 7200
poll: 0
- name: Wait for instance(s) creation to complete
async_status:
jid: "{{ item.ansible_job_id }}"
register: ec2_jobs
until: ec2_jobs.finished
register: digitalocean_jobs
until: digitalocean_jobs.finished
retries: 300
with_items: "{{ server.results }}"
loop: "{{ server.results }}"
# Mandatory configuration for Molecule to function.
- name: Populate instance config dict
set_fact:
instance_conf_dict: {
'instance': "{{ item.instances[0].tags.instance }}",
'address': "{{ item.instances[0].public_ip }}",
'instance': "{{ item.data.droplet.name }}",
'address': "{{ item.data.ip_address }}",
'user': "{{ ssh_user }}",
'port': "{{ ssh_port }}",
'identity_file': "{{ keypair_path }}",
'instance_ids': "{{ item.instance_ids }}", }
with_items: "{{ ec2_jobs.results }}"
'droplet_id': "{{ item.data.droplet.id }}",
'ssh_key_id': "{{ remote_keypair.data.ssh_key.id }}",
}
loop: "{{ digitalocean_jobs.results }}"
register: instance_config_dict
when: server.changed | bool
@ -118,8 +84,4 @@
search_regex: SSH
delay: 10
timeout: 320
with_items: "{{ lookup('file', molecule_instance_config) | molecule_from_yaml }}"
- name: Wait for boot process to finish
pause:
minutes: 2
loop: "{{ lookup('file', molecule_instance_config) | molecule_from_yaml }}"

View File

@ -3,7 +3,7 @@
hosts: localhost
connection: local
gather_facts: false
no_log: "{{ not (lookup('env', 'MOLECULE_DEBUG') | bool or molecule_yml.provisioner.log|default(false) | bool) }}"
no_log: "{{ molecule_no_log }}"
tasks:
- block:
- name: Populate instance config
@ -17,11 +17,12 @@
skip_instances: true
- name: Destroy molecule instance(s)
ec2:
digital_ocean_droplet:
name: "{{ item.instance }}"
id: "{{ item.droplet_id }}"
state: absent
instance_ids: "{{ item.instance_ids }}"
register: server
with_items: "{{ instance_conf }}"
loop: "{{ instance_conf | flatten(levels=1) }}"
when: not skip_instances
async: 7200
poll: 0
@ -29,10 +30,16 @@
- name: Wait for instance(s) deletion to complete
async_status:
jid: "{{ item.ansible_job_id }}"
register: ec2_jobs
until: ec2_jobs.finished
register: digitalocean_jobs
until: digitalocean_jobs.finished
retries: 300
with_items: "{{ server.results }}"
loop: "{{ server.results }}"
- name: Delete remote keypair
digital_ocean_sshkey:
fingerprint: "{{ item.ssh_key_id }}"
state: absent
loop: "{{ instance_conf | flatten(levels=1) }}"
# Mandatory configuration for Molecule to function.
@ -42,6 +49,6 @@
- name: Dump instance config
copy:
content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}"
content: "{{ instance_conf | molecule_to_yaml | molecule_header }}"
dest: "{{ molecule_instance_config }}"
when: server.changed | bool

View File

@ -2,18 +2,20 @@
dependency:
name: galaxy
driver:
name: ec2
name: digitalocean
platforms:
- name: centos7-certbot
region_id: fra1
image_id: centos-7-x64
size_id: s-1vcpu-1gb
lint:
name: yamllint
platforms:
- name: instance
image: ami-a5b196c0
instance_type: t2.micro
vpc_subnet_id: subnet-6456fd1f
enabled: False
provisioner:
name: ansible
lint:
name: ansible-lint
enabled: False
verifier:
name: testinfra
lint:

View File

@ -1,5 +1,9 @@
---
- name: Converge
hosts: all
vars:
certbot_packages_extra:
- epel-release
roles:
- role: xoxys.certbot

View File

@ -4,6 +4,6 @@
gather_facts: false
tasks:
- name: Install python for Ansible
raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal python-zipstream)
raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal)
become: true
changed_when: false

View File

@ -2,13 +2,13 @@ import os
import testinfra.utils.ansible_runner
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
def test_hosts_file(host):
f = host.file('/etc/hosts')
assert f.exists
assert f.user == 'root'
assert f.group == 'root'
def test_certbot_is_installed(host):
certbot = host.package("certbot")
assert certbot.is_installed