mirror of
https://github.com/thegeeklab/ansible-later.git
synced 2024-07-03 23:50:58 +02:00
352 lines
15 KiB
Python
352 lines
15 KiB
Python
# Copyright (c) 2018 Ansible Project
|
|
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
|
|
|
from __future__ import (absolute_import, division, print_function)
|
|
__metaclass__ = type
|
|
|
|
DOCUMENTATION = '''
|
|
name: aws_rds
|
|
plugin_type: inventory
|
|
short_description: rds instance source
|
|
description:
|
|
- Get instances and clusters from Amazon Web Services RDS.
|
|
- Uses a YAML configuration file that ends with aws_rds.(yml|yaml).
|
|
options:
|
|
boto_profile:
|
|
description: The boto profile to use. The plugin will look for an instance role if no credentials
|
|
are provided.
|
|
env:
|
|
- name: AWS_PROFILE
|
|
- name: AWS_DEFAULT_PROFILE
|
|
aws_access_key_id:
|
|
description: The AWS access key to use. If you have specified a profile, you don't need to provide
|
|
an access key/secret key/session token.
|
|
env:
|
|
- name: AWS_ACCESS_KEY_ID
|
|
- name: AWS_ACCESS_KEY
|
|
- name: EC2_ACCESS_KEY
|
|
aws_secret_access_key:
|
|
description: The AWS secret key that corresponds to the access key. If you have specified a profile,
|
|
you don't need to provide an access key/secret key/session token.
|
|
env:
|
|
- name: AWS_SECRET_ACCESS_KEY
|
|
- name: AWS_SECRET_KEY
|
|
- name: EC2_SECRET_KEY
|
|
aws_security_token:
|
|
description: The AWS security token if using temporary access and secret keys.
|
|
env:
|
|
- name: AWS_SECURITY_TOKEN
|
|
- name: AWS_SESSION_TOKEN
|
|
- name: EC2_SECURITY_TOKEN
|
|
regions:
|
|
description: A list of regions in which to describe RDS instances and clusters. Available regions are listed here
|
|
U(https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html)
|
|
default: []
|
|
filters:
|
|
description: A dictionary of filter value pairs. Available filters are listed here
|
|
U(https://docs.aws.amazon.com/cli/latest/reference/rds/describe-db-instances.html#options). If you filter by
|
|
db-cluster-id and I(include_clusters) is True it will apply to clusters as well.
|
|
default: {}
|
|
strict_permissions:
|
|
description: By default if an AccessDenied exception is encountered this plugin will fail. You can set strict_permissions to
|
|
False in the inventory config file which will allow the restrictions to be gracefully skipped.
|
|
type: bool
|
|
default: True
|
|
include_clusters:
|
|
description: Whether or not to query for Aurora clusters as well as instances
|
|
type: bool
|
|
default: False
|
|
statuses:
|
|
description: A list of desired states for instances/clusters to be added to inventory. Set to ['all'] as a shorthand to find everything.
|
|
type: list
|
|
default:
|
|
- creating
|
|
- available
|
|
extends_documentation_fragment:
|
|
- inventory_cache
|
|
- constructed
|
|
requirements:
|
|
- boto3
|
|
- botocore
|
|
author: Sloane Hertel (@s-hertel)
|
|
'''
|
|
|
|
EXAMPLES = '''
|
|
plugin: aws_rds
|
|
regions:
|
|
- us-east-1
|
|
- ca-central-1
|
|
keyed_groups:
|
|
- key: 'db_parameter_groups|json_query("[].db_parameter_group_name")'
|
|
prefix: rds_parameter_group
|
|
- key: engine
|
|
prefix: rds
|
|
- key: tags
|
|
- key: region
|
|
'''
|
|
|
|
from ansible.errors import AnsibleError
|
|
from ansible.module_utils._text import to_native
|
|
from ansible.module_utils.aws.core import is_boto3_error_code
|
|
from ansible.module_utils.ec2 import ansible_dict_to_boto3_filter_list, boto3_tag_list_to_ansible_dict
|
|
from ansible.module_utils.ec2 import camel_dict_to_snake_dict
|
|
from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable
|
|
|
|
try:
|
|
import boto3
|
|
import botocore
|
|
except ImportError:
|
|
raise AnsibleError('The RDS dynamic inventory plugin requires boto3 and botocore.')
|
|
|
|
|
|
class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
|
|
|
|
NAME = 'aws_rds'
|
|
|
|
def __init__(self):
|
|
super(InventoryModule, self).__init__()
|
|
self.credentials = {}
|
|
self.boto_profile = None
|
|
|
|
def _boto3_conn(self, regions):
|
|
'''
|
|
:param regions: A list of regions to create a boto3 client
|
|
|
|
Generator that yields a boto3 client and the region
|
|
'''
|
|
for region in regions:
|
|
try:
|
|
connection = boto3.session.Session(profile_name=self.boto_profile).client('rds', region, **self.credentials)
|
|
except (botocore.exceptions.ProfileNotFound, botocore.exceptions.PartialCredentialsError) as e:
|
|
if self.boto_profile:
|
|
try:
|
|
connection = boto3.session.Session(profile_name=self.boto_profile).client('rds', region)
|
|
except (botocore.exceptions.ProfileNotFound, botocore.exceptions.PartialCredentialsError) as e:
|
|
raise AnsibleError("Insufficient credentials found: %s" % to_native(e))
|
|
else:
|
|
raise AnsibleError("Insufficient credentials found: %s" % to_native(e))
|
|
yield connection, region
|
|
|
|
def _get_hosts_by_region(self, connection, filters, strict):
|
|
|
|
def _add_tags_for_hosts(connection, hosts, strict):
|
|
for host in hosts:
|
|
if 'DBInstanceArn' in host:
|
|
resource_arn = host['DBInstanceArn']
|
|
else:
|
|
resource_arn = host['DBClusterArn']
|
|
|
|
try:
|
|
tags = connection.list_tags_for_resource(ResourceName=resource_arn)['TagList']
|
|
except is_boto3_error_code('AccessDenied') as e:
|
|
if not strict:
|
|
tags = []
|
|
else:
|
|
raise e
|
|
host['Tags'] = tags
|
|
|
|
def wrapper(f, *args, **kwargs):
|
|
try:
|
|
results = f(*args, **kwargs)
|
|
if 'DBInstances' in results:
|
|
results = results['DBInstances']
|
|
else:
|
|
results = results['DBClusters']
|
|
_add_tags_for_hosts(connection, results, strict)
|
|
except is_boto3_error_code('AccessDenied') as e: # pylint: disable=duplicate-except
|
|
if not strict:
|
|
results = []
|
|
else:
|
|
raise AnsibleError("Failed to query RDS: {0}".format(to_native(e)))
|
|
except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: # pylint: disable=duplicate-except
|
|
raise AnsibleError("Failed to query RDS: {0}".format(to_native(e)))
|
|
return results
|
|
return wrapper
|
|
|
|
def _get_all_hosts(self, regions, instance_filters, cluster_filters, strict, statuses, gather_clusters=False):
|
|
'''
|
|
:param regions: a list of regions in which to describe hosts
|
|
:param instance_filters: a list of boto3 filter dicionaries
|
|
:param cluster_filters: a list of boto3 filter dicionaries
|
|
:param strict: a boolean determining whether to fail or ignore 403 error codes
|
|
:param statuses: a list of statuses that the returned hosts should match
|
|
:return A list of host dictionaries
|
|
'''
|
|
all_instances = []
|
|
all_clusters = []
|
|
for connection, region in self._boto3_conn(regions):
|
|
paginator = connection.get_paginator('describe_db_instances')
|
|
all_instances.extend(
|
|
self._get_hosts_by_region(connection, instance_filters, strict)
|
|
(paginator.paginate(Filters=instance_filters).build_full_result)
|
|
)
|
|
if gather_clusters:
|
|
all_clusters.extend(
|
|
self._get_hosts_by_region(connection, cluster_filters, strict)
|
|
(connection.describe_db_clusters, **{'Filters': cluster_filters})
|
|
)
|
|
sorted_hosts = list(
|
|
sorted(all_instances, key=lambda x: x['DBInstanceIdentifier']) +
|
|
sorted(all_clusters, key=lambda x: x['DBClusterIdentifier'])
|
|
)
|
|
return self.find_hosts_with_valid_statuses(sorted_hosts, statuses)
|
|
|
|
def find_hosts_with_valid_statuses(self, hosts, statuses):
|
|
if 'all' in statuses:
|
|
return hosts
|
|
valid_hosts = []
|
|
for host in hosts:
|
|
if host.get('DBInstanceStatus') in statuses:
|
|
valid_hosts.append(host)
|
|
elif host.get('Status') in statuses:
|
|
valid_hosts.append(host)
|
|
return valid_hosts
|
|
|
|
def _populate(self, hosts):
|
|
group = 'aws_rds'
|
|
self.inventory.add_group(group)
|
|
if hosts:
|
|
self._add_hosts(hosts=hosts, group=group)
|
|
self.inventory.add_child('all', group)
|
|
|
|
def _populate_from_source(self, source_data):
|
|
hostvars = source_data.pop('_meta', {}).get('hostvars', {})
|
|
for group in source_data:
|
|
if group == 'all':
|
|
continue
|
|
else:
|
|
self.inventory.add_group(group)
|
|
hosts = source_data[group].get('hosts', [])
|
|
for host in hosts:
|
|
self._populate_host_vars([host], hostvars.get(host, {}), group)
|
|
self.inventory.add_child('all', group)
|
|
|
|
def _get_hostname(self, host):
|
|
if host.get('DBInstanceIdentifier'):
|
|
return host['DBInstanceIdentifier']
|
|
else:
|
|
return host['DBClusterIdentifier']
|
|
|
|
def _format_inventory(self, hosts):
|
|
results = {'_meta': {'hostvars': {}}}
|
|
group = 'aws_rds'
|
|
results[group] = {'hosts': []}
|
|
for host in hosts:
|
|
hostname = self._get_hostname(host)
|
|
results[group]['hosts'].append(hostname)
|
|
h = self.inventory.get_host(hostname)
|
|
results['_meta']['hostvars'][h.name] = h.vars
|
|
return results
|
|
|
|
def _add_hosts(self, hosts, group):
|
|
'''
|
|
:param hosts: a list of hosts to be added to a group
|
|
:param group: the name of the group to which the hosts belong
|
|
'''
|
|
for host in hosts:
|
|
hostname = self._get_hostname(host)
|
|
host = camel_dict_to_snake_dict(host, ignore_list=['Tags'])
|
|
host['tags'] = boto3_tag_list_to_ansible_dict(host.get('tags', []))
|
|
|
|
# Allow easier grouping by region
|
|
if 'availability_zone' in host:
|
|
host['region'] = host['availability_zone'][:-1]
|
|
elif 'availability_zones' in host:
|
|
host['region'] = host['availability_zones'][0][:-1]
|
|
|
|
self.inventory.add_host(hostname, group=group)
|
|
for hostvar, hostval in host.items():
|
|
self.inventory.set_variable(hostname, hostvar, hostval)
|
|
|
|
# Use constructed if applicable
|
|
strict = self.get_option('strict')
|
|
# Composed variables
|
|
self._set_composite_vars(self.get_option('compose'), host, hostname, strict=strict)
|
|
# Complex groups based on jinaj2 conditionals, hosts that meet the conditional are added to group
|
|
self._add_host_to_composed_groups(self.get_option('groups'), host, hostname, strict=strict)
|
|
# Create groups based on variable values and add the corresponding hosts to it
|
|
self._add_host_to_keyed_groups(self.get_option('keyed_groups'), host, hostname, strict=strict)
|
|
|
|
def _set_credentials(self):
|
|
'''
|
|
:param config_data: contents of the inventory config file
|
|
'''
|
|
self.boto_profile = self.get_option('boto_profile')
|
|
aws_access_key_id = self.get_option('aws_access_key_id')
|
|
aws_secret_access_key = self.get_option('aws_secret_access_key')
|
|
aws_security_token = self.get_option('aws_security_token')
|
|
|
|
if not self.boto_profile and not (aws_access_key_id and aws_secret_access_key):
|
|
session = botocore.session.get_session()
|
|
if session.get_credentials() is not None:
|
|
aws_access_key_id = session.get_credentials().access_key
|
|
aws_secret_access_key = session.get_credentials().secret_key
|
|
aws_security_token = session.get_credentials().token
|
|
|
|
if not self.boto_profile and not (aws_access_key_id and aws_secret_access_key):
|
|
raise AnsibleError("Insufficient boto credentials found. Please provide them in your "
|
|
"inventory configuration file or set them as environment variables.")
|
|
|
|
if aws_access_key_id:
|
|
self.credentials['aws_access_key_id'] = aws_access_key_id
|
|
if aws_secret_access_key:
|
|
self.credentials['aws_secret_access_key'] = aws_secret_access_key
|
|
if aws_security_token:
|
|
self.credentials['aws_session_token'] = aws_security_token
|
|
|
|
def verify_file(self, path):
|
|
'''
|
|
:param loader: an ansible.parsing.dataloader.DataLoader object
|
|
:param path: the path to the inventory config file
|
|
:return the contents of the config file
|
|
'''
|
|
if super(InventoryModule, self).verify_file(path):
|
|
if path.endswith(('aws_rds.yml', 'aws_rds.yaml')):
|
|
return True
|
|
return False
|
|
|
|
def parse(self, inventory, loader, path, cache=True):
|
|
super(InventoryModule, self).parse(inventory, loader, path)
|
|
|
|
config_data = self._read_config_data(path)
|
|
self._set_credentials()
|
|
|
|
# get user specifications
|
|
regions = self.get_option('regions')
|
|
filters = self.get_option('filters')
|
|
strict_permissions = self.get_option('strict_permissions')
|
|
statuses = self.get_option('statuses')
|
|
include_clusters = self.get_option('include_clusters')
|
|
instance_filters = ansible_dict_to_boto3_filter_list(filters)
|
|
cluster_filters = []
|
|
if 'db-cluster-id' in filters and include_clusters:
|
|
cluster_filters = ansible_dict_to_boto3_filter_list({'db-cluster-id': filters['db-cluster-id']})
|
|
|
|
cache_key = self.get_cache_key(path)
|
|
# false when refresh_cache or --flush-cache is used
|
|
if cache:
|
|
# get the user-specified directive
|
|
cache = self.get_option('cache')
|
|
|
|
# Generate inventory
|
|
formatted_inventory = {}
|
|
cache_needs_update = False
|
|
if cache:
|
|
try:
|
|
results = self.cache.get(cache_key)
|
|
except KeyError:
|
|
# if cache expires or cache file doesn't exist
|
|
cache_needs_update = True
|
|
else:
|
|
self._populate_from_source(results)
|
|
|
|
if not cache or cache_needs_update:
|
|
results = self._get_all_hosts(regions, instance_filters, cluster_filters, strict_permissions, statuses, include_clusters)
|
|
self._populate(results)
|
|
formatted_inventory = self._format_inventory(results)
|
|
|
|
# If the cache has expired/doesn't exist or if refresh_inventory/flush cache is used
|
|
# when the user is using caching, update the cached inventory
|
|
if cache_needs_update or (not cache and self.get_option('cache')):
|
|
self.cache.set(cache_key, formatted_inventory)
|