summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAndy Grimm <agrimm@redhat.com>2015-03-04 15:51:28 -0500
committerAndy Grimm <agrimm@redhat.com>2015-03-04 15:51:28 -0500
commit9c774bb95baba3da0a8cac56a5e9bf1fb9cf657b (patch)
tree8058114443fbef66aa8f952a524ebedbe87bdfe7
parentebc226eb9580eb0173843c82afec811acca60896 (diff)
downloadopenshift-9c774bb95baba3da0a8cac56a5e9bf1fb9cf657b.tar.gz
openshift-9c774bb95baba3da0a8cac56a5e9bf1fb9cf657b.tar.bz2
openshift-9c774bb95baba3da0a8cac56a5e9bf1fb9cf657b.tar.xz
openshift-9c774bb95baba3da0a8cac56a5e9bf1fb9cf657b.zip
Sync ec2.py with upstream
-rw-r--r--[-rwxr-xr-x]inventory/aws/ec2.py300
1 files changed, 239 insertions, 61 deletions
diff --git a/inventory/aws/ec2.py b/inventory/aws/ec2.py
index 84841d3f0..0f7c19857 100755..100644
--- a/inventory/aws/ec2.py
+++ b/inventory/aws/ec2.py
@@ -123,6 +123,7 @@ from boto import ec2
from boto import rds
from boto import route53
import ConfigParser
+from collections import defaultdict
try:
import json
@@ -222,6 +223,21 @@ class Ec2Inventory(object):
self.route53_excluded_zones.extend(
config.get('ec2', 'route53_excluded_zones', '').split(','))
+ # Include RDS instances?
+ self.rds_enabled = True
+ if config.has_option('ec2', 'rds'):
+ self.rds_enabled = config.getboolean('ec2', 'rds')
+
+ # Return all EC2 and RDS instances (if RDS is enabled)
+ if config.has_option('ec2', 'all_instances'):
+ self.all_instances = config.getboolean('ec2', 'all_instances')
+ else:
+ self.all_instances = False
+ if config.has_option('ec2', 'all_rds_instances') and self.rds_enabled:
+ self.all_rds_instances = config.getboolean('ec2', 'all_rds_instances')
+ else:
+ self.all_rds_instances = False
+
# Cache related
cache_dir = os.path.expanduser(config.get('ec2', 'cache_path'))
if not os.path.exists(cache_dir):
@@ -230,8 +246,66 @@ class Ec2Inventory(object):
self.cache_path_cache = cache_dir + "/ansible-ec2.cache"
self.cache_path_index = cache_dir + "/ansible-ec2.index"
self.cache_max_age = config.getint('ec2', 'cache_max_age')
-
+ # Configure nested groups instead of flat namespace.
+ if config.has_option('ec2', 'nested_groups'):
+ self.nested_groups = config.getboolean('ec2', 'nested_groups')
+ else:
+ self.nested_groups = False
+
+ # Configure which groups should be created.
+ group_by_options = [
+ 'group_by_instance_id',
+ 'group_by_region',
+ 'group_by_availability_zone',
+ 'group_by_ami_id',
+ 'group_by_instance_type',
+ 'group_by_key_pair',
+ 'group_by_vpc_id',
+ 'group_by_security_group',
+ 'group_by_tag_keys',
+ 'group_by_tag_none',
+ 'group_by_route53_names',
+ 'group_by_rds_engine',
+ 'group_by_rds_parameter_group',
+ ]
+ for option in group_by_options:
+ if config.has_option('ec2', option):
+ setattr(self, option, config.getboolean('ec2', option))
+ else:
+ setattr(self, option, True)
+
+ # Do we need to just include hosts that match a pattern?
+ try:
+ pattern_include = config.get('ec2', 'pattern_include')
+ if pattern_include and len(pattern_include) > 0:
+ self.pattern_include = re.compile(pattern_include)
+ else:
+ self.pattern_include = None
+ except ConfigParser.NoOptionError, e:
+ self.pattern_include = None
+
+ # Do we need to exclude hosts that match a pattern?
+ try:
+ pattern_exclude = config.get('ec2', 'pattern_exclude');
+ if pattern_exclude and len(pattern_exclude) > 0:
+ self.pattern_exclude = re.compile(pattern_exclude)
+ else:
+ self.pattern_exclude = None
+ except ConfigParser.NoOptionError, e:
+ self.pattern_exclude = None
+
+ # Instance filters (see boto and EC2 API docs). Ignore invalid filters.
+ self.ec2_instance_filters = defaultdict(list)
+ if config.has_option('ec2', 'instance_filters'):
+ for instance_filter in config.get('ec2', 'instance_filters', '').split(','):
+ instance_filter = instance_filter.strip()
+ if not instance_filter or '=' not in instance_filter:
+ continue
+ filter_key, filter_value = [x.strip() for x in instance_filter.split('=', 1)]
+ if not filter_key:
+ continue
+ self.ec2_instance_filters[filter_key].append(filter_value)
def parse_cli_args(self):
''' Command line argument processing '''
@@ -254,7 +328,8 @@ class Ec2Inventory(object):
for region in self.regions:
self.get_instances_by_region(region)
- self.get_rds_instances_by_region(region)
+ if self.rds_enabled:
+ self.get_rds_instances_by_region(region)
self.write_to_cache(self.inventory, self.cache_path_cache)
self.write_to_cache(self.index, self.cache_path_index)
@@ -275,12 +350,18 @@ class Ec2Inventory(object):
if conn is None:
print("region name: %s likely not supported, or AWS is down. connection to region failed." % region)
sys.exit(1)
-
- reservations = conn.get_all_instances()
+
+ reservations = []
+ if self.ec2_instance_filters:
+ for filter_key, filter_values in self.ec2_instance_filters.iteritems():
+ reservations.extend(conn.get_all_instances(filters = { filter_key : filter_values }))
+ else:
+ reservations = conn.get_all_instances()
+
for reservation in reservations:
for instance in reservation.instances:
self.add_instance(instance, region)
-
+
except boto.exception.BotoServerError, e:
if not self.eucalyptus:
print "Looks like AWS is down again:"
@@ -288,7 +369,7 @@ class Ec2Inventory(object):
sys.exit(1)
def get_rds_instances_by_region(self, region):
- ''' Makes an AWS API call to the list of RDS instances in a particular
+ ''' Makes an AWS API call to the list of RDS instances in a particular
region '''
try:
@@ -321,64 +402,122 @@ class Ec2Inventory(object):
for instance in reservation.instances:
return instance
-
def add_instance(self, instance, region):
''' Adds an instance to the inventory and index, as long as it is
addressable '''
- # Only want running instances
- if instance.state != 'running':
+ # Only want running instances unless all_instances is True
+ if not self.all_instances and instance.state != 'running':
return
# Select the best destination address
if instance.subnet_id:
- dest = getattr(instance, self.vpc_destination_variable)
+ dest = getattr(instance, self.vpc_destination_variable, None)
+ if dest is None:
+ dest = getattr(instance, 'tags').get(self.vpc_destination_variable, None)
else:
- dest = getattr(instance, self.destination_variable)
+ dest = getattr(instance, self.destination_variable, None)
+ if dest is None:
+ dest = getattr(instance, 'tags').get(self.destination_variable, None)
if not dest:
# Skip instances we cannot address (e.g. private VPC subnet)
return
+ # if we only want to include hosts that match a pattern, skip those that don't
+ if self.pattern_include and not self.pattern_include.match(dest):
+ return
+
+ # if we need to exclude hosts that match a pattern, skip those
+ if self.pattern_exclude and self.pattern_exclude.match(dest):
+ return
+
# Add to index
self.index[dest] = [region, instance.id]
# Inventory: Group by instance ID (always a group of 1)
- self.inventory[instance.id] = [dest]
+ if self.group_by_instance_id:
+ self.inventory[instance.id] = [dest]
+ if self.nested_groups:
+ self.push_group(self.inventory, 'instances', instance.id)
# Inventory: Group by region
- self.push(self.inventory, region, dest)
+ if self.group_by_region:
+ self.push(self.inventory, region, dest)
+ if self.nested_groups:
+ self.push_group(self.inventory, 'regions', region)
# Inventory: Group by availability zone
- self.push(self.inventory, instance.placement, dest)
+ if self.group_by_availability_zone:
+ self.push(self.inventory, instance.placement, dest)
+ if self.nested_groups:
+ if self.group_by_region:
+ self.push_group(self.inventory, region, instance.placement)
+ self.push_group(self.inventory, 'zones', instance.placement)
+
+ # Inventory: Group by Amazon Machine Image (AMI) ID
+ if self.group_by_ami_id:
+ ami_id = self.to_safe(instance.image_id)
+ self.push(self.inventory, ami_id, dest)
+ if self.nested_groups:
+ self.push_group(self.inventory, 'images', ami_id)
# Inventory: Group by instance type
- self.push(self.inventory, self.to_safe('type_' + instance.instance_type), dest)
+ if self.group_by_instance_type:
+ type_name = self.to_safe('type_' + instance.instance_type)
+ self.push(self.inventory, type_name, dest)
+ if self.nested_groups:
+ self.push_group(self.inventory, 'types', type_name)
# Inventory: Group by key pair
- if instance.key_name:
- self.push(self.inventory, self.to_safe('key_' + instance.key_name), dest)
-
+ if self.group_by_key_pair and instance.key_name:
+ key_name = self.to_safe('key_' + instance.key_name)
+ self.push(self.inventory, key_name, dest)
+ if self.nested_groups:
+ self.push_group(self.inventory, 'keys', key_name)
+
+ # Inventory: Group by VPC
+ if self.group_by_vpc_id and instance.vpc_id:
+ vpc_id_name = self.to_safe('vpc_id_' + instance.vpc_id)
+ self.push(self.inventory, vpc_id_name, dest)
+ if self.nested_groups:
+ self.push_group(self.inventory, 'vpcs', vpc_id_name)
+
# Inventory: Group by security group
- try:
- for group in instance.groups:
- key = self.to_safe("security_group_" + group.name)
- self.push(self.inventory, key, dest)
- except AttributeError:
- print 'Package boto seems a bit older.'
- print 'Please upgrade boto >= 2.3.0.'
- sys.exit(1)
+ if self.group_by_security_group:
+ try:
+ for group in instance.groups:
+ key = self.to_safe("security_group_" + group.name)
+ self.push(self.inventory, key, dest)
+ if self.nested_groups:
+ self.push_group(self.inventory, 'security_groups', key)
+ except AttributeError:
+ print 'Package boto seems a bit older.'
+ print 'Please upgrade boto >= 2.3.0.'
+ sys.exit(1)
# Inventory: Group by tag keys
- for k, v in instance.tags.iteritems():
- key = self.to_safe("tag_" + k + "=" + v)
- self.push(self.inventory, key, dest)
+ if self.group_by_tag_keys:
+ for k, v in instance.tags.iteritems():
+ key = self.to_safe("tag_" + k + "=" + v)
+ self.push(self.inventory, key, dest)
+ if self.nested_groups:
+ self.push_group(self.inventory, 'tags', self.to_safe("tag_" + k))
+ self.push_group(self.inventory, self.to_safe("tag_" + k), key)
# Inventory: Group by Route53 domain names if enabled
- if self.route53_enabled:
+ if self.route53_enabled and self.group_by_route53_names:
route53_names = self.get_instance_route53_names(instance)
for name in route53_names:
self.push(self.inventory, name, dest)
+ if self.nested_groups:
+ self.push_group(self.inventory, 'route53', name)
+
+ # Global Tag: instances without tags
+ if self.group_by_tag_none and len(instance.tags) == 0:
+ self.push(self.inventory, 'tag_none', dest)
+ if self.nested_groups:
+ self.push_group(self.inventory, 'tags', 'tag_none')
# Global Tag: tag all EC2 instances
self.push(self.inventory, 'ec2', dest)
@@ -390,15 +529,11 @@ class Ec2Inventory(object):
''' Adds an RDS instance to the inventory and index, as long as it is
addressable '''
- # Only want available instances
- if instance.status != 'available':
+ # Only want available instances unless all_rds_instances is True
+ if not self.all_rds_instances and instance.status != 'available':
return
# Select the best destination address
- #if instance.subnet_id:
- #dest = getattr(instance, self.vpc_destination_variable)
- #else:
- #dest = getattr(instance, self.destination_variable)
dest = instance.endpoint[0]
if not dest:
@@ -409,36 +544,70 @@ class Ec2Inventory(object):
self.index[dest] = [region, instance.id]
# Inventory: Group by instance ID (always a group of 1)
- self.inventory[instance.id] = [dest]
+ if self.group_by_instance_id:
+ self.inventory[instance.id] = [dest]
+ if self.nested_groups:
+ self.push_group(self.inventory, 'instances', instance.id)
# Inventory: Group by region
- self.push(self.inventory, region, dest)
+ if self.group_by_region:
+ self.push(self.inventory, region, dest)
+ if self.nested_groups:
+ self.push_group(self.inventory, 'regions', region)
# Inventory: Group by availability zone
- self.push(self.inventory, instance.availability_zone, dest)
-
+ if self.group_by_availability_zone:
+ self.push(self.inventory, instance.availability_zone, dest)
+ if self.nested_groups:
+ if self.group_by_region:
+ self.push_group(self.inventory, region, instance.availability_zone)
+ self.push_group(self.inventory, 'zones', instance.availability_zone)
+
# Inventory: Group by instance type
- self.push(self.inventory, self.to_safe('type_' + instance.instance_class), dest)
-
+ if self.group_by_instance_type:
+ type_name = self.to_safe('type_' + instance.instance_class)
+ self.push(self.inventory, type_name, dest)
+ if self.nested_groups:
+ self.push_group(self.inventory, 'types', type_name)
+
+ # Inventory: Group by VPC
+ if self.group_by_vpc_id and instance.subnet_group and instance.subnet_group.vpc_id:
+ vpc_id_name = self.to_safe('vpc_id_' + instance.subnet_group.vpc_id)
+ self.push(self.inventory, vpc_id_name, dest)
+ if self.nested_groups:
+ self.push_group(self.inventory, 'vpcs', vpc_id_name)
+
# Inventory: Group by security group
- try:
- if instance.security_group:
- key = self.to_safe("security_group_" + instance.security_group.name)
- self.push(self.inventory, key, dest)
- except AttributeError:
- print 'Package boto seems a bit older.'
- print 'Please upgrade boto >= 2.3.0.'
- sys.exit(1)
+ if self.group_by_security_group:
+ try:
+ if instance.security_group:
+ key = self.to_safe("security_group_" + instance.security_group.name)
+ self.push(self.inventory, key, dest)
+ if self.nested_groups:
+ self.push_group(self.inventory, 'security_groups', key)
+
+ except AttributeError:
+ print 'Package boto seems a bit older.'
+ print 'Please upgrade boto >= 2.3.0.'
+ sys.exit(1)
# Inventory: Group by engine
- self.push(self.inventory, self.to_safe("rds_" + instance.engine), dest)
+ if self.group_by_rds_engine:
+ self.push(self.inventory, self.to_safe("rds_" + instance.engine), dest)
+ if self.nested_groups:
+ self.push_group(self.inventory, 'rds_engines', self.to_safe("rds_" + instance.engine))
# Inventory: Group by parameter group
- self.push(self.inventory, self.to_safe("rds_parameter_group_" + instance.parameter_group.name), dest)
+ if self.group_by_rds_parameter_group:
+ self.push(self.inventory, self.to_safe("rds_parameter_group_" + instance.parameter_group.name), dest)
+ if self.nested_groups:
+ self.push_group(self.inventory, 'rds_parameter_groups', self.to_safe("rds_parameter_group_" + instance.parameter_group.name))
# Global Tag: all RDS instances
self.push(self.inventory, 'rds', dest)
+ self.inventory["_meta"]["hostvars"][dest] = self.get_host_info_dict_from_instance(instance)
+
def get_route53_records(self):
''' Get and store the map of resource records to domain names that
@@ -522,8 +691,8 @@ class Ec2Inventory(object):
for group in value:
group_ids.append(group.id)
group_names.append(group.name)
- instance_vars["ec2_security_group_ids"] = ','.join(group_ids)
- instance_vars["ec2_security_group_names"] = ','.join(group_names)
+ instance_vars["ec2_security_group_ids"] = ','.join([str(i) for i in group_ids])
+ instance_vars["ec2_security_group_names"] = ','.join([str(i) for i in group_names])
else:
pass
# TODO Product codes if someone finds them useful
@@ -544,7 +713,7 @@ class Ec2Inventory(object):
# try updating the cache
self.do_api_calls_update_cache()
if not self.args.host in self.index:
- # host migh not exist anymore
+ # host might not exist anymore
return self.json_format_dict({}, True)
(region, instance_id) = self.index[self.args.host]
@@ -553,14 +722,23 @@ class Ec2Inventory(object):
return self.json_format_dict(self.get_host_info_dict_from_instance(instance), True)
def push(self, my_dict, key, element):
- ''' Pushed an element onto an array that may not have been defined in
+ ''' Push an element onto an array that may not have been defined in
the dict '''
-
- if key in my_dict:
- my_dict[key].append(element);
+ group_info = my_dict.setdefault(key, [])
+ if isinstance(group_info, dict):
+ host_list = group_info.setdefault('hosts', [])
+ host_list.append(element)
else:
- my_dict[key] = [element]
-
+ group_info.append(element)
+
+ def push_group(self, my_dict, key, element):
+ ''' Push a group as a child of another group. '''
+ parent_group = my_dict.setdefault(key, {})
+ if not isinstance(parent_group, dict):
+ parent_group = my_dict[key] = {'hosts': parent_group}
+ child_groups = parent_group.setdefault('children', [])
+ if element not in child_groups:
+ child_groups.append(element)
def get_inventory_from_cache(self):
''' Reads the inventory from the cache file and returns it as a JSON