From 1ab9bfa7cc3566c75d82129138567a85f15bdd45 Mon Sep 17 00:00:00 2001 From: Silviu Dicu Date: Tue, 22 Jan 2013 15:31:51 -0500 Subject: [PATCH 01/36] ec2 module - registers to ansible_facts key --- library/ec2_facts | 95 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 library/ec2_facts diff --git a/library/ec2_facts b/library/ec2_facts new file mode 100644 index 0000000000..d04f320844 --- /dev/null +++ b/library/ec2_facts @@ -0,0 +1,95 @@ +#!/usr/bin/python -tt +# -*- coding: utf-8 -*- + +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + + +DOCUMENTATION = ''' +--- +module: ec2_facts +short_description: Gathers facts about remote hosts within ec2 (aws) +options: {} +description: + - This module fetches data from the metadata servers in ec2 (aws). +notes: + - The module can add parameters to filter ec2_facts based on it. + Some of the facts are not returned ( like mapping of the devices - but + can be add it on). +examples: + - code: ansible all -m ec2_facts --tree /tmp/facts + description: Obtain facts from ec2 metatdata servers. You will need to + run an instance within ec2. +author: Silviu Dicu +''' + +import urllib2 +import socket + +socket.setdefaulttimeout(5) + +class Ec2Metadata(object): + + ec2_metadata_server = 'http://169.254.169.254/latest/meta-data/' + ec2_sshdata_server = 'http://169.254.169.254/latest/meta-data/public-keys/0/openssh-key' + ec2_userdata_server = 'http://169.254.169.254/latest/user-data/' + + def __init__(self, ec2_metadata_server=None, ec2_sshdata_server=None, ec2_userdata_server=None): + self.url_meta = ec2_metadata_server or self.ec2_metadata_server + self.url_user = ec2_userdata_server or self.ec2_userdata_server + self.url_ssh = ec2_sshdata_server or self.ec2_sshdata_server + + def _fetch(self, url): + try: + return urllib2.urlopen(url).read() + except urllib2.HTTPError: + return + except urllib2.URLError: + return + + def run(self, field=None): + data = {} + raw_fields = self._fetch(self.url_meta) + if not raw_fields: + return data + fields = raw_fields.split('\n') + for field in fields: + if field.endswith('/'): continue # deal with this later + field_data = self._fetch(self.url_meta + field) + if field == 'security-groups': + sg_fields = ",".join(field_data.split('\n')) + data['ansible_ec2_%s' % field] = sg_fields + else: + data['ansible_ec2_%s' % field] = field_data + data['ansible_ec2_%s' % 'user-data'] = self._fetch(self.url_user) + data['ensible_ec2_%s' % 'public-keys'] = self._fetch(self.url_ssh) + return data + + +def main(): + ec2_facts = Ec2Metadata().run() + ec2_facts_result = { + "changed" : True, + "ansible_facts" : ec2_facts + } + module = AnsibleModule( + argument_spec = dict() + ) + module.exit_json(**ec2_facts_result) + +# this is magic, see lib/ansible/module_common.py +#<> + +main() From ce9e9af6a36156113e6ae80f4e06d83a546d83e2 Mon Sep 17 00:00:00 2001 From: Silviu Dicu Date: Wed, 23 Jan 2013 09:39:26 -0500 Subject: [PATCH 02/36] ec2 facts moduled - updated --- library/ec2_facts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/library/ec2_facts b/library/ec2_facts index d04f320844..c9f85dd04a 100644 --- a/library/ec2_facts +++ b/library/ec2_facts @@ -25,9 +25,9 @@ options: {} description: - This module fetches data from the metadata servers in ec2 (aws). notes: - - The module can add parameters to filter ec2_facts based on it. + - Parameters to filter on ec2_facts may be added later. Some of the facts are not returned ( like mapping of the devices - but - can be add it on). + may be added later). examples: - code: ansible all -m ec2_facts --tree /tmp/facts description: Obtain facts from ec2 metatdata servers. You will need to @@ -42,14 +42,14 @@ socket.setdefaulttimeout(5) class Ec2Metadata(object): - ec2_metadata_server = 'http://169.254.169.254/latest/meta-data/' - ec2_sshdata_server = 'http://169.254.169.254/latest/meta-data/public-keys/0/openssh-key' - ec2_userdata_server = 'http://169.254.169.254/latest/user-data/' + ec2_metadata_url = 'http://169.254.169.254/latest/meta-data/' + ec2_sshdata_url = 'http://169.254.169.254/latest/meta-data/public-keys/0/openssh-key' + ec2_userdata_url = 'http://169.254.169.254/latest/user-data/' - def __init__(self, ec2_metadata_server=None, ec2_sshdata_server=None, ec2_userdata_server=None): - self.url_meta = ec2_metadata_server or self.ec2_metadata_server - self.url_user = ec2_userdata_server or self.ec2_userdata_server - self.url_ssh = ec2_sshdata_server or self.ec2_sshdata_server + def __init__(self, ec2_metadata_url=None, ec2_sshdata_url=None, ec2_userdata_url=None): + self.url_meta = ec2_metadata_url or self.ec2_metadata_url + self.url_user = ec2_userdata_url or self.ec2_userdata_url + self.url_ssh = ec2_sshdata_url or self.ec2_sshdata_url def _fetch(self, url): try: @@ -81,7 +81,7 @@ class Ec2Metadata(object): def main(): ec2_facts = Ec2Metadata().run() ec2_facts_result = { - "changed" : True, + "changed" : False, "ansible_facts" : ec2_facts } module = AnsibleModule( From 3ce0c04b713efe625a20b5601b68a6f3d744ef8b Mon Sep 17 00:00:00 2001 From: Michel Blanc Date: Wed, 23 Jan 2013 21:06:00 +0100 Subject: [PATCH 03/36] Update packaging/arch/PKGBUILD Uploaded new version to AUR and bumped version. --- packaging/arch/PKGBUILD | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/arch/PKGBUILD b/packaging/arch/PKGBUILD index db8230c1e2..3bcfd770e3 100644 --- a/packaging/arch/PKGBUILD +++ b/packaging/arch/PKGBUILD @@ -1,7 +1,7 @@ #Maintainer: Michel Blanc pkgname=ansible-git -pkgver=20130109 -pkgrel=2 +pkgver=20130123 +pkgrel=1 pkgdesc="A radically simple deployment, model-driven configuration management, and command execution framework" arch=('any') url="http://ansible.cc" From 2eedec777ff335345d57d37654c88a1eea1bcea9 Mon Sep 17 00:00:00 2001 From: Shaun Zinck Date: Wed, 23 Jan 2013 22:09:07 -0600 Subject: [PATCH 04/36] added first version of pkgin, just does install, remove (for SmartOS) --- library/pkgin | 142 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100755 library/pkgin diff --git a/library/pkgin b/library/pkgin new file mode 100755 index 0000000000..0d67341b2e --- /dev/null +++ b/library/pkgin @@ -0,0 +1,142 @@ +#!/usr/bin/python -tt +# -*- coding: utf-8 -*- + +# (c) 2013, Shaun Zinck +# Written by Shaun Zinck +# Based on pacman module written by Afterburn +# that was based on apt module written by Matthew Williams +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see . + + +DOCUMENTATION = ''' +--- +module: pkgin +short_description: Package manager for SmartOS +description: + - Manages SmartOS packages + +version_added: "1.0" +options: + name: + description: + - name of package to install/remove + required: true + + state: + description: + - state of the package installed or absent. + required: false + +author: Shaun Zinck +notes: [] +examples: + - code: "pkgin: name=foo state=installed" + description: install package foo" + - code: "pkgin: name=foo state=absent" + description: remove package foo + - code: "pkgin: name=foo,bar state=absent + description: remove packages foo and bar + +''' + + +import json +import shlex +import os +import sys + +PKGIN_PATH = "/opt/local/bin/pkgin" + +def query_package(module, name, state="installed"): + + if state == "installed": + + rc = os.system("%s list | grep ^%s" % (PKGIN_PATH, name)) + + if rc == 0: + return True + + return False + + +def remove_packages(module, packages): + + remove_c = 0 + # Using a for loop incase of error, we can report the package that failed + for package in packages: + # Query the package first, to see if we even need to remove + if not query_package(module, package): + continue + + rc = os.system("%s -y remove %s" % (PKGIN_PATH, package)) + + if rc != 0: + module.fail_json(msg="failed to remove %s" % (package)) + + remove_c += 1 + + if remove_c > 0: + + module.exit_json(changed=True, msg="removed %s package(s)" % remove_c) + + module.exit_json(changed=False, msg="package(s) already absent") + + +def install_packages(module, packages): + + install_c = 0 + + for package in packages: + if query_package(module, package): + continue + + rc = os.system("%s -y install %s" % (PKGIN_PATH, package)) + + if rc != 0: + module.fail_json(msg="failed to install %s" % (package)) + + install_c += 1 + + if install_c > 0: + module.exit_json(changed=True, msg="installed %s package(s)" % (install_c)) + + module.exit_json(changed=False, msg="package(s) already installed") + + + +def main(): + module = AnsibleModule( + argument_spec = dict( + state = dict(default="installed", choices=["installed","absent"]), + name = dict(aliases=["pkg"], required=True))) + + + if not os.path.exists(PKGIN_PATH): + module.fail_json(msg="cannot find pkgin, looking for %s" % (PKGIN_PATH)) + + p = module.params + + pkgs = p["name"].split(",") + + if p["state"] == "installed": + install_packages(module, pkgs) + + elif p["state"] == "absent": + remove_packages(module, pkgs) + +# this is magic, see lib/ansible/module_common.py +#<> + +main() From 4a9201ad65fc89e5a8bbb0281da0979c75d0224c Mon Sep 17 00:00:00 2001 From: Shaun Zinck Date: Wed, 23 Jan 2013 22:12:13 -0600 Subject: [PATCH 05/36] add pkgin to list of package managers in setup --- library/setup | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/library/setup b/library/setup index 07e4cdaf46..f4fa950e3e 100644 --- a/library/setup +++ b/library/setup @@ -82,10 +82,11 @@ class Facts(object): # A list of dicts. If there is a platform with more than one # package manager, put the preferred one last. If there is an # ansible module, use that as the value for the 'name' key. - PKG_MGRS = [ { 'path' : '/usr/bin/yum', 'name' : 'yum' }, - { 'path' : '/usr/bin/apt-get', 'name' : 'apt' }, - { 'path' : '/usr/bin/zypper', 'name' : 'zypper' }, - { 'path' : '/usr/bin/pacman', 'name' : 'pacman' } ] + PKG_MGRS = [ { 'path' : '/usr/bin/yum', 'name' : 'yum' }, + { 'path' : '/usr/bin/apt-get', 'name' : 'apt' }, + { 'path' : '/usr/bin/zypper', 'name' : 'zypper' }, + { 'path' : '/usr/bin/pacman', 'name' : 'pacman' }, + { 'path' : '/opt/local/bin/pkgin', 'name' : 'pkgin' } ] def __init__(self): self.facts = {} From 2a893ab0bb5f0d1a6edb6f5f018f300a87093e0a Mon Sep 17 00:00:00 2001 From: Silviu Dicu Date: Thu, 24 Jan 2013 19:14:32 -0500 Subject: [PATCH 06/36] ec2 facts module - updated as per comments --- library/ec2_facts | 80 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 25 deletions(-) diff --git a/library/ec2_facts b/library/ec2_facts index c9f85dd04a..40984f44c7 100644 --- a/library/ec2_facts +++ b/library/ec2_facts @@ -24,32 +24,36 @@ short_description: Gathers facts about remote hosts within ec2 (aws) options: {} description: - This module fetches data from the metadata servers in ec2 (aws). + Eucalyptus cloud provides a similar service and this module should + work this cloud provider as well. notes: - Parameters to filter on ec2_facts may be added later. - Some of the facts are not returned ( like mapping of the devices - but - may be added later). examples: - code: ansible all -m ec2_facts --tree /tmp/facts description: Obtain facts from ec2 metatdata servers. You will need to run an instance within ec2. -author: Silviu Dicu + +author: Silviu Dicu: silviudicu@gmail.com ''' import urllib2 import socket +import re socket.setdefaulttimeout(5) class Ec2Metadata(object): - ec2_metadata_url = 'http://169.254.169.254/latest/meta-data/' - ec2_sshdata_url = 'http://169.254.169.254/latest/meta-data/public-keys/0/openssh-key' - ec2_userdata_url = 'http://169.254.169.254/latest/user-data/' + ec2_metadata_uri = 'http://169.254.169.254/latest/meta-data/' + ec2_sshdata_uri = 'http://169.254.169.254/latest/meta-data/public-keys/0/openssh-key' + ec2_userdata_uri = 'http://169.254.169.254/latest/user-data/' - def __init__(self, ec2_metadata_url=None, ec2_sshdata_url=None, ec2_userdata_url=None): - self.url_meta = ec2_metadata_url or self.ec2_metadata_url - self.url_user = ec2_userdata_url or self.ec2_userdata_url - self.url_ssh = ec2_sshdata_url or self.ec2_sshdata_url + def __init__(self, ec2_metadata_uri=None, ec2_sshdata_uri=None, ec2_userdata_uri=None): + self.uri_meta = ec2_metadata_uri or self.ec2_metadata_uri + self.uri_user = ec2_userdata_uri or self.ec2_userdata_uri + self.uri_ssh = ec2_sshdata_uri or self.ec2_sshdata_uri + self._data = {} + self._prefix = 'ansible_ec2_%s' def _fetch(self, url): try: @@ -59,22 +63,48 @@ class Ec2Metadata(object): except urllib2.URLError: return - def run(self, field=None): - data = {} - raw_fields = self._fetch(self.url_meta) - if not raw_fields: - return data - fields = raw_fields.split('\n') - for field in fields: - if field.endswith('/'): continue # deal with this later - field_data = self._fetch(self.url_meta + field) - if field == 'security-groups': - sg_fields = ",".join(field_data.split('\n')) - data['ansible_ec2_%s' % field] = sg_fields + def _mangle_fields(self, fields, uri, filter_patterns=['public-keys-0']): + new_fields = {} + for key, value in fields.iteritems(): + split_fields = key[len(uri):].split('/') + if len(split_fields) > 1 and split_fields[1]: + new_key = "-".join(split_fields) + new_fields[self._prefix % new_key] = value else: - data['ansible_ec2_%s' % field] = field_data - data['ansible_ec2_%s' % 'user-data'] = self._fetch(self.url_user) - data['ensible_ec2_%s' % 'public-keys'] = self._fetch(self.url_ssh) + new_key = "".join(split_fields) + new_fields[self._prefix % new_key] = value + for pattern in filter_patterns: + for key in new_fields.keys(): + match = re.search(pattern, key) + if match: new_fields.pop(key) + return new_fields + + def fetch(self, uri, recurse=True): + raw_subfields = self._fetch(uri) + if not raw_subfields: + return + subfields = raw_subfields.split('\n') + for field in subfields: + if field.endswith('/') and recurse: + self.fetch(uri + field) + if uri.endswith('/'): + new_uri = uri + field + else: + new_uri = uri + '/' + field + if new_uri not in self._data and not new_uri.endswith('/'): + content = self._fetch(new_uri) + if field == 'security-groups': + sg_fields = ",".join(content.split('\n')) + self._data['%s' % (new_uri)] = sg_fields + else: + self._data['%s' % (new_uri)] = content + + def run(self): + self.fetch(self.uri_meta) # populate _data + data = self._mangle_fields(self._data, + self.uri_meta) + data[self._prefix % 'user-data'] = self._fetch(self.uri_user) + data[self._prefix % 'public-key'] = self._fetch(self.uri_ssh) return data From e34eaa423d768930630bc9fc34f7534ac95be9ff Mon Sep 17 00:00:00 2001 From: Shaun Zinck Date: Thu, 24 Jan 2013 19:58:31 -0600 Subject: [PATCH 07/36] pkgin: change install/remove to not use return code of pkgin pkgin always returns 0 so can't be used to tell if the install or remove worked. Instead this just queries the installed packages after performing an operation. --- library/pkgin | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library/pkgin b/library/pkgin index 0d67341b2e..acbc55e046 100755 --- a/library/pkgin +++ b/library/pkgin @@ -82,7 +82,7 @@ def remove_packages(module, packages): rc = os.system("%s -y remove %s" % (PKGIN_PATH, package)) - if rc != 0: + if query_package(module, package): module.fail_json(msg="failed to remove %s" % (package)) remove_c += 1 @@ -104,7 +104,7 @@ def install_packages(module, packages): rc = os.system("%s -y install %s" % (PKGIN_PATH, package)) - if rc != 0: + if not query_package(module, package): module.fail_json(msg="failed to install %s" % (package)) install_c += 1 From 05f5b5b27954cbdc199fb6345ac977680291cdef Mon Sep 17 00:00:00 2001 From: Juha Litola Date: Fri, 25 Jan 2013 13:29:39 +0200 Subject: [PATCH 08/36] Fixed add_key stalling indefinitely, and test code leaking into production setting --- library/apt_key | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/library/apt_key b/library/apt_key index dcc4d12717..8e521a4751 100644 --- a/library/apt_key +++ b/library/apt_key @@ -105,7 +105,7 @@ def download_key(url): def add_key(key): - return call("apt-key add -", shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE) + p = Popen("apt-key add -", shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE) (_, _) = p.communicate(key) return p.returncode == 0 @@ -163,12 +163,12 @@ if 'ANSIBLE_TEST_APT_KEY' in environ: return extra -if environ.get('ANSIBLE_TEST_APT_KEY') == 'none': - def key_present(key_id): - return False -else: - def key_present(key_id): - return key_id == environ['ANSIBLE_TEST_APT_KEY'] + if environ.get('ANSIBLE_TEST_APT_KEY') == 'none': + def key_present(key_id): + return False + else: + def key_present(key_id): + return key_id == environ['ANSIBLE_TEST_APT_KEY'] def main(): From 2796603c9206d70affb972064c9fe66612329aee Mon Sep 17 00:00:00 2001 From: Juha Litola Date: Fri, 25 Jan 2013 17:10:04 +0200 Subject: [PATCH 09/36] Removed apt_key tests, as they didn't test the real functionality. Tests used heavily mocked version of the apt_key code, which meant that it didn't properly test real life scenario. --- library/apt_key | 50 -------------------------------------- test/TestRunner.py | 60 ---------------------------------------------- 2 files changed, 110 deletions(-) diff --git a/library/apt_key b/library/apt_key index 8e521a4751..14d1b734fd 100644 --- a/library/apt_key +++ b/library/apt_key @@ -121,56 +121,6 @@ def return_values(tb=False): else: return {} - -# use cues from the environment to mock out functions for testing -if 'ANSIBLE_TEST_APT_KEY' in environ: - orig_download_key = download_key - KEY_ADDED=0 - KEY_REMOVED=0 - KEY_DOWNLOADED=0 - - - def download_key(url): - global KEY_DOWNLOADED - KEY_DOWNLOADED += 1 - return orig_download_key(url) - - - def find_missing_binaries(): - return [] - - - def add_key(key): - global KEY_ADDED - KEY_ADDED += 1 - return True - - - def remove_key(key_id): - global KEY_REMOVED - KEY_REMOVED += 1 - return True - - - def return_values(tb=False): - extra = dict( - added=KEY_ADDED, - removed=KEY_REMOVED, - downloaded=KEY_DOWNLOADED - ) - if tb: - extra['exception'] = format_exc() - return extra - - - if environ.get('ANSIBLE_TEST_APT_KEY') == 'none': - def key_present(key_id): - return False - else: - def key_present(key_id): - return key_id == environ['ANSIBLE_TEST_APT_KEY'] - - def main(): module = AnsibleModule( argument_spec=dict( diff --git a/test/TestRunner.py b/test/TestRunner.py index 9526dbbdee..6a191821ff 100644 --- a/test/TestRunner.py +++ b/test/TestRunner.py @@ -290,63 +290,3 @@ class TestRunner(unittest.TestCase): print result assert result['changed'] == False - def test_apt_key(self): - try: - key_file = self._get_test_file("apt_key.gpg") - key_file_url = 'file://' + urllib2.quote(key_file) - key_id = '473041FA' - - os.environ['ANSIBLE_TEST_APT_KEY'] = 'none' - # key missing, should download and add - result = self._run('apt_key', ['state=present', 'url=' + key_file_url]) - assert 'failed' not in result - assert result['added'] == 1 - assert result['downloaded'] == 1 - assert result['removed'] == 0 - assert result['changed'] - - os.environ["ANSIBLE_TEST_APT_KEY"] = key_id - # key missing, shouldn't download, no changes - result = self._run('apt_key', ['id=12345678', 'state=absent', 'url=' + key_file_url]) - assert 'failed' not in result - assert result['added'] == 0 - assert result['downloaded'] == 0 - assert result['removed'] == 0 - assert not result['changed'] - # key missing, should download and fail sanity check, no changes - result = self._run('apt_key', ['id=12345678', 'state=present', 'url=' + key_file_url]) - assert 'failed' in result - assert result['added'] == 0 - assert result['downloaded'] == 1 - assert result['removed'] == 0 - # key present, shouldn't download, no changes - result = self._run('apt_key', ['id=' + key_id, 'state=present', 'url=' + key_file_url]) - assert 'failed' not in result - assert result['added'] == 0 - assert result['downloaded'] == 0 - assert result['removed'] == 0 - assert not result['changed'] - # key present, should download to get key id - result = self._run('apt_key', ['state=present', 'url=' + key_file_url]) - assert 'failed' not in result - assert result['added'] == 0 - assert result['downloaded'] == 1 - assert result['removed'] == 0 - assert not result['changed'] - # key present, should download to get key id and remove - result = self._run('apt_key', ['state=absent', 'url=' + key_file_url]) - assert 'failed' not in result - assert result['added'] == 0 - assert result['downloaded'] == 1 - assert result['removed'] == 1 - assert result['changed'] - # key present, should remove but not download - result = self._run('apt_key', ['id=' + key_id, 'state=absent', 'url=' + key_file_url]) - assert 'failed' not in result - assert result['added'] == 0 - assert result['downloaded'] == 0 - assert result['removed'] == 1 - assert result['changed'] - finally: - # always clean up the environment - os.environ.pop('ANSIBLE_TEST_APT_KEY', None) From e9ca960558b8ffc391e38c9bdf2d0d8374e3c533 Mon Sep 17 00:00:00 2001 From: Shaun Zinck Date: Fri, 25 Jan 2013 16:48:58 -0600 Subject: [PATCH 10/36] pkgin: use module.run_command to run stuff This also fixes an issue where some console output for packages I was installing was creating invalid JSON because it contained single-quotes. --- library/pkgin | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/library/pkgin b/library/pkgin index acbc55e046..02fac16f27 100755 --- a/library/pkgin +++ b/library/pkgin @@ -63,7 +63,7 @@ def query_package(module, name, state="installed"): if state == "installed": - rc = os.system("%s list | grep ^%s" % (PKGIN_PATH, name)) + (rc, stdout, stderr) = module.run_command("%s list | grep ^%s" % (PKGIN_PATH, name)) if rc == 0: return True @@ -80,7 +80,7 @@ def remove_packages(module, packages): if not query_package(module, package): continue - rc = os.system("%s -y remove %s" % (PKGIN_PATH, package)) + module.run_command("%s -y remove %s" % (PKGIN_PATH, package)) if query_package(module, package): module.fail_json(msg="failed to remove %s" % (package)) @@ -102,7 +102,7 @@ def install_packages(module, packages): if query_package(module, package): continue - rc = os.system("%s -y install %s" % (PKGIN_PATH, package)) + module.run_command("%s -y install %s" % (PKGIN_PATH, package)) if not query_package(module, package): module.fail_json(msg="failed to install %s" % (package)) From b2aed1a4baf9830f75f7f2f20ffad8e24fb54fff Mon Sep 17 00:00:00 2001 From: Shaun Zinck Date: Fri, 25 Jan 2013 16:58:29 -0600 Subject: [PATCH 11/36] pkgin: add stdout to error message when shell commands fail --- library/pkgin | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/library/pkgin b/library/pkgin index 02fac16f27..eec541135b 100755 --- a/library/pkgin +++ b/library/pkgin @@ -63,7 +63,7 @@ def query_package(module, name, state="installed"): if state == "installed": - (rc, stdout, stderr) = module.run_command("%s list | grep ^%s" % (PKGIN_PATH, name)) + rc, out, err = module.run_command("%s list | grep ^%s" % (PKGIN_PATH, name)) if rc == 0: return True @@ -80,10 +80,10 @@ def remove_packages(module, packages): if not query_package(module, package): continue - module.run_command("%s -y remove %s" % (PKGIN_PATH, package)) + rc, out, err = module.run_command("%s -y remove %s" % (PKGIN_PATH, package)) if query_package(module, package): - module.fail_json(msg="failed to remove %s" % (package)) + module.fail_json(msg="failed to remove %s: %s" % (package, out)) remove_c += 1 @@ -102,10 +102,10 @@ def install_packages(module, packages): if query_package(module, package): continue - module.run_command("%s -y install %s" % (PKGIN_PATH, package)) + rc, out, err = module.run_command("%s -y install %s" % (PKGIN_PATH, package)) if not query_package(module, package): - module.fail_json(msg="failed to install %s" % (package)) + module.fail_json(msg="failed to install %s: %s" % (package, out)) install_c += 1 From d683c2431e14769095d99c5ec7cb105e94490ce6 Mon Sep 17 00:00:00 2001 From: igor Date: Thu, 24 Jan 2013 16:16:23 +0100 Subject: [PATCH 12/36] add support for user:password syntax in urls to get_url --- library/get_url | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/library/get_url b/library/get_url index 933c4445b2..1cbc4a12a8 100644 --- a/library/get_url +++ b/library/get_url @@ -100,6 +100,29 @@ def url_do_get(module, url, dest): USERAGENT = 'ansible-httpget' info = dict(url=url, dest=dest) r = None + parsed = urlparse.urlparse(url) + if '@' in parsed.netloc: + credentials = parsed.netloc.split('@')[0] + if ':' in credentials: + username, password = credentials.split(':') + netloc = parsed.netloc.split('@')[1] + parsed = list(parsed) + parsed[1] = netloc + + passman = urllib2.HTTPPasswordMgrWithDefaultRealm() + # this creates a password manager + passman.add_password(None, netloc, username, password) + # because we have put None at the start it will always + # use this username/password combination for urls + # for which `theurl` is a super-url + + authhandler = urllib2.HTTPBasicAuthHandler(passman) + # create the AuthHandler + + opener = urllib2.build_opener(authhandler) + urllib2.install_opener(opener) + #reconstruct url without credentials + url = urlparse.urlunparse(parsed) request = urllib2.Request(url) request.add_header('User-agent', USERAGENT) From 005f864068fd9937e02d7f4f4ae14c6c5ef7c9c3 Mon Sep 17 00:00:00 2001 From: igor Date: Thu, 24 Jan 2013 21:19:14 +0100 Subject: [PATCH 13/36] minor doc changes --- library/get_url | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/library/get_url b/library/get_url index 1cbc4a12a8..4e896070c9 100644 --- a/library/get_url +++ b/library/get_url @@ -35,7 +35,8 @@ version_added: "0.6" options: url: description: - - HTTP, HTTPS, or FTP URL + - HTTP, HTTPS, or FTP URL. + (http|https|ftp)://[user[:pass]]@host.domain[:port]/path required: true default: null aliases: [] @@ -63,18 +64,18 @@ examples: - code: "get_url: url=http://example.com/path/file.conf dest=/etc/foo.conf mode=0440" description: "Example from Ansible Playbooks" notes: - - This module doesn't yet support configuration for proxies or passwords. + - This module doesn't yet support configuration for proxies. # informational: requirements for nodes requirements: [ urllib2, urlparse ] author: Jan-Piet Mens ''' -HAS_URLLIB2=True +HAS_URLLIB2 = True try: import urllib2 except ImportError: - HAS_URLLIB2=False -HAS_URLPARSE=True + HAS_URLLIB2 = False +HAS_URLPARSE = True try: import urlparse @@ -255,8 +256,8 @@ def main(): # Mission complete module.exit_json(url=url, dest=dest, src=tmpsrc, md5sum=md5sum_src, - changed=changed, msg=info.get('msg',''), - daisychain="file", daisychain_args=info.get('daisychain_args','')) + changed=changed, msg=info.get('msg', ''), + daisychain="file", daisychain_args=info.get('daisychain_args', '')) # this is magic, see lib/ansible/module_common.py #<> From b0ecfbb26d391d8c32b34fd3a8d4631030521ca7 Mon Sep 17 00:00:00 2001 From: Lester Wade Date: Fri, 25 Jan 2013 15:02:53 +0000 Subject: [PATCH 14/36] Update library/ec2 I've uncommented and added a very little supporting stuff based on skvidal's work to allow us to launch more than one instance. --- library/ec2 | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/library/ec2 b/library/ec2 index 034da936af..4a79bf0b2a 100644 --- a/library/ec2 +++ b/library/ec2 @@ -66,7 +66,7 @@ options: aliases: [] ec2_url: description: - - url to use to connect to ec2 or your cloud (for example U(https://ec2.amazonaws.com) when using Amazon ec2 directly and not Eucalyptus) + - url to use to connect to ec2 or your Eucalyptus cloud (for example (https://ec2.amazonaws.com) when using Amazon ec2 directly and not Eucalyptus) required: False default: null aliases: [] @@ -82,6 +82,12 @@ options: required: False default: null aliases: [] + count: + description: + - number of instances to launch + required: False + default: 1 + aliases: [] user_data: version_added: "0.9" description: @@ -90,10 +96,10 @@ options: default: null aliases: [] examples: - - code: "local_action: ec2 keypair=admin instance_type=m1.large image=emi-40603AD1 wait=true group=webserver" + - code: "local_action: ec2 keypair=admin instance_type=m1.large image=emi-40603AD1 wait=true group=webserver count=3" description: "Examples from Ansible Playbooks" requirements: [ "boto" ] -author: Seth Vidal, Tim Gerla +author: Seth Vidal, Tim Gerla, Lester Wade ''' import sys @@ -113,7 +119,7 @@ def main(): instance_type = dict(aliases=['type']), image = dict(required=True), kernel = dict(), - #count = dict(default='1'), # maybe someday + count = dict(default='1'), ramdisk = dict(), wait = dict(choices=BOOLEANS, default=False), ec2_url = dict(aliases=['EC2_URL']), @@ -127,7 +133,7 @@ def main(): group = module.params.get('group') instance_type = module.params.get('instance_type') image = module.params.get('image') - #count = module.params.get('count') + count = module.params.get('count') kernel = module.params.get('kernel') ramdisk = module.params.get('ramdisk') wait = module.params.get('wait') @@ -148,10 +154,13 @@ def main(): ec2 = boto.connect_ec2_endpoint(ec2_url, ec2_access_key, ec2_secret_key) else: # otherwise it's Amazon. ec2 = boto.connect_ec2(ec2_access_key, ec2_secret_key) + +# Note min_count is static in value. Since we aren't interested in addressing an autoscaling use-case. +# Autoscaling means more instances are launched on a triggered event, so this is post-play/run stuff. try: res = ec2.run_instances(image, key_name = key_name, - min_count = 1, max_count = 1, + min_count = 1, max_count = count, security_groups = [group], instance_type = instance_type, kernel_id = kernel, @@ -171,9 +180,8 @@ def main(): res_list = res.connection.get_all_instances(instids) this_res = res_list[0] num_running = len([ i for i in this_res.instances if i.state=='running' ]) - time.sleep(2) + time.sleep(5) - # there's only one - but maybe one day there could be more instances = [] for inst in this_res.instances: d = { From de833b3ae0137b6619f89358ee3a641bdc236195 Mon Sep 17 00:00:00 2001 From: Lester Wade Date: Fri, 25 Jan 2013 16:48:27 +0000 Subject: [PATCH 15/36] Update library/ec2 bumping up min_count value. --- library/ec2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/ec2 b/library/ec2 index 4a79bf0b2a..1fd7b098dd 100644 --- a/library/ec2 +++ b/library/ec2 @@ -160,7 +160,7 @@ def main(): try: res = ec2.run_instances(image, key_name = key_name, - min_count = 1, max_count = count, + min_count = count, max_count = count, security_groups = [group], instance_type = instance_type, kernel_id = kernel, From e5b64b2f74d311ed85a28fec83460ecb3869cdfc Mon Sep 17 00:00:00 2001 From: Yeukhon Wong Date: Fri, 25 Jan 2013 22:51:20 -0500 Subject: [PATCH 16/36] Added hg module to the core. --- library/hg | 267 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100644 library/hg diff --git a/library/hg b/library/hg new file mode 100644 index 0000000000..04d74c44fd --- /dev/null +++ b/library/hg @@ -0,0 +1,267 @@ +#!/usr/bin/python +#-*- coding: utf-8 -*- + +# (c) 2013, Yeukhon Wong +# +# This module was originally inspired by Brad Olson's ansible-module-mercurial +# . +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +import os +import shutil +import ConfigParser +from subprocess import Popen, PIPE + +DOCUMENTATION = ''' +--- +module: hg +short_description: Manages Mercurial (hg) repositories. +description: + - Manages Mercurial (hg) repositories. Supports SSH, HTTP/S and local address. +version_added: "1.0" +author: Yeukhon Wong +options: + repo: + description: + - The repository location. + required: true + default: null + dest: + description: + - Absolute path of where the repository should be cloned to. + required: true + default: null + state: + description: + - C(hg clone) is performed when state is set to C(present). C(hg pull) and C(hg update) is + performed when state is set to C(latest). If you want the latest copy of the repository, + just rely on C(present). C(latest) assumes the repository is already on disk. + required: false + default: present + choices: [ "present", "absent", "latest" ] + revision: + description: + - Equivalent C(-r) option in hg command, which can either be a changeset number or a branch + name. + required: false + default: "default" + force: + description: + - Whether to discard uncommitted changes and remove untracked files or not. Basically, it + combines C(hg up -C) and C(hg purge). + required: false + default: "yes" + choices: [ "yes", "no" ] + +examples: + - code: "hg: repo=https://bitbucket.org/user/repo_name dest=/home/user/repo_name" + description: Clone the default branch of repo_name. + + - code: "hg: repo=https://bitbucket.org/user/repo_name dest=/home/user/repo_name force=yes state=latest" + description: Ensure the repository at dest is latest and discard any uncommitted and/or untracked files. + +notes: + - If the task seems to be hanging, first verify remote host is in C(known_hosts). + SSH will prompt user to authorize the first contact with a remote host. One solution is to add + C(StrictHostKeyChecking no) in C(.ssh/config) which will accept and authorize the connection + on behalf of the user. However, if you run as a different user such as setting sudo to True), + for example, root will not look at the user .ssh/config setting. + +requirements: [ ] +''' + +class HgError(Exception): + """ Custom exception class to report hg command error. """ + + def __init__(self, msg, stderr=''): + self.msg = msg + \ + "\n\nExtra information on this error: \n" + \ + stderr + def __str__(self): + return self.msg + +def _set_hgrc(hgrc, vals): + # val is a list of triple-tuple of the form [(section, option, value),...] + parser = ConfigParser.SafeConfigParser() + parser.read(hgrc) + + for each in vals: + section,option, value = each + if not parser.has_section(section): + parser.add_section(section) + parser.set(section, option, value) + + f = open(hgrc, 'w') + parser.write(f) + f.close() + +def _undo_hgrc(hgrc, vals): + parser = ConfigParser.SafeConfigParser() + parser.read(hgrc) + + for each in vals: + section, option, value = each + if parser.has_section(section): + parser.remove_option(section, option) + + f = open(hgrc, 'w') + parser.write(f) + f.close() + +def _hg_command(args_list): + cmd = ['hg'] + args_list + p = Popen(cmd, stdout=PIPE, stderr=PIPE) + out, err = p.communicate() + return out, err, p.returncode + +def _hg_discard(dest): + out, err, code = _hg_command(['up', '-C', '-R', dest]) + if code != 0: + raise HgError(err) + +def _hg_purge(dest): + hgrc = os.path.join(dest, '.hg/hgrc') + purge_option = [('extensions', 'purge', '')] + _set_hgrc(hgrc, purge_option) + out, err, code = _hg_command(['purge', '-R', dest]) + if code == 0: + _undo_hgrc(hgrc, purge_option) + else: + raise HgError(err) + +def _hg_verify(dest): + error1 = "hg verify failed." + error2 = "{dest} is not a repository.".format(dest=dest) + + out, err, code = _hg_command(['verify', '-R', dest]) + if code == 1: + raise HgError(error1, stderr=err) + elif code == 255: + raise HgError(error2, stderr=err) + elif code == 0: + return True + +def _post_op_hg_revision_check(dest, revision): + """ + Verify the tip is the same as `revision`. + + This function is usually called after some hg operations + such as `clone`. However, this check is skipped if `revision` + is the string `default` since it will result an error. + Instead, pull is performed. + + """ + + err1 = "Unable to perform hg tip." + err2 = "tip is different from %s. See below for extended summary." % revision + + if revision == 'default': + out, err, code = _hg_command(['pull', '-R', dest]) + if "no changes found" in out: + return True + else: + raise HgError(err2, stderr=out) + else: + out, err, code = _hg_command(['tip', '-R', dest]) + if revision in out: # revision should be part of the output (changeset: $revision ...) + return True + else: + if code != 0: # something went wrong with hg tip + raise HgError(err1, stderr=err) + else: # hg tip is fine, but tip != revision + raise HgError(err2, stderr=out) + +def force_and_clean(dest): + _hg_discard(dest) + _hg_purge(dest) + +def pull_and_update(repo, dest, revision, force): + if force == 'yes': + force_and_clean(dest) + + if _hg_verify(dest): + cmd1 = ['pull', '-R', dest, '-r', revision] + out, err, code = _hg_command(cmd1) + + if code == 1: + raise HgError("Unable to perform pull on %s" % dest, stderr=err) + elif code == 0: + cmd2 = ['update', '-R', dest, '-r', revision] + out, err, code = _hg_command(cmd2) + if code == 1: + raise HgError("There are unresolved files in %s" % dest, stderr=err) + elif code == 0: + # so far pull and update seems to be working, check revision and $revision are equal + _post_op_hg_revision_check(dest, revision) + return True + # when code aren't 1 or 0 in either command + raise HgError("", stderr=err) + +def clone(repo, dest, revision, force): + if os.path.exists(dest): + if _hg_verify(dest): # make sure it's a real repo + if _post_op_hg_revision_check(dest, revision): # make sure revision and $revision are equal + if force == 'yes': + force_and_clean(dest) + return False + + cmd = ['clone', repo, dest, '-r', revision] + out, err, code = _hg_command(cmd) + if code == 0: + _hg_verify(dest) + _post_op_hg_revision_check(dest, revision) + return True + else: + raise HgError(err, stderr='') + +def main(): + module = AnsibleModule( + argument_spec = dict( + repo = dict(required=True), + dest = dict(required=True), + state = dict(default='present', choices=['present', 'absent', 'latest']), + revision = dict(default="default"), + force = dict(default='yes', choices=['yes', 'no']), + ), + ) + repo = module.params['repo'] + state = module.params['state'] + dest = module.params['dest'] + revision = module.params['revision'] + force = module.params['force'] + + try: + if state == 'absent': + if not os.path.exists(dest): + shutil.rmtree(dest) + changed = True + elif state == 'present': + changed = clone(repo, dest, revision, force) + elif state == 'latest': + changed = pull_and_update(repo, dest, revision, force) + + module.exit_json(dest=dest, changed=changed) + #except HgError as e: + # module.fail_json(msg=str(e), params=module.params) + #except IOError as e: + # module.fail_json(msg=str(e), params=module.params) + except Exception as e: + module.fail_json(msg=str(e), params=module.params) + +# include magic from lib/ansible/module_common.py +#<> +main() From 2da3a54e29c44e9ac3d8976985ce8a096956d03a Mon Sep 17 00:00:00 2001 From: Lester Wade Date: Sat, 26 Jan 2013 11:26:51 +0000 Subject: [PATCH 17/36] Update library/ec2 updated indentation and redundant comment. --- library/ec2 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/library/ec2 b/library/ec2 index 1fd7b098dd..03d1cbc6ff 100644 --- a/library/ec2 +++ b/library/ec2 @@ -155,9 +155,8 @@ def main(): else: # otherwise it's Amazon. ec2 = boto.connect_ec2(ec2_access_key, ec2_secret_key) -# Note min_count is static in value. Since we aren't interested in addressing an autoscaling use-case. -# Autoscaling means more instances are launched on a triggered event, so this is post-play/run stuff. - + # Both min_count and max_count equal count parameter. This means the launch request is explicit (we want count, or fail) in how many instances we want. + try: res = ec2.run_instances(image, key_name = key_name, min_count = count, max_count = count, From 5e4d29565cb4b8ac65ffd542713f1af65b951e99 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Sat, 26 Jan 2013 12:18:41 -0500 Subject: [PATCH 18/36] Standardize the hg command execution around our run command function. --- library/hg | 66 +++++++++++++++++++++++++----------------------------- 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/library/hg b/library/hg index 04d74c44fd..2559d5a5d2 100644 --- a/library/hg +++ b/library/hg @@ -122,32 +122,30 @@ def _undo_hgrc(hgrc, vals): parser.write(f) f.close() -def _hg_command(args_list): - cmd = ['hg'] + args_list - p = Popen(cmd, stdout=PIPE, stderr=PIPE) - out, err = p.communicate() - return out, err, p.returncode +def _hg_command(module, args_list): + (rc, out, err) = module.run_command(['hg'] + args_list) + return (out, err, rc) -def _hg_discard(dest): - out, err, code = _hg_command(['up', '-C', '-R', dest]) +def _hg_discard(module, dest): + out, err, code = _hg_command(module, ['up', '-C', '-R', dest]) if code != 0: raise HgError(err) -def _hg_purge(dest): +def _hg_purge(module, dest): hgrc = os.path.join(dest, '.hg/hgrc') purge_option = [('extensions', 'purge', '')] _set_hgrc(hgrc, purge_option) - out, err, code = _hg_command(['purge', '-R', dest]) + out, err, code = _hg_command(module, ['purge', '-R', dest]) if code == 0: _undo_hgrc(hgrc, purge_option) else: raise HgError(err) -def _hg_verify(dest): +def _hg_verify(module, dest): error1 = "hg verify failed." error2 = "{dest} is not a repository.".format(dest=dest) - out, err, code = _hg_command(['verify', '-R', dest]) + out, err, code = _hg_command(module, ['verify', '-R', dest]) if code == 1: raise HgError(error1, stderr=err) elif code == 255: @@ -155,7 +153,7 @@ def _hg_verify(dest): elif code == 0: return True -def _post_op_hg_revision_check(dest, revision): +def _post_op_hg_revision_check(module, dest, revision): """ Verify the tip is the same as `revision`. @@ -170,13 +168,13 @@ def _post_op_hg_revision_check(dest, revision): err2 = "tip is different from %s. See below for extended summary." % revision if revision == 'default': - out, err, code = _hg_command(['pull', '-R', dest]) + out, err, code = _hg_command(module, ['pull', '-R', dest]) if "no changes found" in out: return True else: raise HgError(err2, stderr=out) else: - out, err, code = _hg_command(['tip', '-R', dest]) + out, err, code = _hg_command(module, ['tip', '-R', dest]) if revision in out: # revision should be part of the output (changeset: $revision ...) return True else: @@ -185,45 +183,45 @@ def _post_op_hg_revision_check(dest, revision): else: # hg tip is fine, but tip != revision raise HgError(err2, stderr=out) -def force_and_clean(dest): - _hg_discard(dest) - _hg_purge(dest) +def force_and_clean(module, dest): + _hg_discard(module, dest) + _hg_purge(module, dest) -def pull_and_update(repo, dest, revision, force): +def pull_and_update(module, repo, dest, revision, force): if force == 'yes': - force_and_clean(dest) + force_and_clean(module, dest) - if _hg_verify(dest): + if _hg_verify(module, dest): cmd1 = ['pull', '-R', dest, '-r', revision] - out, err, code = _hg_command(cmd1) + out, err, code = _hg_command(module, cmd1) if code == 1: raise HgError("Unable to perform pull on %s" % dest, stderr=err) elif code == 0: cmd2 = ['update', '-R', dest, '-r', revision] - out, err, code = _hg_command(cmd2) + out, err, code = _hg_command(module, cmd2) if code == 1: raise HgError("There are unresolved files in %s" % dest, stderr=err) elif code == 0: # so far pull and update seems to be working, check revision and $revision are equal - _post_op_hg_revision_check(dest, revision) + _post_op_hg_revision_check(module, dest, revision) return True # when code aren't 1 or 0 in either command raise HgError("", stderr=err) -def clone(repo, dest, revision, force): +def clone(module, repo, dest, revision, force): if os.path.exists(dest): - if _hg_verify(dest): # make sure it's a real repo - if _post_op_hg_revision_check(dest, revision): # make sure revision and $revision are equal + if _hg_verify(module, dest): # make sure it's a real repo + if _post_op_hg_revision_check(module, dest, revision): # make sure revision and $revision are equal if force == 'yes': - force_and_clean(dest) + force_and_clean(module, dest) return False cmd = ['clone', repo, dest, '-r', revision] - out, err, code = _hg_command(cmd) + out, err, code = _hg_command(module, cmd) if code == 0: - _hg_verify(dest) - _post_op_hg_revision_check(dest, revision) + _hg_verify(module, dest) + _post_op_hg_revision_check(module, dest, revision) return True else: raise HgError(err, stderr='') @@ -250,15 +248,11 @@ def main(): shutil.rmtree(dest) changed = True elif state == 'present': - changed = clone(repo, dest, revision, force) + changed = clone(module, repo, dest, revision, force) elif state == 'latest': - changed = pull_and_update(repo, dest, revision, force) + changed = pull_and_update(module, repo, dest, revision, force) module.exit_json(dest=dest, changed=changed) - #except HgError as e: - # module.fail_json(msg=str(e), params=module.params) - #except IOError as e: - # module.fail_json(msg=str(e), params=module.params) except Exception as e: module.fail_json(msg=str(e), params=module.params) From 925c1a461115030ff21f3e6f5e146a780c7704f9 Mon Sep 17 00:00:00 2001 From: fdavis Date: Sat, 26 Jan 2013 09:25:35 -0800 Subject: [PATCH 19/36] add env vars for plugins action,callback,connection,lookup,vars,filter --- lib/ansible/constants.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/ansible/constants.py b/lib/ansible/constants.py index 117b9a1e91..7bd75777f2 100644 --- a/lib/ansible/constants.py +++ b/lib/ansible/constants.py @@ -94,12 +94,12 @@ DEFAULT_KEEP_REMOTE_FILES = get_config(p, DEFAULTS, 'keep_remote_files', 'ANSIBL DEFAULT_SUDO_EXE = get_config(p, DEFAULTS, 'sudo_exe', 'ANSIBLE_SUDO_EXE', 'sudo') DEFAULT_HASH_BEHAVIOUR = get_config(p, DEFAULTS, 'hash_behaviour', 'ANSIBLE_HASH_BEHAVIOUR', 'replace') -DEFAULT_ACTION_PLUGIN_PATH = shell_expand_path(get_config(p, DEFAULTS, 'action_plugins', None, '/usr/share/ansible_plugins/action_plugins')) -DEFAULT_CALLBACK_PLUGIN_PATH = shell_expand_path(get_config(p, DEFAULTS, 'callback_plugins', None, '/usr/share/ansible_plugins/callback_plugins')) -DEFAULT_CONNECTION_PLUGIN_PATH = shell_expand_path(get_config(p, DEFAULTS, 'connection_plugins', None, '/usr/share/ansible_plugins/connection_plugins')) -DEFAULT_LOOKUP_PLUGIN_PATH = shell_expand_path(get_config(p, DEFAULTS, 'lookup_plugins', None, '/usr/share/ansible_plugins/lookup_plugins')) -DEFAULT_VARS_PLUGIN_PATH = shell_expand_path(get_config(p, DEFAULTS, 'vars_plugins', None, '/usr/share/ansible_plugins/vars_plugins')) -DEFAULT_FILTER_PLUGIN_PATH = shell_expand_path(get_config(p, DEFAULTS, 'filter_plugins', None, '/usr/share/ansible_plugins/filter_plugins')) +DEFAULT_ACTION_PLUGIN_PATH = shell_expand_path(get_config(p, DEFAULTS, 'action_plugins', 'ANSIBLE_ACTION_PLUGINS', '/usr/share/ansible_plugins/action_plugins')) +DEFAULT_CALLBACK_PLUGIN_PATH = shell_expand_path(get_config(p, DEFAULTS, 'callback_plugins', 'ANSIBLE_CALLBACK_PLUGINS', '/usr/share/ansible_plugins/callback_plugins')) +DEFAULT_CONNECTION_PLUGIN_PATH = shell_expand_path(get_config(p, DEFAULTS, 'connection_plugins', 'ANSIBLE_CONNECTION_PLUGINS', '/usr/share/ansible_plugins/connection_plugins')) +DEFAULT_LOOKUP_PLUGIN_PATH = shell_expand_path(get_config(p, DEFAULTS, 'lookup_plugins', 'ANSIBLE_LOOKUP_PLUGINS', '/usr/share/ansible_plugins/lookup_plugins')) +DEFAULT_VARS_PLUGIN_PATH = shell_expand_path(get_config(p, DEFAULTS, 'vars_plugins', 'ANSIBLE_VARS_PLUGINS', '/usr/share/ansible_plugins/vars_plugins')) +DEFAULT_FILTER_PLUGIN_PATH = shell_expand_path(get_config(p, DEFAULTS, 'filter_plugins', 'ANSIBLE_FILTER_PLUGINS', '/usr/share/ansible_plugins/filter_plugins')) # non-configurable things DEFAULT_SUDO_PASS = None From 36066d8664077b77830ce8fcbc2a790950892b92 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Sat, 26 Jan 2013 12:34:30 -0500 Subject: [PATCH 20/36] Fixup ec2_facts docs parsing --- bin/ansible-doc | 9 ++++++--- lib/ansible/utils/module_docs.py | 1 - library/ec2_facts | 13 +++++-------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/bin/ansible-doc b/bin/ansible-doc index 7648e44178..9e8878376c 100755 --- a/bin/ansible-doc +++ b/bin/ansible-doc @@ -33,6 +33,7 @@ from ansible import errors from ansible.utils import module_docs import ansible.constants as C from ansible.utils import version +import traceback MODULEDIR = C.DEFAULT_MODULE_PATH @@ -162,7 +163,8 @@ def main(): desc = desc + '...' print "%-20s %-60.60s" % (module, desc) except: - sys.stderr.write("ERROR: module %s missing documentation\n" % module) + traceback.print_exc() + sys.stderr.write("ERROR: module %s has a documentation error formatting or is missing documentation\n" % module) pass sys.exit() @@ -184,10 +186,11 @@ def main(): try: doc = module_docs.get_docstring(filename) except: - sys.stderr.write("ERROR: module %s missing documentation\n" % module) + traceback.print_exc() + sys.stderr.write("ERROR: module %s has a documentation error formatting or is missing documentation\n" % module) continue - if not doc is None: + if doc is not None: all_keys = [] for (k,v) in doc['options'].iteritems(): diff --git a/lib/ansible/utils/module_docs.py b/lib/ansible/utils/module_docs.py index 014c5dbcfd..221cb4ad4e 100755 --- a/lib/ansible/utils/module_docs.py +++ b/lib/ansible/utils/module_docs.py @@ -43,7 +43,6 @@ def get_docstring(filename, verbose=False): if isinstance(child, ast.Assign): if 'DOCUMENTATION' in (t.id for t in child.targets): doc = yaml.load(child.value.s) - except: if verbose == True: traceback.print_exc() diff --git a/library/ec2_facts b/library/ec2_facts index 40984f44c7..ef1628fac8 100644 --- a/library/ec2_facts +++ b/library/ec2_facts @@ -16,8 +16,7 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see . - -DOCUMENTATION = ''' +DOCUMENTATION=""" --- module: ec2_facts short_description: Gathers facts about remote hosts within ec2 (aws) @@ -29,12 +28,10 @@ description: notes: - Parameters to filter on ec2_facts may be added later. examples: - - code: ansible all -m ec2_facts --tree /tmp/facts - description: Obtain facts from ec2 metatdata servers. You will need to - run an instance within ec2. - -author: Silviu Dicu: silviudicu@gmail.com -''' + - code: ansible all -m ec2_facts + description: Obtain facts from ec2 metatdata servers. You will need to run an instance within ec2. +author: "Silviu Dicu " +""" import urllib2 import socket From 970ae584c9129a32a95f72233d0e0ed0d8789960 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Sat, 26 Jan 2013 12:38:08 -0500 Subject: [PATCH 21/36] fix documentation formatting for get_url --- library/get_url | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/library/get_url b/library/get_url index 4e896070c9..8d6e85f0d8 100644 --- a/library/get_url +++ b/library/get_url @@ -35,8 +35,7 @@ version_added: "0.6" options: url: description: - - HTTP, HTTPS, or FTP URL. - (http|https|ftp)://[user[:pass]]@host.domain[:port]/path + - HTTP, HTTPS, or FTP URL in the form (http|https|ftp)://[user[:pass]]@host.domain[:port]/path required: true default: null aliases: [] From e9d5cf3bc2d0b7c01dbd11cffb828f17ea38cd5e Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Sat, 26 Jan 2013 12:44:43 -0500 Subject: [PATCH 22/36] Fix documentation YAML for pkgin module --- Makefile | 4 ++++ library/pkgin | 24 +++++++++++------------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 1d7a0ff4a7..4e5d1ee4f6 100644 --- a/Makefile +++ b/Makefile @@ -180,3 +180,7 @@ modulejs: webdocs: (cd docsite; make docs) +# just for quick testing of all the module docs +webdocs2: + (cd docsite; make modules) + diff --git a/library/pkgin b/library/pkgin index eec541135b..05891d3f63 100755 --- a/library/pkgin +++ b/library/pkgin @@ -26,29 +26,27 @@ module: pkgin short_description: Package manager for SmartOS description: - Manages SmartOS packages - version_added: "1.0" options: name: description: - name of package to install/remove required: true - state: description: - - state of the package installed or absent. + - state of the package + choices: [ 'present', 'absent' ] required: false - + default: present author: Shaun Zinck notes: [] examples: - - code: "pkgin: name=foo state=installed" + - code: "pkgin: name=foo state=present" description: install package foo" - code: "pkgin: name=foo state=absent" description: remove package foo - - code: "pkgin: name=foo,bar state=absent + - code: "pkgin: name=foo,bar state=absent" description: remove packages foo and bar - ''' @@ -59,9 +57,9 @@ import sys PKGIN_PATH = "/opt/local/bin/pkgin" -def query_package(module, name, state="installed"): +def query_package(module, name, state="present"): - if state == "installed": + if state == "present": rc, out, err = module.run_command("%s list | grep ^%s" % (PKGIN_PATH, name)) @@ -110,16 +108,16 @@ def install_packages(module, packages): install_c += 1 if install_c > 0: - module.exit_json(changed=True, msg="installed %s package(s)" % (install_c)) + module.exit_json(changed=True, msg="present %s package(s)" % (install_c)) - module.exit_json(changed=False, msg="package(s) already installed") + module.exit_json(changed=False, msg="package(s) already present") def main(): module = AnsibleModule( argument_spec = dict( - state = dict(default="installed", choices=["installed","absent"]), + state = dict(default="present", choices=["present","absent"]), name = dict(aliases=["pkg"], required=True))) @@ -130,7 +128,7 @@ def main(): pkgs = p["name"].split(",") - if p["state"] == "installed": + if p["state"] == "present": install_packages(module, pkgs) elif p["state"] == "absent": From 25f52d79d1661a34854c651a678e9bd357fea59e Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Sat, 26 Jan 2013 12:58:12 -0500 Subject: [PATCH 23/36] Tweak sysctl docs so they'll web-render --- bin/ansible-doc | 1 + library/sysctl | 26 +++++++++++--------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/bin/ansible-doc b/bin/ansible-doc index 9e8878376c..7618115d21 100755 --- a/bin/ansible-doc +++ b/bin/ansible-doc @@ -76,6 +76,7 @@ def print_man(doc): opt_leadin = "-" print "%s %s" % (opt_leadin, o) + desc = "".join(opt['description']) if 'choices' in opt: diff --git a/library/sysctl b/library/sysctl index b49e7dcd21..6231f240e3 100644 --- a/library/sysctl +++ b/library/sysctl @@ -24,47 +24,43 @@ DOCUMENTATION = ''' module: sysctl short_description: Permit to handle sysctl.conf entries description: - - This module handle the entries in C(/etc/sysctl.conf), - and perform a I(/sbin/sysctl -p) after any change + - This module manipulates sysctl entries and performs a I(/sbin/sysctl -p) after changing them. version_added: "0.6" options: name: description: - - | - also known as "key", - this is the short path, point separated to the sysctl entry eg: C(vm.swappiness)" + - "this is the short path, decimal seperated, to the sysctl entry, ex: C(vm.swappiness)" required: true default: null aliases: [ 'key' ] value: description: - - "value to affect to the sysctl entry, to not provide if state=absent" + - set the sysctl value to this entry required: false default: null aliases: [ 'val' ] state: description: - - state=present the entry is added if not exist, or updated if exist - state=absent the entry is removed if exist + - whether the entry should be present or absent choices: [ "present", "absent" ] default: present checks: description: - - C(checks)=I(none) no smart/facultative checks will be made - C(checks)=I(before) some checks performed before any update (ie. does the sysctl key is writable ?) - C(checks)=I(after) some checks performed after an update (ie. does kernel give back the setted value ?) - C(checks)=I(both) all the smart checks I(before and after) are performed + - if C(checks)=I(none) no smart/facultative checks will be made + - if C(checks)=I(before) some checks performed before any update (ie. does the sysctl key is writable ?) + - if C(checks)=I(after) some checks performed after an update (ie. does kernel give back the setted value ?) + - if C(checks)=I(both) all the smart checks I(before and after) are performed choices: [ "none", "before", "after", "both" ] default: both reload: description: - - C(reload=yes) perform a I(/sbin/sysctl -p) if C(sysctl_file) updated ! - C(reload=no) do not reload I(sysctl) even if C(sysctl_file) updated ! + - if C(reload=yes), performs a I(/sbin/sysctl -p) if the C(sysctl_file) is updated + - if C(reload=no), does not reload I(sysctl) even if the C(sysctl_file) is updated choices: [ yes, no ] default: yes sysctl_file: description: - - specify the absolute path to C(/etc/sysctl.conf) + - specifies the absolute path to C(sysctl.conf), if not /etc/sysctl.conf required: false default: /etc/sysctl.conf examples: From b749af0b103c57c5a6f13719f6e25b7e35c1d8b8 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Sat, 26 Jan 2013 13:07:06 -0500 Subject: [PATCH 24/36] New changelog additions from today's merging --- CHANGELOG.md | 11 +++++++++++ library/sysctl | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d97db17cd..c11f3a610a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,17 @@ Ansible Changes By Release * "template override" ?? * lots of documentation tweaks * handle tilde shell character for --private-key +* get_url module can now send user/password authorization +* new ec2_facts module +* hg module now in core +* ec2 module can now deploy multiple simultaneous instances +* fix for apt_key modules stalling in some situations +* hash merging policy is now selectable in the config file, can choose to override or merge +* ec2 module is now powered by Boto +* fix to enable Jinja2 {% include %} to work again +* added pkgin module for Joyent SmartOS +* setup module can now detect if package manager is using pacman + * ... diff --git a/library/sysctl b/library/sysctl index 6231f240e3..45e948d224 100644 --- a/library/sysctl +++ b/library/sysctl @@ -29,7 +29,7 @@ version_added: "0.6" options: name: description: - - "this is the short path, decimal seperated, to the sysctl entry, ex: C(vm.swappiness)" + - this is the short path, decimal seperated, to the sysctl entry required: true default: null aliases: [ 'key' ] From 811c711328567a0103209d68c160f11776e51b8e Mon Sep 17 00:00:00 2001 From: Blair Zajac Date: Sat, 26 Jan 2013 10:19:48 -0800 Subject: [PATCH 25/36] library/apt: consistently use underscores in examples. To be consistent with the table showing available options, use underscores in the example tasks, not hyphens, as the table doesn't list hyphenated versions of option names, so it looks like the examples could have typos in them. --- library/apt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/library/apt b/library/apt index b15580070b..653ad6bb0d 100644 --- a/library/apt +++ b/library/apt @@ -70,7 +70,7 @@ options: author: Matthew Williams notes: [] examples: - - code: "apt: pkg=foo update-cache=yes" + - code: "apt: pkg=foo update_cache=yes" description: Update repositories cache and install C(foo) package - code: "apt: pkg=foo state=removed" description: Remove C(foo) package @@ -78,9 +78,9 @@ examples: description: Install the package C(foo) - code: "apt: pkg=foo=1.00 state=installed" description: Install the version '1.00' of package C(foo) - - code: "apt: pkg=nginx state=latest default-release=squeeze-backports update-cache=yes" + - code: "apt: pkg=nginx state=latest default_release=squeeze-backports update_cache=yes" description: Update the repository cache and update package C(ngnix) to latest version using default release C(squeeze-backport) - - code: "apt: pkg=openjdk-6-jdk state=latest install-recommends=no" + - code: "apt: pkg=openjdk-6-jdk state=latest install_recommends=no" description: Install latest version of C(openjdk-6-jdk) ignoring C(install-reccomends) ''' From 5f86f7d0bae9a9e470f2aabfe26f785a54d0de0a Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Sat, 26 Jan 2013 13:30:38 -0500 Subject: [PATCH 26/36] Reorganizing changelog, some cleanup still needed, but now it's easier to tell what is in the release :) --- CHANGELOG.md | 71 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c11f3a610a..d5085bff06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,63 +3,74 @@ Ansible Changes By Release 1.0 "Eruption" -- release pending -- changes unsorted for now -* default_sudo_exe parameter can be set in config to use sudo alternatives +New modules: + * new sysctl module * new pacman module (Arch linux) -* added when_failed and when_changed -* when_set and when_unset can take more than one var (when_set: $a and $b and $c) * new apt_key module +* hg module now in core +* new ec2_facts module +* added pkgin module for Joyent SmartOS + +New config settings: + +* default_sudo_exe parameter can be set in config to use sudo alternatives +* added when_failed and when_changed + +New playbook/language features: + +* task includes can now be of infinite depth +* when_set and when_unset can take more than one var (when_set: $a and $b and $c) +* added the with_sequence lookup plugin +* can override "connection:" on an indvidual task +* parameterized playbook includes can now define complex variables (not just all on one line) +* making inventory variables available for use in vars_files paths +* messages when skipping plays are now more clear + +Module fixes and new flags: + +* ability to use raw module without python on remote system * fix for service status checking on Ubuntu * service module now responds to additional exit code for SERVICE_UNAVAILABLE -* usage of run_command standardized between module implementations * fix for raw module with '-c local' -* fixes to git module +* various fixes to git module * ec2 module now reports the public DNS name -* added the with_sequence lookup plugin -* various fixes for variable resolution in playbooks -* task includes can now be of infinite depth * can pass executable= to the raw module to specify alternative shells -* fixes for handling of "~" in some paths -* can override "connection:" on an indvidual task * fix for postgres module when user contains a "-" -* various other database module fixes * added additional template variables -- $template_fullpath and $template_run_date * raise errors on invalid arguments used with a task include statement -* making inventory variables available for use in vars_files paths -* various fixes to DWIM'ing of relative paths -* ability to use raw module without python on remote system * shell/command module takes a executable= parameter to specify a different shell than /bin/sh * added return code and error output to the raw module * added support for @reboot to the cron module -* hostname patterns in the inventory file can now use alphabetic ranges -* whitespace is now allowed around group variables in the inventory file -* parameterized playbook includes can now define complex variables (not just all on one line) * misc fixes to the pip module -* inventory scripts can now define groups of groups and group vars (need example for docs?) * nagios module can schedule downtime for all services on the host -* various patterns can now take a regex vs a glob if they start with "~" (need docs on which!) -* /bin/ansible now takes a --list-hosts just like ansible-playbook did * various subversion module improvements * various mail module improvements -* allow intersecting host patterns by using "&" ("webservers:!debian:&datacenter1") -* messages when skipping plays are now more clear * SELinux fix for files created by authorized_key module * "template override" ?? -* lots of documentation tweaks -* handle tilde shell character for --private-key * get_url module can now send user/password authorization -* new ec2_facts module -* hg module now in core * ec2 module can now deploy multiple simultaneous instances * fix for apt_key modules stalling in some situations -* hash merging policy is now selectable in the config file, can choose to override or merge +* fix to enable Jinja2 {% include %} to work again in template * ec2 module is now powered by Boto -* fix to enable Jinja2 {% include %} to work again -* added pkgin module for Joyent SmartOS * setup module can now detect if package manager is using pacman +Core fixes and new behaviors: -* ... +* various fixes for variable resolution in playbooks +* fixes for handling of "~" in some paths +* various fixes to DWIM'ing of relative paths +* /bin/ansible now takes a --list-hosts just like ansible-playbook did +* various patterns can now take a regex vs a glob if they start with "~" (need docs on which!) - also /usr/bin/ansible +* allow intersecting host patterns by using "&" ("webservers:!debian:&datacenter1") +* handle tilde shell character for --private-key +* hash merging policy is now selectable in the config file, can choose to override or merge + +Inventory files/scripts: + +* hostname patterns in the inventory file can now use alphabetic ranges +* whitespace is now allowed around group variables in the inventory file +* inventory scripts can now define groups of groups and group vars (need example for docs?) 0.9 "Dreams" -- Nov 30 2012 From 07b056fb7873b7f6148ea6492ddfeb395fbfea08 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Sat, 26 Jan 2013 13:32:55 -0500 Subject: [PATCH 27/36] Add env vars to changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5085bff06..18aef02c8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,7 @@ Core fixes and new behaviors: * allow intersecting host patterns by using "&" ("webservers:!debian:&datacenter1") * handle tilde shell character for --private-key * hash merging policy is now selectable in the config file, can choose to override or merge +* environment variables now available for setting all plugin paths (ANSIBLE_CALLBACK_PLUGINS, etc) Inventory files/scripts: From 0030a2bd099c40d3b5c5f93ec03d3d6bea16859e Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Sun, 27 Jan 2013 10:40:29 -0500 Subject: [PATCH 28/36] Allow handler files to be empty lists. --- lib/ansible/playbook/play.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/ansible/playbook/play.py b/lib/ansible/playbook/play.py index 608691c712..f414ec5553 100644 --- a/lib/ansible/playbook/play.py +++ b/lib/ansible/playbook/play.py @@ -101,6 +101,10 @@ class Play(object): ''' handle task and handler include statements ''' results = [] + if tasks is None: + # support empty handler files, and the like. + tasks = [] + for x in tasks: task_vars = self.vars.copy() task_vars.update(vars) From f4eed3710ba24e61ea0ec6617ed52fbc615d7e2d Mon Sep 17 00:00:00 2001 From: Les Aker Date: Sun, 27 Jan 2013 12:39:35 -0500 Subject: [PATCH 29/36] fixed check for required_together --- lib/ansible/module_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/module_common.py b/lib/ansible/module_common.py index b7e750f61b..4ed02f732a 100644 --- a/lib/ansible/module_common.py +++ b/lib/ansible/module_common.py @@ -481,7 +481,7 @@ class AnsibleModule(object): if spec is None: return for check in spec: - counts = [ self.count_terms([field]) for field in check ] + counts = [ self._count_terms([field]) for field in check ] non_zero = [ c for c in counts if c > 0 ] if len(non_zero) > 0: if 0 in counts: From a2aacaaec8dc1f15ca019730a7722e9a623dda46 Mon Sep 17 00:00:00 2001 From: junyoung Date: Mon, 28 Jan 2013 10:46:34 +0900 Subject: [PATCH 30/36] whitespaces in FULL_PATH cause an error to get ANSIBLE_PATH --- hacking/env-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hacking/env-setup b/hacking/env-setup index bd0a268fe7..78b05dd483 100755 --- a/hacking/env-setup +++ b/hacking/env-setup @@ -11,7 +11,7 @@ fi # The below is an alternative to readlink -fn which doesn't exist on OS X # Source: http://stackoverflow.com/a/1678636 FULL_PATH=`python -c "import os; print(os.path.realpath('$HACKING_DIR'))"` -ANSIBLE_HOME=`dirname $FULL_PATH` +ANSIBLE_HOME=`dirname "$FULL_PATH"` PREFIX_PYTHONPATH="$ANSIBLE_HOME/lib" PREFIX_PATH="$ANSIBLE_HOME/bin" From 819da94ced971b72f524fb6d3add48e2751d2718 Mon Sep 17 00:00:00 2001 From: Les Aker Date: Mon, 28 Jan 2013 00:33:18 -0500 Subject: [PATCH 31/36] adjusted assemble to use new file attribute handling --- library/assemble | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/library/assemble b/library/assemble index 70f429c564..0e0eadc05a 100644 --- a/library/assemble +++ b/library/assemble @@ -91,12 +91,12 @@ def main(): module = AnsibleModule( # not checking because of daisy chain to file module - check_invalid_arguments = False, argument_spec = dict( src = dict(required=True), dest = dict(required=True), backup=dict(default=False, choices=BOOLEANS), - ) + ), + add_file_common_args=True ) changed=False @@ -124,11 +124,11 @@ def main(): shutil.copy(path, dest) changed = True - + file_args = module.load_file_common_arguments(module.params) + changed = module.set_file_attributes_if_different(file_args, changed) # Mission complete module.exit_json(src=src, dest=dest, md5sum=destmd5, - changed=changed, msg="OK", - daisychain="file", daisychain_args=module.params) + changed=changed, msg="OK") # this is magic, see lib/ansible/module_common.py #<> From 133874a5775f0e948cd78d957a77633875b067b8 Mon Sep 17 00:00:00 2001 From: Daniel Hokka Zakrisson Date: Mon, 28 Jan 2013 12:40:18 +0100 Subject: [PATCH 32/36] Fix module.run_command usage in fireball --- library/fireball | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library/fireball b/library/fireball index 6eb363723e..11a0b61188 100644 --- a/library/fireball +++ b/library/fireball @@ -143,7 +143,7 @@ def daemonize_self(module, password, port, minutes): os.dup2(dev_null.fileno(), sys.stderr.fileno()) log("daemonizing successful (%s,%s)" % (password, port)) -def command(data): +def command(module, data): if 'cmd' not in data: return dict(failed=True, msg='internal error: cmd is required') if 'tmp_path' not in data: @@ -220,7 +220,7 @@ def serve(module, password, port, minutes): response = {} if mode == 'command': - response = command(data) + response = command(module, data) elif mode == 'put': response = put(data) elif mode == 'fetch': From 5268d49badb83716c08861dde0d0fb788132ad7d Mon Sep 17 00:00:00 2001 From: Daniel Hokka Zakrisson Date: Mon, 28 Jan 2013 17:41:43 +0100 Subject: [PATCH 33/36] Add sudo_flags configuration option to alter sudo behaviour Default to -H to set $HOME appropriately. --- CHANGELOG.md | 5 +++-- examples/ansible.cfg | 3 +++ lib/ansible/constants.py | 1 + lib/ansible/utils/__init__.py | 5 +++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18aef02c8a..92cbdab82b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,11 +14,12 @@ New modules: New config settings: -* default_sudo_exe parameter can be set in config to use sudo alternatives -* added when_failed and when_changed +* sudo_exe parameter can be set in config to use sudo alternatives +* sudo_flags parameter can alter the flags used with sudo New playbook/language features: +* added when_failed and when_changed * task includes can now be of infinite depth * when_set and when_unset can take more than one var (when_set: $a and $b and $c) * added the with_sequence lookup plugin diff --git a/examples/ansible.cfg b/examples/ansible.cfg index 0ba9957f64..ab8dd20c4f 100644 --- a/examples/ansible.cfg +++ b/examples/ansible.cfg @@ -76,6 +76,9 @@ remote_port=22 sudo_exe=sudo +# the default flags passed to sudo +# sudo_flags=-H + # how to handle hash defined in several places # hash can be merged, or replaced # if you use replace, and have multiple hashes named 'x', the last defined diff --git a/lib/ansible/constants.py b/lib/ansible/constants.py index 7bd75777f2..b43986eb09 100644 --- a/lib/ansible/constants.py +++ b/lib/ansible/constants.py @@ -92,6 +92,7 @@ DEFAULT_MANAGED_STR = get_config(p, DEFAULTS, 'ansible_managed', None, DEFAULT_SYSLOG_FACILITY = get_config(p, DEFAULTS, 'syslog_facility', 'ANSIBLE_SYSLOG_FACILITY', 'LOG_USER') DEFAULT_KEEP_REMOTE_FILES = get_config(p, DEFAULTS, 'keep_remote_files', 'ANSIBLE_KEEP_REMOTE_FILES', '0') DEFAULT_SUDO_EXE = get_config(p, DEFAULTS, 'sudo_exe', 'ANSIBLE_SUDO_EXE', 'sudo') +DEFAULT_SUDO_FLAGS = get_config(p, DEFAULTS, 'sudo_flags', 'ANSIBLE_SUDO_FLAGS', '-H') DEFAULT_HASH_BEHAVIOUR = get_config(p, DEFAULTS, 'hash_behaviour', 'ANSIBLE_HASH_BEHAVIOUR', 'replace') DEFAULT_ACTION_PLUGIN_PATH = shell_expand_path(get_config(p, DEFAULTS, 'action_plugins', 'ANSIBLE_ACTION_PLUGINS', '/usr/share/ansible_plugins/action_plugins')) diff --git a/lib/ansible/utils/__init__.py b/lib/ansible/utils/__init__.py index 139e1584e9..53b9b989e5 100644 --- a/lib/ansible/utils/__init__.py +++ b/lib/ansible/utils/__init__.py @@ -590,6 +590,7 @@ def make_sudo_cmd(sudo_user, executable, cmd): # the -p option. randbits = ''.join(chr(random.randint(ord('a'), ord('z'))) for x in xrange(32)) prompt = '[sudo via ansible, key=%s] password: ' % randbits - sudocmd = '%s -k && %s -S -p "%s" -u %s %s -c %s' % ( - C.DEFAULT_SUDO_EXE, C.DEFAULT_SUDO_EXE, prompt, sudo_user, executable or '$SHELL', pipes.quote(cmd)) + sudocmd = '%s -k && %s %s -S -p "%s" -u %s %s -c %s' % ( + C.DEFAULT_SUDO_EXE, C.DEFAULT_SUDO_EXE, C.DEFAULT_SUDO_FLAGS, + prompt, sudo_user, executable or '$SHELL', pipes.quote(cmd)) return ('/bin/sh -c ' + pipes.quote(sudocmd), prompt) From 0bfec51a04193d445c5217470f10d6a4579abf99 Mon Sep 17 00:00:00 2001 From: Daniel Hokka Zakrisson Date: Mon, 28 Jan 2013 17:46:35 +0100 Subject: [PATCH 34/36] Remove last remnants of daisychaining --- library/get_url | 3 +-- library/ini_file | 14 +++++--------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/library/get_url b/library/get_url index 8d6e85f0d8..2db31c6d7c 100644 --- a/library/get_url +++ b/library/get_url @@ -255,8 +255,7 @@ def main(): # Mission complete module.exit_json(url=url, dest=dest, src=tmpsrc, md5sum=md5sum_src, - changed=changed, msg=info.get('msg', ''), - daisychain="file", daisychain_args=info.get('daisychain_args', '')) + changed=changed, msg=info.get('msg', '')) # this is magic, see lib/ansible/module_common.py #<> diff --git a/library/ini_file b/library/ini_file index ea78bff191..cc9771380d 100644 --- a/library/ini_file +++ b/library/ini_file @@ -153,8 +153,6 @@ def do_ini(module, filename, section=None, option=None, value=None, state='prese def main(): module = AnsibleModule( - # not checking because of daisy chain to file module - check_invalid_arguments = False, argument_spec = dict( dest = dict(required=True), section = dict(required=True), @@ -162,7 +160,8 @@ def main(): value = dict(required=False), backup = dict(default='no', choices=BOOLEANS), state = dict(default='present', choices=['present', 'absent']) - ) + ), + add_file_common_args = True ) info = dict() @@ -176,14 +175,11 @@ def main(): changed = do_ini(module, dest, section, option, value, state, backup) - info['daisychain_args'] = module.params - info['daisychain_args']['state'] = 'file' - info['daisychain_args']['dest'] = dest + file_args = module.load_file_common_arguments(module.params) + changed = module.set_file_attributes_if_different(file_args, changed) # Mission complete - module.exit_json(dest=dest, - changed=changed, msg="OK", - daisychain="file", daisychain_args=info.get('daisychain_args','')) + module.exit_json(dest=dest, changed=changed, msg="OK") # this is magic, see lib/ansible/module_common.py #<> From 4d8f3b09240c6a4adeca82decb9566b30613a9e2 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Mon, 28 Jan 2013 15:41:46 -0500 Subject: [PATCH 35/36] This standardizes the apt_key module some * improves error handling and reporting * uses run_command to reduce code * fails quicker on errors as opposed to return codes and tracebacks * can now also specify the key as data versus needing to wget it from a file --- lib/ansible/module_common.py | 12 ++- library/apt_key | 173 +++++++++++++++-------------------- 2 files changed, 82 insertions(+), 103 deletions(-) diff --git a/lib/ansible/module_common.py b/lib/ansible/module_common.py index 4ed02f732a..5c4732b678 100644 --- a/lib/ansible/module_common.py +++ b/lib/ansible/module_common.py @@ -677,7 +677,7 @@ class AnsibleModule(object): self.set_context_if_different(src, context, False) os.rename(src, dest) - def run_command(self, args, check_rc=False, close_fds=False, executable=None): + def run_command(self, args, check_rc=False, close_fds=False, executable=None, data=None): ''' Execute a command, returns rc, stdout, and stderr. args is the command to run @@ -700,12 +700,20 @@ class AnsibleModule(object): self.fail_json(rc=257, cmd=args, msg=msg) rc = 0 msg = None + st_in = None + if data: + st_in = subprocess.PIPE try: cmd = subprocess.Popen(args, executable=executable, shell=shell, close_fds=close_fds, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdin=st_in, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + if data: + cmd.stdin.write(data) + cmd.stdin.write('\\n') out, err = cmd.communicate() rc = cmd.returncode except (OSError, IOError), e: diff --git a/library/apt_key b/library/apt_key index 14d1b734fd..e2e66cd441 100644 --- a/library/apt_key +++ b/library/apt_key @@ -22,7 +22,7 @@ DOCUMENTATION = ''' --- module: apt_key -author: Jayson Vantuyl +author: Jayson Vantuyl & others version_added: 1.0 short_description: Add or remove an apt key description: @@ -59,145 +59,116 @@ examples: description: Remove a Apt specific signing key ''' +# FIXME: standardize into module_common from urllib2 import urlopen, URLError from traceback import format_exc -from subprocess import Popen, PIPE, call from re import compile as re_compile +# FIXME: standardize into module_common from distutils.spawn import find_executable from os import environ from sys import exc_info +import traceback match_key = re_compile("^gpg:.*key ([0-9a-fA-F]+):.*$") REQUIRED_EXECUTABLES=['gpg', 'grep', 'apt-key'] -def find_missing_binaries(): - return [missing for missing in REQUIRED_EXECUTABLES if not find_executable(missing)] +def check_missing_binaries(module): + missing = [e for e in REQUIRED_EXECUTABLES if not find_executable(e)] + if len(missing): + module.fail_json(msg="binaries are missing", names=all) +def all_keys(module): + (rc, out, err) = module.run_command("apt-key list") + results = [] + lines = out.split('\n') + for line in lines: + if line.startswith("pub"): + tokens = line.split() + code = tokens[1] + (len_type, real_code) = code.split("/") + results.append(real_code) + return results -def get_key_ids(key_data): - p = Popen("gpg --list-only --import -", shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE) - (stdo, stde) = p.communicate(key_data) +def key_present(module, key_id): + (rc, out, err) = module.run_command("apt-key list | 2>&1 grep -q %s" % key_id) + return rc == 0 - if p.returncode > 0: - raise Exception("error running GPG to retrieve keys") - - output = stdo + stde - - for line in output.split('\n'): - match = match_key.match(line) - if match: - yield match.group(1) - - -def key_present(key_id): - return call("apt-key list | 2>&1 grep -q %s" % key_id, shell=True) == 0 - - -def download_key(url): +def download_key(module, url): + # FIXME: move get_url code to common, allow for in-memory D/L, support proxies + # and reuse here if url is None: - raise Exception("Needed URL but none specified") - connection = urlopen(url) - if connection is None: - raise Exception("error connecting to download key from %r" % url) - return connection.read() + module.fail_json(msg="needed a URL but was not specified") + try: + connection = urlopen(url) + if connection is None: + module.fail_json("error connecting to download key from url") + data = connection.read() + return data + except: + module.fail_json(msg="error getting key id from url", traceback=format_exc()) -def add_key(key): - p = Popen("apt-key add -", shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE) - (_, _) = p.communicate(key) - - return p.returncode == 0 - +def add_key(module, key): + cmd = "apt-key add -" + (rc, out, err) = module.run_command(cmd, data=key, check_rc=True) + return True def remove_key(key_id): - return call('apt-key del %s' % key_id, shell=True) == 0 - - -def return_values(tb=False): - if tb: - return {'exception': format_exc()} - else: - return {} + # FIXME: use module.run_command, fail at point of error and don't discard useful stdin/stdout + cmd = 'apt-key del %s' + (rc, out, err) = module.run_command(cmd, check_rc=True) + return True def main(): module = AnsibleModule( argument_spec=dict( id=dict(required=False, default=None), url=dict(required=False), + data=dict(required=False), + key=dict(required=False), state=dict(required=False, choices=['present', 'absent'], default='present') - ) + ), ) - expected_key_id = module.params['id'] - url = module.params['url'] - state = module.params['state'] - changed = False + key_id = module.params['id'] + url = module.params['url'] + data = module.params['data'] + state = module.params['state'] + changed = False + + # FIXME: I think we have a common facility for this, if not, want + check_missing_binaries(module) - missing = find_missing_binaries() - - if missing: - module.fail_json(msg="can't find needed binaries to run", missing=missing, - **return_values()) + keys = all_keys(module) if state == 'present': - if expected_key_id and key_present(expected_key_id): - # key is present, nothing to do - pass + if key_id and key_id in keys: + module.exit_json(changed=False) else: - # download key - try: - key = download_key(url) - (key_id,) = tuple(get_key_ids(key)) # TODO: support multiple key ids? - except Exception: - module.fail_json( - msg="error getting key id from url", - **return_values(True) - ) - - # sanity check downloaded key - if expected_key_id and key_id != expected_key_id: - module.fail_json( - msg="expected key id %s, got key id %s" % (expected_key_id, key_id), - **return_values() - ) - - # actually add key - if key_present(key_id): - changed=False - elif add_key(key): - changed=True + if not data: + data = download_key(module, url) + if key_id and key_id in keys: + module.exit_json(changed=False) else: - module.fail_json( - msg="failed to add key id %s" % key_id, - **return_values() - ) + add_key(module, data) + changed=False + keys2 = all_keys(module) + if len(keys) != len(keys2): + changed=True + if key_id and not key_id in keys2: + module.fail_json(msg="key does not seem to have been added", id=key_id) + module.exit_json(changed=changed) elif state == 'absent': - # optionally download the key and get the id - if not expected_key_id: - try: - key = download_key(url) - (key_id,) = tuple(get_key_ids(key)) # TODO: support multiple key ids? - except Exception: - module.fail_json( - msg="error getting key id from url", - **return_values(True) - ) - else: - key_id = expected_key_id - - # actually remove key - if key_present(key_id): + if not key_id: + module.fail_json(msg="key is required") + if key_id in keys: if remove_key(key_id): changed=True else: + # FIXME: module.fail_json or exit-json immediately at point of failure module.fail_json(msg="error removing key_id", **return_values(True)) - else: - module.fail_json( - msg="unexpected state: %s" % state, - **return_values() - ) module.exit_json(changed=changed, **return_values()) From e166f71fcc5f78ad2cd6739349a02892191b1233 Mon Sep 17 00:00:00 2001 From: Daniel Hokka Zakrisson Date: Tue, 29 Jan 2013 00:38:07 +0100 Subject: [PATCH 36/36] Get output before receiving return code Fixes hangs observed with large amounts of output, as it would get into a dead-lock. --- lib/ansible/runner/connection_plugins/paramiko_ssh.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/ansible/runner/connection_plugins/paramiko_ssh.py b/lib/ansible/runner/connection_plugins/paramiko_ssh.py index 0f329bf7e5..813abfca6e 100644 --- a/lib/ansible/runner/connection_plugins/paramiko_ssh.py +++ b/lib/ansible/runner/connection_plugins/paramiko_ssh.py @@ -142,7 +142,9 @@ class Connection(object): except socket.timeout: raise errors.AnsibleError('ssh timed out waiting for sudo.\n' + sudo_output) - return (chan.recv_exit_status(), chan.makefile('wb', bufsize), chan.makefile('rb', bufsize), chan.makefile_stderr('rb', bufsize)) + stdout = ''.join(chan.makefile('rb', bufsize)) + stderr = ''.join(chan.makefile_stderr('rb', bufsize)) + return (chan.recv_exit_status(), '', stdout, stderr) def put_file(self, in_path, out_path): ''' transfer a file from local to remote '''