Merge pull request #6976 from clconway/gce-snapshots

Adds support for snapshots and extra persistent disks to the gce modules

test/integration/Makefile

@@ -56,6 +56,12 @@ cloud_cleanup: amazon_cleanup rackspace_cleanup
 amazon_cleanup:
 	python cleanup_ec2.py -y --match="^$(CLOUD_RESOURCE_PREFIX)"
 
+gce_setup:
+	python setup_gce.py "$(CLOUD_RESOURCE_PREFIX)"
+
+gce_cleanup:
+	python cleanup_gce.py -y --match="^$(CLOUD_RESOURCE_PREFIX)"
+
 rackspace_cleanup:
 	@echo "FIXME - cleanup_rax.py not yet implemented"
 	@# python cleanup_rax.py -y --match="^$(CLOUD_RESOURCE_PREFIX)"
@@ -70,6 +76,13 @@ amazon: $(CREDENTIALS_FILE)
     CLOUD_RESOURCE_PREFIX="$(CLOUD_RESOURCE_PREFIX)" make amazon_cleanup ; \
     exit $$RC;
 
+gce: $(CREDENTIALS_FILE)
+	CLOUD_RESOURCE_PREFIX="$(CLOUD_RESOURCE_PREFIX)" make gce_setup ; \
+    ansible-playbook gce.yml -i $(INVENTORY) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -e "resource_prefix=$(CLOUD_RESOURCE_PREFIX)" -v $(TEST_FLAGS) ; \
+    RC=$$? ; \
+    CLOUD_RESOURCE_PREFIX="$(CLOUD_RESOURCE_PREFIX)" make gce_cleanup ; \
+    exit $$RC;
+
 rackspace: $(CREDENTIALS_FILE)
 	ansible-playbook rackspace.yml -i $(INVENTORY) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -e "resource_prefix=$(CLOUD_RESOURCE_PREFIX)" -v $(TEST_FLAGS) ; \
     RC=$$? ; \

test/integration/cleanup_gce.py

@@ -0,0 +1,77 @@
+'''
+Find and delete GCE resources matching the provided --match string.  Unless
+--yes|-y is provided, the prompt for confirmation prior to deleting resources.
+Please use caution, you can easily delete your *ENTIRE* GCE infrastructure.
+'''
+
+import os
+import re
+import sys
+import optparse
+import yaml
+
+try:
+    from libcloud.compute.types import Provider
+    from libcloud.compute.providers import get_driver
+    from libcloud.common.google import GoogleBaseError, QuotaExceededError, \
+            ResourceExistsError, ResourceInUseError, ResourceNotFoundError
+    _ = Provider.GCE
+except ImportError:
+    print("failed=True " + \
+        "msg='libcloud with GCE support (0.13.3+) required for this module'")
+    sys.exit(1)
+
+import gce_credentials
+
+
+def delete_gce_resources(get_func, attr, opts):
+    for item in get_func():
+        val = getattr(item, attr)
+        if re.search(opts.match_re, val, re.IGNORECASE):
+            prompt_and_delete(item, "Delete matching %s? [y/n]: " % (item,), opts.assumeyes)
+
+def prompt_and_delete(item, prompt, assumeyes):
+    if not assumeyes:
+        assumeyes = raw_input(prompt).lower() == 'y'
+    assert hasattr(item, 'destroy'), "Class <%s> has no delete attribute" % item.__class__
+    if assumeyes:
+        item.destroy()
+        print ("Deleted %s" % item)
+
+def parse_args():
+    parser = optparse.OptionParser(usage="%s [options]" % (sys.argv[0],),
+                description=__doc__)
+    gce_credentials.add_credentials_options(parser)
+    parser.add_option("--yes", "-y",
+        action="store_true", dest="assumeyes",
+        default=False,
+        help="Don't prompt for confirmation")
+    parser.add_option("--match",
+        action="store", dest="match_re",
+        default="^ansible-testing-",
+        help="Regular expression used to find GCE resources (default: %default)")
+
+    (opts, args) = parser.parse_args()
+    gce_credentials.check_required(opts, parser)
+    return (opts, args)
+
+if __name__ == '__main__':
+
+    (opts, args) = parse_args()
+
+    # Connect to GCE
+    gce = gce_credentials.get_gce_driver(opts)
+
+    try:
+      # Delete matching instances
+      delete_gce_resources(gce.list_nodes, 'name', opts)
+      # Delete matching snapshots
+      def get_snapshots():
+        for volume in gce.list_volumes():
+          for snapshot in gce.list_volume_snapshots(volume):
+            yield snapshot
+      delete_gce_resources(get_snapshots, 'name', opts)
+      # Delete matching disks
+      delete_gce_resources(gce.list_volumes, 'name', opts)
+    except KeyboardInterrupt, e:
+        print "\nExiting on user command."

test/integration/credentials.template

@@ -3,5 +3,10 @@
 ec2_access_key:
 ec2_secret_key:
 
+# GCE Credentials
+service_account_email:
+pem_file:
+project_id:
+
 # GITHUB SSH private key - a path to a SSH private key for use with github.com
 github_ssh_private_key: "{{ lookup('env','HOME') }}/.ssh/id_rsa"

test/integration/gce.yml

@@ -0,0 +1,6 @@
+- hosts: testhost
+  gather_facts: true
+  roles:
+    - { role: test_gce, tags: test_gce }
+    - { role: test_gce_pd, tags: test_gce_pd }
+    # TODO: tests for gce_lb, gce_net, gc_storage

test/integration/gce_credentials.py

@@ -0,0 +1,51 @@
+import collections
+import os
+import yaml
+
+try:
+    from libcloud.compute.types import Provider
+    from libcloud.compute.providers import get_driver
+    _ = Provider.GCE
+except ImportError:
+    print("failed=True " + \
+        "msg='libcloud with GCE support (0.13.3+) required for this module'")
+    sys.exit(1)
+
+
+def add_credentials_options(parser):
+    default_service_account_email=None
+    default_pem_file=None
+    default_project_id=None
+
+    # Load details from credentials.yml
+    if os.path.isfile('credentials.yml'):
+        credentials = yaml.load(open('credentials.yml', 'r'))
+        default_service_account_email = credentials['gce_service_account_email']
+        default_pem_file = credentials['gce_pem_file']
+        default_project_id = credentials['gce_project_id']
+
+    parser.add_option("--service_account_email",
+        action="store", dest="service_account_email",
+        default=default_service_account_email,
+        help="GCE service account email. Default is loaded from credentials.yml.")
+    parser.add_option("--pem_file",
+        action="store", dest="pem_file",
+        default=default_pem_file,
+        help="GCE client key. Default is loaded from credentials.yml.")
+    parser.add_option("--project_id",
+        action="store", dest="project_id",
+        default=default_project_id,
+        help="Google Cloud project ID. Default is loaded from credentials.yml.")
+
+
+def check_required(opts, parser):
+    for required in ['service_account_email', 'pem_file', 'project_id']:
+        if getattr(opts, required) is None:
+            parser.error("Missing required parameter: --%s" % required)
+
+
+def get_gce_driver(opts):
+    # Connect to GCE
+    gce_cls = get_driver(Provider.GCE)
+    return gce_cls(
+        opts.service_account_email, opts.pem_file, project=opts.project_id)

test/integration/roles/test_gce/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+# defaults file for test_gce
+instance_name: "{{ resource_prefix|lower }}"
+service_account_email: "{{ gce_service_account_email }}"
+pem_file: "{{ gce_pem_file }}"
+project_id: "{{ gce_project_id }}"

test/integration/roles/test_gce/tasks/main.yml

@@ -0,0 +1,211 @@
+# TODO: lots of attributes not covered: machine_type, zone, metadata, tags, etc.
+#
+# ============================================================
+- name: test with no parameters
+  gce:
+  register: result
+  ignore_errors: true
+
+- name: assert failure when called with no parameters
+  assert:
+    that:
+       - 'result.failed'
+       - 'result.msg == "Missing GCE connection parameters in libcloud secrets file."'
+
+# ============================================================
+- name: test missing name
+  gce:
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+  register: result
+  ignore_errors: true
+
+- name: assert failure when called with no parameters
+  assert:
+    that:
+       - 'result.failed'
+       - 'result.msg == "Must specify a \"name\" or \"instance_names\""'
+
+# ============================================================
+- name: test state=present (expected changed=true)
+  gce:
+    name: "{{ instance_name }}"
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+    state: present
+  register: result
+
+- name: assert state=present (expected changed=true)
+  assert:
+    that:
+       - 'result.changed'
+       - 'result.name == "{{ instance_name }}"'
+       - 'result.state == "present"'
+
+# ============================================================
+- name: test state=present (expected changed=false)
+  gce:
+    name: "{{ instance_name }}"
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+    state: present
+  register: result
+
+- name: assert state=present (expected changed=false)
+  assert:
+    that:
+       - 'not result.changed'
+       - 'result.name == "{{ instance_name }}"'
+       - 'result.state == "present"'
+
+# ============================================================
+- name: test state=absent (expected changed=true)
+  gce:
+    name: "{{ instance_name }}"
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+    state: absent
+  register: result
+
+- name: assert state=absent (expected changed=true)
+  assert:
+    that:
+       - 'result.changed'
+       - 'result.name == "{{ instance_name }}"'
+       - 'result.state == "absent"'
+
+# ============================================================
+- name: test state=absent (expected changed=false)
+  gce:
+    name: "{{ instance_name }}"
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+    state: absent
+  register: result
+
+- name: assert state=absent (expected changed=false)
+  assert:
+    that:
+       - 'not result.changed'
+       - 'result.name == "{{ instance_name }}"'
+       - 'result.state == "absent"'
+
+# ============================================================
+- name: test disks given (expected changed=true)
+  gce:
+    name: "{{ instance_name }}"
+    disks:
+       - "{{ instance_name }}-base"
+       - "{{ instance_name }}-extra"
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+    state: present
+  register: result
+
+- name: assert disks given
+  assert:
+    that:
+       - 'result.changed'
+       - 'result.instance_data[0].disks == ["{{ instance_name }}-base", "{{ instance_name }}-extra"]'
+       - 'result.state == "present"'
+
+# ============================================================
+- name: test disks given (expected changed=false)
+  gce:
+    name: "{{ instance_name }}"
+    disks:
+       - "{{ instance_name }}-base"
+       - "{{ instance_name }}-extra"
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+    state: present
+  register: result
+
+- name: assert disks given
+  assert:
+    that:
+       - 'not result.changed'
+       - 'result.instance_data[0].disks == ["{{ instance_name }}-base", "{{ instance_name }}-extra"]'
+       - 'result.state == "present"'
+
+# ============================================================
+- name: test disks in the wrong order
+  gce:
+    name: "{{ instance_name }}"
+    disks:
+       - "{{ instance_name }}-extra"
+       - "{{ instance_name }}-base"
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+  register: result
+  ignore_errors: true
+
+- name: assert disks in the wrong order
+  assert:
+    that:
+       - 'result.failed'
+       - '{{ result.msg | match("Disk at index 0 does not match:.*") }}'
+
+# ============================================================
+- name: test disks given with name and mode
+  gce:
+    name: "{{ instance_name }}"
+    disks:
+       - { name: "{{ instance_name }}-base", mode: "READ_WRITE" }
+       - { name: "{{ instance_name }}-extra", mode: "READ_ONLY" }
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+  register: result
+
+- name: assert disks given
+  assert:
+    that:
+       - 'not result.changed'
+       - 'result.state == "present"'
+
+# ============================================================
+- name: test disks given with name and wrong mode
+  gce:
+    name: "{{ instance_name }}"
+    disks:
+       - { name: "{{ instance_name }}-base", mode: "READ_ONLY" }
+       - "{{ instance_name }}-extra"
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+  register: result
+  ignore_errors: true
+
+- name: assert disks given
+  assert:
+    that:
+       - 'result.failed'
+       - '{{ result.msg | match("Disk at index 0 is in the wrong mode:.*") }}'
+
+# ============================================================
+- name: test disks given, state absent (expected changed=true)
+  gce:
+    name: "{{ instance_name }}"
+    disks:
+       - "{{ instance_name }}-base"
+       - "{{ instance_name }}-extra"
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+    state: absent
+  register: result
+
+- name: assert disks given, state absent (expected changed=true)
+  assert:
+    that:
+       - 'result.changed'
+       - 'result.state == "absent"'

test/integration/roles/test_gce_pd/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+# defaults file for test_gce
+instance_name: "{{ resource_prefix|lower }}"
+service_account_email: "{{ gce_service_account_email }}"
+pem_file: "{{ gce_pem_file }}"
+project_id: "{{ gce_project_id }}"

test/integration/roles/test_gce_pd/tasks/main.yml

@@ -0,0 +1,220 @@
+# TODO: need tests for read/write mode.
+
+# ============================================================
+- name: test missing name
+  gce_pd:
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+  register: result
+  ignore_errors: true
+
+- name: assert failure when called with no parameters
+  assert:
+    that:
+       - 'result.failed'
+       - 'result.msg == "missing required arguments: name"'
+
+# ============================================================
+- name: test state=present (expected changed=true)
+  gce_pd:
+    name: "{{ instance_name }}"
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+    state: present
+  register: result
+
+- name: assert state=present (expected changed=true)
+  assert:
+    that:
+       - 'result.changed'
+       - 'result.name == "{{ instance_name }}"'
+       - 'result.size_gb == 10'  # default size
+       - 'result.zone == "us-central1-b"'  # default zone
+       - 'result.state == "present"'
+
+# ============================================================
+- name: test state=present (expected changed=false)
+  gce_pd:
+    name: "{{ instance_name }}"
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+    state: present
+  register: result
+
+- name: assert state=present (expected changed=false)
+  assert:
+    that:
+       - 'not result.changed'
+       - 'result.name == "{{ instance_name }}"'
+       - 'result.state == "present"'
+
+# ============================================================
+- name: test state=absent (expected changed=true)
+  gce_pd:
+    name: "{{ instance_name }}"
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+    state: absent
+  register: result
+
+- name: assert state=absent (expected changed=true)
+  assert:
+    that:
+       - 'result.changed'
+       - 'result.name == "{{ instance_name }}"'
+       - 'result.state == "absent"'
+
+# ============================================================
+- name: test state=absent (expected changed=false)
+  gce_pd:
+    name: "{{ instance_name }}"
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+    state: absent
+  register: result
+
+- name: assert state=absent (expected changed=false)
+  assert:
+    that:
+       - 'not result.changed'
+       - 'result.name == "{{ instance_name }}"'
+       - 'result.state == "absent"'
+
+# ============================================================
+- name: test non-default size/zone
+  gce_pd:
+    name: "{{ instance_name }}"
+    size_gb: 5
+    zone: us-central1-a
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+    state: present
+  register: result
+
+- name: assert non-default size/zone
+  assert:
+    that:
+       - 'result.changed'
+       - 'result.name == "{{ instance_name }}"'
+       - 'result.size_gb == 5'
+       - 'result.zone == "us-central1-a"'
+       - 'result.state == "present"'
+
+# ============================================================
+- name: test non-default size/zone (state=absent)
+  gce_pd:
+    name: "{{ instance_name }}"
+    size_gb: 5
+    zone: us-central1-a
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+    state: absent
+  register: result
+
+- name: assert non-default size/zone (state=absent)
+  assert:
+    that:
+       - 'result.changed'
+       - 'result.name == "{{ instance_name }}"'
+       - 'result.state == "absent"'
+
+# ============================================================
+- name: test image given (state=present)
+  gce_pd:
+    name: "{{ instance_name }}"
+    image: debian-7
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+    state: present
+  register: result
+
+- name: assert image given (state=present)
+  assert:
+    that:
+       - 'result.changed'
+       - 'result.name == "{{ instance_name }}"'
+       - 'result.image == "debian-7"'
+       - 'result.state == "present"'
+
+# ============================================================
+- name: test image given (state=absent)
+  gce_pd:
+    name: "{{ instance_name }}"
+    image: debian-7
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+    state: absent
+  register: result
+
+- name: assert image given (state=absent)
+  assert:
+    that:
+       - 'result.changed'
+       - 'result.name == "{{ instance_name }}"'
+       - 'result.state == "absent"'
+
+# ============================================================
+- name: test snapshot given (state=present)
+  gce_pd:
+    name: "{{ instance_name }}"
+    snapshot: "{{ instance_name }}-snapshot"
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+    state: present
+  register: result
+
+- name: assert image given (state=present)
+  assert:
+    that:
+       - 'result.changed'
+       - 'result.name == "{{ instance_name }}"'
+       - 'result.snapshot == "{{ instance_name }}-snapshot"'
+       - 'result.state == "present"'
+
+# ============================================================
+- name: test snapshot given (state=absent)
+  gce_pd:
+    name: "{{ instance_name }}"
+    snapshot: "{{ instance_name }}-snapshot"
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+    state: absent
+  register: result
+
+- name: assert image given (state=absent)
+  assert:
+    that:
+       - 'result.changed'
+       - 'result.name == "{{ instance_name }}"'
+       - 'result.state == "absent"'
+
+# ============================================================
+- name: test both image and snapshot given
+  gce_pd:
+    name: "{{ instance_name }}"
+    image: "debian-7"
+    snapshot: "{{ instance_name }}-snapshot"
+    service_account_email: "{{ service_account_email }}"
+    pem_file: "{{ pem_file }}"
+    project_id: "{{ project_id }}"
+    state: present
+  register: result
+  ignore_errors: true
+
+- name: assert image given (state=present)
+  assert:
+    that:
+       - 'result.failed'
+       - 'result.msg == "Cannot give both image (debian-7) and snapshot ({{ instance_name }}-snapshot)"'
+

test/integration/setup_gce.py

@@ -0,0 +1,42 @@
+'''
+Create GCE resources for use in integration tests.
+
+Takes a prefix as a command-line argument and creates two persistent disks named
+${prefix}-base and ${prefix}-extra and a snapshot of the base disk named
+${prefix}-snapshot. prefix will be forced to lowercase, to ensure the names are
+legal GCE resource names.
+'''
+
+import sys
+import optparse
+
+import gce_credentials
+
+
+def parse_args():
+    parser = optparse.OptionParser(
+        usage="%s [options] <prefix>" % (sys.argv[0],), description=__doc__)
+    gce_credentials.add_credentials_options(parser)
+    parser.add_option("--prefix",
+        action="store", dest="prefix",
+        help="String used to prefix GCE resource names (default: %default)")
+
+    (opts, args) = parser.parse_args()
+    gce_credentials.check_required(opts, parser)
+    if not args:
+      parser.error("Missing required argument: name prefix")
+    return (opts, args)
+
+if __name__ == '__main__':
+
+    (opts, args) = parse_args()
+    gce = gce_credentials.get_gce_driver(opts)
+    prefix = args[0].lower()
+    try:
+      base_volume = gce.create_volume(
+          size=10, name=prefix+'-base', location='us-central1-a')
+      gce.create_volume_snapshot(base_volume, name=prefix+'-snapshot')
+      gce.create_volume(
+          size=10, name=prefix+'-extra', location='us-central1-a')
+    except KeyboardInterrupt, e:
+        print "\nExiting on user command."