ML Engine: Models in Ansible (#313)

Signed-off-by: Modular Magician <magic-modules@google.com>

plugins/modules/gcp_mlengine_model.py

@@ -0,0 +1,342 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2017 Google
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+# ----------------------------------------------------------------------------
+#
+#     ***     AUTO GENERATED CODE    ***    AUTO GENERATED CODE     ***
+#
+# ----------------------------------------------------------------------------
+#
+#     This file is automatically generated by Magic Modules and manual
+#     changes will be clobbered when the file is regenerated.
+#
+#     Please read more about how to change this file at
+#     https://www.github.com/GoogleCloudPlatform/magic-modules
+#
+# ----------------------------------------------------------------------------
+
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+################################################################################
+# Documentation
+################################################################################
+
+ANSIBLE_METADATA = {'metadata_version': '1.1', 'status': ["preview"], 'supported_by': 'community'}
+
+DOCUMENTATION = '''
+---
+module: gcp_mlengine_model
+description:
+- Represents a machine learning solution.
+- A model can have multiple versions, each of which is a deployed, trained model ready
+  to receive prediction requests. The model itself is just a container.
+short_description: Creates a GCP Model
+version_added: 2.9
+author: Google Inc. (@googlecloudplatform)
+requirements:
+- python >= 2.6
+- requests >= 2.18.4
+- google-auth >= 1.3.0
+options:
+  state:
+    description:
+    - Whether the given object should exist in GCP
+    choices:
+    - present
+    - absent
+    default: present
+  name:
+    description:
+    - The name specified for the model.
+    required: true
+    type: str
+  description:
+    description:
+    - The description specified for the model when it was created.
+    required: false
+    type: str
+  default_version:
+    description:
+    - The default version of the model. This version will be used to handle prediction
+      requests that do not specify a version.
+    required: false
+    type: dict
+    suboptions:
+      name:
+        description:
+        - The name specified for the version when it was created.
+        required: false
+        type: str
+  regions:
+    description:
+    - The list of regions where the model is going to be deployed.
+    - Currently only one region per model is supported .
+    required: false
+    type: list
+  online_prediction_logging:
+    description:
+    - If true, online prediction access logs are sent to StackDriver Logging.
+    required: false
+    type: bool
+  online_prediction_console_logging:
+    description:
+    - If true, online prediction nodes send stderr and stdout streams to Stackdriver
+      Logging.
+    required: false
+    type: bool
+  labels:
+    description:
+    - One or more labels that you can add, to organize your models.
+    required: false
+    type: dict
+extends_documentation_fragment: gcp
+'''
+
+EXAMPLES = '''
+- name: create a model
+  gcp_mlengine_model:
+    name: "{{ resource_name | replace('-', '_') }}"
+    description: My model
+    regions:
+    - us-central1
+    project: test_project
+    auth_kind: serviceaccount
+    service_account_file: "/tmp/auth.pem"
+    state: present
+'''
+
+RETURN = '''
+name:
+  description:
+  - The name specified for the model.
+  returned: success
+  type: str
+description:
+  description:
+  - The description specified for the model when it was created.
+  returned: success
+  type: str
+defaultVersion:
+  description:
+  - The default version of the model. This version will be used to handle prediction
+    requests that do not specify a version.
+  returned: success
+  type: complex
+  contains:
+    name:
+      description:
+      - The name specified for the version when it was created.
+      returned: success
+      type: str
+regions:
+  description:
+  - The list of regions where the model is going to be deployed.
+  - Currently only one region per model is supported .
+  returned: success
+  type: list
+onlinePredictionLogging:
+  description:
+  - If true, online prediction access logs are sent to StackDriver Logging.
+  returned: success
+  type: bool
+onlinePredictionConsoleLogging:
+  description:
+  - If true, online prediction nodes send stderr and stdout streams to Stackdriver
+    Logging.
+  returned: success
+  type: bool
+labels:
+  description:
+  - One or more labels that you can add, to organize your models.
+  returned: success
+  type: dict
+'''
+
+################################################################################
+# Imports
+################################################################################
+
+from ansible.module_utils.gcp_utils import navigate_hash, GcpSession, GcpModule, GcpRequest, remove_nones_from_dict, replace_resource_dict
+import json
+
+################################################################################
+# Main
+################################################################################
+
+
+def main():
+    """Main function"""
+
+    module = GcpModule(
+        argument_spec=dict(
+            state=dict(default='present', choices=['present', 'absent'], type='str'),
+            name=dict(required=True, type='str'),
+            description=dict(type='str'),
+            default_version=dict(type='dict', options=dict(name=dict(type='str'))),
+            regions=dict(type='list', elements='str'),
+            online_prediction_logging=dict(type='bool'),
+            online_prediction_console_logging=dict(type='bool'),
+            labels=dict(type='dict'),
+        )
+    )
+
+    if not module.params['scopes']:
+        module.params['scopes'] = ['https://www.googleapis.com/auth/cloud-platform']
+
+    state = module.params['state']
+
+    fetch = fetch_resource(module, self_link(module))
+    changed = False
+
+    if fetch:
+        if state == 'present':
+            if is_different(module, fetch):
+                update(module, self_link(module))
+                fetch = fetch_resource(module, self_link(module))
+                changed = True
+        else:
+            delete(module, self_link(module))
+            fetch = {}
+            changed = True
+    else:
+        if state == 'present':
+            fetch = create(module, collection(module))
+            changed = True
+        else:
+            fetch = {}
+
+    fetch.update({'changed': changed})
+
+    module.exit_json(**fetch)
+
+
+def create(module, link):
+    auth = GcpSession(module, 'mlengine')
+    return return_if_object(module, auth.post(link, resource_to_request(module)))
+
+
+def update(module, link):
+    delete(module, self_link(module))
+    create(module, collection(module))
+
+
+def delete(module, link):
+    auth = GcpSession(module, 'mlengine')
+    return return_if_object(module, auth.delete(link))
+
+
+def resource_to_request(module):
+    request = {
+        u'name': module.params.get('name'),
+        u'description': module.params.get('description'),
+        u'defaultVersion': ModelDefaultversion(module.params.get('default_version', {}), module).to_request(),
+        u'regions': module.params.get('regions'),
+        u'onlinePredictionLogging': module.params.get('online_prediction_logging'),
+        u'onlinePredictionConsoleLogging': module.params.get('online_prediction_console_logging'),
+        u'labels': module.params.get('labels'),
+    }
+    return_vals = {}
+    for k, v in request.items():
+        if v or v is False:
+            return_vals[k] = v
+
+    return return_vals
+
+
+def fetch_resource(module, link, allow_not_found=True):
+    auth = GcpSession(module, 'mlengine')
+    return return_if_object(module, auth.get(link), allow_not_found)
+
+
+def self_link(module):
+    return "https://ml.googleapis.com/v1/projects/{project}/models/{name}".format(**module.params)
+
+
+def collection(module):
+    return "https://ml.googleapis.com/v1/projects/{project}/models".format(**module.params)
+
+
+def return_if_object(module, response, allow_not_found=False):
+    # If not found, return nothing.
+    if allow_not_found and response.status_code == 404:
+        return None
+
+    # If no content, return nothing.
+    if response.status_code == 204:
+        return None
+
+    try:
+        module.raise_for_status(response)
+        result = response.json()
+    except getattr(json.decoder, 'JSONDecodeError', ValueError):
+        module.fail_json(msg="Invalid JSON response with error: %s" % response.text)
+
+    result = decode_response(result, module)
+
+    if navigate_hash(result, ['error', 'errors']):
+        module.fail_json(msg=navigate_hash(result, ['error', 'errors']))
+
+    return result
+
+
+def is_different(module, response):
+    request = resource_to_request(module)
+    response = response_to_hash(module, response)
+    request = decode_response(request, module)
+
+    # Remove all output-only from response.
+    response_vals = {}
+    for k, v in response.items():
+        if k in request:
+            response_vals[k] = v
+
+    request_vals = {}
+    for k, v in request.items():
+        if k in response:
+            request_vals[k] = v
+
+    return GcpRequest(request_vals) != GcpRequest(response_vals)
+
+
+# Remove unnecessary properties from the response.
+# This is for doing comparisons with Ansible's current parameters.
+def response_to_hash(module, response):
+    return {
+        u'name': response.get(u'name'),
+        u'description': response.get(u'description'),
+        u'defaultVersion': ModelDefaultversion(response.get(u'defaultVersion', {}), module).from_response(),
+        u'regions': response.get(u'regions'),
+        u'onlinePredictionLogging': response.get(u'onlinePredictionLogging'),
+        u'onlinePredictionConsoleLogging': response.get(u'onlinePredictionConsoleLogging'),
+        u'labels': response.get(u'labels'),
+    }
+
+
+# Short names are given (and expected) by the API
+# but are returned as full names.
+def decode_response(response, module):
+    response['name'] = response['name'].split('/')[-1]
+    return response
+
+
+class ModelDefaultversion(object):
+    def __init__(self, request, module):
+        self.module = module
+        if request:
+            self.request = request
+        else:
+            self.request = {}
+
+    def to_request(self):
+        return remove_nones_from_dict({u'name': self.request.get('name')})
+
+    def from_response(self):
+        return remove_nones_from_dict({u'name': self.request.get(u'name')})
+
+
+if __name__ == '__main__':
+    main()

plugins/modules/gcp_mlengine_model_facts.py

@@ -0,0 +1,165 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2017 Google
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+# ----------------------------------------------------------------------------
+#
+#     ***     AUTO GENERATED CODE    ***    AUTO GENERATED CODE     ***
+#
+# ----------------------------------------------------------------------------
+#
+#     This file is automatically generated by Magic Modules and manual
+#     changes will be clobbered when the file is regenerated.
+#
+#     Please read more about how to change this file at
+#     https://www.github.com/GoogleCloudPlatform/magic-modules
+#
+# ----------------------------------------------------------------------------
+
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+################################################################################
+# Documentation
+################################################################################
+
+ANSIBLE_METADATA = {'metadata_version': '1.1', 'status': ["preview"], 'supported_by': 'community'}
+
+DOCUMENTATION = '''
+---
+module: gcp_mlengine_model_facts
+description:
+- Gather facts for GCP Model
+short_description: Gather facts for GCP Model
+version_added: 2.9
+author: Google Inc. (@googlecloudplatform)
+requirements:
+- python >= 2.6
+- requests >= 2.18.4
+- google-auth >= 1.3.0
+options: {}
+extends_documentation_fragment: gcp
+'''
+
+EXAMPLES = '''
+- name: " a model facts"
+  gcp_mlengine_model_facts:
+    project: test_project
+    auth_kind: serviceaccount
+    service_account_file: "/tmp/auth.pem"
+    state: facts
+'''
+
+RETURN = '''
+resources:
+  description: List of resources
+  returned: always
+  type: complex
+  contains:
+    name:
+      description:
+      - The name specified for the model.
+      returned: success
+      type: str
+    description:
+      description:
+      - The description specified for the model when it was created.
+      returned: success
+      type: str
+    defaultVersion:
+      description:
+      - The default version of the model. This version will be used to handle prediction
+        requests that do not specify a version.
+      returned: success
+      type: complex
+      contains:
+        name:
+          description:
+          - The name specified for the version when it was created.
+          returned: success
+          type: str
+    regions:
+      description:
+      - The list of regions where the model is going to be deployed.
+      - Currently only one region per model is supported .
+      returned: success
+      type: list
+    onlinePredictionLogging:
+      description:
+      - If true, online prediction access logs are sent to StackDriver Logging.
+      returned: success
+      type: bool
+    onlinePredictionConsoleLogging:
+      description:
+      - If true, online prediction nodes send stderr and stdout streams to Stackdriver
+        Logging.
+      returned: success
+      type: bool
+    labels:
+      description:
+      - One or more labels that you can add, to organize your models.
+      returned: success
+      type: dict
+'''
+
+################################################################################
+# Imports
+################################################################################
+from ansible.module_utils.gcp_utils import navigate_hash, GcpSession, GcpModule, GcpRequest
+import json
+
+################################################################################
+# Main
+################################################################################
+
+
+def main():
+    module = GcpModule(argument_spec=dict())
+
+    if not module.params['scopes']:
+        module.params['scopes'] = ['https://www.googleapis.com/auth/cloud-platform']
+
+    items = fetch_list(module, collection(module))
+    if items.get('models'):
+        items = items.get('models')
+    else:
+        items = []
+    return_value = {'resources': items}
+    module.exit_json(**return_value)
+
+
+def collection(module):
+    return "https://ml.googleapis.com/v1/projects/{project}/models".format(**module.params)
+
+
+def fetch_list(module, link):
+    auth = GcpSession(module, 'mlengine')
+    response = auth.get(link)
+    return return_if_object(module, response)
+
+
+def return_if_object(module, response):
+    # If not found, return nothing.
+    if response.status_code == 404:
+        return None
+
+    # If no content, return nothing.
+    if response.status_code == 204:
+        return None
+
+    try:
+        module.raise_for_status(response)
+        result = response.json()
+    except getattr(json.decoder, 'JSONDecodeError', ValueError) as inst:
+        module.fail_json(msg="Invalid JSON response with error: %s" % inst)
+
+    if navigate_hash(result, ['error', 'errors']):
+        module.fail_json(msg=navigate_hash(result, ['error', 'errors']))
+
+    return result
+
+
+if __name__ == "__main__":
+    main()

tests/integration/gcp_mlengine_model/aliases

@@ -0,0 +1,2 @@
+cloud/gcp
+unsupported

tests/integration/gcp_mlengine_model/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+resource_name: "{{ resource_prefix }}"

tests/integration/gcp_mlengine_model/tasks/main.yml

@@ -0,0 +1,113 @@
+---
+# ----------------------------------------------------------------------------
+#
+#     ***     AUTO GENERATED CODE    ***    AUTO GENERATED CODE     ***
+#
+# ----------------------------------------------------------------------------
+#
+#     This file is automatically generated by Magic Modules and manual
+#     changes will be clobbered when the file is regenerated.
+#
+#     Please read more about how to change this file at
+#     https://www.github.com/GoogleCloudPlatform/magic-modules
+#
+# ----------------------------------------------------------------------------
+# Pre-test setup
+- name: delete a model
+  gcp_mlengine_model:
+    name: "{{ resource_name | replace('-', '_') }}"
+    description: My model
+    regions:
+    - us-central1
+    project: "{{ gcp_project }}"
+    auth_kind: "{{ gcp_cred_kind }}"
+    service_account_file: "{{ gcp_cred_file }}"
+    state: absent
+#----------------------------------------------------------
+- name: create a model
+  gcp_mlengine_model:
+    name: "{{ resource_name | replace('-', '_') }}"
+    description: My model
+    regions:
+    - us-central1
+    project: "{{ gcp_project }}"
+    auth_kind: "{{ gcp_cred_kind }}"
+    service_account_file: "{{ gcp_cred_file }}"
+    state: present
+  register: result
+- name: assert changed is true
+  assert:
+    that:
+      - result.changed == true
+- name: verify that model was created
+  gcp_mlengine_model_facts:
+      project: "{{ gcp_project }}"
+      auth_kind: "{{ gcp_cred_kind }}"
+      service_account_file: "{{ gcp_cred_file }}"
+      scopes:
+        - https://www.googleapis.com/auth/cloud-platform
+  register: results
+- name: verify that command succeeded
+  assert:
+    that:
+      - results['resources'] | map(attribute='name') | select("match", ".*{{ resource_name | replace('-', '_') }}.*") | list | length == 1
+# ----------------------------------------------------------------------------
+- name: create a model that already exists
+  gcp_mlengine_model:
+    name: "{{ resource_name | replace('-', '_') }}"
+    description: My model
+    regions:
+    - us-central1
+    project: "{{ gcp_project }}"
+    auth_kind: "{{ gcp_cred_kind }}"
+    service_account_file: "{{ gcp_cred_file }}"
+    state: present
+  register: result
+- name: assert changed is false
+  assert:
+    that:
+      - result.changed == false
+#----------------------------------------------------------
+- name: delete a model
+  gcp_mlengine_model:
+    name: "{{ resource_name | replace('-', '_') }}"
+    description: My model
+    regions:
+    - us-central1
+    project: "{{ gcp_project }}"
+    auth_kind: "{{ gcp_cred_kind }}"
+    service_account_file: "{{ gcp_cred_file }}"
+    state: absent
+  register: result
+- name: assert changed is true
+  assert:
+    that:
+      - result.changed == true
+- name: verify that model was deleted
+  gcp_mlengine_model_facts:
+      project: "{{ gcp_project }}"
+      auth_kind: "{{ gcp_cred_kind }}"
+      service_account_file: "{{ gcp_cred_file }}"
+      scopes:
+        - https://www.googleapis.com/auth/cloud-platform
+  register: results
+- name: verify that command succeeded
+  assert:
+    that:
+      - results['resources'] | map(attribute='name') | select("match", ".*{{ resource_name | replace('-', '_') }}.*") | list | length == 0
+# ----------------------------------------------------------------------------
+- name: delete a model that does not exist
+  gcp_mlengine_model:
+    name: "{{ resource_name | replace('-', '_') }}"
+    description: My model
+    regions:
+    - us-central1
+    project: "{{ gcp_project }}"
+    auth_kind: "{{ gcp_cred_kind }}"
+    service_account_file: "{{ gcp_cred_file }}"
+    state: absent
+  register: result
+- name: assert changed is false
+  assert:
+    that:
+      - result.changed == false