New inventory module: Proxmox (#545)

* This commit adds proxmox inventory module and proxmox_snap for snapshot management

* Fixed pylint errors

* Missed this one..

* This should fix the doc errors

* Remove proxmox_snap to allow for single module per PR

* Changes as suggested by felixfontein in #535

* Reverted back to AnsibleError as module.fail_json broke it. Need to investigate further

* Made importerror behave similar to docker_swarm and gitlab_runner

* FALSE != False

* Added myself as author

* Added a requested feature from a colleague to also sort VMs based on their running state

* Prevent VM templates from being added to the inventory

* Processed feedback

* Updated my email and included version

* Processed doc feedback

* More feedback processed

* Shortened this line of documentation, it is a duplicate and it was causing a sanity error (> 160 characters)

* Added test from PR #736 to check what needs to be changed to make it work

* Changed some tests around

* Remove some tests, first get these working

* Disabled all tests, except the one I am hacking together now

* Added mocker, still trying to figure this out

* Am I looking in the right direction?

* Processed docs feedback

* Fixed bot feedback

* Removed all other tests, started with basic ones (borrowed from cobbler)

* Removed all other tests, started with basic ones (borrowed from cobbler)

* Removed all other tests, started with basic ones (borrowed from cobbler)

* Removed init_cache test as it is implemented on a different way in the original foreman/satellite inventory (and thus also this one)

* This actually passes! Need to check if I need to add asserts as well

* Made bot happy again?

* Added some assertions

* Added note about PVE API version

* Mocked only get_json, the rest functions as-is

* Fixed sanity errors

* Fixed version bump (again...) ;-)

* Processed feedback

plugins/inventory/proxmox.py

@@ -0,0 +1,348 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2016 Guido Günther <agx@sigxcpu.org>, Daniel Lobato Garcia <dlobatog@redhat.com>
+# Copyright (c) 2018 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = '''
+    name: proxmox
+    plugin_type: inventory
+    short_description: Proxmox inventory source
+    version_added: "1.2.0"
+    author:
+        - Jeffrey van Pelt (@Thulium-Drake) <jeff@vanpelt.one>
+    requirements:
+        - requests >= 1.1
+    description:
+        - Get inventory hosts from a Proxmox PVE cluster.
+        - "Uses a configuration file as an inventory source, it must end in C(.proxmox.yml) or C(.proxmox.yaml)"
+        - Will retrieve the first network interface with an IP for Proxmox nodes.
+        - Can retrieve LXC/QEMU configuration as facts.
+    extends_documentation_fragment:
+        - inventory_cache
+    options:
+      plugin:
+        description: The name of this plugin, it should always be set to C(community.general.proxmox) for this plugin to recognize it as it's own.
+        required: yes
+        choices: ['community.general.proxmox']
+        type: str
+      url:
+        description: URL to Proxmox cluster.
+        default: 'http://localhost:8006'
+        type: str
+      user:
+        description: Proxmox authentication user.
+        required: yes
+        type: str
+      password:
+        description: Proxmox authentication password.
+        required: yes
+        type: str
+      validate_certs:
+        description: Verify SSL certificate if using HTTPS.
+        type: boolean
+        default: yes
+      group_prefix:
+        description: Prefix to apply to Proxmox groups.
+        default: proxmox_
+        type: str
+      facts_prefix:
+        description: Prefix to apply to LXC/QEMU config facts.
+        default: proxmox_
+        type: str
+      want_facts:
+        description: Gather LXC/QEMU configuration facts.
+        default: no
+        type: bool
+'''
+
+EXAMPLES = '''
+# my.proxmox.yml
+plugin: community.general.proxmox
+url: http://localhost:8006
+user: ansible@pve
+password: secure
+validate_certs: no
+'''
+
+import re
+
+from ansible.module_utils.common._collections_compat import MutableMapping
+from distutils.version import LooseVersion
+
+from ansible.errors import AnsibleError
+from ansible.plugins.inventory import BaseInventoryPlugin, Cacheable
+from ansible.module_utils.six.moves.urllib.parse import urlencode
+
+# 3rd party imports
+try:
+    import requests
+    if LooseVersion(requests.__version__) < LooseVersion('1.1.0'):
+        raise ImportError
+    HAS_REQUESTS = True
+except ImportError:
+    HAS_REQUESTS = False
+
+
+class InventoryModule(BaseInventoryPlugin, Cacheable):
+    ''' Host inventory parser for ansible using Proxmox as source. '''
+
+    NAME = 'community.general.proxmox'
+
+    def __init__(self):
+
+        super(InventoryModule, self).__init__()
+
+        # from config
+        self.proxmox_url = None
+
+        self.session = None
+        self.cache_key = None
+        self.use_cache = None
+
+    def verify_file(self, path):
+
+        valid = False
+        if super(InventoryModule, self).verify_file(path):
+            if path.endswith(('proxmox.yaml', 'proxmox.yml')):
+                valid = True
+            else:
+                self.display.vvv('Skipping due to inventory source not ending in "proxmox.yaml" nor "proxmox.yml"')
+        return valid
+
+    def _get_session(self):
+        if not self.session:
+            self.session = requests.session()
+            self.session.verify = self.get_option('validate_certs')
+        return self.session
+
+    def _get_auth(self):
+        credentials = urlencode({'username': self.proxmox_user, 'password': self.proxmox_password, })
+
+        a = self._get_session()
+        ret = a.post('%s/api2/json/access/ticket' % self.proxmox_url, data=credentials)
+
+        json = ret.json()
+
+        self.credentials = {
+            'ticket': json['data']['ticket'],
+            'CSRFPreventionToken': json['data']['CSRFPreventionToken'],
+        }
+
+    def _get_json(self, url, ignore_errors=None):
+
+        if not self.use_cache or url not in self._cache.get(self.cache_key, {}):
+
+            if self.cache_key not in self._cache:
+                self._cache[self.cache_key] = {'url': ''}
+
+            data = []
+            s = self._get_session()
+            while True:
+                headers = {'Cookie': 'PVEAuthCookie={0}'.format(self.credentials['ticket'])}
+                ret = s.get(url, headers=headers)
+                if ignore_errors and ret.status_code in ignore_errors:
+                    break
+                ret.raise_for_status()
+                json = ret.json()
+
+                # process results
+                # FIXME: This assumes 'return type' matches a specific query,
+                #        it will break if we expand the queries and they dont have different types
+                if 'data' not in json:
+                    # /hosts/:id does not have a 'data' key
+                    data = json
+                    break
+                elif isinstance(json['data'], MutableMapping):
+                    # /facts are returned as dict in 'data'
+                    data = json['data']
+                    break
+                else:
+                    # /hosts 's 'results' is a list of all hosts, returned is paginated
+                    data = data + json['data']
+                    break
+
+            self._cache[self.cache_key][url] = data
+
+        return self._cache[self.cache_key][url]
+
+    def _get_nodes(self):
+        return self._get_json("%s/api2/json/nodes" % self.proxmox_url)
+
+    def _get_pools(self):
+        return self._get_json("%s/api2/json/pools" % self.proxmox_url)
+
+    def _get_lxc_per_node(self, node):
+        return self._get_json("%s/api2/json/nodes/%s/lxc" % (self.proxmox_url, node))
+
+    def _get_qemu_per_node(self, node):
+        return self._get_json("%s/api2/json/nodes/%s/qemu" % (self.proxmox_url, node))
+
+    def _get_members_per_pool(self, pool):
+        ret = self._get_json("%s/api2/json/pools/%s" % (self.proxmox_url, pool))
+        return ret['members']
+
+    def _get_node_ip(self, node):
+        ret = self._get_json("%s/api2/json/nodes/%s/network" % (self.proxmox_url, node))
+
+        for iface in ret:
+            try:
+                return iface['address']
+            except Exception:
+                return None
+
+    def _get_vm_config(self, node, vmid, vmtype, name):
+        ret = self._get_json("%s/api2/json/nodes/%s/%s/%s/config" % (self.proxmox_url, node, vmtype, vmid))
+
+        vmid_key = 'vmid'
+        vmid_key = self.to_safe('%s%s' % (self.get_option('facts_prefix'), vmid_key.lower()))
+        self.inventory.set_variable(name, vmid_key, vmid)
+
+        vmtype_key = 'vmtype'
+        vmtype_key = self.to_safe('%s%s' % (self.get_option('facts_prefix'), vmtype_key.lower()))
+        self.inventory.set_variable(name, vmtype_key, vmtype)
+
+        for config in ret:
+            key = config
+            key = self.to_safe('%s%s' % (self.get_option('facts_prefix'), key.lower()))
+            value = ret[config]
+            try:
+                # fixup disk images as they have no key
+                if config == 'rootfs' or config.startswith(('virtio', 'sata', 'ide', 'scsi')):
+                    value = ('disk_image=' + value)
+
+                if isinstance(value, int) or ',' not in value:
+                    value = value
+                # split off strings with commas to a dict
+                else:
+                    # skip over any keys that cannot be processed
+                    try:
+                        value = dict(key.split("=") for key in value.split(","))
+                    except Exception:
+                        continue
+
+                self.inventory.set_variable(name, key, value)
+            except NameError:
+                return None
+
+    def _get_vm_status(self, node, vmid, vmtype, name):
+        ret = self._get_json("%s/api2/json/nodes/%s/%s/%s/status/current" % (self.proxmox_url, node, vmtype, vmid))
+
+        status = ret['status']
+        status_key = 'status'
+        status_key = self.to_safe('%s%s' % (self.get_option('facts_prefix'), status_key.lower()))
+        self.inventory.set_variable(name, status_key, status)
+
+    def to_safe(self, word):
+        '''Converts 'bad' characters in a string to underscores so they can be used as Ansible groups
+        #> ProxmoxInventory.to_safe("foo-bar baz")
+        'foo_barbaz'
+        '''
+        regex = r"[^A-Za-z0-9\_]"
+        return re.sub(regex, "_", word.replace(" ", ""))
+
+    def _populate(self):
+
+        self._get_auth()
+
+        # gather vm's on nodes
+        for node in self._get_nodes():
+            # FIXME: this can probably be cleaner
+            # create groups
+            lxc_group = 'all_lxc'
+            lxc_group = self.to_safe('%s%s' % (self.get_option('group_prefix'), lxc_group.lower()))
+            self.inventory.add_group(lxc_group)
+            qemu_group = 'all_qemu'
+            qemu_group = self.to_safe('%s%s' % (self.get_option('group_prefix'), qemu_group.lower()))
+            self.inventory.add_group(qemu_group)
+            nodes_group = 'nodes'
+            nodes_group = self.to_safe('%s%s' % (self.get_option('group_prefix'), nodes_group.lower()))
+            self.inventory.add_group(nodes_group)
+            running_group = 'all_running'
+            running_group = self.to_safe('%s%s' % (self.get_option('group_prefix'), running_group.lower()))
+            self.inventory.add_group(running_group)
+            stopped_group = 'all_stopped'
+            stopped_group = self.to_safe('%s%s' % (self.get_option('group_prefix'), stopped_group.lower()))
+            self.inventory.add_group(stopped_group)
+
+            if node.get('node'):
+                self.inventory.add_host(node['node'])
+
+                if node['type'] == 'node':
+                    self.inventory.add_child(nodes_group, node['node'])
+
+                # get node IP address
+                ip = self._get_node_ip(node['node'])
+                self.inventory.set_variable(node['node'], 'ansible_host', ip)
+
+                # get LXC containers for this node
+                node_lxc_group = self.to_safe('%s%s' % (self.get_option('group_prefix'), ('%s_lxc' % node['node']).lower()))
+                self.inventory.add_group(node_lxc_group)
+                for lxc in self._get_lxc_per_node(node['node']):
+                    self.inventory.add_host(lxc['name'])
+                    self.inventory.add_child(lxc_group, lxc['name'])
+                    self.inventory.add_child(node_lxc_group, lxc['name'])
+
+                    # get LXC status when want_facts == True
+                    if self.get_option('want_facts'):
+                        self._get_vm_status(node['node'], lxc['vmid'], 'lxc', lxc['name'])
+                        if lxc['status'] == 'stopped':
+                            self.inventory.add_child(stopped_group, lxc['name'])
+                        elif lxc['status'] == 'running':
+                            self.inventory.add_child(running_group, lxc['name'])
+
+                    # get LXC config for facts
+                    if self.get_option('want_facts'):
+                        self._get_vm_config(node['node'], lxc['vmid'], 'lxc', lxc['name'])
+
+                # get QEMU vm's for this node
+                node_qemu_group = self.to_safe('%s%s' % (self.get_option('group_prefix'), ('%s_qemu' % node['node']).lower()))
+                self.inventory.add_group(node_qemu_group)
+                for qemu in self._get_qemu_per_node(node['node']):
+                    if not qemu['template']:
+                        self.inventory.add_host(qemu['name'])
+                        self.inventory.add_child(qemu_group, qemu['name'])
+                        self.inventory.add_child(node_qemu_group, qemu['name'])
+
+                    # get QEMU status
+                    self._get_vm_status(node['node'], qemu['vmid'], 'qemu', qemu['name'])
+                    if qemu['status'] == 'stopped':
+                        self.inventory.add_child(stopped_group, qemu['name'])
+                    elif qemu['status'] == 'running':
+                        self.inventory.add_child(running_group, qemu['name'])
+
+                    # get QEMU config for facts
+                    if self.get_option('want_facts'):
+                        self._get_vm_config(node['node'], qemu['vmid'], 'qemu', qemu['name'])
+
+        # gather vm's in pools
+        for pool in self._get_pools():
+            if pool.get('poolid'):
+                pool_group = 'pool_' + pool['poolid']
+                pool_group = self.to_safe('%s%s' % (self.get_option('group_prefix'), pool_group.lower()))
+                self.inventory.add_group(pool_group)
+
+                for member in self._get_members_per_pool(pool['poolid']):
+                    if member.get('name'):
+                        self.inventory.add_child(pool_group, member['name'])
+
+    def parse(self, inventory, loader, path, cache=True):
+        if not HAS_REQUESTS:
+            raise AnsibleError('This module requires Python Requests 1.1.0 or higher: '
+                               'https://github.com/psf/requests.')
+
+        super(InventoryModule, self).parse(inventory, loader, path)
+
+        # read config from file, this sets 'options'
+        self._read_config_data(path)
+
+        # get connection host
+        self.proxmox_url = self.get_option('url')
+        self.proxmox_user = self.get_option('user')
+        self.proxmox_password = self.get_option('password')
+        self.cache_key = self.get_cache_key(path)
+        self.use_cache = cache and self.get_option('cache')
+
+        # actually populate inventory
+        self._populate()

tests/unit/plugins/inventory/test_proxmox.py

@@ -0,0 +1,187 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Jeffrey van Pelt <jeff@vanpelt.one>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+#
+# The API responses used in these tests were recorded from PVE version 6.2.
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import pytest
+
+from ansible.errors import AnsibleError, AnsibleParserError
+from ansible.inventory.data import InventoryData
+from ansible_collections.community.general.plugins.inventory.proxmox import InventoryModule
+
+
+@pytest.fixture(scope="module")
+def inventory():
+    r = InventoryModule()
+    r.inventory = InventoryData()
+    return r
+
+
+def test_verify_file_bad_config(inventory):
+    assert inventory.verify_file('foobar.proxmox.yml') is False
+
+
+def get_auth():
+    return True
+
+
+# NOTE: when updating/adding replies to this function,
+# be sure to only add only the _contents_ of the 'data' dict in the API reply
+def get_json(url):
+    if url == "https://localhost:8006/api2/json/nodes":
+        # _get_nodes
+        return [{"type": "node",
+                 "cpu": 0.01,
+                 "maxdisk": 500,
+                 "mem": 500,
+                 "node": "testnode",
+                 "id": "node/testnode",
+                 "maxcpu": 1,
+                 "status": "online",
+                 "ssl_fingerprint": "xx",
+                 "disk": 1000,
+                 "maxmem": 1000,
+                 "uptime": 10000,
+                 "level": ""}]
+    elif url == "https://localhost:8006/api2/json/pools":
+        # _get_pools
+        return [{"poolid": "test"}]
+    elif url == "https://localhost:8006/api2/json/nodes/testnode/lxc":
+        # _get_lxc_per_node
+        return [{"cpus": 1,
+                 "name": "test-lxc",
+                 "cpu": 0.01,
+                 "diskwrite": 0,
+                 "lock": "",
+                 "maxmem": 1000,
+                 "template": "",
+                 "diskread": 0,
+                 "mem": 1000,
+                 "swap": 0,
+                 "type": "lxc",
+                 "maxswap": 0,
+                 "maxdisk": "1000",
+                 "netout": 1000,
+                 "pid": "1000",
+                 "netin": 1000,
+                 "status": "running",
+                 "vmid": "100",
+                 "disk": "1000",
+                 "uptime": 1000}]
+    elif url == "https://localhost:8006/api2/json/nodes/testnode/qemu":
+        # _get_qemu_per_node
+        return [{"name": "test-qemu",
+                "cpus": 1,
+                 "mem": 1000,
+                 "template": "",
+                 "diskread": 0,
+                 "cpu": 0.01,
+                 "maxmem": 1000,
+                 "diskwrite": 0,
+                 "netout": 1000,
+                 "pid": "1001",
+                 "netin": 1000,
+                 "maxdisk": 1000,
+                 "vmid": "101",
+                 "uptime": 1000,
+                 "disk": 0,
+                 "status": "running"}]
+    elif url == "https://localhost:8006/api2/json/pools/test":
+        # _get_members_per_pool
+        return {"members": [{"uptime": 1000,
+                            "template": 0,
+                             "id": "qemu/101",
+                             "mem": 1000,
+                             "status": "running",
+                             "cpu": 0.01,
+                             "maxmem": 1000,
+                             "diskwrite": 1000,
+                             "name": "test-qemu",
+                             "netout": 1000,
+                             "netin": 1000,
+                             "vmid": 101,
+                             "node": "testnode",
+                             "maxcpu": 1,
+                             "type": "qemu",
+                             "maxdisk": 1000,
+                             "disk": 0,
+                             "diskread": 1000}]}
+    elif url == "https://localhost:8006/api2/json/nodes/testnode/network":
+        # _get_node_ip
+        return [{"families": ["inet"],
+                 "priority": 3,
+                 "active": 1,
+                 "cidr": "10.1.1.2/24",
+                 "iface": "eth0",
+                 "method": "static",
+                 "exists": 1,
+                 "type": "eth",
+                 "netmask": "24",
+                 "gateway": "10.1.1.1",
+                 "address": "10.1.1.2",
+                 "method6": "manual",
+                 "autostart": 1},
+                {"method6": "manual",
+                 "autostart": 1,
+                 "type": "OVSPort",
+                 "exists": 1,
+                 "method": "manual",
+                 "iface": "eth1",
+                 "ovs_bridge": "vmbr0",
+                 "active": 1,
+                 "families": ["inet"],
+                 "priority": 5,
+                 "ovs_type": "OVSPort"},
+                {"type": "OVSBridge",
+                 "method": "manual",
+                 "iface": "vmbr0",
+                 "families": ["inet"],
+                 "priority": 4,
+                 "ovs_ports": "eth1",
+                 "ovs_type": "OVSBridge",
+                 "method6": "manual",
+                 "autostart": 1,
+                 "active": 1}]
+
+
+def get_vm_status(node, vmtype, vmid, name):
+    return True
+
+
+def get_option(option):
+    if option == 'group_prefix':
+        return 'proxmox_'
+    else:
+        return False
+
+
+def test_populate(inventory, mocker):
+    # module settings
+    inventory.proxmox_user = 'root@pam'
+    inventory.proxmox_password = 'password'
+    inventory.proxmox_url = 'https://localhost:8006'
+
+    # bypass authentication and API fetch calls
+    inventory._get_auth = mocker.MagicMock(side_effect=get_auth)
+    inventory._get_json = mocker.MagicMock(side_effect=get_json)
+    inventory._get_vm_status = mocker.MagicMock(side_effect=get_vm_status)
+    inventory.get_option = mocker.MagicMock(side_effect=get_option)
+    inventory._populate()
+
+    # get different hosts
+    host_qemu = inventory.inventory.get_host('test-qemu')
+    host_lxc = inventory.inventory.get_host('test-lxc')
+    host_node = inventory.inventory.get_host('testnode')
+
+    # check if qemu-test is in the proxmox_pool_test group
+    assert 'proxmox_pool_test' in inventory.inventory.groups
+    group_qemu = inventory.inventory.groups['proxmox_pool_test']
+    assert group_qemu.hosts == [host_qemu]
+
+    # check if lxc-test has been discovered correctly
+    group_lxc = inventory.inventory.groups['proxmox_all_lxc']
+    assert group_lxc.hosts == [host_lxc]