New module: routeros — manage MikroTik RouterOS (#41155)

* Implement initial RouterOS support

* Correct matchers for license prompts

* Documentation updates & mild refactor

* Remove one last Cisco function

* Sanity test fixes

* Move imports to the beginning

* Remove authorize property

* Handle ANSI codes

* Revert to_lines function

* CR fixes

* test(routeros): add unit tests

* Added another test (with ANSI colors and banner in fixture).

* Ignore CRLF line endings in system_package_print file

* fix: review by ganeshrn

lib/ansible/module_utils/network/routeros/routeros.py

@@ -0,0 +1,156 @@
+# This code is part of Ansible, but is an independent component.
+# This particular file snippet, and this file snippet only, is BSD licensed.
+# Modules you write using this snippet, which is embedded dynamically by Ansible
+# still belong to the author of the module, and may assign their own license
+# to the complete work.
+#
+# (c) 2016 Red Hat Inc.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright notice,
+#      this list of conditions and the following disclaimer in the documentation
+#      and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+import json
+from ansible.module_utils._text import to_text
+from ansible.module_utils.basic import env_fallback, return_values
+from ansible.module_utils.network.common.utils import to_list, ComplexList
+from ansible.module_utils.connection import Connection
+
+_DEVICE_CONFIGS = {}
+
+routeros_provider_spec = {
+    'host': dict(),
+    'port': dict(type='int'),
+    'username': dict(fallback=(env_fallback, ['ANSIBLE_NET_USERNAME'])),
+    'password': dict(fallback=(env_fallback, ['ANSIBLE_NET_PASSWORD']), no_log=True),
+    'ssh_keyfile': dict(fallback=(env_fallback, ['ANSIBLE_NET_SSH_KEYFILE']), type='path'),
+    'timeout': dict(type='int')
+}
+routeros_argument_spec = {}
+
+
+def get_provider_argspec():
+    return routeros_provider_spec
+
+
+def get_connection(module):
+    if hasattr(module, '_routeros_connection'):
+        return module._routeros_connection
+
+    capabilities = get_capabilities(module)
+    network_api = capabilities.get('network_api')
+    if network_api == 'cliconf':
+        module._routeros_connection = Connection(module._socket_path)
+    else:
+        module.fail_json(msg='Invalid connection type %s' % network_api)
+
+    return module._routeros_connection
+
+
+def get_capabilities(module):
+    if hasattr(module, '_routeros_capabilities'):
+        return module._routeros_capabilities
+
+    capabilities = Connection(module._socket_path).get_capabilities()
+    module._routeros_capabilities = json.loads(capabilities)
+    return module._routeros_capabilities
+
+
+def get_defaults_flag(module):
+    connection = get_connection(module)
+
+    try:
+        out = connection.get('/system default-configuration print')
+    except ConnectionError as exc:
+        module.fail_json(msg=to_text(exc, errors='surrogate_then_replace'))
+
+    out = to_text(out, errors='surrogate_then_replace')
+
+    commands = set()
+    for line in out.splitlines():
+        if line.strip():
+            commands.add(line.strip().split()[0])
+
+    if 'all' in commands:
+        return ['all']
+    else:
+        return ['full']
+
+
+def get_config(module, flags=None):
+    flag_str = ' '.join(to_list(flags))
+
+    try:
+        return _DEVICE_CONFIGS[flag_str]
+    except KeyError:
+        connection = get_connection(module)
+
+        try:
+            out = connection.get_config(flags=flags)
+        except ConnectionError as exc:
+            module.fail_json(msg=to_text(exc, errors='surrogate_then_replace'))
+
+        cfg = to_text(out, errors='surrogate_then_replace').strip()
+        _DEVICE_CONFIGS[flag_str] = cfg
+        return cfg
+
+
+def to_commands(module, commands):
+    spec = {
+        'command': dict(key=True),
+        'prompt': dict(),
+        'answer': dict()
+    }
+    transform = ComplexList(spec, module)
+    return transform(commands)
+
+
+def run_commands(module, commands, check_rc=True):
+    responses = list()
+    connection = get_connection(module)
+
+    for cmd in to_list(commands):
+        if isinstance(cmd, dict):
+            command = cmd['command']
+            prompt = cmd['prompt']
+            answer = cmd['answer']
+        else:
+            command = cmd
+            prompt = None
+            answer = None
+
+        try:
+            out = connection.get(command, prompt, answer)
+        except ConnectionError as exc:
+            module.fail_json(msg=to_text(exc, errors='surrogate_then_replace'))
+
+        try:
+            out = to_text(out, errors='surrogate_or_strict')
+        except UnicodeError:
+            module.fail_json(
+                msg=u'Failed to decode output from %s: %s' % (cmd, to_text(out)))
+
+        responses.append(out)
+
+    return responses
+
+
+def load_config(module, commands):
+    connection = get_connection(module)
+
+    out = connection.edit_config(commands)

lib/ansible/modules/network/routeros/routeros_command.py

@@ -0,0 +1,205 @@
+#!/usr/bin/python
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+ANSIBLE_METADATA = {'metadata_version': '1.1',
+                    'status': ['preview'],
+                    'supported_by': 'community'}
+
+
+DOCUMENTATION = """
+---
+module: routeros_command
+version_added: "2.7"
+author: "Egor Zaitsev (@heuels)"
+short_description: Run commands on remote devices running MikroTik RouterOS
+description:
+  - Sends arbitrary commands to an RouterOS node and returns the results
+    read from the device. This module includes an
+    argument that will cause the module to wait for a specific condition
+    before returning or timing out if the condition is not met.
+options:
+  commands:
+    description:
+      - List of commands to send to the remote RouterOS device over the
+        configured provider. The resulting output from the command
+        is returned. If the I(wait_for) argument is provided, the
+        module is not returned until the condition is satisfied or
+        the number of retries has expired.
+    required: true
+  wait_for:
+    description:
+      - List of conditions to evaluate against the output of the
+        command. The task will wait for each condition to be true
+        before moving forward. If the conditional is not true
+        within the configured number of retries, the task fails.
+        See examples.
+    required: false
+    default: null
+  match:
+    description:
+      - The I(match) argument is used in conjunction with the
+        I(wait_for) argument to specify the match policy.  Valid
+        values are C(all) or C(any).  If the value is set to C(all)
+        then all conditionals in the wait_for must be satisfied.  If
+        the value is set to C(any) then only one of the values must be
+        satisfied.
+    required: false
+    default: all
+    choices: ['any', 'all']
+  retries:
+    description:
+      - Specifies the number of retries a command should by tried
+        before it is considered failed. The command is run on the
+        target device every retry and evaluated against the
+        I(wait_for) conditions.
+    required: false
+    default: 10
+  interval:
+    description:
+      - Configures the interval in seconds to wait between retries
+        of the command. If the command does not pass the specified
+        conditions, the interval indicates how long to wait before
+        trying the command again.
+    required: false
+    default: 1
+"""
+
+EXAMPLES = """
+tasks:
+  - name: run command on remote devices
+    routeros_command:
+      commands: /system routerboard print
+
+  - name: run command and check to see if output contains routeros
+    routeros_command:
+      commands: /system resource print
+      wait_for: result[0] contains MikroTik
+
+  - name: run multiple commands on remote nodes
+    routeros_command:
+      commands:
+        - /system routerboard print
+        - /system identity print
+
+  - name: run multiple commands and evaluate the output
+    routeros_command:
+      commands:
+        - /system routerboard print
+        - /interface ethernet print
+      wait_for:
+        - result[0] contains x86
+        - result[1] contains ether1
+"""
+
+RETURN = """
+stdout:
+  description: The set of responses from the commands
+  returned: always apart from low level errors (such as action plugin)
+  type: list
+  sample: ['...', '...']
+stdout_lines:
+  description: The value of stdout split into a list
+  returned: always apart from low level errors (such as action plugin)
+  type: list
+  sample: [['...', '...'], ['...'], ['...']]
+failed_conditions:
+  description: The list of conditionals that have failed
+  returned: failed
+  type: list
+  sample: ['...', '...']
+"""
+
+import re
+import time
+
+from ansible.module_utils.network.routeros.routeros import run_commands
+from ansible.module_utils.network.routeros.routeros import routeros_argument_spec
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.network.common.utils import ComplexList
+from ansible.module_utils.network.common.parsing import Conditional
+from ansible.module_utils.six import string_types
+
+
+def to_lines(stdout):
+    for item in stdout:
+        if isinstance(item, string_types):
+            item = str(item).split('\n')
+        yield item
+
+
+def main():
+    """main entry point for module execution
+    """
+    argument_spec = dict(
+        commands=dict(type='list', required=True),
+
+        wait_for=dict(type='list'),
+        match=dict(default='all', choices=['all', 'any']),
+
+        retries=dict(default=10, type='int'),
+        interval=dict(default=1, type='int')
+    )
+
+    argument_spec.update(routeros_argument_spec)
+
+    module = AnsibleModule(argument_spec=argument_spec,
+                           supports_check_mode=True)
+
+    result = {'changed': False}
+
+    wait_for = module.params['wait_for'] or list()
+    conditionals = [Conditional(c) for c in wait_for]
+
+    retries = module.params['retries']
+    interval = module.params['interval']
+    match = module.params['match']
+
+    while retries > 0:
+        responses = run_commands(module, module.params['commands'])
+
+        for item in list(conditionals):
+            if item(responses):
+                if match == 'any':
+                    conditionals = list()
+                    break
+                conditionals.remove(item)
+
+        if not conditionals:
+            break
+
+        time.sleep(interval)
+        retries -= 1
+
+    if conditionals:
+        failed_conditions = [item.raw for item in conditionals]
+        msg = 'One or more conditional statements have not been satisfied'
+        module.fail_json(msg=msg, failed_conditions=failed_conditions)
+
+    result.update({
+        'changed': False,
+        'stdout': responses,
+        'stdout_lines': list(to_lines(responses))
+    })
+
+    module.exit_json(**result)
+
+
+if __name__ == '__main__':
+    main()

lib/ansible/plugins/cliconf/routeros.py

@@ -0,0 +1,78 @@
+#
+# (c) 2017 Red Hat Inc.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import re
+import json
+
+from itertools import chain
+
+from ansible.module_utils._text import to_bytes, to_text
+from ansible.module_utils.network.common.utils import to_list
+from ansible.plugins.cliconf import CliconfBase, enable_mode
+
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
+
+class Cliconf(CliconfBase):
+
+    def get_device_info(self):
+        device_info = {}
+        device_info['network_os'] = 'RouterOS'
+
+        resource = self.get(b'/system resource print')
+        data = to_text(resource, errors='surrogate_or_strict').strip()
+        match = re.search(r'version: (\S+)', data)
+        if match:
+            device_info['network_os_version'] = match.group(1)
+
+        routerboard = self.get(b'/system routerboard print')
+        data = to_text(routerboard, errors='surrogate_or_strict').strip()
+        match = re.search(r'model: (.+)$', data, re.M)
+        if match:
+            device_info['network_os_model'] = match.group(1)
+
+        identity = self.get(b'/system identity print')
+        data = to_text(identity, errors='surrogate_or_strict').strip()
+        match = re.search(r'name: (.+)$', data, re.M)
+        if match:
+            device_info['network_os_hostname'] = match.group(1)
+
+        return device_info
+
+    def get_config(self, source='running', format='text', flags=None):
+        return
+
+    def edit_config(self, command):
+        return
+
+    def get(self, command, prompt=None, answer=None, sendonly=False):
+        return self.send_command(command, prompt=prompt, answer=answer, sendonly=sendonly)
+
+    def get_capabilities(self):
+        result = {}
+        result['rpc'] = self.get_base_rpc()
+        result['network_api'] = 'cliconf'
+        result['device_info'] = self.get_device_info()
+        return json.dumps(result)

lib/ansible/plugins/terminal/routeros.py

@@ -0,0 +1,72 @@
+#
+# (c) 2016 Red Hat Inc.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+import re
+
+from ansible.errors import AnsibleConnectionFailure
+from ansible.module_utils._text import to_text, to_bytes
+from ansible.plugins.terminal import TerminalBase
+
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
+
+class TerminalModule(TerminalBase):
+
+    ansi_re = [
+        # check ECMA-48 Section 5.4 (Control Sequences)
+        re.compile(br'(\x1b\[\?1h\x1b=)'),
+        re.compile(br'((?:\x9b|\x1b\x5b)[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e])'),
+        re.compile(br'\x08.')
+    ]
+
+    terminal_initial_prompt = [
+        br'\x1bZ',
+    ]
+
+    terminal_initial_answer = b'\x1b/Z'
+
+    terminal_stdout_re = [
+        re.compile(br"\x1b<"),
+        re.compile(br"\[\w+\@[\w\-\.]+\] ?> ?$"),
+        re.compile(br"Please press \"Enter\" to continue!"),
+        re.compile(br"Do you want to see the software license\? \[Y\/n\]: ?"),
+    ]
+
+    terminal_stderr_re = [
+        re.compile(br"\nbad command name"),
+        re.compile(br"\nno such item"),
+        re.compile(br"\ninvalid value for"),
+    ]
+
+    def on_open_shell(self):
+        prompt = self._get_prompt()
+        try:
+            if prompt.strip().endswith(b':'):
+                self._exec_cli_command(b' ')
+            if prompt.strip().endswith(b'!'):
+                self._exec_cli_command(b'\n')
+        except AnsibleConnectionFailure:
+            raise AnsibleConnectionFailure('unable to bypass license prompt')