Add netconf_get module (#39869)

* Add netconf_get module

Implements part-1 of proposal #104
https://github.com/ansible/proposals/issues/104

*  Add netconf_get module
*  Refactor `get`, `get_config`, `lock`, `unlock`
   and `discard_changes` netconf plugin api's
*  Add netconf module_utils file which netconf module
   related common functions
*  Refactor junos and iosxr netconf plugins

* Fix source option handling

* Fix review comments

* Update botmeta file

* Update review comments and add support for lock

* Lock update fix

* Fix CI issue

* Add integration test and minor fixes

* Fix review comments

* Fix CI failure

* Fix CI issues

* Fix CI issues

* Fix review comments and update integration test

* Fix review comments

* Fix review comments

* Fix review comments

Fix reveiw comments

test/integration/targets/netconf_get/defaults/main.yaml

@@ -0,0 +1,2 @@
+---
+testcase: "*"

test/integration/targets/netconf_get/meta/main.yml

@@ -0,0 +1,4 @@
+---
+dependencies:
+  - { role: prepare_junos_tests, when: ansible_network_os == 'junos' }
+  - { role: prepare_iosxr_tests, when: ansible_network_os == 'iosxr' }

test/integration/targets/netconf_get/tasks/iosxr.yaml

@@ -0,0 +1,16 @@
+---
+- name: collect all netconf test cases
+  find:
+    paths: "{{ role_path }}/tests/iosxr"
+    patterns: "{{ testcase }}.yaml"
+  register: test_cases
+  connection: local
+
+- name: set test_items
+  set_fact: test_items="{{ test_cases.files | map(attribute='path') | list }}"
+
+- name: run test case (connection=netconf)
+  include: "{{ test_case_to_run }}"
+  with_items: "{{ test_items }}"
+  loop_control:
+    loop_var: test_case_to_run

test/integration/targets/netconf_get/tasks/junos.yaml

@@ -0,0 +1,16 @@
+---
+- name: collect all netconf test cases
+  find:
+    paths: "{{ role_path }}/tests/junos"
+    patterns: "{{ testcase }}.yaml"
+  register: test_cases
+  connection: local
+
+- name: set test_items
+  set_fact: test_items="{{ test_cases.files | map(attribute='path') | list }}"
+
+- name: run test case (connection=netconf)
+  include: "{{ test_case_to_run }} ansible_connection=netconf"
+  with_items: "{{ test_items }}"
+  loop_control:
+    loop_var: test_case_to_run

test/integration/targets/netconf_get/tasks/main.yaml

@@ -0,0 +1,3 @@
+---
+- { include: junos.yaml, when: ansible_network_os == 'junos', tags: ['netconf'] }
+- { include: iosxr.yaml, when: ansible_network_os == 'iosxr', tags: ['netconf'] }

test/integration/targets/netconf_get/tests/iosxr/basic.yaml

@@ -0,0 +1,163 @@
+---
+- debug: msg="START netconf_get iosxr/basic.yaml on connection={{ ansible_connection }}"
+
+- name: setup interface
+  iosxr_config:
+    commands:
+      - description this is test interface Loopback999
+      - no shutdown
+    parents:
+      - interface Loopback999
+    match: none
+  connection: network_cli
+
+- name: get running interface confiugration with filter
+  netconf_get:
+    source: running
+    filter: <interface-configurations xmlns="http://cisco.com/ns/yang/Cisco-IOS-XR-ifmgr-cfg"></interface-configurations>
+  register: result
+  connection: netconf
+
+- assert:
+    that:
+      - "'<description>this is test interface Loopback999</description>' in result.stdout"
+      - "'<usernames>' not in result.stdout"
+
+- name: test lock=never, get-config, running interface confiugration with filter without lock
+  netconf_get:
+    source: running
+    lock: never
+    filter: <interface-configurations xmlns="http://cisco.com/ns/yang/Cisco-IOS-XR-ifmgr-cfg"></interface-configurations>
+  register: result
+  connection: netconf
+
+- assert:
+    that:
+      - "'<description>this is test interface Loopback999</description>' in result.stdout"
+      - "'<usernames>' not in result.stdout"
+
+- name: test lock=if-supported, get-config, running interface confiugration with filter without lock
+  netconf_get:
+    source: running
+    lock: if-supported
+    filter: <interface-configurations xmlns="http://cisco.com/ns/yang/Cisco-IOS-XR-ifmgr-cfg"></interface-configurations>
+  register: result
+  connection: netconf
+
+- assert:
+    that:
+      - "'<description>this is test interface Loopback999</description>' in result.stdout"
+      - "'<usernames>' not in result.stdout"
+
+- name: Failure scenario, get-config information with lock
+  netconf_get:
+    source: running
+    lock: always
+  register: result
+  ignore_errors: True
+  connection: netconf
+
+- assert:
+    that:
+      - "'<bad-element>running</bad-element>' in result.msg"
+
+- name: Failure scenario, fetch config from startup
+  netconf_get:
+    source: startup
+  register: result
+  ignore_errors: True
+  connection: netconf
+
+- assert:
+    that:
+      - "'startup source is not supported' in result.msg"
+
+- name: test get, information from running datastore without lock
+  netconf_get:
+    lock: never
+    filter: <interface-configurations xmlns="http://cisco.com/ns/yang/Cisco-IOS-XR-ifmgr-cfg"></interface-configurations>
+  register: result
+  connection: netconf
+
+- assert:
+    that:
+      - "'<description>this is test interface Loopback999</description>' in result.stdout"
+
+- name: test get, information from running datastore with lock if supported
+  netconf_get:
+    lock: if-supported
+    filter: <interface-configurations xmlns="http://cisco.com/ns/yang/Cisco-IOS-XR-ifmgr-cfg"></interface-configurations>
+  register: result
+  connection: netconf
+
+- assert:
+    that:
+      - "'<description>this is test interface Loopback999</description>' in result.stdout"
+
+- name: Failure scenario, get information from running with lock
+  netconf_get:
+    lock: always
+  register: result
+  ignore_errors: True
+  connection: netconf
+
+- assert:
+    that:
+      - "'<bad-element>running</bad-element>' in result.msg"
+
+- name: get configuration and state data in json format
+  netconf_get:
+    source: running
+    display: json
+  register: result
+  connection: netconf
+
+- assert:
+    that:
+      - "{{ result['output']['rpc-reply']['data']['aaa'] is defined}}"
+
+- name: get configuration data in xml pretty format
+  netconf_get:
+    source: running
+    display: pretty
+  register: result
+  connection: netconf
+
+- assert:
+    that:
+      - "{{ result['output'] is defined}}"
+
+- name: get configuration data in xml with namespace stripped
+  netconf_get:
+    source: running
+    display: xml
+  register: result
+  connection: netconf
+
+- assert:
+    that:
+      - "{{ result['output'] is defined}}"
+      - "{{ 'xmlns' not in result.output }}"
+
+- name: Failure scenario, unsupported filter
+  netconf_get:
+    filter: configuration/state
+  register: result
+  ignore_errors: True
+  connection: netconf
+
+- assert:
+    that:
+      - "'filter value \\'configuration/state\\' of type xpath is not supported' in result.msg"
+
+- name: setup - teardown
+  iosxr_config:
+    commands:
+      - no description
+      - shutdown
+    parents:
+      - interface Loopback999
+    match: none
+  connection: network_cli
+
+- debug: msg="END netconf_get iosxr/basic.yaml on connection={{ ansible_connection }}"

test/integration/targets/netconf_get/tests/junos/basic.yaml

@@ -0,0 +1,126 @@
+---
+- debug: msg="START netconf_get junos/basic.yaml on connection={{ ansible_connection }}"
+
+- name: Configure syslog file - setup
+  junos_config:
+    lines:
+    - set system syslog file test1 any any
+  register: result
+
+- name: Get system configuration data from running datastore state
+  netconf_get:
+    source: running
+    filter: <configuration><system><syslog></syslog></system></configuration>
+  register: result
+
+- assert:
+    that:
+      - "'<name>test1</name>' in result.stdout"
+      - "'<name>any</name>' in result.stdout"
+      - "'<any/>' in result.stdout"
+      - "'<login>' not in result.stdout"
+      - "'<interface>' not in result.stdout"
+
+- name: Failure scenario, fetch config from startup
+  netconf_get:
+    source: startup
+  register: result
+  ignore_errors: True
+
+- assert:
+    that:
+      - "'startup source is not supported' in result.msg"
+
+- name: Failure scenario, fetch config from running with lock
+  netconf_get:
+    lock: always
+    source: running
+  register: result
+  ignore_errors: True
+
+- assert:
+    that:
+      - "'syntax error' in result.msg"
+
+- name: Get system configuration data from running datastore state and lock if-supported
+  netconf_get:
+    source: running
+    filter: <configuration><system><syslog></syslog></system></configuration>
+    lock: if-supported
+  register: result
+
+- assert:
+    that:
+      - "'<name>test1</name>' in result.stdout"
+      - "'<name>any</name>' in result.stdout"
+      - "'<any/>' in result.stdout"
+      - "'<login>' not in result.stdout"
+      - "'<interface>' not in result.stdout"
+
+- name: get configuration and state data in json format
+  netconf_get:
+    source: running
+    display: json
+  register: result
+
+- assert:
+    that:
+      - "{{ result['output']['rpc-reply']['data']['configuration'] is defined}}"
+
+- name: get configuration and state data in xml pretty format
+  netconf_get:
+    source: running
+    display: pretty
+  register: result
+
+- assert:
+    that:
+      - "{{ result['output'] is defined}}"
+
+- name: get configuration data in xml with namespace stripped
+  netconf_get:
+    source: running
+    display: xml
+  register: result
+
+- assert:
+    that:
+      - "{{ result['output'] is defined}}"
+      - "{{ 'xmlns' not in result.output }}"
+
+- name: get configuration and state data without datastore lock
+  netconf_get:
+    lock: never
+  register: result
+
+- assert:
+    that:
+      - "'<database-status-information>' in result.stdout"
+      - "'</configuration>' in result.stdout"
+
+- name: get configuration and state data and lock data-store if supported
+  netconf_get:
+    lock: if-supported
+  register: result
+
+- assert:
+    that:
+      - "'<database-status-information>' in result.stdout"
+      - "'</configuration>' in result.stdout"
+
+- name: Failure scenario, unsupported filter
+  netconf_get:
+    filter: configuration/state
+  register: result
+  ignore_errors: True
+
+- assert:
+    that:
+      - "'filter value \\'configuration/state\\' of type xpath is not supported' in result.msg"
+
+- name: Configure syslog file - teardown
+  junos_config:
+    lines:
+    - delete system syslog file test1 any any
+
+- debug: msg="END netconf_get junos/basic.yaml on connection={{ ansible_connection }}"