Merge b95ab803f6 into 84b5d38c51

changelogs/fragments/10346-jenkins-plugins-fixes.yml

@@ -0,0 +1,6 @@
+bugfixes:
+  - "jenkins_plugin - install latest compatible version instead of latest (https://github.com/ansible-collections/community.general/issues/854, https://github.com/ansible-collections/community.general/pull/10346)."
+  - "jenkins_plugin - separate Jenkins and external URL credentials (https://github.com/ansible-collections/community.general/issues/4419, https://github.com/ansible-collections/community.general/pull/10346)."
+
+minor_changes:
+  - "jenkins_plugin - install dependencies for specific version (https://github.com/ansible-collections/community.general/issue/4995, https://github.com/ansible-collections/community.general/pull/10346)."

plugins/modules/jenkins_plugin.py

@@ -74,6 +74,18 @@ options:
       - A list of base URL(s) to retrieve C(update-center.json), and direct plugin files from.
       - This can be a list since community.general 3.3.0.
     default: ['https://updates.jenkins.io', 'http://mirrors.jenkins.io']
+  updates_url_username:
+    description:
+      - If using a custom O(updates_url), set this as the username of the user with access to the URL.
+      - If the custom O(updates_url) does not require authentication, this can be left empty.
+    type: str
+    version_added: 11.2.0
+  updates_url_password:
+    description:
+      - If using a custom O(updates_url), set this as the password of the user with access to the URL.
+      - If the custom O(updates_url) does not require authentication, this can be left empty.
+    type: str
+    version_added: 11.2.0
   update_json_url_segment:
     type: list
     elements: str
@@ -81,6 +93,13 @@ options:
       - A list of URL segment(s) to retrieve the update center JSON file from.
     default: ['update-center.json', 'updates/update-center.json']
     version_added: 3.3.0
+  plugin_versions_url_segment:
+    type: list
+    elements: str
+    description:
+      - A list of URL segment(s) to retrieve the plugin versions JSON file from.
+    default: ['plugin-versions.json', 'current/plugin-versions.json']
+    version_added: 11.2.0
   latest_plugins_url_segments:
     type: list
     elements: str
@@ -112,7 +131,8 @@ options:
   with_dependencies:
     description:
       - Defines whether to install plugin dependencies.
-      - This option takes effect only if the O(version) is not defined.
+      - In earlier versions, this option had no effect when a specific O(version) was set.
+        Since community.general 11.2.0, dependencies are also installed for versioned plugins.
     type: bool
     default: true
 
@@ -124,6 +144,9 @@ notes:
   - Pinning works only if the plugin is installed and Jenkins service was successfully restarted after the plugin installation.
   - It is not possible to run the module remotely by changing the O(url) parameter to point to the Jenkins server. The module
     must be used on the host where Jenkins runs as it needs direct access to the plugin files.
+  - If using a custom O(updates_url), ensure that the URL provides a C(plugin-versions.json) file.
+    This file must include metadata for all available plugin versions to support version compatibility resolution.
+    The file should be in the same format as the one provided by Jenkins update center (https://updates.jenkins.io/current/plugin-versions.json).
 extends_documentation_fragment:
   - ansible.builtin.url
   - ansible.builtin.files
@@ -315,11 +338,13 @@ import io
 import json
 import os
 import tempfile
+import time
+from collections import OrderedDict
 
 from ansible.module_utils.basic import AnsibleModule, to_bytes
 from ansible.module_utils.six.moves import http_cookiejar as cookiejar
 from ansible.module_utils.six.moves.urllib.parse import urlencode
-from ansible.module_utils.urls import fetch_url, url_argument_spec
+from ansible.module_utils.urls import fetch_url, url_argument_spec, basic_auth_header
 from ansible.module_utils.six import text_type, binary_type
 from ansible.module_utils.common.text.converters import to_native
 
@@ -340,14 +365,24 @@ class JenkinsPlugin(object):
         self.url = self.params['url']
         self.timeout = self.params['timeout']
 
+        # Authentication for non-Jenkins calls
+        self.updates_url_credentials = {}
+        if self.params.get('updates_url_username') and self.params.get('updates_url_password'):
+            self.updates_url_credentials["Authorization"] = basic_auth_header(self.params['updates_url_username'], self.params['updates_url_password'])
+
         # Crumb
         self.crumb = {}
+
+        # Authentication for Jenkins calls
+        if self.params.get('url_username') and self.params.get('url_password'):
+            self.crumb["Authorization"] = basic_auth_header(self.params['url_username'], self.params['url_password'])
+
         # Cookie jar for crumb session
         self.cookies = None
 
         if self._csrf_enabled():
             self.cookies = cookiejar.LWPCookieJar()
-            self.crumb = self._get_crumb()
+            self._get_crumb()
 
         # Get list of installed plugins
         self._get_installed_plugins()
@@ -390,10 +425,14 @@ class JenkinsPlugin(object):
             err_msg = None
             try:
                 self.module.debug("fetching url: %s" % url)
+
+                is_jenkins_call = url.startswith(self.url)
+                self.module.params['force_basic_auth'] = is_jenkins_call
+
                 response, info = fetch_url(
                     self.module, url, timeout=self.timeout, cookies=self.cookies,
-                    headers=self.crumb, **kwargs)
-
+                    headers=self.crumb if is_jenkins_call else self.updates_url_credentials or self.crumb,
+                    **kwargs)
                 if info['status'] == 200:
                     return response
                 else:
@@ -422,9 +461,13 @@ class JenkinsPlugin(object):
 
         # Get the URL data
         try:
+            is_jenkins_call = url.startswith(self.url)
+            self.module.params['force_basic_auth'] = is_jenkins_call
+
             response, info = fetch_url(
                 self.module, url, timeout=self.timeout, cookies=self.cookies,
-                headers=self.crumb, **kwargs)
+                headers=self.crumb if is_jenkins_call else self.updates_url_credentials or self.crumb,
+                **kwargs)
 
             if info['status'] != 200:
                 if dont_fail:
@@ -444,16 +487,12 @@ class JenkinsPlugin(object):
             "%s/%s" % (self.url, "crumbIssuer/api/json"), 'Crumb')
 
         if 'crumbRequestField' in crumb_data and 'crumb' in crumb_data:
-            ret = {
-                crumb_data['crumbRequestField']: crumb_data['crumb']
-            }
+            self.crumb[crumb_data['crumbRequestField']] = crumb_data['crumb']
         else:
             self.module.fail_json(
                 msg="Required fields not found in the Crum response.",
                 details=crumb_data)
 
-        return ret
-
     def _get_installed_plugins(self):
         plugins_data = self._get_json_data(
             "%s/%s" % (self.url, "pluginManager/api/json?depth=1"),
@@ -467,6 +506,7 @@ class JenkinsPlugin(object):
         self.is_installed = False
         self.is_pinned = False
         self.is_enabled = False
+        self.installed_plugins = plugins_data['plugins']
 
         for p in plugins_data['plugins']:
             if p['shortName'] == self.params['name']:
@@ -480,6 +520,40 @@ class JenkinsPlugin(object):
 
                 break
 
+    def _install_dependencies(self):
+        dependencies = self._get_versioned_dependencies()
+        self.dependencies_states = []
+
+        for dep_name, dep_version in dependencies.items():
+            if not any(p['shortName'] == dep_name and p['version'] == dep_version for p in self.installed_plugins):
+                dep_params = self.params.copy()
+                dep_params['name'] = dep_name
+                dep_params['version'] = dep_version
+                dep_module = AnsibleModule(
+                    argument_spec=self.module.argument_spec,
+                    supports_check_mode=self.module.check_mode
+                )
+                dep_module.params = dep_params
+                dep_plugin = JenkinsPlugin(dep_module)
+                if not dep_plugin.install():
+                    self.dependencies_states.append(
+                        {
+                            'name': dep_name,
+                            'version': dep_version,
+                            'state': 'absent'})
+                else:
+                    self.dependencies_states.append(
+                        {
+                            'name': dep_name,
+                            'version': dep_version,
+                            'state': 'present'})
+            else:
+                self.dependencies_states.append(
+                    {
+                        'name': dep_name,
+                        'version': dep_version,
+                        'state': 'present'})
+
     def _install_with_plugin_manager(self):
         if not self.module.check_mode:
             # Install the plugin (with dependencies)
@@ -540,6 +614,10 @@ class JenkinsPlugin(object):
                     plugin_content = plugin_fh.read()
                 checksum_old = hashlib.sha1(plugin_content).hexdigest()
 
+            # Install dependencies
+            if self.params['with_dependencies']:
+                self._install_dependencies()
+
             if self.params['version'] in [None, 'latest']:
                 # Take latest version
                 plugin_urls = self._get_latest_plugin_urls()
@@ -612,6 +690,58 @@ class JenkinsPlugin(object):
                 urls.append("{0}/{1}/{2}.hpi".format(base_url, update_segment, self.params['name']))
         return urls
 
+    def _get_latest_compatible_plugin_version(self, plugin_name=None):
+        if not hasattr(self, 'jenkins_version'):
+            self.module.params['force_basic_auth'] = True
+            resp, info = fetch_url(self.module, self.url)
+            raw_version = info.get("x-jenkins")
+            self.jenkins_version = self.parse_version(raw_version)
+        name = plugin_name or self.params['name']
+        cache_path = "{}/ansible_jenkins_plugin_cache.json".format(self.params['jenkins_home'])
+        plugin_version_urls = []
+        for base_url in self.params['updates_url']:
+            for update_json in self.params['plugin_versions_url_segment']:
+                plugin_version_urls.append("{}/{}".format(base_url, update_json))
+
+        try:  # Check if file is saved localy
+            if os.path.exists(cache_path):
+                file_mtime = os.path.getmtime(cache_path)
+            else:
+                file_mtime = 0
+
+            now = time.time()
+            if now - file_mtime >= 86400:
+                response = self._get_urls_data(plugin_version_urls, what="plugin-versions.json")
+                plugin_data = json.loads(to_native(response.read()), object_pairs_hook=OrderedDict)
+
+                # Save it to file for next time
+                with open(cache_path, "w") as f:
+                    json.dump(plugin_data, f)
+
+            with open(cache_path, "r") as f:
+                plugin_data = json.load(f)
+
+        except Exception as e:
+            if os.path.exists(cache_path):
+                os.remove(cache_path)
+            self.module.fail_json(msg="Failed to parse plugin-versions.json", details=to_native(e))
+
+        plugin_versions = plugin_data.get("plugins", {}).get(name)
+        if not plugin_versions:
+            self.module.fail_json(msg="Plugin '{}' not found.".format(name))
+
+        sorted_versions = list(reversed(plugin_versions.items()))
+
+        for idx, (version_title, version_info) in enumerate(sorted_versions):
+            required_core = version_info.get("requiredCore", "0.0")
+            if self.parse_version(required_core) <= self.jenkins_version:
+                return 'latest' if idx == 0 else version_title
+
+        self.module.warn(
+            "No compatible version found for plugin '{}'. "
+            "Installing latest version.".format(name))
+        return 'latest'
+
     def _get_versioned_plugin_urls(self):
         urls = []
         for base_url in self.params['updates_url']:
@@ -626,6 +756,18 @@ class JenkinsPlugin(object):
                 urls.append("{0}/{1}".format(base_url, update_json))
         return urls
 
+    def _get_versioned_dependencies(self):
+        # Get dependencies for the specified plugin version
+        plugin_data = self._download_updates()['dependencies']
+
+        dependencies_info = {
+            dep["name"]: self._get_latest_compatible_plugin_version(dep["name"])
+            for dep in plugin_data
+            if not dep.get("optional", False)
+        }
+
+        return dependencies_info
+
     def _download_updates(self):
         try:
             updates_file, download_updates = download_updates_file(self.params['updates_expiration'])
@@ -779,6 +921,10 @@ class JenkinsPlugin(object):
             msg_exception="%s has failed." % msg,
             method="POST")
 
+    @staticmethod
+    def parse_version(version_str):
+        return tuple(int(x) for x in version_str.split('.'))
+
 
 def main():
     # Module arguments
@@ -803,8 +949,12 @@ def main():
         updates_expiration=dict(default=86400, type="int"),
         updates_url=dict(type="list", elements="str", default=['https://updates.jenkins.io',
                                                                'http://mirrors.jenkins.io']),
+        updates_url_username=dict(type="str"),
+        updates_url_password=dict(type="str", no_log=True),
         update_json_url_segment=dict(type="list", elements="str", default=['update-center.json',
                                                                            'updates/update-center.json']),
+        plugin_versions_url_segment=dict(type="list", elements="str", default=['plugin-versions.json',
+                                                                               'current/plugin-versions.json']),
         latest_plugins_url_segments=dict(type="list", elements="str", default=['latest']),
         versioned_plugins_url_segments=dict(type="list", elements="str", default=['download/plugins', 'plugins']),
         url=dict(default='http://localhost:8080'),
@@ -819,9 +969,6 @@ def main():
         supports_check_mode=True,
     )
 
-    # Force basic authentication
-    module.params['force_basic_auth'] = True
-
     # Convert timeout to float
     try:
         module.params['timeout'] = float(module.params['timeout'])
@@ -829,11 +976,17 @@ def main():
         module.fail_json(
             msg='Cannot convert %s to float.' % module.params['timeout'],
             details=to_native(e))
+    # Instantiate the JenkinsPlugin object
+    jp = JenkinsPlugin(module)
 
     # Set version to latest if state is latest
     if module.params['state'] == 'latest':
         module.params['state'] = 'present'
-        module.params['version'] = 'latest'
+        module.params['version'] = jp._get_latest_compatible_plugin_version()
+
+    # Set version to latest compatible version if version is latest
+    if module.params['version'] == 'latest':
+        module.params['version'] = jp._get_latest_compatible_plugin_version()
 
     # Create some shortcuts
     name = module.params['name']
@@ -842,9 +995,6 @@ def main():
     # Initial change state of the task
     changed = False
 
-    # Instantiate the JenkinsPlugin object
-    jp = JenkinsPlugin(module)
-
     # Perform action depending on the requested state
     if state == 'present':
         changed = jp.install()
@@ -860,7 +1010,7 @@ def main():
         changed = jp.disable()
 
     # Print status of the change
-    module.exit_json(changed=changed, plugin=name, state=state)
+    module.exit_json(changed=changed, plugin=name, state=state, dependencies=jp.dependencies_states if hasattr(jp, 'dependencies_states') else None)
 
 
 if __name__ == '__main__':

tests/unit/plugins/modules/test_jenkins_plugin.py

@@ -6,9 +6,16 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 from io import BytesIO
+import json
+from collections import OrderedDict
 
 from ansible_collections.community.general.plugins.modules.jenkins_plugin import JenkinsPlugin
 from ansible.module_utils.common._collections_compat import Mapping
+from ansible_collections.community.internal_test_tools.tests.unit.compat.mock import (
+    MagicMock,
+    patch,
+)
+from ansible.module_utils.urls import basic_auth_header
 
 
 def pass_function(*args, **kwargs):
@@ -190,3 +197,98 @@ def isInList(l, i):
         if item == i:
             return True
     return False
+
+
+@patch("ansible_collections.community.general.plugins.modules.jenkins_plugin.fetch_url")
+def test__get_latest_compatible_plugin_version(fetch_mock, mocker):
+    "test the latest compatible plugin version retrieval"
+
+    params = {
+        "url": "http://fake.jenkins.server",
+        "timeout": 30,
+        "name": "git",
+        "version": "latest",
+        "updates_url": ["https://some.base.url"],
+        "plugin_versions_url_segment": ["plugin-versions.json"],
+        "latest_plugins_url_segments": ["test_latest"],
+        "jenkins_home": "/var/lib/jenkins",
+    }
+    module = mocker.Mock()
+    module.params = params
+
+    jenkins_info = {"x-jenkins": "2.263.1"}
+    jenkins_response = MagicMock()
+    jenkins_response.read.return_value = b"{}"
+
+    plugin_data = {
+        "plugins": {
+            "git": OrderedDict([
+                ("4.8.2", {"requiredCore": "2.263.1"}),
+                ("4.8.3", {"requiredCore": "2.263.1"}),
+                ("4.9.0", {"requiredCore": "2.289.1"}),
+                ("4.9.1", {"requiredCore": "2.289.1"}),
+            ])
+        }
+    }
+    plugin_versions_response = MagicMock()
+    plugin_versions_response.read.return_value = json.dumps(plugin_data).encode("utf-8")
+    plugin_versions_info = {"status": 200}
+
+    def fetch_url_side_effect(module, url, **kwargs):
+        if "plugin-versions.json" in url:
+            return (plugin_versions_response, plugin_versions_info)
+        else:
+            return (jenkins_response, jenkins_info)
+
+    fetch_mock.side_effect = fetch_url_side_effect
+
+    JenkinsPlugin._csrf_enabled = lambda self: False
+    JenkinsPlugin._get_installed_plugins = lambda self: None
+
+    jenkins_plugin = JenkinsPlugin(module)
+    latest_version = jenkins_plugin._get_latest_compatible_plugin_version()
+    assert latest_version == '4.8.3'
+
+
+@patch("ansible_collections.community.general.plugins.modules.jenkins_plugin.fetch_url")
+def test__get_urls_data_sets_correct_headers(fetch_mock, mocker):
+    params = {
+        "url": "http://jenkins.example.com",
+        "timeout": 30,
+        "name": "git",
+        "jenkins_home": "/var/lib/jenkins",
+        "updates_url": ["http://updates.example.com"],
+        "latest_plugins_url_segments": ["latest"],
+        "update_json_url_segment": ["update-center.json"],
+        "versioned_plugins_url_segments": ["plugins"],
+        "url_username": "jenkins_user",
+        "url_password": "jenkins_pass",
+        "updates_url_username": "update_user",
+        "updates_url_password": "update_pass",
+    }
+    module = mocker.Mock()
+    module.params = params
+
+    dummy_response = MagicMock()
+    fetch_mock.return_value = (dummy_response, {"status": 200})
+
+    JenkinsPlugin._csrf_enabled = lambda self: False
+    JenkinsPlugin._get_installed_plugins = lambda self: None
+
+    jp = JenkinsPlugin(module)
+
+    update_url = "http://updates.example.com/plugin-versions.json"
+    jp._get_urls_data([update_url])
+
+    jenkins_url = "http://jenkins.example.com/some-endpoint"
+    jp._get_urls_data([jenkins_url])
+
+    calls = fetch_mock.call_args_list
+
+    dummy, kwargs_2 = calls[1]
+    jenkins_auth = basic_auth_header("jenkins_user", "jenkins_pass")
+    assert kwargs_2["headers"]["Authorization"] == jenkins_auth
+
+    dummy, kwargs_1 = calls[0]
+    updates_auth = basic_auth_header("update_user", "update_pass")
+    assert kwargs_1["headers"]["Authorization"] == updates_auth