diff --git a/changelogs/fragments/10346-jenkins-plugins-fixes.yml b/changelogs/fragments/10346-jenkins-plugins-fixes.yml new file mode 100644 index 0000000000..382fe7aa53 --- /dev/null +++ b/changelogs/fragments/10346-jenkins-plugins-fixes.yml @@ -0,0 +1,6 @@ +bugfixes: + - "jenkins_plugin - install latest compatible version instead of latest (https://github.com/ansible-collections/community.general/issues/854, https://github.com/ansible-collections/community.general/pull/10346)." + - "jenkins_plugin - separate Jenkins and external URL credentials (https://github.com/ansible-collections/community.general/issues/4419, https://github.com/ansible-collections/community.general/pull/10346)." + +minor_changes: + - "jenkins_plugin - install dependencies for specific version (https://github.com/ansible-collections/community.general/issue/4995, https://github.com/ansible-collections/community.general/pull/10346)." diff --git a/plugins/modules/jenkins_plugin.py b/plugins/modules/jenkins_plugin.py index f47dcfe92f..ca06d77a8f 100644 --- a/plugins/modules/jenkins_plugin.py +++ b/plugins/modules/jenkins_plugin.py @@ -74,6 +74,18 @@ options: - A list of base URL(s) to retrieve C(update-center.json), and direct plugin files from. - This can be a list since community.general 3.3.0. default: ['https://updates.jenkins.io', 'http://mirrors.jenkins.io'] + updates_url_username: + description: + - If using a custom O(updates_url), set this as the username of the user with access to the URL. + - If the custom O(updates_url) does not require authentication, this can be left empty. + type: str + version_added: 11.2.0 + updates_url_password: + description: + - If using a custom O(updates_url), set this as the password of the user with access to the URL. + - If the custom O(updates_url) does not require authentication, this can be left empty. + type: str + version_added: 11.2.0 update_json_url_segment: type: list elements: str @@ -81,6 +93,13 @@ options: - A list of URL segment(s) to retrieve the update center JSON file from. default: ['update-center.json', 'updates/update-center.json'] version_added: 3.3.0 + plugin_versions_url_segment: + type: list + elements: str + description: + - A list of URL segment(s) to retrieve the plugin versions JSON file from. + default: ['plugin-versions.json', 'current/plugin-versions.json'] + version_added: 11.2.0 latest_plugins_url_segments: type: list elements: str @@ -112,7 +131,8 @@ options: with_dependencies: description: - Defines whether to install plugin dependencies. - - This option takes effect only if the O(version) is not defined. + - In earlier versions, this option had no effect when a specific O(version) was set. + Since community.general 11.2.0, dependencies are also installed for versioned plugins. type: bool default: true @@ -124,6 +144,9 @@ notes: - Pinning works only if the plugin is installed and Jenkins service was successfully restarted after the plugin installation. - It is not possible to run the module remotely by changing the O(url) parameter to point to the Jenkins server. The module must be used on the host where Jenkins runs as it needs direct access to the plugin files. + - If using a custom O(updates_url), ensure that the URL provides a C(plugin-versions.json) file. + This file must include metadata for all available plugin versions to support version compatibility resolution. + The file should be in the same format as the one provided by Jenkins update center (https://updates.jenkins.io/current/plugin-versions.json). extends_documentation_fragment: - ansible.builtin.url - ansible.builtin.files @@ -315,11 +338,13 @@ import io import json import os import tempfile +import time +from collections import OrderedDict from ansible.module_utils.basic import AnsibleModule, to_bytes from ansible.module_utils.six.moves import http_cookiejar as cookiejar from ansible.module_utils.six.moves.urllib.parse import urlencode -from ansible.module_utils.urls import fetch_url, url_argument_spec +from ansible.module_utils.urls import fetch_url, url_argument_spec, basic_auth_header from ansible.module_utils.six import text_type, binary_type from ansible.module_utils.common.text.converters import to_native @@ -340,14 +365,24 @@ class JenkinsPlugin(object): self.url = self.params['url'] self.timeout = self.params['timeout'] + # Authentication for non-Jenkins calls + self.updates_url_credentials = {} + if self.params.get('updates_url_username') and self.params.get('updates_url_password'): + self.updates_url_credentials["Authorization"] = basic_auth_header(self.params['updates_url_username'], self.params['updates_url_password']) + # Crumb self.crumb = {} + + # Authentication for Jenkins calls + if self.params.get('url_username') and self.params.get('url_password'): + self.crumb["Authorization"] = basic_auth_header(self.params['url_username'], self.params['url_password']) + # Cookie jar for crumb session self.cookies = None if self._csrf_enabled(): self.cookies = cookiejar.LWPCookieJar() - self.crumb = self._get_crumb() + self._get_crumb() # Get list of installed plugins self._get_installed_plugins() @@ -390,10 +425,14 @@ class JenkinsPlugin(object): err_msg = None try: self.module.debug("fetching url: %s" % url) + + is_jenkins_call = url.startswith(self.url) + self.module.params['force_basic_auth'] = is_jenkins_call + response, info = fetch_url( self.module, url, timeout=self.timeout, cookies=self.cookies, - headers=self.crumb, **kwargs) - + headers=self.crumb if is_jenkins_call else self.updates_url_credentials or self.crumb, + **kwargs) if info['status'] == 200: return response else: @@ -422,9 +461,13 @@ class JenkinsPlugin(object): # Get the URL data try: + is_jenkins_call = url.startswith(self.url) + self.module.params['force_basic_auth'] = is_jenkins_call + response, info = fetch_url( self.module, url, timeout=self.timeout, cookies=self.cookies, - headers=self.crumb, **kwargs) + headers=self.crumb if is_jenkins_call else self.updates_url_credentials or self.crumb, + **kwargs) if info['status'] != 200: if dont_fail: @@ -444,16 +487,12 @@ class JenkinsPlugin(object): "%s/%s" % (self.url, "crumbIssuer/api/json"), 'Crumb') if 'crumbRequestField' in crumb_data and 'crumb' in crumb_data: - ret = { - crumb_data['crumbRequestField']: crumb_data['crumb'] - } + self.crumb[crumb_data['crumbRequestField']] = crumb_data['crumb'] else: self.module.fail_json( msg="Required fields not found in the Crum response.", details=crumb_data) - return ret - def _get_installed_plugins(self): plugins_data = self._get_json_data( "%s/%s" % (self.url, "pluginManager/api/json?depth=1"), @@ -467,6 +506,7 @@ class JenkinsPlugin(object): self.is_installed = False self.is_pinned = False self.is_enabled = False + self.installed_plugins = plugins_data['plugins'] for p in plugins_data['plugins']: if p['shortName'] == self.params['name']: @@ -480,6 +520,40 @@ class JenkinsPlugin(object): break + def _install_dependencies(self): + dependencies = self._get_versioned_dependencies() + self.dependencies_states = [] + + for dep_name, dep_version in dependencies.items(): + if not any(p['shortName'] == dep_name and p['version'] == dep_version for p in self.installed_plugins): + dep_params = self.params.copy() + dep_params['name'] = dep_name + dep_params['version'] = dep_version + dep_module = AnsibleModule( + argument_spec=self.module.argument_spec, + supports_check_mode=self.module.check_mode + ) + dep_module.params = dep_params + dep_plugin = JenkinsPlugin(dep_module) + if not dep_plugin.install(): + self.dependencies_states.append( + { + 'name': dep_name, + 'version': dep_version, + 'state': 'absent'}) + else: + self.dependencies_states.append( + { + 'name': dep_name, + 'version': dep_version, + 'state': 'present'}) + else: + self.dependencies_states.append( + { + 'name': dep_name, + 'version': dep_version, + 'state': 'present'}) + def _install_with_plugin_manager(self): if not self.module.check_mode: # Install the plugin (with dependencies) @@ -540,6 +614,10 @@ class JenkinsPlugin(object): plugin_content = plugin_fh.read() checksum_old = hashlib.sha1(plugin_content).hexdigest() + # Install dependencies + if self.params['with_dependencies']: + self._install_dependencies() + if self.params['version'] in [None, 'latest']: # Take latest version plugin_urls = self._get_latest_plugin_urls() @@ -612,6 +690,58 @@ class JenkinsPlugin(object): urls.append("{0}/{1}/{2}.hpi".format(base_url, update_segment, self.params['name'])) return urls + def _get_latest_compatible_plugin_version(self, plugin_name=None): + if not hasattr(self, 'jenkins_version'): + self.module.params['force_basic_auth'] = True + resp, info = fetch_url(self.module, self.url) + raw_version = info.get("x-jenkins") + self.jenkins_version = self.parse_version(raw_version) + name = plugin_name or self.params['name'] + cache_path = "{}/ansible_jenkins_plugin_cache.json".format(self.params['jenkins_home']) + plugin_version_urls = [] + for base_url in self.params['updates_url']: + for update_json in self.params['plugin_versions_url_segment']: + plugin_version_urls.append("{}/{}".format(base_url, update_json)) + + try: # Check if file is saved localy + if os.path.exists(cache_path): + file_mtime = os.path.getmtime(cache_path) + else: + file_mtime = 0 + + now = time.time() + if now - file_mtime >= 86400: + response = self._get_urls_data(plugin_version_urls, what="plugin-versions.json") + plugin_data = json.loads(to_native(response.read()), object_pairs_hook=OrderedDict) + + # Save it to file for next time + with open(cache_path, "w") as f: + json.dump(plugin_data, f) + + with open(cache_path, "r") as f: + plugin_data = json.load(f) + + except Exception as e: + if os.path.exists(cache_path): + os.remove(cache_path) + self.module.fail_json(msg="Failed to parse plugin-versions.json", details=to_native(e)) + + plugin_versions = plugin_data.get("plugins", {}).get(name) + if not plugin_versions: + self.module.fail_json(msg="Plugin '{}' not found.".format(name)) + + sorted_versions = list(reversed(plugin_versions.items())) + + for idx, (version_title, version_info) in enumerate(sorted_versions): + required_core = version_info.get("requiredCore", "0.0") + if self.parse_version(required_core) <= self.jenkins_version: + return 'latest' if idx == 0 else version_title + + self.module.warn( + "No compatible version found for plugin '{}'. " + "Installing latest version.".format(name)) + return 'latest' + def _get_versioned_plugin_urls(self): urls = [] for base_url in self.params['updates_url']: @@ -626,6 +756,18 @@ class JenkinsPlugin(object): urls.append("{0}/{1}".format(base_url, update_json)) return urls + def _get_versioned_dependencies(self): + # Get dependencies for the specified plugin version + plugin_data = self._download_updates()['dependencies'] + + dependencies_info = { + dep["name"]: self._get_latest_compatible_plugin_version(dep["name"]) + for dep in plugin_data + if not dep.get("optional", False) + } + + return dependencies_info + def _download_updates(self): try: updates_file, download_updates = download_updates_file(self.params['updates_expiration']) @@ -779,6 +921,10 @@ class JenkinsPlugin(object): msg_exception="%s has failed." % msg, method="POST") + @staticmethod + def parse_version(version_str): + return tuple(int(x) for x in version_str.split('.')) + def main(): # Module arguments @@ -803,8 +949,12 @@ def main(): updates_expiration=dict(default=86400, type="int"), updates_url=dict(type="list", elements="str", default=['https://updates.jenkins.io', 'http://mirrors.jenkins.io']), + updates_url_username=dict(type="str"), + updates_url_password=dict(type="str", no_log=True), update_json_url_segment=dict(type="list", elements="str", default=['update-center.json', 'updates/update-center.json']), + plugin_versions_url_segment=dict(type="list", elements="str", default=['plugin-versions.json', + 'current/plugin-versions.json']), latest_plugins_url_segments=dict(type="list", elements="str", default=['latest']), versioned_plugins_url_segments=dict(type="list", elements="str", default=['download/plugins', 'plugins']), url=dict(default='http://localhost:8080'), @@ -819,9 +969,6 @@ def main(): supports_check_mode=True, ) - # Force basic authentication - module.params['force_basic_auth'] = True - # Convert timeout to float try: module.params['timeout'] = float(module.params['timeout']) @@ -829,11 +976,17 @@ def main(): module.fail_json( msg='Cannot convert %s to float.' % module.params['timeout'], details=to_native(e)) + # Instantiate the JenkinsPlugin object + jp = JenkinsPlugin(module) # Set version to latest if state is latest if module.params['state'] == 'latest': module.params['state'] = 'present' - module.params['version'] = 'latest' + module.params['version'] = jp._get_latest_compatible_plugin_version() + + # Set version to latest compatible version if version is latest + if module.params['version'] == 'latest': + module.params['version'] = jp._get_latest_compatible_plugin_version() # Create some shortcuts name = module.params['name'] @@ -842,9 +995,6 @@ def main(): # Initial change state of the task changed = False - # Instantiate the JenkinsPlugin object - jp = JenkinsPlugin(module) - # Perform action depending on the requested state if state == 'present': changed = jp.install() @@ -860,7 +1010,7 @@ def main(): changed = jp.disable() # Print status of the change - module.exit_json(changed=changed, plugin=name, state=state) + module.exit_json(changed=changed, plugin=name, state=state, dependencies=jp.dependencies_states if hasattr(jp, 'dependencies_states') else None) if __name__ == '__main__': diff --git a/tests/unit/plugins/modules/test_jenkins_plugin.py b/tests/unit/plugins/modules/test_jenkins_plugin.py index 194cc2d724..5bd14bcbee 100644 --- a/tests/unit/plugins/modules/test_jenkins_plugin.py +++ b/tests/unit/plugins/modules/test_jenkins_plugin.py @@ -6,9 +6,16 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type from io import BytesIO +import json +from collections import OrderedDict from ansible_collections.community.general.plugins.modules.jenkins_plugin import JenkinsPlugin from ansible.module_utils.common._collections_compat import Mapping +from ansible_collections.community.internal_test_tools.tests.unit.compat.mock import ( + MagicMock, + patch, +) +from ansible.module_utils.urls import basic_auth_header def pass_function(*args, **kwargs): @@ -190,3 +197,98 @@ def isInList(l, i): if item == i: return True return False + + +@patch("ansible_collections.community.general.plugins.modules.jenkins_plugin.fetch_url") +def test__get_latest_compatible_plugin_version(fetch_mock, mocker): + "test the latest compatible plugin version retrieval" + + params = { + "url": "http://fake.jenkins.server", + "timeout": 30, + "name": "git", + "version": "latest", + "updates_url": ["https://some.base.url"], + "plugin_versions_url_segment": ["plugin-versions.json"], + "latest_plugins_url_segments": ["test_latest"], + "jenkins_home": "/var/lib/jenkins", + } + module = mocker.Mock() + module.params = params + + jenkins_info = {"x-jenkins": "2.263.1"} + jenkins_response = MagicMock() + jenkins_response.read.return_value = b"{}" + + plugin_data = { + "plugins": { + "git": OrderedDict([ + ("4.8.2", {"requiredCore": "2.263.1"}), + ("4.8.3", {"requiredCore": "2.263.1"}), + ("4.9.0", {"requiredCore": "2.289.1"}), + ("4.9.1", {"requiredCore": "2.289.1"}), + ]) + } + } + plugin_versions_response = MagicMock() + plugin_versions_response.read.return_value = json.dumps(plugin_data).encode("utf-8") + plugin_versions_info = {"status": 200} + + def fetch_url_side_effect(module, url, **kwargs): + if "plugin-versions.json" in url: + return (plugin_versions_response, plugin_versions_info) + else: + return (jenkins_response, jenkins_info) + + fetch_mock.side_effect = fetch_url_side_effect + + JenkinsPlugin._csrf_enabled = lambda self: False + JenkinsPlugin._get_installed_plugins = lambda self: None + + jenkins_plugin = JenkinsPlugin(module) + latest_version = jenkins_plugin._get_latest_compatible_plugin_version() + assert latest_version == '4.8.3' + + +@patch("ansible_collections.community.general.plugins.modules.jenkins_plugin.fetch_url") +def test__get_urls_data_sets_correct_headers(fetch_mock, mocker): + params = { + "url": "http://jenkins.example.com", + "timeout": 30, + "name": "git", + "jenkins_home": "/var/lib/jenkins", + "updates_url": ["http://updates.example.com"], + "latest_plugins_url_segments": ["latest"], + "update_json_url_segment": ["update-center.json"], + "versioned_plugins_url_segments": ["plugins"], + "url_username": "jenkins_user", + "url_password": "jenkins_pass", + "updates_url_username": "update_user", + "updates_url_password": "update_pass", + } + module = mocker.Mock() + module.params = params + + dummy_response = MagicMock() + fetch_mock.return_value = (dummy_response, {"status": 200}) + + JenkinsPlugin._csrf_enabled = lambda self: False + JenkinsPlugin._get_installed_plugins = lambda self: None + + jp = JenkinsPlugin(module) + + update_url = "http://updates.example.com/plugin-versions.json" + jp._get_urls_data([update_url]) + + jenkins_url = "http://jenkins.example.com/some-endpoint" + jp._get_urls_data([jenkins_url]) + + calls = fetch_mock.call_args_list + + dummy, kwargs_2 = calls[1] + jenkins_auth = basic_auth_header("jenkins_user", "jenkins_pass") + assert kwargs_2["headers"]["Authorization"] == jenkins_auth + + dummy, kwargs_1 = calls[0] + updates_auth = basic_auth_header("update_user", "update_pass") + assert kwargs_1["headers"]["Authorization"] == updates_auth