introduce module_utils.urls.fetch_file as a wrapper to download and save files (#19172)

* module_utils.urls: add fetch_file function

* apt: use fetch_file instead of own download()

* unarchive: use fetch_file instead of own codecopy

* apt: add test for deb=http://…

* unarchive: add test for a remote file download and unarchive

* yum: replace fetch_rpm_from_url by fetch_file

* use NamedTemporaryFile

* don't add a dot to fileext, it's already there

lib/ansible/modules/packaging/os/apt.py

@@ -266,7 +266,7 @@ import time
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils._text import to_bytes, to_native
-from ansible.module_utils.urls import fetch_url
+from ansible.module_utils.urls import fetch_file
 
 # APT related constants
 APT_ENV_VARS = dict(
@@ -846,36 +846,6 @@ def upgrade(m, mode="yes", force=False, default_release=None,
     m.exit_json(changed=True, msg=out, stdout=out, stderr=err, diff=diff)
 
 
-def download(module, deb):
-    package = os.path.join(module.tmpdir, to_native(deb.rsplit('/', 1)[1], errors='surrogate_or_strict'))
-    # When downloading a deb, how much of the deb to download before
-    # saving to a tempfile (64k)
-    BUFSIZE = 65536
-
-    try:
-        rsp, info = fetch_url(module, deb, method='GET')
-        if info['status'] != 200:
-            module.fail_json(msg="Failed to download %s, %s" % (deb,
-                                                                info['msg']))
-        # Ensure file is open in binary mode for Python 3
-        f = open(package, 'wb')
-        # Read 1kb at a time to save on ram
-        while True:
-            data = rsp.read(BUFSIZE)
-            data = to_bytes(data, errors='surrogate_or_strict')
-
-            if len(data) < 1:
-                break  # End of file, break while loop
-
-            f.write(data)
-        f.close()
-        deb = package
-    except Exception as e:
-        module.fail_json(msg="Failure downloading %s, %s" % (deb, to_native(e)))
-
-    return deb
-
-
 def get_cache_mtime():
     """Return mtime of a valid apt cache file.
     Stat the apt cache file and if no cache file is found return 0
@@ -1053,7 +1023,7 @@ def main():
             if p['state'] != 'present':
                 module.fail_json(msg="deb only supports state=present")
             if '://' in p['deb']:
-                p['deb'] = download(module, p['deb'])
+                p['deb'] = fetch_file(module, p['deb'])
             install_deb(module, p['deb'], cache,
                         install_recommends=install_recommends,
                         allow_unauthenticated=allow_unauthenticated,

lib/ansible/modules/packaging/os/yum.py

@@ -352,13 +352,11 @@ except ImportError:
     transaction_helpers = False
 
 from contextlib import contextmanager
+from ansible.module_utils.urls import fetch_file
 
 def_qf = "%{epoch}:%{name}-%{version}-%{release}.%{arch}"
 rpmbin = None
 
-# 64k.  Number of bytes to read at a time when manually downloading pkgs via a url
-BUFSIZE = 65536
-
 
 class YumModule(YumDnf):
     """
@@ -384,28 +382,6 @@ class YumModule(YumDnf):
         self.pkg_mgr_name = "yum"
         self.lockfile = '/var/run/yum.pid'
 
-    def fetch_rpm_from_url(self, spec):
-        # FIXME: Remove this once this PR is merged:
-        #   https://github.com/ansible/ansible/pull/19172
-
-        # download package so that we can query it
-        package_name, dummy = os.path.splitext(str(spec.rsplit('/', 1)[1]))
-        package_file = tempfile.NamedTemporaryFile(dir=self.module.tmpdir, prefix=package_name, suffix='.rpm', delete=False)
-        self.module.add_cleanup_file(package_file.name)
-        try:
-            rsp, info = fetch_url(self.module, spec)
-            if not rsp:
-                self.module.fail_json(msg="Failure downloading %s, %s" % (spec, info['msg']))
-            data = rsp.read(BUFSIZE)
-            while data:
-                package_file.write(data)
-                data = rsp.read(BUFSIZE)
-            package_file.close()
-        except Exception as e:
-            self.module.fail_json(msg="Failure downloading %s, %s" % (spec, to_native(e)))
-
-        return package_file.name
-
     def yum_base(self):
         my = yum.YumBase()
         my.preconf.debuglevel = 0
@@ -884,7 +860,7 @@ class YumModule(YumDnf):
 
                 if '://' in spec:
                     with self.set_env_proxy():
-                        package = self.fetch_rpm_from_url(spec)
+                        package = fetch_file(self.module, spec)
                 else:
                     package = spec
 
@@ -1205,7 +1181,7 @@ class YumModule(YumDnf):
                 elif '://' in spec:
                     # download package so that we can check if it's already installed
                     with self.set_env_proxy():
-                        package = self.fetch_rpm_from_url(spec)
+                        package = fetch_file(self.module, spec)
                     envra = self.local_envra(package)
 
                     if envra is None: