ansible-collections/community.general
1
0
Fork 0
mirror of https://github.com/ansible-collections/community.general.git synced 2025-04-24 11:21:25 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Fix for recursive copy slowness

Browse source 
Copy module was walking over files in subdirectories repeatedly (a
directory tree a few levels deep could bring the time spent into the
tens of minutes)

This was traced to the fix for this bug report: https://github.com/ansible/ansible/issues/13013

Fixed #13013 a different way and added an integration test to check for
regressions of #13013 as we optimize this code.

Fixes #21513
This commit is contained in:
Toshio Kuratomi 2017-04-05 17:26:11 -07:00 • committed by Brian Coca
parent ac61f09203
commit 78ced5318f
2 changed files with 46 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

22
lib/ansible/plugins/action/copy.py
View file

@ -113,7 +113,7 @@ class ActionModule(ActionBase):
sz = len(source.rsplit('/', 1)[0]) + 1
# Walk the directory and append the file tuples to source_files.
for base_path, sub_folders, files in os.walk(to_bytes(source)):
for base_path, sub_folders, files in os.walk(to_bytes(source), followlinks=True):
for file in files:
full_path = to_text(os.path.join(base_path, file), errors='surrogate_or_strict')
rel_path = full_path[sz:]
@ -121,10 +121,6 @@ class ActionModule(ActionBase):
rel_path = rel_path[1:]
source_files.append((full_path, rel_path))
# recurse into subdirs
for sf in sub_folders:
source_files += self._get_recursive_files(os.path.join(source, to_text(sf)), sz=sz)
# If it's recursive copy, destination is always a dir,
# explicitly mark it so (note - copy module relies on this).
if not self._connection._shell.path_has_trailing_slash(dest):
@ -326,22 +322,6 @@ class ActionModule(ActionBase):
return result
def _get_recursive_files(self, topdir, sz=0):
''' Recursively create file tuples for sub folders '''
r_files = []
for base_path, sub_folders, files in os.walk(to_bytes(topdir)):
for fname in files:
full_path = to_text(os.path.join(base_path, fname), errors='surrogate_or_strict')
rel_path = full_path[sz:]
if rel_path.startswith('/'):
rel_path = rel_path[1:]
r_files.append((full_path, rel_path))
for sf in sub_folders:
r_files += self._get_recursive_files(os.path.join(topdir, to_text(sf)), sz=sz)
return r_files
def _create_content_tempfile(self, content):
''' Create a tempfile containing defined content '''
fd, content_tempfile = tempfile.mkstemp()