Allow specifying the output encoding in the template module (#42171)

Allow specifying the source and destination files' encodings in the template module

* Added output_encoding to the template module, default to utf-8
* Added documentation for the new variables
* Leveraged the encoding argument on to_text() and to_bytes() to keep the implementation as simple as possible
* Added integration tests with files in utf-8 and windows-1252 encodings, testing all combinations
* fix bad smell test by excluding windows-1252 files from the utf8 checks
* fix bad smell test by excluding valid files from the smart quote test
This commit is contained in:
Julien Champseix 2018-07-25 22:10:40 +02:00 committed by Toshio Kuratomi
commit 19dc267e4c
8 changed files with 44 additions and 1 deletions

View file

@ -58,6 +58,7 @@ class ActionModule(ActionBase):
block_end_string = self._task.args.get('block_end_string', None)
trim_blocks = boolean(self._task.args.get('trim_blocks', True), strict=False)
lstrip_blocks = boolean(self._task.args.get('lstrip_blocks', False), strict=False)
output_encoding = self._task.args.get('output_encoding', 'utf-8') or 'utf-8'
# Option `lstrip_blocks' was added in Jinja2 version 2.7.
if lstrip_blocks:
@ -176,13 +177,14 @@ class ActionModule(ActionBase):
new_task.args.pop('variable_end_string', None)
new_task.args.pop('trim_blocks', None)
new_task.args.pop('lstrip_blocks', None)
new_task.args.pop('output_encoding', None)
local_tempdir = tempfile.mkdtemp(dir=C.DEFAULT_LOCAL_TMP)
try:
result_file = os.path.join(local_tempdir, os.path.basename(source))
with open(to_bytes(result_file, errors='surrogate_or_strict'), 'wb') as f:
f.write(to_bytes(resultant, errors='surrogate_or_strict'))
f.write(to_bytes(resultant, encoding=output_encoding, errors='surrogate_or_strict'))
new_task.args.update(
dict(