diff --git a/lib/ansible/utils/template.py b/lib/ansible/utils/template.py index b01010622f..db6e7fa754 100644 --- a/lib/ansible/utils/template.py +++ b/lib/ansible/utils/template.py @@ -33,15 +33,23 @@ import pwd _LISTRE = re.compile(r"(\w+)\[(\d+)\]") -def _varFindLimitSpace(vars, space, part, depth): - - # TODO: comments +def _varFindLimitSpace(basedir, vars, space, part, depth): + ''' limits the search space of space to part + + basically does space.get(part, None), but with + templating for part and a few more things + ''' + # Previous part couldn't be found, nothing to limit to if space is None: return space + # A part with escaped .s in it is compounded by { and }, remove them if part[0] == '{' and part[-1] == '}': part = part[1:-1] - part = varReplace(part, vars, depth=depth + 1) + # Template part to resolve variables within (${var$var2}) + part = varReplace(basedir, part, vars, depth=depth + 1) + + # Now find it if part in space: space = space[part] elif "[" in part: @@ -55,11 +63,30 @@ def _varFindLimitSpace(vars, space, part, depth): return None else: return None + return space -def _varFind(text, vars, depth=0): +def _varFind(basedir, text, vars, depth=0): + ''' Searches for a variable in text and finds its replacement in vars - # TODO: comments + The variables can have two formats; + - simple, $ followed by alphanumerics and/or underscores + - complex, ${ followed by alphanumerics, underscores, periods, braces and brackets, ended by a } + + Examples: + - $variable: simple variable that will have vars['variable'] as its replacement + - ${variable.complex}: complex variable that will have vars['variable']['complex'] as its replacement + - $variable.complex: simple variable, identical to the first, .complex ignored + + Complex variables are broken into parts by separating on periods, except if enclosed in {}. + ${variable.{fully.qualified.domain}} would be parsed as two parts, variable and fully.qualified.domain, + whereas ${variable.fully.qualified.domain} would be parsed as four parts. + + Returns a dict(replacement=, start=, + end=) + or None if no variable could be found in text. If replacement is None, it should be replaced with the + original data in the caller. + ''' start = text.find("$") if start == -1: @@ -78,13 +105,28 @@ def _varFind(text, vars, depth=0): var_start += 1 else: is_complex = False - brace_level = 0 + brace_level = 1 + # is_lookup is true for $FILE(...) and friends + is_lookup = False + lookup_plugin_name = None end = var_start - part_start = (var_start, brace_level) + # part_start is an index of where the current part started + part_start = var_start space = vars - while end < len(text) and ((is_complex and brace_level > 0) or not is_complex): + while end < len(text) and (((is_lookup or is_complex) and brace_level > 0) or (not is_complex and not is_lookup)): if text[end].isalnum() or text[end] == '_': pass + elif not is_complex and not is_lookup and text[end] == '(' and text[part_start:end].isupper(): + is_lookup = True + lookup_plugin_name = text[part_start:end] + part_start = end + 1 + elif is_lookup and text[end] == '(': + brace_level += 1 + elif is_lookup and text[end] == ')': + brace_level -= 1 + elif is_lookup: + # lookups are allowed arbitrary contents + pass elif is_complex and text[end] == '{': brace_level += 1 elif is_complex and text[end] == '}': @@ -92,23 +134,44 @@ def _varFind(text, vars, depth=0): elif is_complex and text[end] in ('$', '[', ']'): pass elif is_complex and text[end] == '.': - if brace_level == part_start[1]: - space = _varFindLimitSpace(vars, space, text[part_start[0]:end], depth) - part_start = (end + 1, brace_level) + if brace_level == 1: + space = _varFindLimitSpace(basedir, vars, space, text[part_start:end], depth) + part_start = end + 1 else: + # This breaks out of the loop on non-variable name characters break end += 1 var_end = end + # Handle "This has $ in it" + if var_end == part_start: + return {'replacement': None, 'start': start, 'end': end} + + # Handle lookup plugins + if is_lookup: + # When basedir is None, handle lookup plugins later + if basedir is None: + return {'replacement': None, 'start': start, 'end': end} + var_end -= 1 + from ansible import utils + args = text[part_start:var_end] + if lookup_plugin_name == 'LOOKUP': + lookup_plugin_name, args = args.split(",", 1) + args = args.strip() + instance = utils.plugins.lookup_loader.get(lookup_plugin_name.lower(), basedir=basedir) + if instance is not None: + replacement = instance.run(args, inject=vars) + else: + replacement = None + return {'replacement': replacement, 'start': start, 'end': end} + if is_complex: var_end -= 1 if text[var_end] != '}' or brace_level != 0: return None - if var_end == part_start[0]: - return None - space = _varFindLimitSpace(vars, space, text[part_start[0]:var_end], depth) + space = _varFindLimitSpace(basedir, vars, space, text[part_start:var_end], depth) return {'replacement': space, 'start': start, 'end': end} -def varReplace(raw, vars, depth=0, expand_lists=False): +def varReplace(basedir, raw, vars, depth=0, expand_lists=False): ''' Perform variable replacement of $variables in string raw using vars dictionary ''' # this code originally from yum @@ -118,7 +181,7 @@ def varReplace(raw, vars, depth=0, expand_lists=False): done = [] # Completed chunks to return while raw: - m = _varFind(raw, vars, depth) + m = _varFind(basedir, raw, vars, depth) if not m: done.append(raw) break @@ -130,7 +193,7 @@ def varReplace(raw, vars, depth=0, expand_lists=False): if expand_lists and isinstance(replacement, (list, tuple)): replacement = ",".join(replacement) if isinstance(replacement, (str, unicode)): - replacement = varReplace(replacement, vars, depth=depth+1, expand_lists=expand_lists) + replacement = varReplace(basedir, replacement, vars, depth=depth+1, expand_lists=expand_lists) if replacement is None: replacement = raw[m['start']:m['end']] @@ -141,53 +204,11 @@ def varReplace(raw, vars, depth=0, expand_lists=False): return ''.join(done) -_FILEPIPECRE = re.compile(r"\$(?P[A-Z]+)\(([^\)]*)\)") -def _varReplaceLookups(basedir, raw, vars): - from ansible import utils - done = [] # Completed chunks to return - - while raw: - m = _FILEPIPECRE.search(raw) - if not m: - done.append(raw) - break - - # Determine replacement value (if unknown lookup plugin then preserve - # original) - - replacement = m.group() - if m.group(1) == "FILE": - module_name = "file" - args = m.group(2) - elif m.group(1) == "PIPE": - module_name = "pipe" - args = m.group(2) - elif m.group(1) == "LOOKUP": - module_name, args = m.group(2).split(",", 1) - args = args.strip() - else: - module_name = m.group(1).lower() - args = m.group(2) - instance = utils.plugins.lookup_loader.get(module_name, basedir=basedir) - if instance is not None: - replacement = instance.run(args, inject=vars) - if not isinstance(replacement, basestring): - replacement = ",".join(replacement) - else: - replacement = m.group(0) - - start, end = m.span() - done.append(raw[:start]) # Keep stuff leading up to token - done.append(replacement.rstrip()) # Append replacement value - raw = raw[end:] # Continue with remainder of string - - return ''.join(done) - def template_ds(basedir, varname, vars): ''' templates a data structure by traversing it and substituting for other data structures ''' if isinstance(varname, basestring): - m = _varFind(varname, vars) + m = _varFind(basedir, varname, vars) if not m: return varname if m['start'] == 0 and m['end'] == len(varname): @@ -214,11 +235,21 @@ def template(basedir, text, vars, expand_lists=False): text = text.decode('utf-8') except UnicodeEncodeError: pass # already unicode - text = varReplace(unicode(text), vars, expand_lists=expand_lists) - if basedir is not None: - text = _varReplaceLookups(basedir, text, vars) + text = varReplace(basedir, unicode(text), vars, expand_lists=expand_lists) return text +class _jinja2_vars(object): + ''' helper class to template all variable content before jinja2 sees it ''' + def __init__(self, basedir, vars): + self.basedir = basedir + self.vars = vars + def __contains__(self, k): + return k in self.vars + def __getitem__(self, varname): + if varname not in self.vars: + raise KeyError("undefined variable: %s" % varname) + return template_ds(self.basedir, self.vars[varname], self.vars) + def template_from_file(basedir, path, vars): ''' run a file through the templating engine ''' @@ -257,7 +288,11 @@ def template_from_file(basedir, path, vars): vars['ansible_managed'] = time.strftime(managed_str, time.localtime(os.path.getmtime(realpath))) - res = t.render(vars) + # This line performs deep Jinja2 magic that uses the _jinja2_vars object for vars + # Ideally, this could use some API where setting shared=True and the object won't get + # passed through dict(o), but I have not found that yet. + res = jinja2.utils.concat(t.root_render_func(t.new_context(_jinja2_vars(basedir, vars), shared=True))) + if data.endswith('\n') and not res.endswith('\n'): res = res + '\n' return template(basedir, res, vars) diff --git a/test/TestPlayBook.py b/test/TestPlayBook.py index 55e318cb66..9ff679973d 100644 --- a/test/TestPlayBook.py +++ b/test/TestPlayBook.py @@ -202,7 +202,7 @@ class TestPlaybook(unittest.TestCase): assert utils.jsonify(expected, format=True) == utils.jsonify(actual,format=True) - assert len(EVENTS) == 44 + assert len(EVENTS) == 60 def test_includes(self): pb = os.path.join(self.test_dir, 'playbook-includer.yml') diff --git a/test/TestUtils.py b/test/TestUtils.py index b8a2bfa6ef..1cf6e61bf2 100644 --- a/test/TestUtils.py +++ b/test/TestUtils.py @@ -15,14 +15,14 @@ class TestUtils(unittest.TestCase): 'who': 'world', } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'hello world' def test_varReplace_trailing_dollar(self): template = '$what $who $' vars = dict(what='hello', who='world') - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'hello world $' def test_varReplace_multiple(self): @@ -32,7 +32,7 @@ class TestUtils(unittest.TestCase): 'who': 'world', } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'hello world' @@ -42,7 +42,7 @@ class TestUtils(unittest.TestCase): 'whoVar': 'world', } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) print res assert res == 'hello world' @@ -52,7 +52,7 @@ class TestUtils(unittest.TestCase): 'who': 'world', } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'hello world!' @@ -62,7 +62,7 @@ class TestUtils(unittest.TestCase): 'who': 'world', } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'hello world' @@ -72,7 +72,7 @@ class TestUtils(unittest.TestCase): 'who': 'world', } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'hello world}' @@ -82,7 +82,7 @@ class TestUtils(unittest.TestCase): 'who': 'world', } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == template @@ -92,7 +92,7 @@ class TestUtils(unittest.TestCase): 'who': 'world', } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'hello world }' @@ -104,7 +104,7 @@ class TestUtils(unittest.TestCase): }, } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) print res assert res == template @@ -117,7 +117,7 @@ class TestUtils(unittest.TestCase): }, } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'hello world' @@ -130,7 +130,7 @@ class TestUtils(unittest.TestCase): 'what': 'hello', } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'hello 2' @@ -140,7 +140,7 @@ class TestUtils(unittest.TestCase): 'who': u'wórld', } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == u'hello wórld' @@ -150,7 +150,7 @@ class TestUtils(unittest.TestCase): 'data': [ 'no-one', 'world' ] } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'hello world' @@ -160,7 +160,7 @@ class TestUtils(unittest.TestCase): 'data': [ 'no-one', 'world' ] } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == template @@ -170,7 +170,7 @@ class TestUtils(unittest.TestCase): 'data': [ 'no-one', 'world' ] } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == template @@ -180,7 +180,7 @@ class TestUtils(unittest.TestCase): 'data': { 'no-one': 0, 'world': 1 } } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == template @@ -190,7 +190,7 @@ class TestUtils(unittest.TestCase): 'data': [ 'no-one', {'msg': [ 'world'] } ] } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'hello world' @@ -201,7 +201,7 @@ class TestUtils(unittest.TestCase): } template = '${foo}${bar}' - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'foobar' def test_varReplace_escape_dot(self): @@ -214,7 +214,7 @@ class TestUtils(unittest.TestCase): } template = '${hostvars.{test.example.com}.foo}' - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'bar' def test_varReplace_list_join(self): @@ -227,7 +227,7 @@ class TestUtils(unittest.TestCase): } template = 'yum pkg=${list} state=installed' - res = ansible.utils.varReplace(template, vars, expand_lists=True) + res = ansible.utils.varReplace(None, template, vars, expand_lists=True) assert res == 'yum pkg=foo,bar,baz state=installed' def test_varReplace_escaped_var(self): @@ -235,7 +235,7 @@ class TestUtils(unittest.TestCase): 'foo': 'bar', } template = 'action \$foo' - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'action $foo' def test_varReplace_var_part(self): @@ -246,7 +246,7 @@ class TestUtils(unittest.TestCase): 'key': 'bar', } template = 'test ${foo.$key}' - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'test result' def test_varReplace_var_partial_part(self): @@ -257,7 +257,7 @@ class TestUtils(unittest.TestCase): 'key': 'bar', } template = 'test ${foo.${key}baz}' - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'test result' def test_template_varReplace_iterated(self): @@ -274,14 +274,14 @@ class TestUtils(unittest.TestCase): def test_varReplace_include(self): template = 'hello $FILE(world) $LOOKUP(file, world)' - res = ansible.utils.template("test", template, {}) + res = ansible.utils.template("test", template, {}, expand_lists=True) assert res == u'hello world world' def test_varReplace_include_script(self): template = 'hello $PIPE(echo world) $LOOKUP(pipe, echo world)' - res = ansible.utils.template("test", template, {}) + res = ansible.utils.template("test", template, {}, expand_lists=True) assert res == u'hello world world'