Fix docs syntax highlighting errors (#50836)

* Add support for [WARNING]: ...

* Fix unreachable/failed output lexing.

* Detecting retry/--limit lines.

* Removing strange (invisible) characters which cause lexing problems.

* Using better-fitting lexers.

* Improve YAML lexing: don't accept quotes in keys.

* Add Django lexer (unchanged) from Pygments.

* Add support for != and % operators.

docs/docsite/_extensions/pygments_lexer.py

@@ -37,9 +37,11 @@
 from __future__ import absolute_import, print_function
 
 from pygments.lexer import LexerContext, ExtendedRegexLexer, DelegatingLexer, RegexLexer, bygroups, include
-from pygments.lexers import DjangoLexer, DiffLexer
+from pygments.lexers import DiffLexer
 from pygments import token
 
+import re
+
 
 class AnsibleYamlLexerContext(LexerContext):
     """Indentation context for the YAML lexer."""
@@ -255,7 +257,7 @@ class AnsibleYamlLexer(ExtendedRegexLexer):
             # whitespaces separating tokens
             (r'[ ]+', token.Text),
             # key with colon
-            (r'([^,:?\[\]{}\n]+)(:)(?=[ ]|$)',
+            (r'''([^,:?\[\]{}"'\n]+)(:)(?=[ ]|$)''',
              bygroups(token.Name.Tag, set_indent(token.Punctuation, implicit=True))),
             # tags, anchors and aliases,
             include('descriptors'),
@@ -334,7 +336,7 @@ class AnsibleYamlLexer(ExtendedRegexLexer):
         # a flow mapping indicated by '{' and '}'
         'flow-mapping': [
             # key with colon
-            (r'([^,:?\[\]{}\n]+)(:)(?=[ ]|$)',
+            (r'''([^,:?\[\]{}"'\n]+)(:)(?=[ ]|$)''',
              bygroups(token.Name.Tag, token.Punctuation)),
             # include flow collection rules
             include('flow-collection'),
@@ -458,6 +460,89 @@ class AnsibleYamlLexer(ExtendedRegexLexer):
         return super(AnsibleYamlLexer, self).get_tokens_unprocessed(text, context)
 
 
+class AnsibleDjangoLexer(RegexLexer):
+    """
+    Generic `django <http://www.djangoproject.com/documentation/templates/>`_
+    and `jinja <http://wsgiarea.pocoo.org/jinja/>`_ template lexer.
+
+    It just highlights django/jinja code between the preprocessor directives,
+    other data is left untouched by the lexer.
+    """
+
+    name = 'Django/Jinja'
+    aliases = ['django', 'jinja']
+    mimetypes = ['application/x-django-templating', 'application/x-jinja']
+
+    flags = re.M | re.S
+
+    tokens = {
+        'root': [
+            (r'[^{]+', token.Other),
+            (r'\{\{', token.Comment.Preproc, 'var'),
+            # jinja/django comments
+            (r'\{[*#].*?[*#]\}', token.Comment),
+            # django comments
+            (r'(\{%)(-?\s*)(comment)(\s*-?)(%\})(.*?)'
+             r'(\{%)(-?\s*)(endcomment)(\s*-?)(%\})',
+             bygroups(token.Comment.Preproc, token.Text, token.Keyword, token.Text, token.Comment.Preproc,
+                      token.Comment, token.Comment.Preproc, token.Text, token.Keyword, token.Text,
+                      token.Comment.Preproc)),
+            # raw jinja blocks
+            (r'(\{%)(-?\s*)(raw)(\s*-?)(%\})(.*?)'
+             r'(\{%)(-?\s*)(endraw)(\s*-?)(%\})',
+             bygroups(token.Comment.Preproc, token.Text, token.Keyword, token.Text, token.Comment.Preproc,
+                      token.Text, token.Comment.Preproc, token.Text, token.Keyword, token.Text,
+                      token.Comment.Preproc)),
+            # filter blocks
+            (r'(\{%)(-?\s*)(filter)(\s+)([a-zA-Z_]\w*)',
+             bygroups(token.Comment.Preproc, token.Text, token.Keyword, token.Text, token.Name.Function),
+             'block'),
+            (r'(\{%)(-?\s*)([a-zA-Z_]\w*)',
+             bygroups(token.Comment.Preproc, token.Text, token.Keyword), 'block'),
+            (r'\{', token.Other)
+        ],
+        'varnames': [
+            (r'(\|)(\s*)([a-zA-Z_]\w*)',
+             bygroups(token.Operator, token.Text, token.Name.Function)),
+            (r'(is)(\s+)(not)?(\s+)?([a-zA-Z_]\w*)',
+             bygroups(token.Keyword, token.Text, token.Keyword, token.Text, token.Name.Function)),
+            (r'(_|true|false|none|True|False|None)\b', token.Keyword.Pseudo),
+            (r'(in|as|reversed|recursive|not|and|or|is|if|else|import|'
+             r'with(?:(?:out)?\s*context)?|scoped|ignore\s+missing)\b',
+             token.Keyword),
+            (r'(loop|block|super|forloop)\b', token.Name.Builtin),
+            (r'[a-zA-Z_][\w-]*', token.Name.Variable),
+            (r'\.\w+', token.Name.Variable),
+            (r':?"(\\\\|\\"|[^"])*"', token.String.Double),
+            (r":?'(\\\\|\\'|[^'])*'", token.String.Single),
+            (r'([{}()\[\]+\-*/%,:~]|[><=]=?|!=)', token.Operator),
+            (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|"
+             r"0[xX][0-9a-fA-F]+[Ll]?", token.Number),
+        ],
+        'var': [
+            (r'\s+', token.Text),
+            (r'(-?)(\}\})', bygroups(token.Text, token.Comment.Preproc), '#pop'),
+            include('varnames')
+        ],
+        'block': [
+            (r'\s+', token.Text),
+            (r'(-?)(%\})', bygroups(token.Text, token.Comment.Preproc), '#pop'),
+            include('varnames'),
+            (r'.', token.Punctuation)
+        ]
+    }
+
+    def analyse_text(text):
+        rv = 0.0
+        if re.search(r'\{%\s*(block|extends)', text) is not None:
+            rv += 0.4
+        if re.search(r'\{%\s*if\s*.*?%\}', text) is not None:
+            rv += 0.1
+        if re.search(r'\{\{.*?\}\}', text) is not None:
+            rv += 0.1
+        return rv
+
+
 class AnsibleYamlJinjaLexer(DelegatingLexer):
     """
     Subclass of the `DjangoLexer` that highlights unlexed data with the
@@ -474,7 +559,7 @@ class AnsibleYamlJinjaLexer(DelegatingLexer):
     mimetypes = ['text/x-yaml+jinja']
 
     def __init__(self, **options):
-        super(AnsibleYamlJinjaLexer, self).__init__(AnsibleYamlLexer, DjangoLexer, **options)
+        super(AnsibleYamlJinjaLexer, self).__init__(AnsibleYamlLexer, AnsibleDjangoLexer, **options)
 
 
 class AnsibleOutputPrimaryLexer(RegexLexer):
@@ -555,8 +640,8 @@ class AnsibleOutputPrimaryLexer(RegexLexer):
         ],
 
         'host-error': [
-            (r'(?:( )(UNREACHABLE|FAILED)(!))?',
-                bygroups(token.Text, token.Keyword, token.Punctuation),
+            (r'(?:(:)( )(UNREACHABLE|FAILED)(!))?',
+                bygroups(token.Punctuation, token.Text, token.Keyword, token.Punctuation),
                 'host-postfix'),
             (r'', token.Text, 'host-postfix'),
         ],
@@ -579,9 +664,12 @@ class AnsibleOutputPrimaryLexer(RegexLexer):
             (r'(fatal|ok|changed|skipping)(:)( )',
                 bygroups(token.Keyword, token.Punctuation, token.Text),
                 'host-name'),
+            (r'(\[)(WARNING)(\]:)([^\n]+)',
+                bygroups(token.Punctuation, token.Keyword, token.Punctuation, token.Text)),
             (r'([^ ]+)( +)(:)',
                 bygroups(token.Name, token.Text, token.Punctuation),
                 'host-result'),
+            (r'(\tto retry, use: )(.*)(\n)', bygroups(token.Text, token.Literal.String, token.Text)),
             (r'.*\n', token.Other),
         ],
     }
@@ -609,7 +697,12 @@ def setup(app):
     """ Initializer for Sphinx extension API.
         See http://www.sphinx-doc.org/en/stable/extdev/index.html#dev-extensions.
     """
-    for lexer in [AnsibleYamlLexer(startinline=True), AnsibleYamlJinjaLexer(startinline=True), AnsibleOutputLexer(startinline=True)]:
+    for lexer in [
+        AnsibleDjangoLexer(startinline=True),
+        AnsibleYamlLexer(startinline=True),
+        AnsibleYamlJinjaLexer(startinline=True),
+        AnsibleOutputLexer(startinline=True)
+    ]:
         app.add_lexer(lexer.name, lexer)
         for alias in lexer.aliases:
             app.add_lexer(alias, lexer)