Module deprecation: docs, scheme and tests (#34100)

Enforce module deprecation.
After module has reached the end of it's deprecation cycle we will replace it with a docs stub.

* Replace deprecated modules with docs-only sub
* Use of deprecated past deprecation cycle gives meaningful message (see examples below)
* Enforce documentation.deprecation dict via `schema.py`
* Update `ansible-doc` and web docs to display documentation.deprecation
* Document that structure in `dev_guide`
* Ensure that all modules starting with `_` have a `deprecation:` block
* Ensure `deprecation:` block is only used on modules that start with `_`
* `removed_in` A string which represents when this module needs **deleting**
* CHANGELOG.md and porting_guide_2.5.rst list removed modules as well as alternatives
* CHANGELOG.md links to porting guide index

To ensure that meaningful messages are given to the user if they try to use a module at the end of it's deprecation cycle we enforce the module to contain:
```python
if __name__ == '__main__':
    removed_module()
```

test/sanity/validate-modules/ignore.txt

@@ -427,7 +427,7 @@ lib/ansible/modules/network/ordnance/ordnance_facts.py E322
 lib/ansible/modules/network/panos/panos_nat_rule.py E322
 lib/ansible/modules/network/panos/panos_sag.py E322
 lib/ansible/modules/network/panos/panos_sag.py E323
-lib/ansible/modules/network/panos/panos_security_policy.py E322
+lib/ansible/modules/network/panos/_panos_security_policy.py E322
 lib/ansible/modules/network/panos/panos_security_rule.py E322
 lib/ansible/modules/network/radware/vdirect_commit.py E321
 lib/ansible/modules/network/radware/vdirect_file.py E321

test/sanity/validate-modules/main.py

@@ -505,7 +505,14 @@ class ModuleValidator(Validator):
 
         return min(linenos)
 
-    def _find_main_call(self):
+    def _find_main_call(self, look_for="main"):
+        """ Ensure that the module ends with:
+            if __name__ == '__main__':
+                main()
+        OR, in the case of modules that are in the docs-only deprecation phase
+            if __name__ == '__main__':
+                removed_module()
+        """
         lineno = False
         if_bodies = []
         for child in self.ast.body:
@@ -547,13 +554,13 @@ class ModuleValidator(Validator):
             if isinstance(child, ast.Expr):
                 if isinstance(child.value, ast.Call):
                     if (isinstance(child.value.func, ast.Name) and
-                            child.value.func.id == 'main'):
+                            child.value.func.id == look_for):
                         lineno = child.lineno
                         if lineno < self.length - 1:
                             self.reporter.error(
                                 path=self.object_path,
                                 code=104,
-                                msg='Call to main() not the last line',
+                                msg=('Call to %s() not the last line' % look_for),
                                 line=lineno
                             )
 
@@ -561,7 +568,7 @@ class ModuleValidator(Validator):
             self.reporter.error(
                 path=self.object_path,
                 code=103,
-                msg='Did not find a call to main'
+                msg=('Did not find a call to %s()' % look_for)
             )
 
         return lineno or 0
@@ -1220,10 +1227,18 @@ class ModuleValidator(Validator):
                 )
             return
 
+        end_of_deprecation_should_be_docs_only = False
         if self._python_module():
             doc_info, docs = self._validate_docs()
 
-        if self._python_module() and not self._just_docs():
+            # See if current version => deprecated.removed_in, ie, should be docs only
+            if 'deprecated' in docs and docs['deprecated'] is not None:
+                removed_in = docs.get('deprecated')['removed_in']
+                strict_ansible_version = StrictVersion('.'.join(ansible_version.split('.')[:2]))
+                end_of_deprecation_should_be_docs_only = strict_ansible_version >= removed_in
+                # FIXME if +2 then file should be empty? - maybe add this only in the future
+
+        if self._python_module() and not self._just_docs() and not end_of_deprecation_should_be_docs_only:
             self._validate_argument_spec(docs)
             self._check_for_sys_exit()
             self._find_blacklist_imports()
@@ -1239,11 +1254,14 @@ class ModuleValidator(Validator):
             self._find_ps_docs_py_file()
 
         self._check_gpl3_header()
-        if not self._just_docs():
+        if not self._just_docs() and not end_of_deprecation_should_be_docs_only:
             self._check_interpreter(powershell=self._powershell_module())
             self._check_type_instead_of_isinstance(
                 powershell=self._powershell_module()
             )
+        if end_of_deprecation_should_be_docs_only:
+            # Ensure that `if __name__ == '__main__':` calls `removed_module()` which ensure that the module has no code in
+            main = self._find_main_call('removed_module')
 
 
 class PythonPackageValidator(Validator):

test/sanity/validate-modules/schema.py

@@ -83,23 +83,51 @@ def return_schema(data):
     )
 
 
+def deprecation_schema():
+
+    deprecation_schema_dict = {
+        # Only list branches that are deprecated or may have docs stubs in
+        # Deprecation cycle changed at 2.4 (though not retroactively)
+        # 2.3 -> removed_in: "2.5" + n for docs stub
+        # 2.4 -> removed_in: "2.8" + n for docs stub
+        Required('removed_in'): Any("2.2", "2.3", "2.4", "2.5", "2.8", "2.9"),
+        Required('why'): Any(*string_types),
+        Required('alternative'): Any(*string_types),
+        'removed': Any(True),
+    }
+    return Schema(
+        deprecation_schema_dict,
+        extra=PREVENT_EXTRA
+    )
+
+
 def doc_schema(module_name):
+    deprecated_module = False
+
     if module_name.startswith('_'):
         module_name = module_name[1:]
+        deprecated_module = True
+    doc_schema_dict = {
+        Required('module'): module_name,
+        Required('short_description'): Any(*string_types),
+        Required('description'): Any(list_string_types, *string_types),
+        Required('version_added'): Any(float, *string_types),
+        Required('author'): Any(None, list_string_types, *string_types),
+        'notes': Any(None, list_string_types),
+        'requirements': list_string_types,
+        'todo': Any(None, list_string_types, *string_types),
+        'options': Any(None, *list_dict_option_schema),
+        'extends_documentation_fragment': Any(list_string_types, *string_types)
+    }
+
+    if deprecated_module:
+        deprecation_required_scheme = {
+            Required('deprecated'): Any(deprecation_schema()),
+        }
+
+        doc_schema_dict.update(deprecation_required_scheme)
     return Schema(
-        {
-            Required('module'): module_name,
-            'deprecated': Any(*string_types),
-            Required('short_description'): Any(*string_types),
-            Required('description'): Any(list_string_types, *string_types),
-            Required('version_added'): Any(float, *string_types),
-            Required('author'): Any(None, list_string_types, *string_types),
-            'notes': Any(None, list_string_types),
-            'requirements': list_string_types,
-            'todo': Any(None, list_string_types, *string_types),
-            'options': Any(None, *list_dict_option_schema),
-            'extends_documentation_fragment': Any(list_string_types, *string_types)
-        },
+        doc_schema_dict,
         extra=PREVENT_EXTRA
     )
 

test/sanity/validate-modules/skip.txt

@@ -1,2 +1 @@
 lib/ansible/modules/utilities/logic/async_status.py
-lib/ansible/modules/utilities/helper/_accelerate.py