From b3b356480da93d9266a9a846c364b2a74f4d0085 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Mon, 27 Oct 2014 15:52:56 -0700 Subject: [PATCH 01/21] added the ability to keep aliased and deprecated modules prefixed with '_', they will be loaded after non prefixed modules are checked they can be full modules or symlinks to existing ones (alias) also updated ansible doc to ignore these, will eventually add selective display --- bin/ansible-doc | 4 +++- lib/ansible/utils/plugins.py | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/ansible-doc b/bin/ansible-doc index d5143f33a1..8a7faadb24 100755 --- a/bin/ansible-doc +++ b/bin/ansible-doc @@ -225,11 +225,13 @@ def main(): # list all modules paths = utils.plugins.module_finder._get_paths() module_list = [] + deprecated_list = [] + module_aliases = {} for path in paths: # os.system("ls -C %s" % (path)) if os.path.isdir(path): for module in os.listdir(path): - if any(module.endswith(x) for x in BLACKLIST_EXTS): + if module.startswith('_') or any(module.endswith(x) for x in BLACKLIST_EXTS): continue module_list.append(module) diff --git a/lib/ansible/utils/plugins.py b/lib/ansible/utils/plugins.py index faf5b5f26f..0d050fd13d 100644 --- a/lib/ansible/utils/plugins.py +++ b/lib/ansible/utils/plugins.py @@ -178,6 +178,9 @@ class PluginLoader(object): self._plugin_path_cache[full_name] = path return path + if not name.startswith('_'): + return self.find_plugin('_' + name, suffixes, transport) + return None def has_plugin(self, name): From 5ab4467708d95777abbac8b9e74f99965da2f4aa Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Tue, 28 Oct 2014 08:36:31 -0700 Subject: [PATCH 02/21] module formatter skips modules with leading underscore to avoid documenting them. Soon will be patched to recognize them as either deprecated or an alias --- hacking/module_formatter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hacking/module_formatter.py b/hacking/module_formatter.py index f7d8570e93..53c2616533 100755 --- a/hacking/module_formatter.py +++ b/hacking/module_formatter.py @@ -128,6 +128,9 @@ def list_modules(module_dir): files2 = glob.glob("%s/*" % d) for f in files2: + if os.path.basename(f).startswith("_"): # skip deprecated/aliases for now + continue + if not f.endswith(".py") or f.endswith('__init__.py'): # windows powershell modules have documentation stubs in python docstring # format (they are not executed) so skip the ps1 format files From 617b6323e23128c938a28c01f55ab254ffdd183d Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Tue, 28 Oct 2014 08:43:18 -0700 Subject: [PATCH 03/21] added info about new deprecated/alias plugin loading --- docsite/rst/developing_modules.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docsite/rst/developing_modules.rst b/docsite/rst/developing_modules.rst index 608ac7185b..355f402835 100644 --- a/docsite/rst/developing_modules.rst +++ b/docsite/rst/developing_modules.rst @@ -465,6 +465,23 @@ a github pull request to the `extras Date: Wed, 29 Oct 2014 10:32:17 -0400 Subject: [PATCH 04/21] fixed typo --- docsite/rst/developing_modules.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docsite/rst/developing_modules.rst b/docsite/rst/developing_modules.rst index 355f402835..4a331626db 100644 --- a/docsite/rst/developing_modules.rst +++ b/docsite/rst/developing_modules.rst @@ -466,7 +466,7 @@ Included modules will ship with ansible, and also have a change to be promoted t gives them slightly higher development priority (though they'll work in exactly the same way). -Deprecating and makingm module aliases +Deprecating and making module aliases `````````````````````````````````````` Starting in 1.8 you can deprecate modules by renaming them with a preceeding _, i.e. old_cloud.py to From 1b70ef6cbaa23fa2399204689a489d39be7a76fb Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Wed, 29 Oct 2014 22:33:31 -0400 Subject: [PATCH 05/21] Several changes to ansible-doc added display of deprecated to ansible-doc now it does better job of using tty columns fixed indication truncation of desc with trailing ... removed extension from module list, also fixed matching exlusion blacklist --- bin/ansible-doc | 45 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/bin/ansible-doc b/bin/ansible-doc index 8a7faadb24..3c4f84964a 100755 --- a/bin/ansible-doc +++ b/bin/ansible-doc @@ -164,7 +164,11 @@ def get_snippet_text(doc): return "\n".join(text) def get_module_list_text(module_list): + columns = max(60, int(os.popen('stty size', 'r').read().split()[1])) + displace = max(len(x) for x in module_list) + linelimit = columns - displace - 5 text = [] + deprecated = [] for module in sorted(set(module_list)): if module in module_docs.BLACKLIST_MODULES: @@ -181,13 +185,22 @@ def get_module_list_text(module_list): try: doc, plainexamples = module_docs.get_docstring(filename) - desc = tty_ify(doc.get('short_description', '?')) - if len(desc) > 55: - desc = desc + '...' - text.append("%-20s %-60.60s" % (module, desc)) + desc = tty_ify(doc.get('short_description', '?')).strip() + if len(desc) > linelimit: + desc = desc[:linelimit] + '...' + + if module.startswith('_'): # Handle replecated + module = module[1:] + deprecated.append("%-*s %-*.*s" % (displace, module, linelimit, len(desc), desc)) + else: + text.append("%-*s %-*.*s" % (displace, module, linelimit, len(desc), desc)) except: traceback.print_exc() sys.stderr.write("ERROR: module %s has a documentation error formatting or is missing documentation\n" % module) + + if len(deprecated) > 0: + text.append("\nDEPRECATED:") + text.extend(deprecated) return "\n".join(text) def main(): @@ -208,6 +221,11 @@ def main(): default=False, dest='list_dir', help='List available modules') + p.add_option("-c", "--list-columns", + action="store_true", + default=False, + dest='list_columns', + help='List modules in columns') p.add_option("-s", "--snippet", action="store_true", default=False, @@ -221,20 +239,25 @@ def main(): for i in options.module_path.split(os.pathsep): utils.plugins.module_finder.add_directory(i) - if options.list_dir: - # list all modules + if options.list_dir or options.list_deprecated: + # list modules paths = utils.plugins.module_finder._get_paths() module_list = [] - deprecated_list = [] - module_aliases = {} for path in paths: - # os.system("ls -C %s" % (path)) if os.path.isdir(path): for module in os.listdir(path): - if module.startswith('_') or any(module.endswith(x) for x in BLACKLIST_EXTS): + if any(module.endswith(x) for x in BLACKLIST_EXTS): continue - module_list.append(module) + elif module.startswith('__'): + continue + elif module.startswith('_'): + fullpath = '/'.join([path,module]) + if os.path.islink(fullpath): # avoids aliases + continue + module = os.path.splitext(module)[0] # removes the extension + module_list.append(module) + pager(get_module_list_text(module_list)) sys.exit() From e41bcc41d335996d7ff73eb84d8376f19372c297 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Thu, 30 Oct 2014 08:56:41 -0400 Subject: [PATCH 06/21] removed 'column display' options as there is no code handling this function --- bin/ansible-doc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/bin/ansible-doc b/bin/ansible-doc index 3c4f84964a..d399e4668e 100755 --- a/bin/ansible-doc +++ b/bin/ansible-doc @@ -221,11 +221,6 @@ def main(): default=False, dest='list_dir', help='List available modules') - p.add_option("-c", "--list-columns", - action="store_true", - default=False, - dest='list_columns', - help='List modules in columns') p.add_option("-s", "--snippet", action="store_true", default=False, From 27d741102c4009b89938fe32d8ec50b44b3c8a03 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Thu, 30 Oct 2014 11:26:43 -0400 Subject: [PATCH 07/21] Created Deprecated module category that only appears when there is something to show --- hacking/module_formatter.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/hacking/module_formatter.py b/hacking/module_formatter.py index 53c2616533..345c84ca04 100755 --- a/hacking/module_formatter.py +++ b/hacking/module_formatter.py @@ -121,28 +121,33 @@ def write_data(text, options, outputname, module): def list_modules(module_dir): ''' returns a hash of categories, each category being a hash of module names to file paths ''' - categories = dict(all=dict()) + categories = dict(all=dict(),deprecated=dict()) files = glob.glob("%s/*/*" % module_dir) for d in files: if os.path.isdir(d): files2 = glob.glob("%s/*" % d) for f in files2: - if os.path.basename(f).startswith("_"): # skip deprecated/aliases for now - continue + module = os.path.splitext(os.path.basename(f))[0] + category = os.path.dirname(f).split("/")[-1] if not f.endswith(".py") or f.endswith('__init__.py'): # windows powershell modules have documentation stubs in python docstring # format (they are not executed) so skip the ps1 format files continue + elif module.startswith("_"): # Handle deprecated modules + if not os.path.islink(f): # ignores aliases + categories['deprecated'][module] = f + continue + elif module in categories['deprecated']: # Removes dupes + categories['deprecated'].pop(module, None) - tokens = f.split("/") - module = tokens[-1].replace(".py","") - category = tokens[-2] if not category in categories: categories[category] = {} categories[category][module] = f categories['all'][module] = f + if not len(categories['deprecated']) > 0: + categories.pop('deprecated', None) return categories ##################################################################################### From 8b5b97d0667186b6adb4e8ba76c62dc9fa01b85f Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Thu, 30 Oct 2014 13:29:54 -0400 Subject: [PATCH 08/21] now docs handle deprecated modules but still ignore aliases --- hacking/module_formatter.py | 12 ++++++++++-- hacking/templates/rst.j2 | 7 +++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/hacking/module_formatter.py b/hacking/module_formatter.py index 345c84ca04..61de1ea136 100755 --- a/hacking/module_formatter.py +++ b/hacking/module_formatter.py @@ -202,17 +202,23 @@ def process_module(module, options, env, template, outputname, module_map): fname = module_map[module] + basename = os.path.basename(fname) + deprecated = False # ignore files with extensions - if not os.path.basename(fname).endswith(".py"): + if not basename.endswith(".py"): return + elif basename.startswith("_"): + if os.path.islink(fname): # alias + return + deprecated = True # use ansible core library to parse out doc metadata YAML and plaintext examples doc, examples = ansible.utils.module_docs.get_docstring(fname, verbose=options.verbose) # crash if module is missing documentation and not explicitly hidden from docs index if doc is None and module not in ansible.utils.module_docs.BLACKLIST_MODULES: - sys.stderr.write("*** ERROR: CORE MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module)) + sys.stderr.write("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module)) sys.exit(1) if doc is None: @@ -254,6 +260,8 @@ def process_module(module, options, env, template, outputname, module_map): doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d') doc['ansible_version'] = options.ansible_version doc['plainexamples'] = examples #plain text + if deprecated and 'deprecated' not in doc: + doc['deprecated'] = "This module is deprecated, as such it's use is discouraged." # here is where we build the table of contents... diff --git a/hacking/templates/rst.j2 b/hacking/templates/rst.j2 index fbedae566a..8d6dc1c89b 100644 --- a/hacking/templates/rst.j2 +++ b/hacking/templates/rst.j2 @@ -21,6 +21,13 @@ # --------------------------------------------#} +{% if deprecated is defined -%} +DEPRECATED +---------- + +@{ deprecated }@ +{% endif %} + Synopsis -------- From 0fb0548d0b04cf2a1d9b6755697b7dca45d2dbf8 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Thu, 30 Oct 2014 13:40:34 -0400 Subject: [PATCH 09/21] removed no unused var that was not cleaned up properlly --- bin/ansible-doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ansible-doc b/bin/ansible-doc index d399e4668e..e4c7d19522 100755 --- a/bin/ansible-doc +++ b/bin/ansible-doc @@ -234,7 +234,7 @@ def main(): for i in options.module_path.split(os.pathsep): utils.plugins.module_finder.add_directory(i) - if options.list_dir or options.list_deprecated: + if options.list_dir: # list modules paths = utils.plugins.module_finder._get_paths() module_list = [] From 94a732fb1a122e6018713b0b7cc3f359e62a88c1 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Thu, 30 Oct 2014 13:50:34 -0400 Subject: [PATCH 10/21] fixed typo in comments --- bin/ansible-doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ansible-doc b/bin/ansible-doc index e4c7d19522..5a708a421c 100755 --- a/bin/ansible-doc +++ b/bin/ansible-doc @@ -189,7 +189,7 @@ def get_module_list_text(module_list): if len(desc) > linelimit: desc = desc[:linelimit] + '...' - if module.startswith('_'): # Handle replecated + if module.startswith('_'): # Handle deprecated module = module[1:] deprecated.append("%-*s %-*.*s" % (displace, module, linelimit, len(desc), desc)) else: From 0317e7b91029ce90079802f125155c1944cf279c Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Thu, 30 Oct 2014 13:53:05 -0400 Subject: [PATCH 11/21] avoid modifying module var by just passing the substring to the append --- bin/ansible-doc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/ansible-doc b/bin/ansible-doc index 5a708a421c..aed7d4d23c 100755 --- a/bin/ansible-doc +++ b/bin/ansible-doc @@ -190,8 +190,7 @@ def get_module_list_text(module_list): desc = desc[:linelimit] + '...' if module.startswith('_'): # Handle deprecated - module = module[1:] - deprecated.append("%-*s %-*.*s" % (displace, module, linelimit, len(desc), desc)) + deprecated.append("%-*s %-*.*s" % (displace, module[1:], linelimit, len(desc), desc)) else: text.append("%-*s %-*.*s" % (displace, module, linelimit, len(desc), desc)) except: From 86de59235f0f18f397bab1637167fdb278803931 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Fri, 31 Oct 2014 14:18:18 -0400 Subject: [PATCH 12/21] bypass core/extras text when module is deprecated --- hacking/templates/rst.j2 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hacking/templates/rst.j2 b/hacking/templates/rst.j2 index 8d6dc1c89b..1d55a0452b 100644 --- a/hacking/templates/rst.j2 +++ b/hacking/templates/rst.j2 @@ -109,7 +109,8 @@ Examples {% endif %} -{% if core %} +{% if not deprecated %} + {% if core %} This is a Core Module --------------------- @@ -124,7 +125,7 @@ Documentation updates for this module can also be edited directly by submitting This is a "core" ansible module, which means it will receive slightly higher priority for all requests than those in the "extras" repos. -{% else %} + {% else %} This is an Extras Module ------------------------ @@ -140,6 +141,7 @@ Documentation updates for this module can also be edited directly by submitting Note that this module is designated a "extras" module. Non-core modules are still fully usable, but may receive slightly lower response rates for issues and pull requests. Popular "extras" modules may be promoted to core modules over time. + {% endif %} {% endif %} For help in developing on modules, should you be so inclined, please read :doc:`community`, :doc:`developing_test_pr` and :doc:`developing_modules`. From 44f0279d0a2440172a97353ec47dc17eebbee98a Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Fri, 31 Oct 2014 14:20:26 -0400 Subject: [PATCH 13/21] Now adds flags for non core and deprecated modules in listing --- hacking/module_formatter.py | 79 +++++++++++++++++++++++-------------- 1 file changed, 50 insertions(+), 29 deletions(-) diff --git a/hacking/module_formatter.py b/hacking/module_formatter.py index 61de1ea136..6392c83ac6 100755 --- a/hacking/module_formatter.py +++ b/hacking/module_formatter.py @@ -59,6 +59,8 @@ _MODULE = re.compile(r"M\(([^)]+)\)") _URL = re.compile(r"U\(([^)]+)\)") _CONST = re.compile(r"C\(([^)]+)\)") +DEPRECATED = " (D)" +NOTCORE = " (E)" ##################################################################################### def rst_ify(text): @@ -121,7 +123,7 @@ def write_data(text, options, outputname, module): def list_modules(module_dir): ''' returns a hash of categories, each category being a hash of module names to file paths ''' - categories = dict(all=dict(),deprecated=dict()) + categories = dict(all=dict()) files = glob.glob("%s/*/*" % module_dir) for d in files: if os.path.isdir(d): @@ -135,19 +137,14 @@ def list_modules(module_dir): # windows powershell modules have documentation stubs in python docstring # format (they are not executed) so skip the ps1 format files continue - elif module.startswith("_"): # Handle deprecated modules - if not os.path.islink(f): # ignores aliases - categories['deprecated'][module] = f + elif module.startswith("_") and os.path.islink(f): # ignores aliases continue - elif module in categories['deprecated']: # Removes dupes - categories['deprecated'].pop(module, None) if not category in categories: categories[category] = {} categories[category][module] = f categories['all'][module] = f - if not len(categories['deprecated']) > 0: - categories.pop('deprecated', None) + return categories ##################################################################################### @@ -198,9 +195,6 @@ def jinja2_environment(template_dir, typ): def process_module(module, options, env, template, outputname, module_map): - print "rendering: %s" % module - - fname = module_map[module] basename = os.path.basename(fname) deprecated = False @@ -208,21 +202,28 @@ def process_module(module, options, env, template, outputname, module_map): # ignore files with extensions if not basename.endswith(".py"): return - elif basename.startswith("_"): - if os.path.islink(fname): # alias - return + elif module.startswith("_"): + if os.path.islink(fname): + return # ignore, its an alias deprecated = True + module = module.replace("_","",1) + + print "rendering: %s" % module # use ansible core library to parse out doc metadata YAML and plaintext examples doc, examples = ansible.utils.module_docs.get_docstring(fname, verbose=options.verbose) # crash if module is missing documentation and not explicitly hidden from docs index - if doc is None and module not in ansible.utils.module_docs.BLACKLIST_MODULES: - sys.stderr.write("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module)) - sys.exit(1) - if doc is None: - return "SKIPPED" + if module in ansible.utils.module_docs.BLACKLIST_MODULES: + return "SKIPPED" + else: + sys.stderr.write("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module)) + sys.exit(1) + + if deprecated and 'deprecated' not in doc: + sys.stderr.write("*** ERROR: DEPRECATED MODULE MISSING 'deprecated' DOCUMENTATION: %s, %s ***\n" % (fname, module)) + sys.exit(1) if "/core/" in fname: doc['core'] = True @@ -252,21 +253,21 @@ def process_module(module, options, env, template, outputname, module_map): for (k,v) in doc['options'].iteritems(): all_keys.append(k) - all_keys = sorted(all_keys) - doc['option_keys'] = all_keys + all_keys = sorted(all_keys) + + doc['option_keys'] = all_keys doc['filename'] = fname doc['docuri'] = doc['module'].replace('_', '-') doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d') doc['ansible_version'] = options.ansible_version doc['plainexamples'] = examples #plain text - if deprecated and 'deprecated' not in doc: - doc['deprecated'] = "This module is deprecated, as such it's use is discouraged." # here is where we build the table of contents... text = template.render(doc) write_data(text, options, outputname, module) + return doc['short_description'] ##################################################################################### @@ -283,7 +284,19 @@ def process_category(category, categories, options, env, template, outputname): category = category.replace("_"," ") category = category.title() - modules = module_map.keys() + modules = [] + deprecated = [] + core = [] + for module in module_map.keys(): + + if module.startswith("_"): + module = module.replace("_","",1) + deprecated.append(module) + elif '/core/' in module_map[module]: + core.append(module) + + modules.append(module) + modules.sort() category_header = "%s Modules" % (category.title()) @@ -293,16 +306,24 @@ def process_category(category, categories, options, env, template, outputname): %s %s -.. toctree:: - :maxdepth: 1 +.. toctree:: :maxdepth: 1 """ % (category_header, underscores)) for module in modules: - result = process_module(module, options, env, template, outputname, module_map) - if result != "SKIPPED": - category_file.write(" %s_module\n" % module) + modstring = module + modname = module + if module in deprecated: + modstring = modstring + DEPRECATED + modname = "_" + module + elif module not in core: + modstring = modstring + NOTCORE + + result = process_module(modname, options, env, template, outputname, module_map) + + if result != "SKIPPED": + category_file.write(" %s - %s <%s_module>\n" % (modstring, result, module)) category_file.close() From 023f5fd7e0c959fe09d26c49c534e966f3e82fb5 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Fri, 31 Oct 2014 15:06:00 -0400 Subject: [PATCH 14/21] Added note explaning the module tagging --- hacking/module_formatter.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hacking/module_formatter.py b/hacking/module_formatter.py index 6392c83ac6..51bea3e135 100755 --- a/hacking/module_formatter.py +++ b/hacking/module_formatter.py @@ -325,6 +325,11 @@ def process_category(category, categories, options, env, template, outputname): if result != "SKIPPED": category_file.write(" %s - %s <%s_module>\n" % (modstring, result, module)) + category_file.write("""\n\n +.. note:: + - %s: Denotes that this module is not part of core, it can be found in the extras repo + - %s: This marks a module as deprecated, kept for backwards compatibility but use is discouraged +""" % (DEPRECATED, NOTCORE)) category_file.close() # TODO: end a new category file From f6d9aa7a8ffcd97bb4cdd22871735a694ea7024a Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Fri, 31 Oct 2014 16:05:22 -0400 Subject: [PATCH 15/21] corrected text/flag --- hacking/module_formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hacking/module_formatter.py b/hacking/module_formatter.py index 51bea3e135..1218b85e71 100755 --- a/hacking/module_formatter.py +++ b/hacking/module_formatter.py @@ -327,8 +327,8 @@ def process_category(category, categories, options, env, template, outputname): category_file.write("""\n\n .. note:: - - %s: Denotes that this module is not part of core, it can be found in the extras repo - %s: This marks a module as deprecated, kept for backwards compatibility but use is discouraged + - %s: Denotes that this module is not part of core, it can be found in the extras or some other external repo """ % (DEPRECATED, NOTCORE)) category_file.close() From 7a5e7db2df04c6c673b9d715b052503e49cdb6cf Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Sat, 1 Nov 2014 01:17:42 -0400 Subject: [PATCH 16/21] ansible doc now finds modules recursively more intelligent about ignoring files that are clearly not modules --- bin/ansible-doc | 46 ++++++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/bin/ansible-doc b/bin/ansible-doc index aed7d4d23c..0ba84b9a30 100755 --- a/bin/ansible-doc +++ b/bin/ansible-doc @@ -34,6 +34,7 @@ import traceback MODULEDIR = C.DEFAULT_MODULE_PATH BLACKLIST_EXTS = ('.pyc', '.swp', '.bak', '~', '.rpm') +IGNORE_FILES = [ "COPYING", "CONTRIBUTING", "LICENSE", "README" ] _ITALIC = re.compile(r"I\(([^)]+)\)") _BOLD = re.compile(r"B\(([^)]+)\)") @@ -94,7 +95,7 @@ def get_man_text(doc): desc = " ".join(doc['description']) text.append("%s\n" % textwrap.fill(tty_ify(desc), initial_indent=" ", subsequent_indent=" ")) - + if 'option_keys' in doc and len(doc['option_keys']) > 0: text.append("Options (= is mandatory):\n") @@ -202,6 +203,28 @@ def get_module_list_text(module_list): text.extend(deprecated) return "\n".join(text) +def find_modules(path, module_list): + + if os.path.isdir(path): + for module in os.listdir(path): + if module.startswith('.'): + continue + elif os.path.isdir(module): + find_modules(module, module_list) + elif any(module.endswith(x) for x in BLACKLIST_EXTS): + continue + elif module.startswith('__'): + continue + elif module in IGNORE_FILES: + continue + elif module.startswith('_'): + fullpath = '/'.join([path,module]) + if os.path.islink(fullpath): # avoids aliases + continue + + module = os.path.splitext(module)[0] # removes the extension + module_list.append(module) + def main(): p = optparse.OptionParser( @@ -238,26 +261,14 @@ def main(): paths = utils.plugins.module_finder._get_paths() module_list = [] for path in paths: - if os.path.isdir(path): - for module in os.listdir(path): - if any(module.endswith(x) for x in BLACKLIST_EXTS): - continue - elif module.startswith('__'): - continue - elif module.startswith('_'): - fullpath = '/'.join([path,module]) - if os.path.islink(fullpath): # avoids aliases - continue + find_modules(path, module_list) - module = os.path.splitext(module)[0] # removes the extension - module_list.append(module) - pager(get_module_list_text(module_list)) sys.exit() if len(args) == 0: p.print_help() - + def print_paths(finder): ''' Returns a string suitable for printing of the search path ''' @@ -267,14 +278,13 @@ def main(): if i not in ret: ret.append(i) return os.pathsep.join(ret) - + text = '' for module in args: filename = utils.plugins.module_finder.find_plugin(module) if filename is None: - sys.stderr.write("module %s not found in %s\n" % (module, - print_paths(utils.plugins.module_finder))) + sys.stderr.write("module %s not found in %s\n" % (module, print_paths(utils.plugins.module_finder))) continue if any(filename.endswith(x) for x in BLACKLIST_EXTS): From 80b1365d53fe480776c2b84d61cacbc54a5fb3dc Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Sat, 1 Nov 2014 23:19:25 -0400 Subject: [PATCH 17/21] now correctly processes modules when in subdirs of cloud --- hacking/module_formatter.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hacking/module_formatter.py b/hacking/module_formatter.py index 1218b85e71..fe0da35ed8 100755 --- a/hacking/module_formatter.py +++ b/hacking/module_formatter.py @@ -127,11 +127,12 @@ def list_modules(module_dir): files = glob.glob("%s/*/*" % module_dir) for d in files: if os.path.isdir(d): - files2 = glob.glob("%s/*" % d) + files2 = glob.glob("%s/*" % d) + glob.glob("%s/*/*" % d) for f in files2: - module = os.path.splitext(os.path.basename(f))[0] - category = os.path.dirname(f).split("/")[-1] + category = "cloud" + if os.path.dirname(f).split("/")[-2] != "cloud": + category = os.path.dirname(f).split("/")[-1] if not f.endswith(".py") or f.endswith('__init__.py'): # windows powershell modules have documentation stubs in python docstring From 7bd2c945a76a1ca921b53f10a4bd4afbee5feeab Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Mon, 3 Nov 2014 08:15:26 -0500 Subject: [PATCH 18/21] now doc generation does not ignore subdirs of cloud --- hacking/module_formatter.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hacking/module_formatter.py b/hacking/module_formatter.py index fe0da35ed8..ee7ee45327 100755 --- a/hacking/module_formatter.py +++ b/hacking/module_formatter.py @@ -129,9 +129,11 @@ def list_modules(module_dir): if os.path.isdir(d): files2 = glob.glob("%s/*" % d) + glob.glob("%s/*/*" % d) for f in files2: + module = os.path.splitext(os.path.basename(f))[0] - category = "cloud" - if os.path.dirname(f).split("/")[-2] != "cloud": + if os.path.dirname(f).split("/")[-2] == "cloud": + category = "cloud" + else: category = os.path.dirname(f).split("/")[-1] if not f.endswith(".py") or f.endswith('__init__.py'): From 650048f7dd06d6704255c0ae6abd7d22ac88dc07 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Mon, 3 Nov 2014 22:02:13 -0500 Subject: [PATCH 19/21] now displays subcategories correctly --- hacking/module_formatter.py | 101 +++++++++++++++++++++++++----------- hacking/templates/rst.j2 | 4 ++ 2 files changed, 76 insertions(+), 29 deletions(-) diff --git a/hacking/module_formatter.py b/hacking/module_formatter.py index ee7ee45327..f182550aff 100755 --- a/hacking/module_formatter.py +++ b/hacking/module_formatter.py @@ -120,33 +120,52 @@ def write_data(text, options, outputname, module): ##################################################################################### -def list_modules(module_dir): +def list_modules(module_dir, depth=0): ''' returns a hash of categories, each category being a hash of module names to file paths ''' - categories = dict(all=dict()) - files = glob.glob("%s/*/*" % module_dir) - for d in files: - if os.path.isdir(d): - files2 = glob.glob("%s/*" % d) + glob.glob("%s/*/*" % d) - for f in files2: + categories = dict(all=dict(),_aliases=dict()) + if depth <= 3: # limit # of subdirs - module = os.path.splitext(os.path.basename(f))[0] - if os.path.dirname(f).split("/")[-2] == "cloud": - category = "cloud" + files = glob.glob("%s/*" % module_dir) + for d in files: + + category = os.path.splitext(os.path.basename(d))[0] + if os.path.isdir(d): + + res = list_modules(d, depth + 1) + for key in res.keys(): + if key in categories: + categories[key].update(res[key]) + res.pop(key, None) + + if depth < 2: + categories.update(res) else: - category = os.path.dirname(f).split("/")[-1] - - if not f.endswith(".py") or f.endswith('__init__.py'): + category = module_dir.split("/")[-1] + if not category in categories: + categories[category] = res + else: + categories[category].update(res) + else: + module = category + category = os.path.basename(module_dir) + if not d.endswith(".py") or d.endswith('__init__.py'): # windows powershell modules have documentation stubs in python docstring # format (they are not executed) so skip the ps1 format files continue - elif module.startswith("_") and os.path.islink(f): # ignores aliases + elif module.startswith("_") and os.path.islink(d): + source = os.path.splitext(os.path.basename(os.path.realpath(d)))[0] + module = module.replace("_","",1) + if not d in categories['_aliases']: + categories['_aliases'][source] = [module] + else: + categories['_aliases'][source].update(module) continue if not category in categories: categories[category] = {} - categories[category][module] = f - categories['all'][module] = f + categories[category][module] = d + categories['all'][module] = d return categories @@ -196,9 +215,12 @@ def jinja2_environment(template_dir, typ): ##################################################################################### -def process_module(module, options, env, template, outputname, module_map): +def process_module(module, options, env, template, outputname, module_map, aliases): fname = module_map[module] + if isinstance(fname, dict): + return "SKIPPED" + basename = os.path.basename(fname) deprecated = False @@ -233,6 +255,8 @@ def process_module(module, options, env, template, outputname, module_map): else: doc['core'] = False + if module in aliases: + doc['aliases'] = aliases[module] all_keys = [] @@ -274,10 +298,28 @@ def process_module(module, options, env, template, outputname, module_map): ##################################################################################### +def print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map, aliases): + modstring = module + modname = module + if module in deprecated: + modstring = modstring + DEPRECATED + modname = "_" + module + elif module not in core: + modstring = modstring + NOTCORE + + result = process_module(modname, options, env, template, outputname, module_map, aliases) + + if result != "SKIPPED": + category_file.write(" %s - %s <%s_module>\n" % (modstring, result, module)) + def process_category(category, categories, options, env, template, outputname): module_map = categories[category] + aliases = {} + if '_aliases' in categories: + aliases = categories['_aliases'] + category_file_path = os.path.join(options.output_dir, "list_of_%s_modules.rst" % category) category_file = open(category_file_path, "w") print "*** recording category %s in %s ***" % (category, category_file_path) @@ -312,21 +354,20 @@ def process_category(category, categories, options, env, template, outputname): .. toctree:: :maxdepth: 1 """ % (category_header, underscores)) - + sections = [] for module in modules: + if module in module_map and isinstance(module_map[module], dict): + sections.append(module) + continue + else: + print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map, aliases) - modstring = module - modname = module - if module in deprecated: - modstring = modstring + DEPRECATED - modname = "_" + module - elif module not in core: - modstring = modstring + NOTCORE + for section in sections: + category_file.write("%s/\n%s\n\n" % (section,'-' * len(section))) + category_file.write(".. toctree:: :maxdepth: 1\n\n") - result = process_module(modname, options, env, template, outputname, module_map) - - if result != "SKIPPED": - category_file.write(" %s - %s <%s_module>\n" % (modstring, result, module)) + for module in module_map[section]: + print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map[section], aliases) category_file.write("""\n\n .. note:: @@ -377,6 +418,8 @@ def main(): category_list_file.write(" :maxdepth: 1\n\n") for category in category_names: + if category.startswith("_"): + continue category_list_file.write(" list_of_%s_modules\n" % category) process_category(category, categories, options, env, template, outputname) diff --git a/hacking/templates/rst.j2 b/hacking/templates/rst.j2 index 1d55a0452b..232d97a731 100644 --- a/hacking/templates/rst.j2 +++ b/hacking/templates/rst.j2 @@ -21,6 +21,10 @@ # --------------------------------------------#} +{% if aliases is defined -%} +Aliases: @{ ','.join(aliases) }@ +{% endif %} + {% if deprecated is defined -%} DEPRECATED ---------- From 5f1ad79cd30ae0069ce4dcb449763e15677a24b1 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Mon, 3 Nov 2014 23:14:22 -0500 Subject: [PATCH 20/21] now correctly flags and sorts subcategory modules --- hacking/module_formatter.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/hacking/module_formatter.py b/hacking/module_formatter.py index f182550aff..03b8827d48 100755 --- a/hacking/module_formatter.py +++ b/hacking/module_formatter.py @@ -334,11 +334,19 @@ def process_category(category, categories, options, env, template, outputname): core = [] for module in module_map.keys(): - if module.startswith("_"): - module = module.replace("_","",1) - deprecated.append(module) - elif '/core/' in module_map[module]: - core.append(module) + if isinstance(module_map[module], dict): + for mod in module_map[module].keys(): + if mod.startswith("_"): + mod = mod.replace("_","",1) + deprecated.append(mod) + elif '/core/' in module_map[module][mod]: + core.append(mod) + else: + if module.startswith("_"): + module = module.replace("_","",1) + deprecated.append(module) + elif '/core/' in module_map[module]: + core.append(module) modules.append(module) @@ -362,11 +370,15 @@ def process_category(category, categories, options, env, template, outputname): else: print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map, aliases) + sections.sort() for section in sections: - category_file.write("%s/\n%s\n\n" % (section,'-' * len(section))) + category_file.write("%s\n%s\n\n" % (section,'-' * len(section))) category_file.write(".. toctree:: :maxdepth: 1\n\n") - for module in module_map[section]: + section_modules = module_map[section].keys() + section_modules.sort() + #for module in module_map[section]: + for module in section_modules: print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map[section], aliases) category_file.write("""\n\n From 12393a4b47f05fbf384ab7bb3bd7afa2fcf0b930 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Tue, 4 Nov 2014 08:44:39 -0500 Subject: [PATCH 21/21] subcategories are now Title case and _ gets changed to a space --- hacking/module_formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hacking/module_formatter.py b/hacking/module_formatter.py index 03b8827d48..73729da4d6 100755 --- a/hacking/module_formatter.py +++ b/hacking/module_formatter.py @@ -372,7 +372,7 @@ def process_category(category, categories, options, env, template, outputname): sections.sort() for section in sections: - category_file.write("%s\n%s\n\n" % (section,'-' * len(section))) + category_file.write("%s\n%s\n\n" % (section.replace("_"," ").title(),'-' * len(section))) category_file.write(".. toctree:: :maxdepth: 1\n\n") section_modules = module_map[section].keys()