d47e15a1 |
#!/usr/bin/env python
# (c) 2012, Jan-Piet Mens <jpmens () gmail.com> |
bceb0026 |
# (c) 2012-2014, Michael DeHaan <michael@ansible.com> and others |
d47e15a1 |
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
#
|
b8c9391d |
from __future__ import print_function |
b27c424f |
|
d47e15a1 |
import os |
391fb98e |
import glob |
d47e15a1 |
import sys
import yaml
import re |
2786149b |
import optparse |
d47e15a1 |
import datetime |
7681b1ce |
import cgi |
6ba706f7 |
import warnings |
b27c424f |
from collections import defaultdict
|
fe2d00d9 |
from jinja2 import Environment, FileSystemLoader |
823677b4 |
from six import iteritems |
fe2d00d9 |
|
30101905 |
from ansible.utils import module_docs
from ansible.utils.vars import merge_hash |
5cd3f717 |
from ansible.utils.unicode import to_bytes |
a78fdde3 |
from ansible.errors import AnsibleError |
31a4fe41 |
|
fe2d00d9 |
#####################################################################################
# constants and paths
|
40429ee6 |
# if a module is added in a version of Ansible older than this, don't print the version added information
# in the module documentation because everyone is assumed to be running something newer than this already. |
6ba706f7 |
TO_OLD_TO_BE_NOTABLE = 1.3 |
40429ee6 |
|
60f06c36 |
# Get parent directory of the directory this script lives in
MODULEDIR=os.path.abspath(os.path.join( |
bceb0026 |
os.path.dirname(os.path.realpath(__file__)), os.pardir, 'lib', 'ansible', 'modules' |
31d0060d |
))
# The name of the DOCUMENTATION template |
60f06c36 |
EXAMPLE_YAML=os.path.abspath(os.path.join( |
31d0060d |
os.path.dirname(os.path.realpath(__file__)), os.pardir, 'examples', 'DOCUMENTATION.yml'
)) |
d47e15a1 |
_ITALIC = re.compile(r"I\(([^)]+)\)")
_BOLD = re.compile(r"B\(([^)]+)\)")
_MODULE = re.compile(r"M\(([^)]+)\)")
_URL = re.compile(r"U\(([^)]+)\)")
_CONST = re.compile(r"C\(([^)]+)\)")
|
44f0279d |
DEPRECATED = " (D)"
NOTCORE = " (E)" |
fe2d00d9 |
##################################################################################### |
d47e15a1 |
def rst_ify(text): |
fe2d00d9 |
''' convert symbols like I(this is in italics) to valid restructured text ''' |
d47e15a1 |
|
a78fdde3 |
try:
t = _ITALIC.sub(r'*' + r"\1" + r"*", text)
t = _BOLD.sub(r'**' + r"\1" + r"**", t)
t = _MODULE.sub(r':ref:`' + r"\1 <\1>" + r"`", t)
t = _URL.sub(r"\1", t)
t = _CONST.sub(r'``' + r"\1" + r"``", t)
except Exception as e:
raise AnsibleError("Could not process (%s) : %s" % (str(text), str(e))) |
d47e15a1 |
return t
|
fe2d00d9 |
##################################################################################### |
7681b1ce |
|
fe2d00d9 |
def html_ify(text):
''' convert symbols like I(this is in italics) to valid HTML ''' |
d4f89122 |
|
7681b1ce |
t = cgi.escape(text) |
fe2d00d9 |
t = _ITALIC.sub("<em>" + r"\1" + "</em>", t)
t = _BOLD.sub("<b>" + r"\1" + "</b>", t)
t = _MODULE.sub("<span class='module'>" + r"\1" + "</span>", t)
t = _URL.sub("<a href='" + r"\1" + "'>" + r"\1" + "</a>", t)
t = _CONST.sub("<code>" + r"\1" + "</code>", t) |
d4f89122 |
return t
|
fe2d00d9 |
#####################################################################################
|
d47e15a1 |
def rst_fmt(text, fmt): |
fe2d00d9 |
''' helper for Jinja2 to do format strings '''
|
d47e15a1 |
return fmt % (text)
|
fe2d00d9 |
#####################################################################################
|
d47e15a1 |
def rst_xline(width, char="="): |
fe2d00d9 |
''' return a restructured text line of a given length '''
|
d47e15a1 |
return char * width
|
fe2d00d9 |
##################################################################################### |
d47e15a1 |
|
35ec9f81 |
def write_data(text, options, outputname, module): |
fe2d00d9 |
''' dumps module output to a file or the screen, as requested '''
|
2786149b |
if options.output_dir is not None: |
bceb0026 |
fname = os.path.join(options.output_dir, outputname % module)
fname = fname.replace(".py","")
f = open(fname, 'w') |
29aaa5e6 |
f.write(text.encode('utf-8')) |
ee679c01 |
f.close()
else: |
b8c9391d |
print(text) |
ee679c01 |
|
fe2d00d9 |
#####################################################################################
|
bceb0026 |
|
650048f7 |
def list_modules(module_dir, depth=0): |
fe2d00d9 |
''' returns a hash of categories, each category being a hash of module names to file paths '''
|
b27c424f |
categories = dict()
module_info = dict()
aliases = defaultdict(set)
# * windows powershell modules have documentation stubs in python docstring
# format (they are not executed) so skip the ps1 format files
# * One glob level for every module level that we're going to traverse
files = glob.glob("%s/*.py" % module_dir) + glob.glob("%s/*/*.py" % module_dir) + glob.glob("%s/*/*/*.py" % module_dir) + glob.glob("%s/*/*/*/*.py" % module_dir)
for module_path in files:
if module_path.endswith('__init__.py'):
continue
category = categories
mod_path_only = os.path.dirname(module_path[len(module_dir) + 1:])
# Start at the second directory because we don't want the "vendor"
# directories (core, extras)
for new_cat in mod_path_only.split('/')[1:]:
if new_cat not in category:
category[new_cat] = dict()
category = category[new_cat]
module = os.path.splitext(os.path.basename(module_path))[0]
if module in module_docs.BLACKLIST_MODULES:
# Do not list blacklisted modules
continue
if module.startswith("_") and os.path.islink(module_path):
source = os.path.splitext(os.path.basename(os.path.realpath(module_path)))[0]
module = module.replace("_","",1)
aliases[source].add(module)
continue
category[module] = module_path
module_info[module] = module_path
# keep module tests out of becoming module docs |
5719912e |
if 'test' in categories:
del categories['test']
|
b27c424f |
return module_info, categories, aliases |
ee679c01 |
|
fe2d00d9 |
#####################################################################################
def generate_parser():
''' generate an optparse parser ''' |
d47e15a1 |
|
2786149b |
p = optparse.OptionParser(
version='%prog 1.0',
usage='usage: %prog [options] arg1 arg2', |
fe2d00d9 |
description='Generate module documentation from metadata', |
2786149b |
)
|
fe2d00d9 |
p.add_option("-A", "--ansible-version", action="store", dest="ansible_version", default="unknown", help="Ansible version number")
p.add_option("-M", "--module-dir", action="store", dest="module_dir", default=MODULEDIR, help="Ansible library path")
p.add_option("-T", "--template-dir", action="store", dest="template_dir", default="hacking/templates", help="directory containing Jinja2 templates") |
95d102f5 |
p.add_option("-t", "--type", action='store', dest='type', choices=['rst'], default='rst', help="Document type") |
af1f8db5 |
p.add_option("-v", "--verbose", action='store_true', default=False, help="Verbose") |
fe2d00d9 |
p.add_option("-o", "--output-dir", action="store", dest="output_dir", default=None, help="Output directory for module files")
p.add_option("-I", "--includes-file", action="store", dest="includes_file", default=None, help="Create a file containing list of processed modules") |
60f06c36 |
p.add_option('-V', action='version', help='Show version number and exit') |
fe2d00d9 |
return p |
2786149b |
|
fe2d00d9 |
##################################################################################### |
d47e15a1 |
|
fe2d00d9 |
def jinja2_environment(template_dir, typ): |
60f06c36 |
|
fe2d00d9 |
env = Environment(loader=FileSystemLoader(template_dir), |
62d038dc |
variable_start_string="@{",
variable_end_string="}@", |
e4338d0c |
trim_blocks=True, |
626203a7 |
) |
62d038dc |
env.globals['xline'] = rst_xline |
83f277cf |
|
fe2d00d9 |
if typ == 'rst': |
10009b0d |
env.filters['convert_symbols_to_format'] = rst_ify |
83f277cf |
env.filters['html_ify'] = html_ify |
d47e15a1 |
env.filters['fmt'] = rst_fmt
env.filters['xline'] = rst_xline
template = env.get_template('rst.j2') |
35ec9f81 |
outputname = "%s_module.rst" |
fe2d00d9 |
else:
raise Exception("unknown module format type: %s" % typ) |
eb8a1123 |
|
fe2d00d9 |
return env, template, outputname |
d47e15a1 |
|
fe2d00d9 |
##################################################################################### |
6ba706f7 |
def too_old(added):
if not added:
return False
try:
added_tokens = str(added).split(".")
readded = added_tokens[0] + "." + added_tokens[1]
added_float = float(readded)
except ValueError as e:
warnings.warn("Could not parse %s: %s" % (added, str(e)))
return False
return (added_float < TO_OLD_TO_BE_NOTABLE) |
391fb98e |
|
650048f7 |
def process_module(module, options, env, template, outputname, module_map, aliases): |
fe2d00d9 |
fname = module_map[module] |
650048f7 |
if isinstance(fname, dict):
return "SKIPPED"
|
8b5b97d0 |
basename = os.path.basename(fname)
deprecated = False |
fe2d00d9 |
# ignore files with extensions |
8b5b97d0 |
if not basename.endswith(".py"): |
fe2d00d9 |
return |
44f0279d |
elif module.startswith("_"):
if os.path.islink(fname):
return # ignore, its an alias |
8b5b97d0 |
deprecated = True |
44f0279d |
module = module.replace("_","",1)
|
b8c9391d |
print("rendering: %s" % module) |
fe2d00d9 |
# use ansible core library to parse out doc metadata YAML and plaintext examples |
30101905 |
doc, examples, returndocs = module_docs.get_docstring(fname, verbose=options.verbose) |
fe2d00d9 |
# crash if module is missing documentation and not explicitly hidden from docs index
if doc is None: |
b27c424f |
sys.stderr.write("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module))
sys.exit(1) |
44f0279d |
if deprecated and 'deprecated' not in doc:
sys.stderr.write("*** ERROR: DEPRECATED MODULE MISSING 'deprecated' DOCUMENTATION: %s, %s ***\n" % (fname, module))
sys.exit(1) |
fe2d00d9 |
|
ffee9a8f |
if "/core/" in fname: |
e8fe306c |
doc['core'] = True
else:
doc['core'] = False
|
650048f7 |
if module in aliases:
doc['aliases'] = aliases[module] |
e8fe306c |
|
fe2d00d9 |
all_keys = [] |
391fb98e |
|
fe2d00d9 |
if not 'version_added' in doc:
sys.stderr.write("*** ERROR: missing version_added in: %s ***\n" % module)
sys.exit(1)
added = 0
if doc['version_added'] == 'historical':
del doc['version_added']
else:
added = doc['version_added']
# don't show version added information if it's too old to be called out |
6ba706f7 |
if too_old(added):
del doc['version_added'] |
fe2d00d9 |
|
a78fdde3 |
if 'options' in doc and doc['options']: |
823677b4 |
for (k,v) in iteritems(doc['options']): |
6ba706f7 |
# don't show version added information if it's too old to be called out
if 'version_added' in doc['options'][k] and too_old(doc['options'][k]['version_added']):
del doc['options'][k]['version_added'] |
6414c967 |
if not 'description' in doc['options'][k]:
raise AnsibleError("Missing required description for option %s in %s " % (k, module))
if not isinstance(doc['options'][k]['description'],list):
doc['options'][k]['description'] = [doc['options'][k]['description']]
|
30101905 |
all_keys.append(k) |
44f0279d |
|
fe2d00d9 |
all_keys = sorted(all_keys)
|
44f0279d |
doc['option_keys'] = all_keys |
fe2d00d9 |
doc['filename'] = fname
doc['docuri'] = doc['module'].replace('_', '-')
doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d')
doc['ansible_version'] = options.ansible_version
doc['plainexamples'] = examples #plain text |
c3076b84 |
if returndocs: |
3a0bf55a |
try:
doc['returndocs'] = yaml.safe_load(returndocs)
except:
print("could not load yaml: %s" % returndocs)
raise |
c3076b84 |
else:
doc['returndocs'] = None |
fe2d00d9 |
# here is where we build the table of contents...
|
a78fdde3 |
try:
text = template.render(doc)
except Exception as e:
raise AnsibleError("Failed to render doc for %s: %s" % (fname, str(e))) |
35ec9f81 |
write_data(text, options, outputname, module) |
44f0279d |
return doc['short_description'] |
391fb98e |
|
fe2d00d9 |
##################################################################################### |
5f18a535 |
|
650048f7 |
def print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map, aliases):
modstring = module |
b27c424f |
if modstring.startswith('_'):
modstring = module[1:]
modname = modstring |
650048f7 |
if module in deprecated:
modstring = modstring + DEPRECATED
elif module not in core:
modstring = modstring + NOTCORE
|
b27c424f |
category_file.write(" %s - %s <%s_module>\n" % (to_bytes(modstring), to_bytes(rst_ify(module_map[module][1])), to_bytes(modname))) |
650048f7 |
|
fe2d00d9 |
def process_category(category, categories, options, env, template, outputname): |
5f18a535 |
|
b27c424f |
### FIXME:
# We no longer conceptually deal with a mapping of category names to
# modules to file paths. Instead we want several different records:
# (1) Mapping of module names to file paths (what's presently used
# as categories['all']
# (2) Mapping of category names to lists of module names (what you'd
# presently get from categories[category_name][subcategory_name].keys()
# (3) aliases (what's presently in categories['_aliases']
#
# list_modules() now returns those. Need to refactor this function and
# main to work with them.
|
fe2d00d9 |
module_map = categories[category] |
b27c424f |
module_info = categories['all'] |
391fb98e |
|
650048f7 |
aliases = {}
if '_aliases' in categories:
aliases = categories['_aliases']
|
35ec9f81 |
category_file_path = os.path.join(options.output_dir, "list_of_%s_modules.rst" % category)
category_file = open(category_file_path, "w") |
b8c9391d |
print("*** recording category %s in %s ***" % (category, category_file_path)) |
35ec9f81 |
|
6ba706f7 |
# start a new category file |
391fb98e |
|
fe2d00d9 |
category = category.replace("_"," ")
category = category.title()
|
44f0279d |
modules = []
deprecated = []
core = []
for module in module_map.keys():
|
5f1ad79c |
if isinstance(module_map[module], dict): |
b27c424f |
for mod in (m for m in module_map[module].keys() if m in module_info): |
5f1ad79c |
if mod.startswith("_"):
deprecated.append(mod) |
b27c424f |
elif '/core/' in module_info[mod][0]: |
5f1ad79c |
core.append(mod)
else: |
b27c424f |
if module not in module_info:
continue |
5f1ad79c |
if module.startswith("_"):
deprecated.append(module) |
b27c424f |
elif '/core/' in module_info[module][0]: |
5f1ad79c |
core.append(module) |
44f0279d |
modules.append(module)
|
b27c424f |
modules.sort(key=lambda k: k[1:] if k.startswith('_') else k) |
fe2d00d9 |
|
35ec9f81 |
category_header = "%s Modules" % (category.title())
underscores = "`" * len(category_header)
|
7965d331 |
category_file.write("""\
%s
%s
|
44f0279d |
.. toctree:: :maxdepth: 1 |
7965d331 |
""" % (category_header, underscores)) |
650048f7 |
sections = [] |
fe2d00d9 |
for module in modules: |
650048f7 |
if module in module_map and isinstance(module_map[module], dict):
sections.append(module)
continue
else: |
b27c424f |
print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_info, aliases) |
35ec9f81 |
|
5f1ad79c |
sections.sort() |
650048f7 |
for section in sections: |
2ba5c3c6 |
category_file.write("\n%s\n%s\n\n" % (section.replace("_"," ").title(),'-' * len(section))) |
650048f7 |
category_file.write(".. toctree:: :maxdepth: 1\n\n") |
44f0279d |
|
5f1ad79c |
section_modules = module_map[section].keys() |
b27c424f |
section_modules.sort(key=lambda k: k[1:] if k.startswith('_') else k) |
5f1ad79c |
#for module in module_map[section]: |
b27c424f |
for module in (m for m in section_modules if m in module_info):
print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_info, aliases) |
35ec9f81 |
|
023f5fd7 |
category_file.write("""\n\n
.. note:: |
c551fe8b |
- %s: This marks a module as deprecated, which means a module is kept for backwards compatibility but usage is discouraged. The module documentation details page may explain more about this rationale. |
8b278fee |
- %s: This marks a module as 'extras', which means it ships with ansible but may be a newer module and possibly (but not necessarily) less actively maintained than 'core' modules. |
c551fe8b |
- Tickets filed on modules are filed to different repos than those on the main open source project. Core module tickets should be filed at `ansible/ansible-modules-core on GitHub <http://github.com/ansible/ansible-modules-core>`_, extras tickets to `ansible/ansible-modules-extras on GitHub <http://github.com/ansible/ansible-modules-extras>`_ |
023f5fd7 |
""" % (DEPRECATED, NOTCORE)) |
35ec9f81 |
category_file.close() |
fe2d00d9 |
# TODO: end a new category file
#####################################################################################
def validate_options(options):
''' validate option parser options '''
if not options.module_dir: |
b8c9391d |
print("--module-dir is required", file=sys.stderr) |
fe2d00d9 |
sys.exit(1)
if not os.path.exists(options.module_dir): |
b8c9391d |
print("--module-dir does not exist: %s" % options.module_dir, file=sys.stderr) |
fe2d00d9 |
sys.exit(1)
if not options.template_dir: |
b8c9391d |
print("--template-dir must be specified") |
fe2d00d9 |
sys.exit(1) |
391fb98e |
|
fe2d00d9 |
##################################################################################### |
d47e15a1 |
|
fe2d00d9 |
def main(): |
d47e15a1 |
|
fe2d00d9 |
p = generate_parser() |
0c855a85 |
|
fe2d00d9 |
(options, args) = p.parse_args()
validate_options(options) |
d47e15a1 |
|
fe2d00d9 |
env, template, outputname = jinja2_environment(options.template_dir, options.type) |
ee679c01 |
|
b27c424f |
mod_info, categories, aliases = list_modules(options.module_dir)
categories['all'] = mod_info
categories['_aliases'] = aliases
category_names = [c for c in categories.keys() if not c.startswith('_')] |
fe2d00d9 |
category_names.sort() |
af1f8db5 |
|
b27c424f |
# Write master category list |
35ec9f81 |
category_list_path = os.path.join(options.output_dir, "modules_by_category.rst") |
b27c424f |
with open(category_list_path, "w") as category_list_file:
category_list_file.write("Module Index\n")
category_list_file.write("============\n")
category_list_file.write("\n\n")
category_list_file.write(".. toctree::\n")
category_list_file.write(" :maxdepth: 1\n\n")
for category in category_names:
category_list_file.write(" list_of_%s_modules\n" % category)
#
# Import all the docs into memory
#
module_map = mod_info.copy()
skipped_modules = set()
for modname in module_map:
result = process_module(modname, options, env, template, outputname, module_map, aliases)
if result == 'SKIPPED':
del categories['all'][modname]
else:
categories['all'][modname] = (categories['all'][modname], result) |
af1f8db5 |
|
b27c424f |
#
# Render all the docs to rst via category pages
# |
fe2d00d9 |
for category in category_names:
process_category(category, categories, options, env, template, outputname) |
d47e15a1 |
if __name__ == '__main__':
main() |