Implement a framework for having common code for release scripts (#55893)

* Implement a framework for having common code for release scripts

* Release scripts will go through hacking/build-ansible.  build-ansible is
  a pluggable script which will set a directory that has common code for
  non-enduser scripts.  It will then invoke the plugin which implements
  that subcommand.  Uses straight.plugin for loading each sub-command.

* We're going to add tools which are needed to test ansible (the changelog
  generation, for instance) so we need to include the pieces relevant to
  that in the tarball.

* Add straight.plugin to the sanity test requirements for the same
  reason

* Skip compile test just for build-ansible plugins which won't be run as
  part of sanity tests.

hacking/build_library/build_ansible/command_plugins/porting_guide.py

@@ -0,0 +1,142 @@
+# coding: utf-8
+# Copyright: (c) 2019, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+import argparse
+import os.path
+import sys
+
+from jinja2 import Environment, DictLoader
+
+# Pylint doesn't understand Python3 namespace modules.
+from ..commands import Command  # pylint: disable=relative-beyond-top-level
+
+
+PORTING_GUIDE_TEMPLATE = """
+.. _porting_{{ ver }}_guide:
+
+*************************
+Ansible {{ ver }} Porting Guide
+*************************
+
+This section discusses the behavioral changes between Ansible {{ prev_ver }} and Ansible {{ ver }}.
+
+It is intended to assist in updating your playbooks, plugins and other parts of your Ansible infrastructure so they will work with this version of Ansible.
+
+We suggest you read this page along with `Ansible Changelog for {{ ver }} <https://github.com/ansible/ansible/blob/devel/changelogs/CHANGELOG-v{{ ver }}.rst>`_ to understand what updates you may need to make.
+
+This document is part of a collection on porting. The complete list of porting guides can be found at :ref:`porting guides <porting_guides>`.
+
+.. contents:: Topics
+
+
+Playbook
+========
+
+No notable changes
+
+
+Command Line
+============
+
+No notable changes
+
+
+Deprecated
+==========
+
+No notable changes
+
+
+Modules
+=======
+
+No notable changes
+
+
+Modules removed
+---------------
+
+The following modules no longer exist:
+
+* No notable changes
+
+
+Deprecation notices
+-------------------
+
+No notable changes
+
+
+Noteworthy module changes
+-------------------------
+
+No notable changes
+
+
+Plugins
+=======
+
+No notable changes
+
+
+Porting custom scripts
+======================
+
+No notable changes
+
+
+Networking
+==========
+
+No notable changes
+
+"""  # noqa for E501 (line length).
+# jinja2 is horrid about getting rid of extra newlines so we have to have a single line per
+# paragraph for proper wrapping to occur
+
+JINJA_ENV = Environment(
+    loader=DictLoader({'porting_guide': PORTING_GUIDE_TEMPLATE,
+                       }),
+    extensions=['jinja2.ext.i18n'],
+    trim_blocks=True,
+    lstrip_blocks=True,
+)
+
+
+def generate_porting_guide(version):
+    template = JINJA_ENV.get_template('porting_guide')
+
+    version_list = version.split('.')
+    version_list[-1] = str(int(version_list[-1]) - 1)
+    previous_version = '.'.join(version_list)
+
+    content = template.render(ver=version, prev_ver=previous_version)
+    return content
+
+
+def write_guide(version, guide_content):
+    filename = f'porting_guide_{version}.rst'
+    with open(filename, 'w') as out_file:
+        out_file.write(guide_content)
+
+
+class PortingGuideCommand(Command):
+    name = 'porting-guide'
+
+    @classmethod
+    def init_parser(cls, add_parser):
+        parser = add_parser(cls.name, description="Generate a fresh porting guide template")
+        parser.add_argument("--version", dest="version", type=str, required=True, action='store',
+                            help="Version of Ansible to write the porting guide for")
+
+    @staticmethod
+    def main(args):
+        guide_content = generate_porting_guide(args.version)
+        write_guide(args.version, guide_content)
+        return 0

hacking/build_library/build_ansible/command_plugins/release_announcement.py

@@ -0,0 +1,306 @@
+# coding: utf-8
+# Copyright: (c) 2019, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+import argparse
+import asyncio
+import datetime
+import hashlib
+import os.path
+import sys
+from collections import UserString
+from distutils.version import LooseVersion
+
+import aiohttp
+from jinja2 import Environment, DictLoader
+
+# Pylint doesn't understand Python3 namespace modules.
+from ..commands import Command  # pylint: disable=relative-beyond-top-level
+
+
+# pylint: disable=
+VERSION_FRAGMENT = """
+{%- if versions | length > 1 %}
+  {% for version in versions %}
+    {% if loop.last %}and {{ version }}{% else %}
+      {% if versions | length == 2 %}{{ version }} {% else %}{{ version }}, {% endif -%}
+    {% endif -%}
+  {% endfor -%}
+{%- else %}{{ versions[0] }}{% endif -%}
+"""
+
+LONG_TEMPLATE = """
+{% set plural = False if versions | length == 1 else True %}
+{% set latest_ver = (versions | sort(attribute='ver_obj'))[-1] %}
+
+To: ansible-devel@googlegroups.com, ansible-project@googlegroups.com, ansible-announce@googlegroups.com
+Subject: New Ansible release{% if plural %}s{% endif %} {{ version_str }}
+
+{% filter wordwrap %}
+Hi all- we're happy to announce that the general release of Ansible {{ version_str }}{% if plural %} are{%- else %} is{%- endif %} now available!
+{% endfilter %}
+
+
+
+How do you get it?
+------------------
+
+{% for version in versions %}
+$ pip install ansible=={{ version }} --user
+{% if not loop.last %}
+or
+{% endif %}
+{% endfor %}
+
+The tar.gz of the release{% if plural %}s{% endif %} can be found here:
+
+{% for version in versions %}
+* {{ version }}
+  https://releases.ansible.com/ansible/ansible-{{ version }}.tar.gz
+  SHA256: {{ hashes[version] }}
+{% endfor %}
+
+
+What's new in {{ version_str }}
+{{ '-' * (14 + version_str | length) }}
+
+{% filter wordwrap %}
+{% if plural %}These releases are{% else %}This release is a{% endif %} maintenance release{% if plural %}s{% endif %} containing numerous bugfixes. The full {% if plural %} changelogs are{% else %} changelog is{% endif %} at:
+{% endfilter %}
+
+
+{% for version in versions %}
+* {{ version }}
+  https://github.com/ansible/ansible/blob/stable-{{ version.split('.')[:2] | join('.') }}/changelogs/CHANGELOG-v{{ version.split('.')[:2] | join('.') }}.rst
+{% endfor %}
+
+
+What's the schedule for future maintenance releases?
+----------------------------------------------------
+
+{% filter wordwrap %}
+Future maintenance releases will occur approximately every 3 weeks.  So expect the next one around {{ next_release.strftime('%Y-%m-%d') }}.
+{% endfilter %}
+
+
+
+Porting Help
+------------
+
+{% filter wordwrap %}
+We've published a porting guide at
+https://docs.ansible.com/ansible/devel/porting_guides/porting_guide_{{ latest_ver.split('.')[:2] | join('.') }}.html to help migrate your content to {{ latest_ver.split('.')[:2] | join('.') }}.
+{% endfilter %}
+
+
+
+{% filter wordwrap %}
+If you discover any errors or if any of your working playbooks break when you upgrade to {{ latest_ver }}, please use the following link to report the regression:
+{% endfilter %}
+
+
+  https://github.com/ansible/ansible/issues/new/choose
+
+{% filter wordwrap %}
+In your issue, be sure to mention the Ansible version that works and the one that doesn't.
+{% endfilter %}
+
+
+Thanks!
+
+-{{ name }}
+
+"""  # noqa for E501 (line length).
+# jinja2 is horrid about getting rid of extra newlines so we have to have a single per paragraph for
+# proper wrapping to occur
+
+SHORT_TEMPLATE = """
+{% set plural = False if versions | length == 1 else True %}
+@ansible
+{{ version_str }}
+{% if plural %}
+  have
+{% else %}
+  has
+{% endif %}
+been released! Get
+{% if plural %}
+them
+{% else %}
+it
+{% endif %}
+on PyPI: pip install ansible=={{ (versions|sort(attribute='ver_obj'))[-1] }},
+https://releases.ansible.com/ansible/, the Ansible PPA on Launchpad, or GitHub.  Happy automating!
+"""  # noqa for E501 (line length).
+# jinja2 is horrid about getting rid of extra newlines so we have to have a single per paragraph for
+# proper wrapping to occur
+
+JINJA_ENV = Environment(
+    loader=DictLoader({'long': LONG_TEMPLATE,
+                       'short': SHORT_TEMPLATE,
+                       'version_string': VERSION_FRAGMENT,
+                       }),
+    extensions=['jinja2.ext.i18n'],
+    trim_blocks=True,
+    lstrip_blocks=True,
+)
+
+
+class VersionStr(UserString):
+    def __init__(self, string):
+        super().__init__(string.strip())
+        self.ver_obj = LooseVersion(string)
+
+
+def transform_args(args):
+    # Make it possible to sort versions in the jinja2 templates
+    new_versions = []
+    for version in args.versions:
+        new_versions.append(VersionStr(version))
+    args.versions = new_versions
+
+    return args
+
+
+async def calculate_hash_from_tarball(session, version):
+    tar_url = f'https://releases.ansible.com/ansible/ansible-{version}.tar.gz'
+    tar_task = asyncio.create_task(session.get(tar_url))
+    tar_response = await tar_task
+
+    tar_hash = hashlib.sha256()
+    while True:
+        chunk = await tar_response.content.read(1024)
+        if not chunk:
+            break
+        tar_hash.update(chunk)
+
+    return tar_hash.hexdigest()
+
+
+async def parse_hash_from_file(session, version):
+    filename = f'ansible-{version}.tar.gz'
+    hash_url = f'https://releases.ansible.com/ansible/{filename}.sha'
+    hash_task = asyncio.create_task(session.get(hash_url))
+    hash_response = await hash_task
+
+    hash_content = await hash_response.read()
+    precreated_hash, precreated_filename = hash_content.split(None, 1)
+    if filename != precreated_filename.strip().decode('utf-8'):
+        raise ValueError(f'Hash file contains hash for a different file: {precreated_filename}')
+
+    return precreated_hash.decode('utf-8')
+
+
+async def get_hash(session, version):
+    calculated_hash = await calculate_hash_from_tarball(session, version)
+    precreated_hash = await parse_hash_from_file(session, version)
+
+    if calculated_hash != precreated_hash:
+        raise ValueError(f'Hash in file ansible-{version}.tar.gz.sha {precreated_hash} does not'
+                         f' match hash of tarball {calculated_hash}')
+
+    return calculated_hash
+
+
+async def get_hashes(versions):
+    hashes = {}
+    requestors = {}
+    async with aiohttp.ClientSession() as aio_session:
+        for version in versions:
+            requestors[version] = asyncio.create_task(get_hash(aio_session, version))
+
+        for version, request in requestors.items():
+            await request
+            hashes[version] = request.result()
+
+    return hashes
+
+
+def next_release_date(weeks=3):
+    days_in_the_future = weeks * 7
+    today = datetime.datetime.now()
+    numeric_today = today.weekday()
+
+    # We release on Thursdays
+    if numeric_today == 3:
+        # 3 is Thursday
+        pass
+    elif numeric_today == 4:
+        # If this is Friday, we can adjust back to Thursday for the next release
+        today -= datetime.timedelta(days=1)
+    elif numeric_today < 3:
+        # Otherwise, slide forward to Thursday
+        today += datetime.timedelta(days=(3 - numeric_today))
+    else:
+        # slightly different formula if it's past Thursday this week.  We need to go forward to
+        # Thursday of next week
+        today += datetime.timedelta(days=(10 - numeric_today))
+
+    next_release = today + datetime.timedelta(days=days_in_the_future)
+    return next_release
+
+
+def generate_long_message(versions, name):
+    hashes = asyncio.run(get_hashes(versions))
+
+    version_template = JINJA_ENV.get_template('version_string')
+    version_str = version_template.render(versions=versions).strip()
+
+    next_release = next_release_date()
+
+    template = JINJA_ENV.get_template('long')
+    message = template.render(versions=versions, version_str=version_str,
+                              name=name, hashes=hashes, next_release=next_release)
+    return message
+
+
+def generate_short_message(versions):
+    version_template = JINJA_ENV.get_template('version_string')
+    version_str = version_template.render(versions=versions).strip()
+
+    template = JINJA_ENV.get_template('short')
+    message = template.render(versions=versions, version_str=version_str)
+    message = ' '.join(message.split()) + '\n'
+    return message
+
+
+def write_message(filename, message):
+    if filename != '-':
+        with open(filename, 'w') as out_file:
+            out_file.write(message)
+    else:
+        sys.stdout.write('\n\n')
+        sys.stdout.write(message)
+
+
+class ReleaseAnnouncementCommand(Command):
+    name = 'release-announcement'
+
+    @classmethod
+    def init_parser(cls, add_parser):
+        parser = add_parser(cls.name,
+                            description="Generate email and twitter announcements from template")
+
+        parser.add_argument("--version", dest="versions", type=str, required=True, action='append',
+                            help="Versions of Ansible to announce")
+        parser.add_argument("--name", type=str, required=True, help="Real name to use on emails")
+        parser.add_argument("--email-out", type=str, default="-",
+                            help="Filename to place the email announcement into")
+        parser.add_argument("--twitter-out", type=str, default="-",
+                            help="Filename to place the twitter announcement into")
+
+    @staticmethod
+    def main(args):
+        args = transform_args(args)
+
+        twitter_message = generate_short_message(args.versions)
+        email_message = generate_long_message(args.versions, args.name)
+
+        write_message(args.twitter_out, twitter_message)
+        write_message(args.email_out, email_message)
+        return 0

hacking/build_library/build_ansible/commands.py

@@ -0,0 +1,50 @@
+# coding: utf-8
+# Copyright: (c) 2019, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+from abc import ABCMeta, abstractmethod, abstractproperty
+
+
+class Command:
+    """
+    Subcommands of :program:`build-ansible`.
+
+    This defines an interface that all subcommands must conform to.  :program:`build-ansible` will
+    require that these things are present in order to proceed.
+    """
+    @staticmethod
+    @abstractproperty
+    def name():
+        """Name of the command.  The same as the string is invoked with"""
+
+    @staticmethod
+    @abstractmethod
+    def init_parser(add_parser):
+        """
+        Initialize and register an argparse ArgumentParser
+
+        :arg add_parser: function which creates an ArgumentParser for the main program.
+
+        Implementations should first create an ArgumentParser using `add_parser` and then populate
+        it with the command line arguments that are needed.
+
+        .. seealso:
+            `add_parser` information in the :py:meth:`ArgumentParser.add_subparsers` documentation.
+        """
+
+    @staticmethod
+    @abstractmethod
+    def main(arguments):
+        """
+        Run the command
+
+        :arg arguments: The **parsed** command line args
+
+        This is the Command's entrypoint.  The command line args are already parsed but from here
+        on, the command can do its work.
+        """