new file mode 100644

@@ -0,0 +1,10 @@

+Sanity Tests » azure-requirements

+=================================

+

+Update the Azure integration test requirements file when changes are made to the Azure packaging requirements file:

+

+.. code-block:: bash

+

+    cp packaging/requirements/requirements-azure.txt test/runner/requirements/integration.cloud.azure.txt

+

+This copy of the requirements file is used when building the ``ansible/ansible:default`` Docker container from ``test/runner/Dockerfile``.

new file mode 100644

deleted file mode 100644

@@ -1,8 +0,0 @@

-- name: install cryptography requirement

-  # preempt the installation of cryptography from requirements-azure.txt to limit the version installed

-  # requests[security] requires cryptography >= 1.3.4

-  # cryptography 2.1 requires pip 8.1.2+

-  command: pip install cryptography>=1.3.4,<2.1

-

-- pip:

-    requirements: '{{ role_path }}/../../../../packaging/requirements/requirements-azure.txt'

@@ -13,6 +13,7 @@ RUN apt-get update -y && \

     libxslt1-dev \

     locales \

     make \

+    openssh-client \

     python2.6-dev \

     python2.7-dev \

     python3.5-dev \

@@ -11,13 +11,60 @@ requirements=()

 for requirement in *.txt; do

     if [ "${requirement}" != "constraints.txt" ]; then

-        requirements+=("-r" "${requirement}")

+        requirements+=("${requirement}")

     fi

 done

 for python_version in "${python_versions[@]}"; do

+    version_requirements=()

+

+    for requirement in "${requirements[@]}"; do

+        case "${python_version}" in

+            "2.6")

+                case "${requirement}" in

+                    "integration.cloud.azure.txt") continue ;;

+                esac

+        esac

+

+        version_requirements+=("${requirement}")

+    done

+

+    echo "==> Installing pip for python ${python_version} ..."

+

     set -x

     "python${python_version}" /tmp/get-pip.py -c constraints.txt

-    "pip${python_version}" install --disable-pip-version-check -c constraints.txt "${requirements[@]}"

     set +x

+

+    echo "==> Installing requirements for python ${python_version} ..."

+

+    for requirement in "${version_requirements[@]}"; do

+        set -x

+        "pip${python_version}" install --disable-pip-version-check -c constraints.txt -r "${requirement}"

+        set +x

+    done

+

+    echo "==> Checking for requirements conflicts for ${python_version} ..."

+

+    after=$("pip${python_version}" list)

+

+    for requirement in "${version_requirements[@]}"; do

+        before="${after}"

+

+        set -x

+        "pip${python_version}" install --disable-pip-version-check -c constraints.txt -r "${requirement}"

+        set +x

+

+        after=$("pip${python_version}" list)

+

+        if [ "${before}" != "${after}" ]; then

+            echo "==> Conflicts detected in requirements for python ${python_version}: ${requirement}"

+            echo ">>> Before"

+            echo "${before}"

+            echo ">>> After"

+            echo "${after}"

+            exit 1

+        fi

+    done

+

+    echo "==> Finished with requirements for python ${python_version}."

 done

@@ -13,6 +13,7 @@ import textwrap

 import functools

 import shutil

 import stat

+import pipes

 import random

 import string

 import atexit

@@ -162,30 +163,78 @@ def install_command_requirements(args):

         if args.junit:

             packages.append('junit-xml')

-    extras = []

+    commands = [generate_pip_install(args.command, packages=packages)]

     if isinstance(args, IntegrationConfig):

-        extras += ['cloud.%s' % cp for cp in get_cloud_platforms(args)]

+        for cloud_platform in get_cloud_platforms(args):

+            commands.append(generate_pip_install('%s.cloud.%s' % (args.command, cloud_platform)))

-    cmd = generate_pip_install(args.command, packages, extras)

+    # only look for changes when more than one requirements file is needed

+    detect_pip_changes = len(commands) > 1

-    if not cmd:

-        return

+    # first pass to install requirements, changes expected unless environment is already set up

+    changes = run_pip_commands(args, commands, detect_pip_changes)

-    try:

-        run_command(args, cmd)

-    except SubprocessError as ex:

-        if ex.status != 2:

-            raise

+    if not changes:

+        return  # no changes means we can stop early

+

+    # second pass to check for conflicts in requirements, changes are not expected here

+    changes = run_pip_commands(args, commands, detect_pip_changes)

+

+    if not changes:

+        return  # no changes means no conflicts

+

+    raise ApplicationError('Conflicts detected in requirements. The following commands reported changes during verification:\n%s' %

+                           '\n'.join((' '.join(pipes.quote(c) for c in cmd) for cmd in changes)))

+

+

+def run_pip_commands(args, commands, detect_pip_changes=False):

+    """

+    :type args: EnvironmentConfig

+    :type commands: list[list[str]]

+    :type detect_pip_changes: bool

+    :rtype: list[list[str]]

+    """

+    changes = []

+

+    after_list = pip_list(args) if detect_pip_changes else None

+

+    for cmd in commands:

+        if not cmd:

+            continue

-        # If pip is too old it won't understand the arguments we passed in, so we'll need to upgrade it.

+        before_list = after_list

-        # Installing "coverage" on ubuntu 16.04 fails with the error:

-        # AttributeError: 'Requirement' object has no attribute 'project_name'

-        # See: https://bugs.launchpad.net/ubuntu/xenial/+source/python-pip/+bug/1626258

-        # Upgrading pip works around the issue.

-        run_command(args, ['pip', 'install', '--upgrade', 'pip'])

-        run_command(args, cmd)

+        try:

+            run_command(args, cmd)

+        except SubprocessError as ex:

+            if ex.status != 2:

+                raise

+

+            # If pip is too old it won't understand the arguments we passed in, so we'll need to upgrade it.

+

+            # Installing "coverage" on ubuntu 16.04 fails with the error:

+            # AttributeError: 'Requirement' object has no attribute 'project_name'

+            # See: https://bugs.launchpad.net/ubuntu/xenial/+source/python-pip/+bug/1626258

+            # Upgrading pip works around the issue.

+            run_command(args, ['pip', 'install', '--upgrade', 'pip'])

+            run_command(args, cmd)

+

+        after_list = pip_list(args) if detect_pip_changes else None

+

+        if before_list != after_list:

+            changes.append(cmd)

+

+    return changes

+

+

+def pip_list(args):

+    """

+    :type args: EnvironmentConfig

+    :rtype: str

+    """

+    stdout, _ = run_command(args, ['pip', 'list'], capture=True, always=True)

+    return stdout

 def generate_egg_info(args):

new file mode 100644

@@ -0,0 +1,16 @@

+packaging

+requests[security]

+azure-mgmt-compute>=2.0.0,<3

+azure-mgmt-network>=1.3.0,<2

+azure-mgmt-storage>=1.2.0,<2

+azure-mgmt-resource>=1.1.0,<2

+azure-storage>=0.35.1,<0.36

+azure-cli-core>=2.0.12,<3

+msrest!=0.4.15

+msrestazure>=0.4.11,<0.5

+azure-mgmt-dns>=1.0.1,<2

+azure-mgmt-keyvault>=0.40.0,<0.41

+azure-mgmt-batch>=4.1.0,<5

+azure-mgmt-sql>=0.7.1,<0.8

+azure-mgmt-web>=0.32.0,<0.33

+azure-mgmt-containerservice>=1.0.0

\ No newline at end of file

@@ -255,6 +255,8 @@ def parse_args():

                                      targets=walk_network_integration_targets,

                                      config=NetworkIntegrationConfig)

+    add_extra_docker_options(network_integration, integration=False)

+

     network_integration.add_argument('--platform',

                                      metavar='PLATFORM',

                                      action='append',

@@ -272,6 +274,8 @@ def parse_args():

                                      targets=walk_windows_integration_targets,

                                      config=WindowsIntegrationConfig)

+    add_extra_docker_options(windows_integration, integration=False)

+

     windows_integration.add_argument('--windows',

                                      metavar='VERSION',

                                      action='append',

new file mode 100755

@@ -0,0 +1,12 @@

+#!/usr/bin/env python

+"""Make sure the Azure requirements files match."""

+

+import filecmp

+

+src = 'packaging/requirements/requirements-azure.txt'

+dst = 'test/runner/requirements/integration.cloud.azure.txt'

+

+if not filecmp.cmp(src, dst):

+    print('Update the Azure integration test requirements with the packaging test requirements:')

+    print('cp %s %s' % (src, dst))

+    exit(1)

@@ -5,6 +5,7 @@ constraints=$(

         | grep -v '(sanity_ok)$' \

         | sed 's/ *;.*$//; s/ #.*$//' \

         | grep -v '/constraints.txt:' \

+        | grep -v '/integration.cloud.azure.txt:' \

         | grep '[<>=]'

 )