Backport/stable 2/pr 452 (#463)

* Sync GHA workflow w/ the collection template (#452)

* Sync GHA workflow w/ the collection template

* Drop the trailing pre-cmd semicolon

* Recover missing `-e` flag of `sed`

* Use relative paths for version configs

* Unquote `env.connector_version_file`

* Use string formatting to fix the substitution problem

(cherry picked from commit 8107530744)

* Backport mysql_version_parts variable assignation

(cherry picked from commit 79046a88cb)

* Add changelog fragment

* Backport flags and variables to differentiate MariaDB from MySQL setup

* Backport issue-28 check for tls support

* Backport tls_requirements simplified and deduplified tests

Co-authored-by: Sviatoslav Sydorenko <wk.cvs.github@sydorenko.org.ua>

.github/workflows/ansible-test-plugins.yml

@@ -15,12 +15,12 @@ on:
 
 
 env:
-  mysql_version_file: "./ansible_collections/community/mysql/tests/integration/targets/setup_mysql/defaults/main.yml"
+  mysql_version_file: "tests/integration/targets/setup_mysql/defaults/main.yml"
-  connector_version_file: "./ansible_collections/community/mysql/tests/integration/targets/setup_mysql/vars/main.yml"
+  connector_version_file: "tests/integration/targets/setup_mysql/vars/main.yml"
 
 jobs:
   sanity:
-    name: "Sanity (Python: ${{ matrix.python }}, Ansible: ${{ matrix.ansible }})"
+    name: "Sanity (Ansible: ${{ matrix.ansible }})"
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -30,48 +30,22 @@ jobs:
           - stable-2.13
           - stable-2.14
           - devel
-        python:
-          - 3.8
-          - 3.9
-        exclude:
-          - python: 3.8
-            ansible: stable-2.13
-          - python: 3.8
-            ansible: stable-2.14
-          - python: 3.8
-            ansible: devel
-          - python: 3.9
-            ansible: stable-2.11
-          - python: 3.9
-            ansible: stable-2.12
     steps:
-
+      - name: Perform sanity testing
-      - name: Check out code
+        uses: ansible-community/ansible-test-gh-action@release/v1
-        uses: actions/checkout@v2
         with:
-          path: ansible_collections/community/mysql
+          ansible-core-version: ${{ matrix.ansible }}
-
+          testing-type: sanity
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python }}
-
-      - name: Install ansible-base (${{ matrix.ansible }})
-        run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check
-
-      - name: Run sanity tests
-        run: ansible-test sanity --docker -v --color
-        working-directory: ./ansible_collections/community/mysql
 
   integration:
-    name: "Integration (Python: ${{ matrix.python }}, Ansible: ${{ matrix.ansible }}, MySQL: ${{ matrix.mysql }}, Connector: ${{ matrix.connector }})"
+    name: "Integration (Python: ${{ matrix.python }}, Ansible: ${{ matrix.ansible }}, MySQL: ${{ matrix.db_engine_version }}, Connector: ${{ matrix.connector }})"
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        mysql:
+        db_engine_version:
-          - 5.7.31
+          - mysql_5.7.31
-          - 8.0.22
+          - mysql_8.0.22
         ansible:
           - stable-2.11
           - stable-2.12
@@ -87,7 +61,7 @@ jobs:
           - pymysql==0.9.3
           - mysqlclient==2.0.1
         exclude:
-          - mysql: 8.0.22
+          - db_engine_version: mysql_8.0.22
             connector: pymysql==0.7.10
           - python: 3.6
             ansible: stable-2.12
@@ -111,37 +85,31 @@ jobs:
             ansible: stable-2.12
 
     steps:
-
+      - name: >-
-      - name: Check out code
+          Perform integration testing against
-        uses: actions/checkout@v2
+          Ansible version ${{ matrix.ansible }}
+          under Python ${{ matrix.python }}
+        uses: ansible-community/ansible-test-gh-action@release/v1
         with:
-          path: ansible_collections/community/mysql
+          ansible-core-version: ${{ matrix.ansible }}
-
+          pre-test-cmd: >-
-      - name: Set up Python
+            DB_ENGINE=$(echo '${{ matrix.db_engine_version }}' | awk -F_ '{print $1}');
-        uses: actions/setup-python@v2
+            DB_VERSION=$(echo '${{ matrix.db_engine_version }}' | awk -F_ '{print $2}');
-        with:
+            DB_ENGINE_PRETTY=$([[ "${DB_ENGINE}" == 'mysql' ]] && echo 'MySQL' || echo 'MariaDB');
-          python-version: ${{ matrix.python }}
+            >&2 echo Matrix factor for the DB is ${{ matrix.db_engine_version }}...;
-
+            >&2 echo Setting ${DB_ENGINE_PRETTY} version to ${DB_VERSION}...;
-      - name: Install ansible-base (${{ matrix.ansible }})
+            sed -i -e "s/^${DB_ENGINE}_version:.*/${DB_ENGINE}_version: $DB_VERSION/g" -e 's/^mariadb_install: false/mariadb_install: true/g' '${{ env.mysql_version_file }}';
-        run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check
+            ${{
-
+              matrix.db_engine_version == 'mariadb_10.8.3'
-      - name: Set MySQL version (${{ matrix.mysql }})
+              && format(
-        run: "sed -i 's/^mysql_version:.*/mysql_version: \"${{ matrix.mysql }}\"/g' ${{ env.mysql_version_file }}"
+                '>&2 echo Set MariaDB v10.8.3 URL sub dir...; sed -i -e "s/^mariadb_url_subdir:.*/mariadb_url_subdir: linux-systemd/g" "{0}";', env.connector_version_file
-
+              )
-      - name: Set Connector version (${{ matrix.connector }})
+              || ''
-        run: "sed -i 's/^python_packages:.*/python_packages: [${{ matrix.connector }}]/' ${{ env.connector_version_file }}"
+            }}
-
+            >&2 echo Setting Connector version to ${{ matrix.connector }}...;
-      - name: Run integration tests
+            sed -i 's/^python_packages:.*/python_packages: [${{ matrix.connector }}]/' ${{ env.connector_version_file }}
-        run: ansible-test integration --docker -v --color --retry-on-error --continue-on-error --python ${{ matrix.python }} --diff --coverage
+          target-python-version: ${{ matrix.python }}
-        working-directory: ./ansible_collections/community/mysql
+          testing-type: integration
-
-      - name: Generate coverage report.
-        run: ansible-test coverage xml -v --requirements --group-by command --group-by version
-        working-directory: ./ansible_collections/community/mysql
-
-      - uses: codecov/codecov-action@v1
-        with:
-          fail_ci_if_error: false
 
   units:
     runs-on: ubuntu-latest
@@ -173,30 +141,11 @@ jobs:
             ansible: stable-2.12
 
     steps:
-      - name: Check out code
+      - name: >-
-        uses: actions/checkout@v2
+          Perform unit testing against
+          Ansible version ${{ matrix.ansible }}
+        uses: ansible-community/ansible-test-gh-action@release/v1
         with:
-          path: ./ansible_collections/community/mysql
+          ansible-core-version: ${{ matrix.ansible }}
-
+          target-python-version: ${{ matrix.python }}
-      - name: Set up Python
+          testing-type: units
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python }}
-
-      - name: Install ansible-base (${{matrix.ansible}})
-        run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check
-
-      # Run the unit tests
-      - name: Run unit test
-        run: ansible-test units -v --color --docker --coverage
-        working-directory: ./ansible_collections/community/mysql
-
-      # ansible-test support producing code coverage date
-      - name: Generate coverage report
-        run: ansible-test coverage xml -v --requirements --group-by command --group-by version
-        working-directory: ./ansible_collections/community/mysql
-
-      # See the reports at https://codecov.io/gh/GITHUBORG/REPONAME
-      - uses: codecov/codecov-action@v1
-        with:
-          fail_ci_if_error: false

changelogs/fragments/backport-stable-2-pr-452.yml

@@ -0,0 +1,5 @@
+---
+
+trivial:
+
+  - Tests - Backport PR-452 Sync GHA workflow w/ the collection template

tests/integration/targets/setup_mysql/defaults/main.yml

@@ -3,11 +3,16 @@ dbdeployer_home_dir: /opt/dbdeployer
 
 home_dir: /root
 
-percona_client_version: 5.7
-
 mariadb_install: false
 
 mysql_version: 8.0.22
 mariadb_version: 10.5.4
 
 mysql_base_port: 3306
+
+percona_client_package: >-
+  {%- if mariadb_install -%}
+    mariadb-client
+  {%- else -%}
+    percona-server-client-5.7
+  {%- endif -%}

tests/integration/targets/setup_mysql/tasks/config.yml

@@ -6,10 +6,10 @@
 
 - name: "{{ role_name }} | config | run unpack tarball"
   shell:
-    cmd: "dbdeployer unpack {{ dbdeployer_sandbox_download_dir }}/{{ install_tarball }}"
+    cmd: "dbdeployer unpack {{ dbdeployer_sandbox_download_dir }}/{{ install_tarball }} --flavor {{ install_type }}"
     creates: "{{ dbdeployer_sandbox_binary_dir }}/{{ install_version }}"
 
 - name: "{{ role_name }} | config | setup replication topology"
   shell:
-    cmd: "dbdeployer deploy multiple {{ install_version }} --base-port {{ mysql_base_port }} --my-cnf-options=\"master_info_repository='TABLE'\" --my-cnf-options=\"relay_log_info_repository='TABLE'\""
+    cmd: "dbdeployer deploy multiple {{ install_version }} --flavor {{ install_type }} --base-port {{ mysql_base_port }} --my-cnf-options=\"master_info_repository='TABLE'\" --my-cnf-options=\"relay_log_info_repository='TABLE'\""
     creates: "{{ dbdeployer_sandbox_home_dir }}/multi_msb_{{ install_version|replace('.','_') }}"

tests/integration/targets/setup_mysql/tasks/install.yml

@@ -4,12 +4,26 @@
     keyserver: keyserver.ubuntu.com
     id: 4D1BB29D63D98E422B2113B19334A25F8507EFA5
     state: present
+  when: install_type == 'mysql'
 
 - name: "{{ role_name }} | install | add percona repositories"
   apt_repository:
-    repo: "{{ item }}"
+    repo: deb http://repo.percona.com/percona/apt {{ ansible_lsb.codename }} main
     state: present
-  loop: "{{ percona_mysql_repos }}"
+  when: install_type == 'mysql'
+
+- name: "{{ role_name }} | install | add apt signing key for mariadb"
+  apt_key:
+    keyserver: keyserver.ubuntu.com
+    id: F1656F24C74CD1D8
+    state: present
+  when: install_type == 'mariadb'
+
+- name: "{{ role_name }} | install | add mariadb repositories"
+  apt_repository:
+    repo: "deb [arch=amd64,arm64] https://downloads.mariadb.com/MariaDB/mariadb-{{ mysql_major_version }}/repo/ubuntu {{ ansible_lsb.codename }} main"
+    state: present
+  when: install_type == 'mariadb'
 
 - name: "{{ role_name }} | install | install packages required by percona"
   apt:

tests/integration/targets/setup_mysql/tasks/setvars.yml

@@ -1,7 +1,12 @@
 ---
 - name: "{{ role_name }} | setvars | split mysql version in parts"
   set_fact:
-    mysql_version_parts: "{{ mysql_version.split('.') }}"
+    mysql_version_parts: >-
+      {%- if mariadb_install -%}
+      {{ mariadb_version.split('.') }}
+      {%- else -%}
+      {{ mysql_version.split('.') }}
+      {%- endif -%}
 
 - name: "{{ role_name }} | setvars | get mysql major version"
   set_fact:

tests/integration/targets/setup_mysql/vars/main.yml

@@ -7,12 +7,8 @@ dbdeployer_sandbox_download_dir: "{{ home_dir }}/downloads"
 dbdeployer_sandbox_binary_dir: "{{ home_dir }}/opt/mysql"
 dbdeployer_sandbox_home_dir: "{{ home_dir }}/sandboxes"
 
-percona_mysql_repos:
-  - deb http://repo.percona.com/apt {{ ansible_lsb.codename }} main
-  - deb-src http://repo.percona.com/apt {{ ansible_lsb.codename }} main
-
 percona_mysql_packages:
-  - percona-server-client-{{ percona_client_version }}
+  - "{{ percona_client_package }}"
 
 python_packages: [pymysql == 0.9.3]
 
@@ -23,10 +19,12 @@ install_prereqs:
 
 install_python_prereqs:
   - python3-dev
+  - python3-cryptography
   - default-libmysqlclient-dev
   - build-essential
 
 mysql_tarball: "mysql-{{ mysql_version }}-linux-glibc2.12-x86_64.tar.{{ mysql_compression_extension }}"
 mysql_src: "https://dev.mysql.com/get/Downloads/MySQL-{{ mysql_major_version }}/{{ mysql_tarball }}"
-mariadb_tarball: "mariadb-{{ mariadb_version }}-linux-x86_64.tar.gz"
+mariadb_url_subdir: "linux"
-mariadb_src: "https://downloads.mariadb.com/MariaDB/mariadb-{{ mariadb_version }}/bintar-linux-x86_64/{{ mariadb_tarball }}"
+mariadb_tarball: "mariadb-{{ mariadb_version }}-{{ mariadb_url_subdir }}-x86_64.tar.gz"
+mariadb_src: "https://downloads.mariadb.com/MariaDB/mariadb-{{ mariadb_version }}/bintar-{{ mariadb_url_subdir }}-x86_64/{{ mariadb_tarball }}"

tests/integration/targets/test_mysql_db/tasks/issue-28.yml

@@ -1,11 +1,17 @@
 ---
+- name: set fact tls_enabled
+  command: "{{ mysql_command }} \"-e SHOW VARIABLES LIKE 'have_ssl';\""
+  register: result
+- set_fact:
+    tls_enabled: "{{ 'YES' in result.stdout | bool | default('false', true) }}"
+
 - vars:
     mysql_parameters: &mysql_params
       login_user: '{{ mysql_user }}'
       login_password: '{{ mysql_password }}'
       login_host: 127.0.0.1
       login_port: '{{ mysql_primary_port }}'
-
+  when: tls_enabled
   block:
 
     # ============================================================

tests/integration/targets/test_mysql_info/tasks/issue-28.yml

@@ -1,11 +1,21 @@
 ---
+- name: alias mysql command to include default options
+  set_fact:
+    mysql_command: "mysql -u{{ mysql_user }} -p{{ mysql_password }} -P{{ mysql_primary_port }} --protocol=tcp"
+
+- name: set fact tls_enabled
+  command: "{{ mysql_command }} \"-e SHOW VARIABLES LIKE 'have_ssl';\""
+  register: result
+- set_fact:
+    tls_enabled: "{{ 'YES' in result.stdout | bool | default('false', true) }}"
+
 - vars:
     mysql_parameters: &mysql_params
       login_user: '{{ mysql_user }}'
       login_password: '{{ mysql_password }}'
       login_host: 127.0.0.1
       login_port: '{{ mysql_primary_port }}'
-
+  when: tls_enabled
   block:
 
     # ============================================================

tests/integration/targets/test_mysql_query/tasks/issue-28.yml

@@ -1,11 +1,21 @@
 ---
+- name: alias mysql command to include default options
+  set_fact:
+    mysql_command: "mysql -u{{ mysql_user }} -p{{ mysql_password }} -P{{ mysql_primary_port }} --protocol=tcp"
+
+- name: set fact tls_enabled
+  command: "{{ mysql_command }} \"-e SHOW VARIABLES LIKE 'have_ssl';\""
+  register: result
+- set_fact:
+    tls_enabled: "{{ 'YES' in result.stdout | bool | default('false', true) }}"
+
 - vars:
     mysql_parameters: &mysql_params
       login_user: '{{ mysql_user }}'
       login_password: '{{ mysql_password }}'
       login_host: 127.0.0.1
       login_port: '{{ mysql_primary_port }}'
-
+  when: tls_enabled
   block:
 
     # ============================================================

tests/integration/targets/test_mysql_replication/tasks/issue-28.yml

@@ -1,11 +1,21 @@
 ---
+- name: alias mysql command to include default options
+  set_fact:
+    mysql_command: "mysql -u{{ mysql_user }} -p{{ mysql_password }} -P{{ mysql_primary_port }} --protocol=tcp"
+
+- name: set fact tls_enabled
+  command: "{{ mysql_command }} \"-e SHOW VARIABLES LIKE 'have_ssl';\""
+  register: result
+- set_fact:
+    tls_enabled: "{{ 'YES' in result.stdout | bool | default('false', true) }}"
+
 - vars:
     mysql_parameters: &mysql_params
       login_user: '{{ mysql_user }}'
       login_password: '{{ mysql_password }}'
       login_host: 127.0.0.1
       login_port: '{{ mysql_primary_port }}'
-
+  when: tls_enabled
   block:
 
     # ============================================================

tests/integration/targets/test_mysql_replication/tasks/main.yml

@@ -14,6 +14,8 @@
 
 # Tests of channel parameter:
 - import_tasks: mysql_replication_channel.yml
+  when:
+    - install_type == 'mysql' # FIXME: mariadb introduces FOR CHANNEL in 10.7
 
 # Tests of resetprimary mode:
 - import_tasks: mysql_replication_resetprimary_mode.yml

tests/integration/targets/test_mysql_replication/tasks/mysql_replication_initial.yml

@@ -24,14 +24,20 @@
         mysql8022_and_higher: true
       when:
       - db.version.major > 8 or (db.version.major == 8 and db.version.minor > 0) or (db.version.major == 8 and db.version.minor == 0 and db.version.release >= 22)
+      - install_type == 'mysql'
 
     - name: alias mysql command to include default options
       set_fact:
         mysql_command: "mysql -u{{ mysql_user }} -p{{ mysql_password }} --protocol=tcp"
 
     # Preparation:
-    - name: Create user for replication
+    - name: Create user for mysql replication
       shell: "echo \"CREATE USER '{{ replication_user }}'@'localhost' IDENTIFIED WITH mysql_native_password BY '{{ replication_pass }}'; GRANT REPLICATION SLAVE ON *.* TO '{{ replication_user }}'@'localhost';\" | {{ mysql_command }} -P{{ mysql_primary_port }}"
+      when: install_type == 'mysql'
+
+    - name: Create user for mariadb replication
+      shell: "echo \"CREATE USER '{{ replication_user }}'@'localhost' IDENTIFIED BY '{{ replication_pass }}'; GRANT REPLICATION SLAVE ON *.* TO '{{ replication_user }}'@'localhost';\" | {{ mysql_command }} -P{{ mysql_primary_port }}"
+      when: install_type == 'mariadb'
 
     - name: Create test database
       mysql_db:

tests/integration/targets/test_mysql_user/tasks/issue-28.yml

@@ -1,11 +1,17 @@
 ---
+- name: set fact tls_enabled
+  command: "{{ mysql_command }} \"-e SHOW VARIABLES LIKE 'have_ssl';\""
+  register: result
+- set_fact:
+    tls_enabled: "{{ 'YES' in result.stdout | bool | default('false', true) }}"
+
 - vars:
     mysql_parameters: &mysql_params
       login_user: '{{ mysql_user }}'
       login_password: '{{ mysql_password }}'
       login_host: 127.0.0.1
       login_port: '{{ mysql_primary_port }}'
-
+  when: tls_enabled
   block:
 
     # ============================================================

tests/integration/targets/test_mysql_user/tasks/tls_requirements.yml

@@ -166,21 +166,13 @@
         password: '{{ user_password_1 }}'
         tls_requires:
 
-    - name: retrieve TLS requirements for users in old database version
+    - name: retrieve TLS requirements for users
-      command: "{{ mysql_command }} -L -N -s -e \"SHOW GRANTS for '{{ user_name_1 }}'@'localhost'\""
-      register: old_result
-      when: db_version.version.major <= 5 and db_version.version.minor <= 6 or db_version.version.major == 10 and db_version.version.minor < 2
-
-    - name: retrieve TLS requirements for users in new database version
       command: "{{ mysql_command }} -L -N -s -e \"SHOW CREATE USER '{{ user_name_1 }}'@'localhost'\""
-      register: new_result
+      register: result
-      when: db_version.version.major == 5 and db_version.version.minor >= 7 or db_version.version.major > 5 and db_version.version.major < 10  or db_version.version.major == 10 and db_version.version.minor >= 2
 
     - name: assert user1 TLS requirements
       assert:
-        that: "'NONE' in reqs"
+        that: "'REQUIRE ' not in result.stdout or 'REQUIRE NONE' in result.stdout"
-      vars:
-        - reqs: "{{(old_result is skipped | ternary(new_result, old_result)).stdout.split('REQUIRE')[1].split(separator)[0].strip() | default('NONE') }}"
 
     - include: remove_user.yml user_name={{user_name_1}} user_password={{ user_password_1 }}
 

tests/integration/targets/test_mysql_variables/tasks/issue-28.yml

@@ -1,11 +1,17 @@
 ---
+- name: set fact tls_enabled
+  command: "{{ mysql_command }} \"-e SHOW VARIABLES LIKE 'have_ssl';\""
+  register: result
+- set_fact:
+    tls_enabled: "{{ 'YES' in result.stdout | bool | default('false', true) }}"
+
 - vars:
     mysql_parameters: &mysql_params
       login_user: '{{ mysql_user }}'
       login_password: '{{ mysql_password }}'
       login_host: 127.0.0.1
       login_port: '{{ mysql_primary_port }}'
-
+  when: tls_enabled
   block:
 
     # ============================================================