mysql_user: fix compatibility issues with various MySQL/MariaDB versions (#45355)

* mysql_user: fix MySQL/MariaDB version check

To handle properly user management, version check needed refacto, as well as the query used to get existing password hash

* mysql_user: break long query in multiple lines

* mysql_user: fix query fetch existing password hash

* mysql_user: MariaDB version check 100.2 != 10.2

* mysql_user: fix existing password fetch

In some cases, both columns (Password and authentication_string) may exist and be populated.
In other cases one exist, but not the second.
This fix should handle properly all situations

* mysql_user: break long queries

* mysql_user: refactor duplicated code

* mysql_user: handle updates from root with empty passwd to new passwd

* mysql_user: GC debug statement and readd trailing new line

* mysql_user: fix pep8 under indentation

* mysql_user: fix privileges management
https://github.com/ansible/ansible/pull/45355#issuecomment-428200244

* mysql_user: raise exception if exception caught doesn't match the one that is managed

* mysql_user: improve plugins output (add msg field with explicit informations)

* mysql_user: fix old / new password hash comparison

* mysql_user: fix reference to old MySQLdb lib

* mysql_user: fix cursor when root password is left empty (mysql DB invisible)

* mysql_user: add changelog

* ALL privileges comparison

* fixed blank line

* added mysql 8 fixes

* fixed version compatibility

* mysql_user: fix MySQL/MariaDB version check

To handle properly user management, version check needed refacto, as well as the query used to get existing password hash

* mysql_user: break long query in multiple lines

* mysql_user: fix query fetch existing password hash

* mysql_user: MariaDB version check 100.2 != 10.2

* mysql_user: fix existing password fetch

In some cases, both columns (Password and authentication_string) may exist and be populated.
In other cases one exist, but not the second.
This fix should handle properly all situations

* mysql_user: break long queries

* mysql_user: refactor duplicated code

* mysql_user: handle updates from root with empty passwd to new passwd

* mysql_user: GC debug statement and readd trailing new line

* mysql_user: fix pep8 under indentation

* mysql_user: fix privileges management
https://github.com/ansible/ansible/pull/45355#issuecomment-428200244

* mysql_user: raise exception if exception caught doesn't match the one that is managed

* mysql_user: improve plugins output (add msg field with explicit informations)

* mysql_user: fix old / new password hash comparison

* mysql_user: fix reference to old MySQLdb lib

* mysql_user: fix cursor when root password is left empty (mysql DB invisible)

* mysql_user: add contrib

* Rename changelogs/fragments/45355-mysql_user-fix-versions-compatibilities to add YML extension

changelogs/fragments/45355-mysql_user-fix-versions-compatibilities.yml

@@ -0,0 +1,2 @@
+bugfixes:
+- "mysql_user: fix compatibility issues with various MySQL/MariaDB versions"

lib/ansible/modules/database/mysql/mysql_user.py

@@ -105,6 +105,7 @@ notes:
 
 author:
 - Jonathan Mainguy (@Jmainguy)
+- Benjamin Malynovytch (@bmalynovytch)
 extends_documentation_fragment: mysql
 '''
 
@@ -239,22 +240,25 @@ class InvalidPrivsError(Exception):
 #
 
 
-# User Authentication Management was change in MySQL 5.7
+# User Authentication Management changed in MySQL 5.7 and MariaDB 10.2.0
-# This is a generic check for if the server version is less than version 5.7
+def use_old_user_mgmt(cursor):
-def server_version_check(cursor):
     cursor.execute("SELECT VERSION()")
     result = cursor.fetchone()
     version_str = result[0]
     version = version_str.split('.')
 
-    # Currently we have no facility to handle new-style password update on
-    # mariadb and the old-style update continues to work
     if 'mariadb' in version_str.lower():
-        return True
+        # Prior to MariaDB 10.2
-    if int(version[0]) <= 5 and int(version[1]) < 7:
+        if int(version[0]) * 1000 + int(version[1]) < 10002:
-        return True
+            return True
+        else:
+            return False
     else:
-        return False
+        # Prior to MySQL 5.7
+        if int(version[0]) * 1000 + int(version[1]) < 5007:
+            return True
+        else:
+            return False
 
 
 def get_mode(cursor):
@@ -270,9 +274,9 @@ def get_mode(cursor):
 
 def user_exists(cursor, user, host, host_all):
     if host_all:
-        cursor.execute("SELECT count(*) FROM user WHERE user = %s", ([user]))
+        cursor.execute("SELECT count(*) FROM mysql.user WHERE user = %s", ([user]))
     else:
-        cursor.execute("SELECT count(*) FROM user WHERE user = %s AND host = %s", (user, host))
+        cursor.execute("SELECT count(*) FROM mysql.user WHERE user = %s AND host = %s", (user, host))
 
     count = cursor.fetchone()
     return count[0] > 0
@@ -308,6 +312,7 @@ def is_hash(password):
 
 def user_mod(cursor, user, host, host_all, password, encrypted, new_priv, append_privs, module):
     changed = False
+    msg = "User unchanged"
     grant_option = False
 
     if host_all:
@@ -319,41 +324,68 @@ def user_mod(cursor, user, host, host_all, password, encrypted, new_priv, append
         # Handle clear text and hashed passwords.
         if bool(password):
             # Determine what user management method server uses
-            old_user_mgmt = server_version_check(cursor)
+            old_user_mgmt = use_old_user_mgmt(cursor)
 
-            if old_user_mgmt:
+            # Get a list of valid columns in mysql.user table to check if Password and/or authentication_string exist
-                cursor.execute("SELECT password FROM user WHERE user = %s AND host = %s", (user, host))
+            cursor.execute("""
-            else:
+                SELECT COLUMN_NAME FROM information_schema.COLUMNS
-                cursor.execute("SELECT authentication_string FROM user WHERE user = %s AND host = %s", (user, host))
+                WHERE TABLE_SCHEMA = 'mysql' AND TABLE_NAME = 'user' AND COLUMN_NAME IN ('Password', 'authentication_string')
-            current_pass_hash = cursor.fetchone()
+                ORDER BY COLUMN_NAME DESC LIMIT 1
+            """)
+            colA = cursor.fetchone()
+
+            cursor.execute("""
+                SELECT COLUMN_NAME FROM information_schema.COLUMNS
+                WHERE TABLE_SCHEMA = 'mysql' AND TABLE_NAME = 'user' AND COLUMN_NAME IN ('Password', 'authentication_string')
+                ORDER BY COLUMN_NAME ASC  LIMIT 1
+            """)
+            colB = cursor.fetchone()
+
+            # Select hash from either Password or authentication_string, depending which one exists and/or is filled
+            cursor.execute("""
+                SELECT COALESCE(
+                        CASE WHEN %s = '' THEN NULL ELSE %s END,
+                        CASE WHEN %s = '' THEN NULL ELSE %s END
+                    )
+                FROM mysql.user WHERE user = %%s AND host = %%s
+                """ % (colA[0], colA[0], colB[0], colB[0]), (user, host))
+            current_pass_hash = cursor.fetchone()[0]
 
             if encrypted:
-                encrypted_string = (password)
+                encrypted_password = password
-                if is_hash(password):
+                if not is_hash(encrypted_password):
-                    if current_pass_hash[0] != encrypted_string:
-                        if module.check_mode:
-                            return True
-                        if old_user_mgmt:
-                            cursor.execute("SET PASSWORD FOR %s@%s = %s", (user, host, password))
-                        else:
-                            cursor.execute("ALTER USER %s@%s IDENTIFIED WITH mysql_native_password AS %s", (user, host, password))
-                        changed = True
-                else:
                     module.fail_json(msg="encrypted was specified however it does not appear to be a valid hash expecting: *SHA1(SHA1(your_password))")
             else:
                 if old_user_mgmt:
                     cursor.execute("SELECT PASSWORD(%s)", (password,))
                 else:
                     cursor.execute("SELECT CONCAT('*', UCASE(SHA1(UNHEX(SHA1(%s)))))", (password,))
-                new_pass_hash = cursor.fetchone()
+                encrypted_password = cursor.fetchone()[0]
-                if current_pass_hash[0] != new_pass_hash[0]:
+
-                    if module.check_mode:
+            if current_pass_hash != encrypted_password:
-                        return True
+                msg = "Password updated"
-                    if old_user_mgmt:
+                if module.check_mode:
-                        cursor.execute("SET PASSWORD FOR %s@%s = PASSWORD(%s)", (user, host, password))
+                    return (True, msg)
-                    else:
+                if old_user_mgmt:
-                        cursor.execute("ALTER USER %s@%s IDENTIFIED WITH mysql_native_password BY %s", (user, host, password))
+                    cursor.execute("SET PASSWORD FOR %s@%s = %s", (user, host, encrypted_password))
-                    changed = True
+                    msg = "Password updated (old style)"
+                else:
+                    try:
+                        cursor.execute("ALTER USER %s@%s IDENTIFIED WITH mysql_native_password AS %s", (user, host, encrypted_password))
+                        msg = "Password updated (new style)"
+                    except (mysql_driver.Error) as e:
+                        # https://stackoverflow.com/questions/51600000/authentication-string-of-root-user-on-mysql
+                        # Replacing empty root password with new authentication mechanisms fails with error 1396
+                        if e.args[0] == 1396:
+                            cursor.execute(
+                                "UPDATE user SET plugin = %s, authentication_string = %s, Password = '' WHERE User = %s AND Host = %s",
+                                ('mysql_native_password', encrypted_password, user, host)
+                            )
+                            cursor.execute("FLUSH PRIVILEGES")
+                            msg = "Password forced update"
+                        else:
+                            raise e
+                changed = True
 
         # Handle privileges
         if new_priv is not None:
@@ -367,8 +399,9 @@ def user_mod(cursor, user, host, host_all, password, encrypted, new_priv, append
                     grant_option = True
                 if db_table not in new_priv:
                     if user != "root" and "PROXY" not in priv and not append_privs:
+                        msg = "Privileges updated"
                         if module.check_mode:
-                            return True
+                            return (True, msg)
                         privileges_revoke(cursor, user, host, db_table, priv, grant_option)
                         changed = True
 
@@ -376,8 +409,9 @@ def user_mod(cursor, user, host, host_all, password, encrypted, new_priv, append
             # we can perform a straight grant operation.
             for db_table, priv in iteritems(new_priv):
                 if db_table not in curr_priv:
+                    msg = "New privileges granted"
                     if module.check_mode:
-                        return True
+                        return (True, msg)
                     privileges_grant(cursor, user, host, db_table, priv)
                     changed = True
 
@@ -387,14 +421,15 @@ def user_mod(cursor, user, host, host_all, password, encrypted, new_priv, append
             for db_table in db_table_intersect:
                 priv_diff = set(new_priv[db_table]) ^ set(curr_priv[db_table])
                 if len(priv_diff) > 0:
+                    msg = "Privileges updated"
                     if module.check_mode:
-                        return True
+                        return (True, msg)
                     if not append_privs:
                         privileges_revoke(cursor, user, host, db_table, curr_priv[db_table], grant_option)
                     privileges_grant(cursor, user, host, db_table, new_priv[db_table])
                     changed = True
 
-    return changed
+    return (changed, msg)
 
 
 def user_delete(cursor, user, host, host_all, check_mode):
@@ -444,7 +479,7 @@ def privileges_get(cursor, user, host):
             return x
 
     for grant in grants:
-        res = re.match("""GRANT (.+) ON (.+) TO (['`"]).*\\3@(['`"]).*\\4( IDENTIFIED BY PASSWORD (['`"]).+\\6)? ?(.*)""", grant[0])
+        res = re.match("""GRANT (.+) ON (.+) TO (['`"]).*\\3@(['`"]).*\\4( IDENTIFIED BY PASSWORD (['`"]).+\5)? ?(.*)""", grant[0])
         if res is None:
             raise InvalidPrivsError('unable to parse the MySQL grant string: %s' % grant[0])
         privileges = res.group(1).split(", ")
@@ -593,7 +628,7 @@ def main():
     ssl_cert = module.params["client_cert"]
     ssl_key = module.params["client_key"]
     ssl_ca = module.params["ca_cert"]
-    db = 'mysql'
+    db = ''
     sql_log_bin = module.params["sql_log_bin"]
 
     if mysql_driver is None:
@@ -632,9 +667,9 @@ def main():
         if user_exists(cursor, user, host, host_all):
             try:
                 if update_password == 'always':
-                    changed = user_mod(cursor, user, host, host_all, password, encrypted, priv, append_privs, module)
+                    changed, msg = user_mod(cursor, user, host, host_all, password, encrypted, priv, append_privs, module)
                 else:
-                    changed = user_mod(cursor, user, host, host_all, None, encrypted, priv, append_privs, module)
+                    changed, msg = user_mod(cursor, user, host, host_all, None, encrypted, priv, append_privs, module)
 
             except (SQLParseError, InvalidPrivsError, mysql_driver.Error) as e:
                 module.fail_json(msg=to_native(e))
@@ -643,14 +678,19 @@ def main():
                 module.fail_json(msg="host_all parameter cannot be used when adding a user")
             try:
                 changed = user_add(cursor, user, host, host_all, password, encrypted, priv, module.check_mode)
+                if changed:
+                    msg = "User added"
+
             except (SQLParseError, InvalidPrivsError, mysql_driver.Error) as e:
                 module.fail_json(msg=to_native(e))
     elif state == "absent":
         if user_exists(cursor, user, host, host_all):
             changed = user_delete(cursor, user, host, host_all, module.check_mode)
+            msg = "User deleted"
         else:
             changed = False
-    module.exit_json(changed=changed, user=user)
+            msg = "User doesn't exist"
+    module.exit_json(changed=changed, user=user, msg=msg)
 
 
 if __name__ == '__main__':