python/samba: Ensure md5 always provided with bytes

To allow code run in both python3 and python2 we have to ensure
that md5 always receives bytes

Signed-off-by: Noel Power <noel.power@suse.com>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>

3 files changed, 13 insertions, 1 deletions

diff --git a/python/samba/netcmd/visualize.py b/python/samba/netcmd/visualize.py
index 6f880ae61e8..576f48d41ef 100644
--- a/python/samba/netcmd/visualize.py
+++ b/python/samba/netcmd/visualize.py
@@ -34,6 +34,7 @@ from ldb import SCOPE_BASE, SCOPE_SUBTREE, LdbError
 import time
 from samba.kcc import KCC
 from samba.kcc.kcc_utils import KCCError
+from samba.compat import text_type
 
 COMMON_OPTIONS = [
     Option("-H", "--URL", help="LDB URL for database or target server",
@@ -161,7 +162,10 @@ def colour_hash(x):
     """Generate a randomish but consistent darkish colour based on the
     given object."""
     from hashlib import md5
-    c = int(md5(str(x)).hexdigest()[:6], base=16) & 0x7f7f7f
+    tmp_str = str(x)
+    if isinstance(tmp_str, text_type):
+        tmp_str = tmp_str.encode('utf8')
+    c = int(md5(tmp_str).hexdigest()[:6], base=16) & 0x7f7f7f
     return '#%06x' % c
 
 
diff --git a/python/samba/tests/password_hash.py b/python/samba/tests/password_hash.py
index 4c4dbcf2fd7..745ac704c9f 100644
--- a/python/samba/tests/password_hash.py
+++ b/python/samba/tests/password_hash.py
@@ -35,6 +35,7 @@ import samba
 import binascii
 from hashlib import md5
 import crypt
+from samba.compat import text_type
 
 
 USER_NAME = "PasswordHashTestUser"
@@ -61,6 +62,9 @@ def get_package(sc, name):
 def calc_digest(user, realm, password):
 
     data = "%s:%s:%s" % (user, realm, password)
+    if isinstance(data, text_type):
+        data = data.encode('utf8')
+
     return md5(data).hexdigest()
 
 
diff --git a/python/samba/tests/samba_tool/user_wdigest.py b/python/samba/tests/samba_tool/user_wdigest.py
index 35283ebfcb3..eddb79f4d18 100644
--- a/python/samba/tests/samba_tool/user_wdigest.py
+++ b/python/samba/tests/samba_tool/user_wdigest.py
@@ -33,6 +33,7 @@ from samba.dcerpc import drsblobs
 from hashlib import md5
 import random
 import string
+from samba.compat import text_type
 
 USER_NAME = "WdigestTestUser"
 # Create a random 32 character password, containing only letters and
@@ -45,6 +46,9 @@ USER_PASS = ''.join(random.choice(string.ascii_uppercase +
 #
 def calc_digest(user, realm, password):
     data = "%s:%s:%s" % (user, realm, password)
+    if isinstance(data, text_type):
+        data = data.encode('utf8')
+
     return "%s:%s:%s" % (user, realm, md5(data).hexdigest())