Convert json strings to unicode when they are unmarshalled.

This patch removes some individual work-arounds of converting strings
to unicode, they only masked the problem. String values are not
passed to the validator or normalizers so things like adding the
realm automatically to services weren't happening.

ticket 941

ipalib/parameters.py

@@ -1021,7 +1021,7 @@ class Int(Number):
         """
         if type(value) in (int, long):
             return value
-        if type(value) in (str, unicode):
+        if type(value) is unicode:
             # permit floating point strings
             if value.find(u'.') >= 0:
                 try:
@@ -1254,14 +1254,6 @@ class Str(Data):
         """
         if type(value) is self.type:
             return value
-        if type(value) is str:
-            try:
-                return value.decode('utf-8')
-            except UnicodeDecodeError:
-                raise ConversionError(
-                    name=self.name, index=index,
-                    error=ugettext(self.scalar_error)
-                )
         if type(value) in (int, float):
             return self.type(value)
         if type(value) in (tuple, list):
@@ -1385,24 +1377,6 @@ class StrEnum(Enum):
 
     type = unicode
 
-    def _convert_scalar(self, value, index=None):
-        """
-        Convert a single scalar value.
-        """
-        if type(value) is self.type:
-            return value
-        if type(value) is str:
-            try:
-                return value.decode('utf-8')
-            except UnicodeDecodeError:
-                raise ConversionError(
-                    name=self.name, index=index,
-                    error=ugettext(self.scalar_error)
-                )
-        raise ConversionError(name=self.name, index=index,
-            error=ugettext(self.type_error),
-        )
-
 
 class List(Param):
     """

ipaserver/rpcserver.py

@@ -27,7 +27,7 @@ from cgi import parse_qs
 from xml.sax.saxutils import escape
 from xmlrpclib import Fault
 from ipalib.backend import Executioner
-from ipalib.errors import PublicError, InternalError, CommandError, JSONError
+from ipalib.errors import PublicError, InternalError, CommandError, JSONError, ConversionError
 from ipalib.request import context, Connection, destroy_context
 from ipalib.rpc import xml_dumps, xml_loads
 from ipalib.util import make_repr
@@ -402,7 +402,16 @@ def json_decode_binary(val):
         del val
         return new_list
     else:
-        return val
+        if isinstance(val, basestring):
+            try:
+                return val.decode('utf-8')
+            except UnicodeDecodeError:
+                raise ConversionError(
+                    name=val,
+                    error='incorrect type'
+                )
+        else:
+            return val
 
 class jsonserver(WSGIExecutioner):
     """

tests/test_ipalib/test_parameters.py

@@ -922,7 +922,7 @@ class test_Str(ClassChecker):
         mthd = o._convert_scalar
         for value in (u'Hello', 42, 1.2, unicode_str):
             assert mthd(value) == unicode(value)
-        bad = [True, dict(one=1)]
+        bad = [True, 'Hello', dict(one=1), utf8_bytes]
         for value in bad:
             e = raises(errors.ConversionError, mthd, value)
             assert e.name == 'my_str'

tests/test_xmlrpc/test_user_plugin.py

@@ -510,8 +510,8 @@ class test_user(Declarative):
             desc='Create %r with a full address' % user1,
             command=(
                 'user_add', [user1], dict(givenname=u'Test', sn=u'User1',
-                street='123 Maple Rd', locality='Anytown', st='MD',
-                telephonenumber='410-555-1212',)
+                street=u'123 Maple Rd', locality=u'Anytown', st=u'MD',
+                telephonenumber=u'410-555-1212',)
             ),
             expected=dict(
                 value=user1,