Intermediate state on the way to fixing both #10 and #16. Builds and tests.

* Added functional level users (which do everything) and groups (which just do one query, because that's all the server does).
* Added automatic throttling of command count per action and action count per batch to the Connection object.
* Added test for throttling, but not any of functional actions yet.

I want to rework the way throttling works to make it more of a queueing system.

Author: brotskydotcom

setup.py

@@ -46,6 +46,7 @@
           'cryptography',
           'PyJWT',
           'six',
+          'enum34'
       ],
       setup_requires=[
           'pytest-runner',

tests/test_actions.py

@@ -61,15 +61,15 @@ def test_execute_single_success():
         mock_post.return_value = MockResponse(200, {"result": "success"})
         conn = Connection(**mock_connection_params)
         action = Action(top="top").append(a="a")
-        assert conn.execute_single(action) is True
+        assert conn.execute_single(action) == (1, 1)
 
 
 def test_execute_single_dofirst_success():
     with mock.patch("umapi_client.connection.requests.post") as mock_post:
         mock_post.return_value = MockResponse(200, {"result": "success"})
         conn = Connection(**mock_connection_params)
         action = Action(top="top").insert(a="a")
-        assert conn.execute_single(action) is True
+        assert conn.execute_single(action) == (1, 1)
 
 
 def test_execute_multiple_success():
@@ -78,7 +78,7 @@ def test_execute_multiple_success():
         conn = Connection(**mock_connection_params)
         action0 = Action(top="top0").append(a="a0").append(b="b")
         action1 = Action(top="top1").append(a="a1")
-        assert conn.execute_multiple([action0, action1]) == 2
+        assert conn.execute_multiple([action0, action1]) == (2, 2)
 
 
 def test_execute_multiple_dofirst_success():
@@ -87,7 +87,7 @@ def test_execute_multiple_dofirst_success():
         conn = Connection(**mock_connection_params)
         action0 = Action(top="top0").append(a="a0").insert(b="b")
         action1 = Action(top="top1").append(a="a1")
-        assert conn.execute_multiple([action0, action1]) == 2
+        assert conn.execute_multiple([action0, action1]) == (2, 2)
 
 
 def test_execute_single_error():
@@ -98,9 +98,8 @@ def test_execute_single_error():
                                                                 "message": "Test error message"}]})
         conn = Connection(**mock_connection_params)
         action = Action(top="top").append(a="a")
-        assert conn.execute_single(action) is False
+        assert conn.execute_single(action) == (1, 0)
         assert action.execution_errors() == [{"command": {"a": "a"},
-                                              "step": 0,
                                               "errorCode": "test.error",
                                               "message": "Test error message"}]
 
@@ -116,13 +115,11 @@ def test_execute_single_multi_error():
                                                                 "message": "message2"}]})
         conn = Connection(**mock_connection_params)
         action = Action(top="top").append(a="a")
-        assert conn.execute_single(action) is False
+        assert conn.execute_single(action) == (1, 0)
         assert action.execution_errors() == [{"command": {"a": "a"},
-                                              "step": 0,
                                               "errorCode": "error1",
                                               "message": "message1"},
                                              {"command": {"a": "a"},
-                                              "step": 0,
                                               "errorCode": "error2",
                                               "message": "message2"}]
 
@@ -135,9 +132,8 @@ def test_execute_single_dofirst_error():
                                                                 "message": "Test error message"}]})
         conn = Connection(**mock_connection_params)
         action = Action(top="top").insert(a="a")
-        assert conn.execute_single(action) is False
+        assert conn.execute_single(action) == (1, 0)
         assert action.execution_errors() == [{"command": {"a": "a"},
-                                              "step": 0,
                                               "errorCode": "test.error",
                                               "message": "Test error message"}]
 
@@ -153,10 +149,9 @@ def test_execute_multiple_error():
         conn = Connection(**mock_connection_params)
         action0 = Action(top="top0").append(a="a0")
         action1 = Action(top="top1").append(a="a1").append(b="b")
-        assert conn.execute_multiple([action0, action1]) == 1
+        assert conn.execute_multiple([action0, action1]) == (2, 1)
         assert action0.execution_errors() == []
         assert action1.execution_errors() == [{"command": {"b": "b"},
-                                               "step": 1,
                                                "errorCode": "test.error",
                                                "message": "Test error message"}]
 
@@ -175,14 +170,12 @@ def test_execute_multiple_multi_error():
         conn = Connection(**mock_connection_params)
         action0 = Action(top="top0").append(a="a0")
         action1 = Action(top="top1").append(a="a1").append(b="b")
-        assert conn.execute_multiple([action0, action1]) == 1
+        assert conn.execute_multiple([action0, action1]) == (2, 1)
         assert action0.execution_errors() == []
         assert action1.execution_errors() == [{"command": {"b": "b"},
-                                               "step": 1,
                                                "errorCode": "error1",
                                                "message": "message1"},
                                               {"command": {"b": "b"},
-                                               "step": 1,
                                                "errorCode": "error2",
                                                "message": "message2"}]
 
@@ -198,9 +191,37 @@ def test_execute_multiple_dofirst_error():
         conn = Connection(**mock_connection_params)
         action0 = Action(top="top0").append(a="a0")
         action1 = Action(top="top1").append(a="a1").insert(b="b")
-        assert conn.execute_multiple([action0, action1]) == 1
+        assert conn.execute_multiple([action0, action1]) == (2, 1)
         assert action0.execution_errors() == []
         assert action1.execution_errors() == [{"command": {"a": "a1"},
-                                               "step": 1,
                                                "errorCode": "test.error",
                                                "message": "Test error message"}]
+
+
+def test_execute_single_throttle_commands():
+    with mock.patch("umapi_client.connection.requests.post") as mock_post:
+        mock_post.return_value = MockResponse(200, {"result": "partial",
+                                                    "completed": 1,
+                                                    "notCompleted": 1,
+                                                    "errors": [{"index": 1, "step": 0, "errorCode": "test"}]})
+        conn = Connection(throttle_commands=2, **mock_connection_params)
+        action = Action(top="top0").append(a="a0").append(a="a1").append(a="a2")
+        assert conn.execute_single(action) == (2, 1)
+        assert action.execution_errors() == [{"command": {"a": "a2"}, "errorCode": "test"}]
+
+
+def test_execute_multiple_throttle_actions():
+    with mock.patch("umapi_client.connection.requests.post") as mock_post:
+        mock_post.side_effect = [MockResponse(200, {"result": "success"}),
+                                 MockResponse(200, {"result": "partial",
+                                                    "completed": 0,
+                                                    "notCompleted": 1,
+                                                    "errors": [{"index": 0, "step": 0, "errorCode": "test"}]})]
+        conn = Connection(throttle_actions=2, **mock_connection_params)
+        action0 = Action(top="top0").append(a="a0")
+        action1 = Action(top="top1").append(a="a1")
+        action2 = Action(top="top2").append(a="a2")
+        assert conn.execute_multiple([action0, action1, action2]) == (3, 2)
+        assert action0.execution_errors() == []
+        assert action1.execution_errors() == []
+        assert action2.execution_errors() == [{"command": {"a": "a2"}, "errorCode": "test"}]

umapi_client/__init__.py

@@ -19,5 +19,7 @@
 # SOFTWARE.
 
 from .connection import Connection
-from .api import Action, UserAction, QueryMultiple, UsersQuery, GroupsQuery, QuerySingle, UserQuery
+from .api import Action, QuerySingle, QueryMultiple
+from .users import UserAction, UserQuery, UsersQuery, IdentityTypes, GroupTypes, RoleTypes
+from .groups import GroupsQuery
 from .error import ClientError, RequestError, ServerError, UnavailableError

umapi_client/api.py

@@ -41,6 +41,23 @@ def __init__(self, **kwargs):
         self.frame = dict(kwargs)
         self.commands = []
         self.errors = []
+        self.split_actions = None
+
+    def split(self, max_commands):
+        """
+        Split this action into an equivalent list of actions, each of which have at most max_commands commands.
+        :param max_commands: max number of commands allowed in any action
+        :return: the list of commands created from this one
+        """
+        prior = Action(**self.frame)
+        prior.commands = list(self.commands)
+        self.split_actions = [prior]
+        while len(prior.commands) > max_commands:
+            next = Action(**self.frame)
+            prior.commands, next.commands = prior.commands[0:max_commands], prior.commands[max_commands:]
+            self.split_actions.append(next)
+            prior = next
+        return self.split_actions
 
     def wire_dict(self):
         """
@@ -94,7 +111,8 @@ def report_command_error(self, error_dict):
         """
         error = dict(error_dict)
         error["command"] = self.commands[error_dict["step"]]
-        del error["index"]  # doesn't matter which action this was in the server-sent batch
+        del error["index"]  # throttling can change which action this was in the batch
+        del error["step"]   # throttling can change which step this was in the action
         self.errors.append(error)
 
     def execution_errors(self):
@@ -104,38 +122,11 @@ def execution_errors(self):
         Each dictionary entry gives the command dictionary and the error dictionary
         :return: list of commands that gave errors, with their error information
         """
-        return [dict(e) for e in self.errors]
-
-
-class UserAction(Action):
-    """
-    An sequence of commands for the UMAPI to perform on a single user.
-    """
-
-    def __init__(self, email=None, username=None, domain=None, **kwargs):
-        """
-        Create an Action for a user identified either by email or by username and domain.
-        There is never a reason to specify both email and username.
-        :param username: string, username in the Adobe domain (might be email)
-        :param domain: string, required if the username is not an email address
-        :param kwargs: other key/value pairs desired to identify this user
-        """
-        if email:
-            if username or domain:
-                ValueError("User create: Email was specified; don't also specify username or domain")
-            if not re.match(r"^[a-zA-Z0-9!#$%&'*+/=?^_`{|}~;-]+([.][a-zA-Z0-9!#$%&'*+/=?^_`{|}~;-]+)*"
-                            r"@"
-                            r"[a-zA-Z0-9-]+([.][a-zA-Z0-9-]+)+$", email):
-                ValueError("Action create: Illegal email format")
-            Action.__init__(user=email, **kwargs)
+        if self.split_actions:
+            # throttling split this action, get errors from the split
+            return [dict(e) for s in self.split_actions for e in s.errors]
         else:
-            if not username or not domain:
-                ValueError("User create: Both username and domain must be specified")
-            if not re.match(r"^[a-zA-Z0-9!#$%&'*+/=?^_`{|}~;-]+([.][a-zA-Z0-9!#$%&'*+/=?^_`{|}~;-]+)*$", username):
-                ValueError("User create: Illegal characters in username")
-            if not re.match(r"^[a-zA-Z0-9-]+([.][a-zA-Z0-9-]+)+$", domain):
-                ValueError("User create: Illegal domain format")
-            Action.__init__(user=username, domain=domain, **kwargs)
+            return [dict(e) for e in self.errors]
 
 
 class QueryMultiple:
@@ -228,37 +219,6 @@ def all_results(self):
         return list(self._results)
 
 
-class UsersQuery(QueryMultiple):
-    """
-    Query for users meeting (optional) criteria
-    """
-
-    def __init__(self, connection, in_group="", in_domain="", identity_type=""):
-        """
-        Create a query for all users, or for those in a group or domain or both
-        :param connection: Connection to run the query against
-        :param in_group: (optional) name of the group to restrict the query to
-        :param in_domain: (optional) name of the domain to restrict the query to
-        """
-        groups = [in_group] if in_group else []
-        params = {}
-        if in_domain: params["domain"] = str(in_domain)
-        if identity_type: params["type"] = str(identity_type)
-        QueryMultiple.__init__(self, connection=connection, object_type="user", url_params=groups, query_params=params)
-
-
-class GroupsQuery(QueryMultiple):
-    """
-    Query for all groups
-    """
-
-    def __init__(self, connection):
-        """
-        Create a query for all groups
-        :param connection: Connection to run the query against
-        """
-        QueryMultiple.__init__(self, connection=connection, object_type="group")
-
 class QuerySingle:
     """
     Look for a single object
@@ -301,16 +261,3 @@ def result(self):
         if self._result is None:
             self._fetch_result()
         return self._result
-
-class UserQuery(QuerySingle):
-    """
-    Query for a single user
-    """
-
-    def __init__(self, connection, email):
-        """
-        Create a query for the user with the given email
-        :param connection: Connection to run the query against
-        :param email: email of user to query for
-        """
-        QuerySingle.__init__(self, connection=connection, object_type="user", url_params=[str(email)])