Implement v2 actions on Cloud Run.

Author: sammccauley117

apiserver/commands.py

@@ -24,23 +24,39 @@ class NavigateArgs(BaseModel):
 
 
 class ClickAtArgs(BaseModel):
-    y: int
     x: int
+    y: int
 
 
 class HoverAtArgs(BaseModel):
-    y: int
     x: int
+    y: int
 
 
 class TypeTextAtArgs(BaseModel):
-    y: int
     x: int
+    y: int
     text: str
+    press_enter: bool
+    clear_before_typing: bool
 
 
 class ScrollDocumentArgs(BaseModel):
-    direction: Literal["up"] | Literal["down"]
+    direction: Literal["up", "down", "left", "right"]
+
+
+class ScrollAtArgs(BaseModel):
+    x: int
+    y: int
+    direction: Literal["up", "down", "left", "right"]
+    magnitude: int
+
+
+class DragAndDropArgs(BaseModel):
+    x: int
+    y: int
+    destination_x: int
+    destination_y: int
 
 
 class KeyCombinationArgs(BaseModel):
@@ -72,6 +88,16 @@ class ScrollDocument(BaseModel):
     args: ScrollDocumentArgs
 
 
+class ScrollAt(BaseModel):
+    name: Literal["scroll_at"]
+    args: ScrollAtArgs
+
+
+class DragAndDrop(BaseModel):
+    name: Literal["drag_and_drop"]
+    args: DragAndDropArgs
+
+
 class Wait5Seconds(BaseModel):
     name: Literal["wait_5_seconds"]
     args: Optional[EmptyJson] = Field(None)
@@ -120,6 +146,8 @@ class Shutdown(BaseModel):
         HoverAt,
         TypeTextAt,
         ScrollDocument,
+        ScrollAt,
+        DragAndDrop,
         GoBack,
         GoForward,
         Search,

apiserver/test_app.py

@@ -114,7 +114,12 @@ def test_get_command_timeout_standard_command():
 
 def test_get_command_timeout_type_text_at_command():
     text = "hello"
-    command = TypeTextAt(name="type_text_at", args=TypeTextAtArgs(y=0, x=0, text=text))
+    command = TypeTextAt(
+        name="type_text_at",
+        args=TypeTextAtArgs(
+            y=0, x=0, text=text, press_enter=False, clear_before_typing=True
+        ),
+    )
     base_timeout = 10
     key_delay = 0.1
     expected_timeout = base_timeout + (len(text) * key_delay)
@@ -124,7 +129,12 @@ def test_get_command_timeout_type_text_at_command():
 
 def test_get_command_timeout_type_text_at_command_zero_delay():
     text = "world"
-    command = TypeTextAt(name="type_text_at", args=TypeTextAtArgs(y=0, x=0, text=text))
+    command = TypeTextAt(
+        name="type_text_at",
+        args=TypeTextAtArgs(
+            y=0, x=0, text=text, press_enter=False, clear_before_typing=True
+        ),
+    )
     base_timeout = 20
     key_delay = 0.0
     expected_timeout = base_timeout

apiserver/test_commands.py

@@ -19,6 +19,8 @@
     HoverAt,
     TypeTextAt,
     ScrollDocument,
+    ScrollAt,
+    DragAndDrop,
     GoBack,
     GoForward,
     Search,
@@ -38,21 +40,27 @@ def test_navigate(self):
         self.assertEqual(command.model_dump(), function_call)
 
     def test_click_at(self):
-        function_call = {"name": "click_at", "args": {"y": 1, "x": 2}}
+        function_call = {"name": "click_at", "args": {"x": 2, "y": 1}}
         command = CommandModel.model_validate(function_call)
         self.assertIsInstance(command.root, ClickAt)
         self.assertEqual(command.model_dump(), function_call)
 
     def test_hover_at(self):
-        function_call = {"name": "hover_at", "args": {"y": 1, "x": 2}}
+        function_call = {"name": "hover_at", "args": {"x": 2, "y": 1}}
         command = CommandModel.model_validate(function_call)
         self.assertIsInstance(command.root, HoverAt)
         self.assertEqual(command.model_dump(), function_call)
 
     def test_type_text_at(self):
         function_call = {
             "name": "type_text_at",
-            "args": {"y": 1, "x": 2, "text": "one"},
+            "args": {
+                "x": 2,
+                "y": 1,
+                "text": "one",
+                "press_enter": True,
+                "clear_before_typing": False,
+            },
         }
         command = CommandModel.model_validate(
             function_call,
@@ -62,7 +70,13 @@ def test_type_text_at(self):
             command.model_dump(),
             {
                 "name": "type_text_at",
-                "args": {"y": 1, "x": 2, "text": "one"},
+                "args": {
+                    "x": 2,
+                    "y": 1,
+                    "text": "one",
+                    "press_enter": True,
+                    "clear_before_typing": False,
+                },
             },
         )
 
@@ -77,6 +91,58 @@ def test_scroll_document(self):
         self.assertIsInstance(command.root, ScrollDocument)
         self.assertEqual(command.model_dump(), function_call)
 
+        function_call = {
+            "name": "scroll_document",
+            "args": {"direction": "down"},
+        }
+        command = CommandModel.model_validate(
+            function_call,
+        )
+        self.assertIsInstance(command.root, ScrollDocument)
+        self.assertEqual(command.model_dump(), function_call)
+
+        function_call = {
+            "name": "scroll_document",
+            "args": {"direction": "left"},
+        }
+        command = CommandModel.model_validate(
+            function_call,
+        )
+        self.assertIsInstance(command.root, ScrollDocument)
+        self.assertEqual(command.model_dump(), function_call)
+
+        function_call = {
+            "name": "scroll_document",
+            "args": {"direction": "right"},
+        }
+        command = CommandModel.model_validate(
+            function_call,
+        )
+        self.assertIsInstance(command.root, ScrollDocument)
+        self.assertEqual(command.model_dump(), function_call)
+
+    def test_scroll_at(self):
+        function_call = {
+            "name": "scroll_at",
+            "args": {"x": 1, "y": 2, "direction": "up", "magnitude": 10},
+        }
+        command = CommandModel.model_validate(
+            function_call,
+        )
+        self.assertIsInstance(command.root, ScrollAt)
+        self.assertEqual(command.model_dump(), function_call)
+
+    def test_drag_and_drop(self):
+        function_call = {
+            "name": "drag_and_drop",
+            "args": {"x": 1, "y": 2, "destination_x": 3, "destination_y": 4},
+        }
+        command = CommandModel.model_validate(
+            function_call,
+        )
+        self.assertIsInstance(command.root, DragAndDrop)
+        self.assertEqual(command.model_dump(), function_call)
+
     def test_go_back(self):
         function_call = {"name": "go_back", "args": {}}
         command = CommandModel.model_validate(function_call)

computers/cloud_run/cloud_run.py

@@ -14,7 +14,7 @@
 import base64
 import termcolor
 import time
-from typing import Any, Optional
+from typing import Any, Optional, Literal
 import requests
 from urllib.parse import urljoin
 from ..computer import (
@@ -110,25 +110,46 @@ def _run_command(
     def open_web_browser(self) -> EnvState:
         return self._run_command("open_web_browser")
 
-    def click_at(self, y, x):
+    def click_at(self, x: int, y: int) -> EnvState:
         return self._run_command("click_at", args={"x": x, "y": y})
 
-    def hover_at(self, y, x):
+    def hover_at(self, x: int, y: int) -> EnvState:
         return self._run_command("hover_at", args={"x": x, "y": y})
 
-    def type_text_at(self, x: int, y: int, text: str) -> EnvState:
+    def type_text_at(
+        self,
+        x: int,
+        y: int,
+        text: str,
+        press_enter: bool,
+        clear_before_typing: bool,
+    ) -> EnvState:
         return self._run_command(
             "type_text_at",
             args={
                 "x": x,
                 "y": y,
                 "text": text,
+                "press_enter": press_enter,
+                "clear_before_typing": clear_before_typing,
             },
         )
 
     def scroll_document(self, direction: str) -> EnvState:
         return self._run_command("scroll_document", args={"direction": direction})
 
+    def scroll_at(
+        self,
+        x: int,
+        y: int,
+        direction: Literal["up", "down", "left", "right"],
+        magnitude: int,
+    ) -> EnvState:
+        return self._run_command(
+            "scroll_at",
+            args={"x": x, "y": y, "direction": direction, "magnitude": magnitude},
+        )
+
     def wait_5_seconds(self) -> EnvState:
         return self._run_command("wait_5_seconds")
 
@@ -147,6 +168,19 @@ def navigate(self, url: str) -> EnvState:
     def key_combination(self, keys: list[str]) -> EnvState:
         return self._run_command("key_combination", args={"keys": "+".join(keys)})
 
+    def drag_and_drop(
+        self, x: int, y: int, destination_x: int, destination_y: int
+    ) -> EnvState:
+        return self._run_command(
+            "drag_and_drop",
+            args={
+                "x": x,
+                "y": y,
+                "destination_x": destination_x,
+                "destination_y": destination_y,
+            },
+        )
+
     def current_state(self) -> EnvState:
         return self._run_command("screenshot")
 

computers/puppeteer/browser.ts

@@ -96,30 +96,64 @@ export class BrowserShell implements ComputerShell {
         break;
       case 'type_text_at':
         await this.page.mouse.click(c.args.x, c.args.y);
+        if (c.args.clear_before_typing) {
+          await this.page.keyboard.down('Control');
+          await this.page.keyboard.press('a');
+          await this.page.keyboard.up('Control');
+          await this.page.keyboard.press('Backspace');
+        }
         // see https://github.com/puppeteer/puppeteer/issues/1648
         for (let i = 0; i < c.args.text.length; i++) {
           await this.page.keyboard.type(c.args.text[i]);
           await new Promise(resolve => setTimeout(resolve, 100));
         }
-        await this.page.keyboard.press('Enter');
+        if (c.args.press_enter) {
+          await this.page.keyboard.press('Enter');
+        }
         break;
       case 'scroll_document':
+        if (c.args.direction === 'up') {
+          await this.page.keyboard.press('PageUp');
+        } else if (c.args.direction === 'down') {
+          await this.page.keyboard.press('PageDown');
+        } else {
+          let deltaX = 0;
+          if (c.args.direction === 'left') {
+            let viewport = this.page.viewport();
+            if (viewport) {
+              deltaX = -viewport.width / 2;
+            }
+          }
+          if (c.args.direction === 'right') {
+            let viewport = this.page.viewport();
+            if (viewport) {
+              deltaX = viewport.width / 2;
+            }
+          }
+          await this.page.mouse.wheel({ deltaX });
+        }
+        break;
+      case 'scroll_at':
+        await this.page.mouse.move(c.args.x, c.args.y);
         let deltaX = 0;
         let deltaY = 0;
-        if (c.args.direction === 'left') {
-          deltaX = -900;
-        }
-        if (c.args.direction === 'right') {
-          deltaX = 900;
-        }
         if (c.args.direction === 'up') {
-          deltaY = -900;
-        }
-        if (c.args.direction === 'down') {
-          deltaY = 900;
+          deltaY = -c.args.magnitude;
+        } else if (c.args.direction === 'down') {
+          deltaY = c.args.magnitude;
+        } else if (c.args.direction === 'left') {
+          deltaX = -c.args.magnitude;
+        } else if (c.args.direction === 'right') {
+          deltaX = c.args.magnitude;
         }
         await this.page.mouse.wheel({deltaY, deltaX});
         break;
+      case 'drag_and_drop':
+        await this.page.mouse.move(c.args.x, c.args.y);
+        await this.page.mouse.down();
+        await this.page.mouse.move(c.args.destination_x, c.args.destination_y);
+        await this.page.mouse.up();
+        break;
       case 'wait_5_seconds':
         await new Promise(resolve => setTimeout(resolve, 5000));
         break;