Implement the v2 actions.

ericpts · ericpts · commit e23bc49a30b3 · 2025-06-12T10:44:05.000-07:00
diff --git a/agent.py b/agent.py
@@ -39,8 +39,8 @@ def __init__(
         browser_computer: Computer,
         query: str,
         model_name: Literal[
-            "computer-use-exp"
-        ] = "computer-use-exp",
+            "computer-use-exp-6-11"
+        ] = "computer-use-exp-6-11",
     ):
         self._browser_computer = browser_computer
         self._query = query
@@ -50,6 +50,10 @@ def __init__(
             vertexai=os.environ.get("USE_VERTEXAI", "0").lower() in ["true", "1"],
             project=os.environ.get("VERTEXAI_PROJECT"),
             location=os.environ.get("VERTEXAI_LOCATION"),
+            http_options=types.HttpOptions(
+                api_version="v1alpha",
+                base_url="https://generativelanguage.googleapis.com",
+                )
         )
         self._contents: list[Content] = [
             Content(
@@ -94,13 +98,32 @@ def handle_action(self, action: types.FunctionCall) -> EnvState:
         elif action.name == "type_text_at":
             x = self.normalize_x(action.args["x"])
             y = self.normalize_y(action.args["y"])
+            press_enter = action.args.get("press_enter", True)
+            clear_before_typing = action.args.get("clear_before_typing", True)
             return self._browser_computer.type_text_at(
                 x=x,
                 y=y,
                 text=action.args["text"],
+                press_enter=press_enter,
+                clear_before_typing=clear_before_typing,
             )
         elif action.name == "scroll_document":
             return self._browser_computer.scroll_document(action.args["direction"])
+        elif action.name == "scroll_at":
+            x = self.normalize_x(action.args["x"])
+            y = self.normalize_y(action.args["y"])
+            magnitude = action.args.get("magnitude", 200)
+            direction = action.args["direction"]
+
+            if direction in ("up", "down"):
+                magnitude = self.normalize_y(magnitude)
+            elif direction in ("left", "right"):
+                magnitude = self.normalize_x(magnitude)
+            else:
+                raise ValueError("Unknown direction: ", direction)
+            return self._browser_computer.scroll_at(
+                x=x, y=y, direction=direction, magnitude=magnitude
+            )
         elif action.name == "wait_5_seconds":
             return self._browser_computer.wait_5_seconds()
         elif action.name == "go_back":
@@ -115,6 +138,17 @@ def handle_action(self, action: types.FunctionCall) -> EnvState:
             return self._browser_computer.key_combination(
                 action.args["keys"].split("+")
             )
+        elif action.name == "drag_and_drop":
+            x = self.normalize_x(action.args["x"])
+            y = self.normalize_y(action.args["y"])
+            destination_x = self.normalize_x(action.args["destination_x"])
+            destination_y = self.normalize_y(action.args["destination_y"])
+            return self._browser_computer.drag_and_drop(
+                x=x,
+                y=y,
+                destination_x=destination_x,
+                destination_y=destination_y,
+            )
         else:
             raise ValueError(f"Unsupported function: {action}")
 
diff --git a/computers/computer.py b/computers/computer.py
@@ -13,6 +13,7 @@
 # limitations under the License.
 import abc
 import pydantic
+from typing import Literal
 
 
 class EnvState(pydantic.BaseModel):
@@ -33,31 +34,54 @@ def open_web_browser(self) -> EnvState:
         """Opens the web browser."""
 
     @abc.abstractmethod
-    def click_at(self, y: int, x: int) -> EnvState:
-        """Clicks at a specific y (0-999), x (0-999) coordinate on the webpage.
+    def click_at(self, x: int, y: int) -> EnvState:
+        """Clicks at a specific x, y  coordinate on the webpage.
 
-        The 'x' and 'y' values are scaled to the height and width of the screen.
+        The 'x' and 'y' values are absolute values, scaled to the height and width of the screen.
         """
 
     @abc.abstractmethod
-    def hover_at(self, y: int, x: int) -> EnvState:
-        """Hovers at a specific y (0-999), x (0-999) coordinate on the webpage.
+    def hover_at(self, x: int, y: int) -> EnvState:
+        """Hovers at a specific x, y coordinate on the webpage.
 
         May be used to explore sub-menus that appear on hover.
-        The 'x' and 'y' values are scaled to the height and width of the screen.
+        The 'x' and 'y' values are absolute values, scaled to the height and width of the screen.
         """
 
     @abc.abstractmethod
-    def type_text_at(self, y: int, x: int, text: str) -> EnvState:
-        """Types text at a specific y (0-999), x (0-999) coordinate.
-
-        The system automatically presses ENTER after typing.
-        The 'x' and 'y' values are scaled to the height and width of the screen.
+    def type_text_at(
+        self,
+        x: int,
+        y: int,
+        text: str,
+        press_enter: bool,
+        clear_before_typing: bool,
+    ) -> EnvState:
+        """Types text at a specific x, y coordinate.
+
+        The system automatically presses ENTER after typing. To disable this, set `press_enter` to False.
+        The system automatically clears any existing content before typing the specified `text`. To disable this, set `clear_before_typing` to False.
+        The 'x' and 'y' values are absolute values, scaled to the height and width of the screen.
         """
 
     @abc.abstractmethod
-    def scroll_document(self, direction: str) -> EnvState:
-        """Scrolls the entire webpage "up" or "down" based on direction."""
+    def scroll_document(
+        self, direction: Literal["up", "down", "left", "right"]
+    ) -> EnvState:
+        """Scrolls the entire webpage "up", "down", "left" or "right" based on direction."""
+
+    @abc.abstractmethod
+    def scroll_at(
+        self,
+        x: int,
+        y: int,
+        direction: Literal["up", "down", "left", "right"],
+        magnitude: int,
+    ) -> EnvState:
+        """Scrolls up, down, right, or left at a x, y coordinate by magnitude.
+
+        The 'x' and 'y' values are absolute values, scaled to the height and width of the screen.
+        """
 
     @abc.abstractmethod
     def wait_5_seconds(self) -> EnvState:
@@ -88,6 +112,14 @@ def navigate(self, url: str) -> EnvState:
     def key_combination(self, keys: list[str]) -> EnvState:
         """Presses keyboard keys and combinations, such as "control+c" or "enter"."""
 
+    @abc.abstractmethod
+    def drag_and_drop(
+        self, x: int, y: int, destination_x: int, destination_y: int
+    ) -> EnvState:
+        """Drag and drop an element from a x, y coordinate to a destination destination_y, destination_x coordinate.
+        The 'x', 'y', 'destination_y' and 'destination_x' values are absolute values, scaled to the height and width of the screen.
+        """
+
     @abc.abstractmethod
     def current_state(self) -> EnvState:
         """Returns the current state of the current webpage."""
diff --git a/computers/playwright/playwright.py b/computers/playwright/playwright.py
@@ -18,6 +18,54 @@
     EnvState,
 )
 from playwright.sync_api import sync_playwright
+from typing import Literal
+
+# Define a mapping from the user-friendly key names to Playwright's expected key names.
+# Playwright is generally good with case-insensitivity for these, but it's best to be canonical.
+# See: https://playwright.dev/docs/api/class-keyboard#keyboard-press
+# Keys like 'a', 'b', '1', '$' are passed directly.
+PLAYWRIGHT_KEY_MAP = {
+    "backspace": "Backspace",
+    "tab": "Tab",
+    "return": "Enter",  # Playwright uses 'Enter'
+    "enter": "Enter",
+    "shift": "Shift",
+    "control": "Control",  # Or 'ControlOrMeta' for cross-platform Ctrl/Cmd
+    "alt": "Alt",
+    "escape": "Escape",
+    "space": "Space",  # Can also just be " "
+    "pageup": "PageUp",
+    "pagedown": "PageDown",
+    "end": "End",
+    "home": "Home",
+    "left": "ArrowLeft",
+    "up": "ArrowUp",
+    "right": "ArrowRight",
+    "down": "ArrowDown",
+    "insert": "Insert",
+    "delete": "Delete",
+    "semicolon": ";",  # For actual character ';'
+    "equals": "=",  # For actual character '='
+    "multiply": "Multiply",  # NumpadMultiply
+    "add": "Add",  # NumpadAdd
+    "separator": "Separator",  # Numpad specific
+    "subtract": "Subtract",  # NumpadSubtract, or just '-' for character
+    "decimal": "Decimal",  # NumpadDecimal, or just '.' for character
+    "divide": "Divide",  # NumpadDivide, or just '/' for character
+    "f1": "F1",
+    "f2": "F2",
+    "f3": "F3",
+    "f4": "F4",
+    "f5": "F5",
+    "f6": "F6",
+    "f7": "F7",
+    "f8": "F8",
+    "f9": "F9",
+    "f10": "F10",
+    "f11": "F11",
+    "f12": "F12",
+    "command": "Meta",  # 'Meta' is Command on macOS, Windows key on Windows
+}
 
 
 class PlaywrightComputer(Computer):
@@ -91,24 +139,86 @@ def hover_at(self, x: int, y: int):
         self._page.wait_for_load_state()
         return self.current_state()
 
-    def type_text_at(self, x: int, y: int, text: str) -> EnvState:
+    def type_text_at(
+        self,
+        x: int,
+        y: int,
+        text: str,
+        press_enter: bool = True,
+        clear_before_typing: bool = True,
+    ) -> EnvState:
         self.highlight_mouse(x, y)
         self._page.mouse.click(x, y)
         self._page.wait_for_load_state()
+
+        if clear_before_typing:
+            self.key_combination(["Control", "A"])
+            self.key_combination(["Delete"])
+
         self._page.keyboard.type(text)
         self._page.wait_for_load_state()
-        self.key_combination(["Enter"])
+
+        if press_enter:
+            self.key_combination(["Enter"])
         self._page.wait_for_load_state()
         return self.current_state()
 
-    def scroll_document(self, direction: str) -> EnvState:
-        if direction.lower() == "down":
+    def _horizontal_document_scroll(
+        self, direction: Literal["left", "right"]
+    ) -> EnvState:
+        # Scroll by 50% of the viewport size.
+        horizontal_scroll_amount = self.screen_size()[0] // 2
+        if direction == "left":
+            sign = "-"
+        else:
+            sign = ""
+        scroll_argument = f"{sign}{horizontal_scroll_amount}"
+        # Scroll using JS.
+        self._page.evaluate(f"window.scrollBy({scroll_argument}, 0); ")
+        self._page.wait_for_load_state()
+        return self.current_state()
+
+    def scroll_document(
+        self, direction: Literal["up", "down", "left", "right"]
+    ) -> EnvState:
+        if direction == "down":
             return self.key_combination(["PageDown"])
-        elif direction.lower() == "up":
+        elif direction == "up":
             return self.key_combination(["PageUp"])
+        elif direction in ("left", "right"):
+            return self._horizontal_document_scroll(direction)
+        else:
+            raise ValueError("Unsupported direction: ", direction)
+
+    def scroll_at(
+        self,
+        x: int,
+        y: int,
+        direction: Literal["up", "down", "left", "right"],
+        magnitude: int,
+    ) -> EnvState:
+        self.highlight_mouse(x, y)
+
+        self._page.mouse.move(x, y)
+        self._page.wait_for_load_state()
+
+        dx = 0
+        dy = 0
+        if direction == "up":
+            dy = -magnitude
+        elif direction == "down":
+            dy = magnitude
+        elif direction == "left":
+            dx = -magnitude
+        elif direction == "right":
+            dx = magnitude
         else:
             raise ValueError("Unsupported direction: ", direction)
 
+        self._page.mouse.wheel(dx, dy)
+        self._page.wait_for_load_state()
+        return self.current_state()
+
     def wait_5_seconds(self) -> EnvState:
         time.sleep(5)
         return self.current_state()
@@ -132,6 +242,9 @@ def navigate(self, url: str) -> EnvState:
         return self.current_state()
 
     def key_combination(self, keys: list[str]) -> EnvState:
+        # Normalize all keys to the Playwright compatible version.
+        keys = [PLAYWRIGHT_KEY_MAP.get(k.lower(), k) for k in keys]
+
         for key in keys[:-1]:
             self._page.keyboard.down(key)
 
@@ -142,6 +255,21 @@ def key_combination(self, keys: list[str]) -> EnvState:
 
         return self.current_state()
 
+    def drag_and_drop(
+        self, x: int, y: int, destination_x: int, destination_y: int
+    ) -> EnvState:
+        self.highlight_mouse(x, y)
+        self._page.mouse.move(x, y)
+        self._page.wait_for_load_state()
+        self._page.mouse.down()
+        self._page.wait_for_load_state()
+
+        self.highlight_mouse(destination_x, destination_y)
+        self._page.mouse.move(destination_x, destination_y)
+        self._page.wait_for_load_state()
+        self._page.mouse.up()
+        return self.current_state()
+
     def current_state(self) -> EnvState:
         self._page.wait_for_load_state()
         # Even if Playwright reports the page as loaded, it may not be so.
@@ -167,7 +295,7 @@ def highlight_mouse(self, x: int, y: int):
             div.style.borderRadius = '50%';
             div.style.width = '20px';
             div.style.height = '20px';
-            div.style.position = 'absolute';
+            div.style.position = 'fixed';
             div.style.zIndex = '9999';
             document.body.appendChild(div);
 
