Add computer use demo

python/composio/py.typed

@@ -0,0 +1,14 @@
+// Copyright 2024 admin
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     https://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+

python/composio/tools/local/anthropicComputerTools/Dockerfile

@@ -0,0 +1,116 @@
+FROM docker.io/ubuntu:22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV DEBIAN_PRIORITY=high
+
+RUN apt-get update && \
+    apt-get -y upgrade && \
+    apt-get -y install \
+    build-essential \
+    # UI Requirements
+    xvfb \
+    xterm \
+    xdotool \
+    scrot \
+    imagemagick \
+    sudo \
+    mutter \
+    x11vnc \
+    # Python/pyenv reqs
+    build-essential \
+    libssl-dev  \
+    zlib1g-dev \
+    libbz2-dev \
+    libreadline-dev \
+    libsqlite3-dev \
+    curl \
+    git \
+    libncursesw5-dev \
+    xz-utils \
+    tk-dev \
+    libxml2-dev \
+    libxmlsec1-dev \
+    libffi-dev \
+    liblzma-dev \
+    # Network tools
+    net-tools \
+    netcat \
+    # PPA req
+    software-properties-common && \
+    # Userland apps
+    sudo add-apt-repository ppa:mozillateam/ppa && \
+    sudo apt-get install -y --no-install-recommends \
+    libreoffice \
+    firefox-esr \
+    x11-apps \
+    xpdf \
+    gedit \
+    xpaint \
+    tint2 \
+    galculator \
+    pcmanfm \
+    unzip && \
+    apt-get clean
+
+# Install noVNC
+RUN git clone --branch v1.5.0 https://github.com/novnc/noVNC.git /opt/noVNC && \
+    git clone --branch v0.12.0 https://github.com/novnc/websockify /opt/noVNC/utils/websockify && \
+    ln -s /opt/noVNC/vnc.html /opt/noVNC/index.html
+
+# setup user
+ENV USERNAME=computeruse
+ENV HOME=/home/$USERNAME
+RUN useradd -m -s /bin/bash -d $HOME $USERNAME
+RUN echo "${USERNAME} ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+USER computeruse
+WORKDIR $HOME
+
+# setup python
+RUN git clone https://github.com/pyenv/pyenv.git ~/.pyenv && \
+    cd ~/.pyenv && src/configure && make -C src && cd .. && \
+    echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc && \
+    echo 'command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc && \
+    echo 'eval "$(pyenv init -)"' >> ~/.bashrc
+ENV PYENV_ROOT="$HOME/.pyenv"
+ENV PATH="$PYENV_ROOT/bin:$PATH"
+ENV PYENV_VERSION_MAJOR=3
+ENV PYENV_VERSION_MINOR=11
+ENV PYENV_VERSION_PATCH=6
+ENV PYENV_VERSION=$PYENV_VERSION_MAJOR.$PYENV_VERSION_MINOR.$PYENV_VERSION_PATCH
+RUN eval "$(pyenv init -)" && \
+    pyenv install $PYENV_VERSION && \
+    pyenv global $PYENV_VERSION && \
+    pyenv rehash
+
+ENV PATH="$HOME/.pyenv/shims:$HOME/.pyenv/bin:$PATH"
+
+RUN python -m pip install --upgrade pip==23.1.2 setuptools==58.0.4 wheel==0.40.0 && \
+    python -m pip config set global.disable-pip-version-check true
+
+# only reinstall if requirements.txt changes
+COPY --chown=$USERNAME:$USERNAME requirements.txt $HOME/anthropicComputerTools/requirements.txt
+RUN python -m pip install -r $HOME/anthropicComputerTools/requirements.txt
+# Install streamlit first, explicitly
+RUN python -m pip install streamlit==1.38.0
+
+# setup desktop env & app
+COPY --chown=$USERNAME:$USERNAME image/ $HOME
+COPY --chown=$USERNAME:$USERNAME . $HOME/anthropicComputerTools/
+
+# RUN cd $HOME && \
+#     python -m pip install -e /home/computeruse/anthropicComputerTools
+# Set PYTHONPATH before using it
+ENV PYTHONPATH=/home/computeruse/anthropicComputerTools
+# Now when we modify it later, it exists
+ENV PYTHONPATH="${PYTHONPATH}:/home/computeruse/anthropicComputerTools"
+ENV COMPOSIO_PATH="/home/computeruse/anthropicComputerTools"
+ENV COMPOSIO_API_KEY="246jb4x8mbv1jdusbis892"
+
+ARG DISPLAY_NUM=1
+ARG HEIGHT=768
+ARG WIDTH=1024
+ENV DISPLAY_NUM=$DISPLAY_NUM
+ENV HEIGHT=$HEIGHT
+ENV WIDTH=$WIDTH
+
+ENTRYPOINT [ "./entrypoint.sh" ]

python/composio/tools/local/anthropicComputerTools/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2024 admin
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     https://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+

python/composio/tools/local/anthropicComputerTools/image/.config/tint2/applications/firefox-custom.desktop

@@ -0,0 +1,8 @@
+[Desktop Entry]
+Name=Firefox Custom
+Comment=Open Firefox with custom URL
+Exec=firefox-esr -new-window
+Icon=firefox-esr
+Terminal=false
+Type=Application
+Categories=Network;WebBrowser;

python/composio/tools/local/anthropicComputerTools/image/.config/tint2/applications/gedit.desktop

@@ -0,0 +1,8 @@
+[Desktop Entry]
+Name=Gedit
+Comment=Open gedit
+Exec=gedit
+Icon=text-editor-symbolic
+Terminal=false
+Type=Application
+Categories=TextEditor;

python/composio/tools/local/anthropicComputerTools/image/.config/tint2/applications/terminal.desktop

@@ -0,0 +1,8 @@
+[Desktop Entry]
+Name=Terminal
+Comment=Open Terminal
+Exec=xterm
+Icon=utilities-terminal
+Terminal=false
+Type=Application
+Categories=System;TerminalEmulator;

python/composio/tools/local/anthropicComputerTools/image/.config/tint2/tint2rc

@@ -0,0 +1,100 @@
+#-------------------------------------
+# Panel
+panel_items = TL
+panel_size = 100% 60
+panel_margin = 0 0
+panel_padding = 2 0 2
+panel_background_id = 1
+wm_menu = 0
+panel_dock = 0
+panel_position = bottom center horizontal
+panel_layer = top
+panel_monitor = all
+panel_shrink = 0
+autohide = 0
+autohide_show_timeout = 0
+autohide_hide_timeout = 0.5
+autohide_height = 2
+strut_policy = follow_size
+panel_window_name = tint2
+disable_transparency = 1
+mouse_effects = 1
+font_shadow = 0
+mouse_hover_icon_asb = 100 0 10
+mouse_pressed_icon_asb = 100 0 0
+scale_relative_to_dpi = 0
+scale_relative_to_screen_height = 0
+
+#-------------------------------------
+# Taskbar
+taskbar_mode = single_desktop
+taskbar_hide_if_empty = 0
+taskbar_padding = 0 0 2
+taskbar_background_id = 0
+taskbar_active_background_id = 0
+taskbar_name = 1
+taskbar_hide_inactive_tasks = 0
+taskbar_hide_different_monitor = 0
+taskbar_hide_different_desktop = 0
+taskbar_always_show_all_desktop_tasks = 0
+taskbar_name_padding = 4 2
+taskbar_name_background_id = 0
+taskbar_name_active_background_id = 0
+taskbar_name_font_color = #e3e3e3 100
+taskbar_name_active_font_color = #ffffff 100
+taskbar_distribute_size = 0
+taskbar_sort_order = none
+task_align = left
+
+#-------------------------------------
+# Launcher
+launcher_padding = 4 8 4
+launcher_background_id = 0
+launcher_icon_background_id = 0
+launcher_icon_size = 48
+launcher_icon_asb = 100 0 0
+launcher_icon_theme_override = 0
+startup_notifications = 1
+launcher_tooltip = 1
+
+#-------------------------------------
+# Launcher icon
+launcher_item_app = /usr/share/applications/libreoffice-calc.desktop
+launcher_item_app = /home/computeruse/.config/tint2/applications/terminal.desktop
+launcher_item_app = /home/computeruse/.config/tint2/applications/firefox-custom.desktop
+launcher_item_app = /usr/share/applications/xpaint.desktop
+launcher_item_app = /usr/share/applications/xpdf.desktop
+launcher_item_app = /home/computeruse/.config/tint2/applications/gedit.desktop
+launcher_item_app = /usr/share/applications/galculator.desktop
+
+#-------------------------------------
+# Background definitions
+# ID 1
+rounded = 0
+border_width = 0
+background_color = #000000 60
+border_color = #000000 30
+
+# ID 2
+rounded = 4
+border_width = 1
+background_color = #777777 20
+border_color = #777777 30
+
+# ID 3
+rounded = 4
+border_width = 1
+background_color = #777777 20
+border_color = #ffffff 40
+
+# ID 4
+rounded = 4
+border_width = 1
+background_color = #aa4400 100
+border_color = #aa7733 100
+
+# ID 5
+rounded = 4
+border_width = 1
+background_color = #aaaa00 100
+border_color = #aaaa00 100

python/composio/tools/local/anthropicComputerTools/image/.streamlit/config.toml

@@ -0,0 +1,6 @@
+[server]
+fileWatcherType = "auto"
+runOnSave = true
+
+[browser]
+gatherUsageStats = false

python/composio/tools/local/anthropicComputerTools/image/entrypoint.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+
+./start_all.sh
+./novnc_startup.sh
+
+python http_server.py > /tmp/server_logs.txt 2>&1 &
+cd $HOME/anthropicComputerTools
+STREAMLIT_SERVER_PORT=8501 python -m streamlit run streamlit_project.py > /tmp/streamlit_stdout.log &
+
+echo "✨ Computer Use Demo is ready!"
+echo "➡️  Open http://localhost:8080 in your browser to begin"
+
+# Keep the container running
+tail -f /dev/null

python/composio/tools/local/anthropicComputerTools/image/http_server.py

@@ -0,0 +1,19 @@
+import os
+import socket
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+
+
+class HTTPServerV6(HTTPServer):
+    address_family = socket.AF_INET6
+
+
+def run_server():
+    os.chdir(os.path.dirname(__file__) + "/static_content")
+    server_address = ("::", 8080)
+    httpd = HTTPServerV6(server_address, SimpleHTTPRequestHandler)
+    print("Starting HTTP server on port 8080...")  # noqa: T201
+    httpd.serve_forever()
+
+
+if __name__ == "__main__":
+    run_server()

python/composio/tools/local/anthropicComputerTools/image/index.html

@@ -0,0 +1,43 @@
+<!doctype html>
+<html>
+    <head>
+        <title>Computer Use Demo</title>
+        <meta name="permissions-policy" content="fullscreen=*" />
+        <style>
+            body {
+                margin: 0;
+                padding: 0;
+                overflow: hidden;
+            }
+            .container {
+                display: flex;
+                height: 100vh;
+                width: 100vw;
+            }
+            .left {
+                flex: 1;
+                border: none;
+                height: 100vh;
+            }
+            .right {
+                flex: 2;
+                border: none;
+                height: 100vh;
+            }
+        </style>
+    </head>
+    <body>
+        <div class="container">
+            <iframe
+                src="http://localhost:8501"
+                class="left"
+                allow="fullscreen"
+            ></iframe>
+            <iframe
+                src="http://localhost:6080/vnc.html?view_only=1&autoconnect=1&resize=scale"
+                class="right"
+                allow="fullscreen"
+            ></iframe>
+        </div>
+    </body>
+</html>

python/composio/tools/local/anthropicComputerTools/image/mutter_startup.sh

@@ -0,0 +1,20 @@
+echo "starting mutter"
+XDG_SESSION_TYPE=x11 mutter --replace --sm-disable 2>/tmp/mutter_stderr.log &
+
+# Wait for tint2 window properties to appear
+timeout=30
+while [ $timeout -gt 0 ]; do
+    if xdotool search --class "mutter" >/dev/null 2>&1; then
+        break
+    fi
+    sleep 1
+    ((timeout--))
+done
+
+if [ $timeout -eq 0 ]; then
+    echo "mutter stderr output:" >&2
+    cat /tmp/mutter_stderr.log >&2
+    exit 1
+fi
+
+rm /tmp/mutter_stderr.log