Add GAIA eval_infer for unified evaluation workflow

.github/workflows/build-gaia-image.yml → .github/workflows/build-gaia-eval-image.yml

@@ -9,14 +9,6 @@ on:
         description: 'Software Agent SDK commit/ref to use'
         required: true
         type: string
-      target:
-        description: 'Build target (default: binary-minimal)'
-        required: false
-        default: 'binary-minimal'
-        type: choice
-        options:
-          - binary-minimal
-          - source-minimal
 
 concurrency:
   group: build-gaia-${{ github.ref }}
@@ -65,8 +57,8 @@ jobs:
           git add vendor/software-agent-sdk
           echo "Updated SDK submodule to $SDK_SHA (from ${{ inputs.sdk-commit }})"
 
-      - name: Set up Docker Buildx with Blacksmith
-        uses: useblacksmith/setup-docker-builder@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
       - name: Log in to GitHub Container Registry
         uses: docker/login-action@v3
@@ -88,7 +80,8 @@ jobs:
         run: |
           set -euo pipefail
 
-          TARGET="${{ inputs.target || 'binary-minimal' }}"
+          # GAIA requires 'binary' target to include Chromium for browser operations
+          TARGET="binary"
 
           CMD="uv run benchmarks/gaia/build_images.py \
             --image ghcr.io/openhands/eval-agent-server \
@@ -101,6 +94,39 @@ jobs:
           DOCKER_BUILDKIT: 1
           BUILDKIT_PROGRESS: plain
 
+      - name: Build and push GAIA image with MCP pre-installed
+        run: |
+          set -euo pipefail
+
+          # Get the SDK commit SHA for tagging
+          SDK_SHA=$(git submodule status vendor/software-agent-sdk | awk '{print $1}' | sed 's/^[+-]//' | cut -c1-7)
+
+          # GAIA requires 'binary' target to include Chromium for browser operations
+          TARGET="binary"
+
+          # Compute base and MCP image tags
+          BASE_IMAGE="ghcr.io/openhands/eval-agent-server:${SDK_SHA}-gaia"
+          MCP_IMAGE="ghcr.io/openhands/eval-agent-server:${SDK_SHA}-gaia-with-mcp"
+
+          echo "Building MCP-enhanced image..."
+          echo "  Base image: ${BASE_IMAGE}"
+          echo "  MCP image:  ${MCP_IMAGE}"
+
+          # Build the derived image with MCP pre-cached
+          docker build \
+            -f benchmarks/gaia/Dockerfile.gaia \
+            --build-arg SDK_IMAGE="${BASE_IMAGE}" \
+            -t "${MCP_IMAGE}" \
+            .
+
+          # Push the image
+          docker push "${MCP_IMAGE}"
+
+          echo "✅ MCP-enhanced image built and pushed: ${MCP_IMAGE}"
+        env:
+          DOCKER_BUILDKIT: 1
+          BUILDKIT_PROGRESS: plain
+
       - name: Archive build logs
         if: always()
         run: |
@@ -157,6 +183,7 @@ jobs:
         run: |
           # Get SDK version
           SDK_SHA=$(git submodule status vendor/software-agent-sdk | awk '{print $1}' | sed 's/^[+-]//')
+          SDK_SHA_SHORT=${SDK_SHA:0:7}
 
           # Read the single manifest file
           MANIFEST_FILE=$(find builds -name "manifest.jsonl" -type f 2>/dev/null | head -1 || true)
@@ -167,18 +194,16 @@ jobs:
           fi
 
           # Extract the image tag from the manifest
-          IMAGE_TAG=$(cat "$MANIFEST_FILE" | python3 -c "
-import sys, json
-data = json.loads(sys.stdin.read())
-tags = data.get('tags', [])
-print(tags[0] if tags else 'unknown')
-          ")
+          IMAGE_TAG=$(cat "$MANIFEST_FILE" | python3 -c "import sys, json; data = json.loads(sys.stdin.read()); tags = data.get('tags', []); print(tags[0] if tags else 'unknown')")
 
           if [ "$IMAGE_TAG" = "unknown" ]; then
             echo "No valid image tag found in manifest"
             exit 0
           fi
 
+          # Construct MCP image tag (always binary for GAIA)
+          MCP_IMAGE_TAG="ghcr.io/openhands/eval-agent-server:${SDK_SHA_SHORT}-gaia-with-mcp"
+
           # Determine trigger source
           if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
             TRIGGER="Manual trigger (workflow_dispatch)"
@@ -188,22 +213,23 @@ print(tags[0] if tags else 'unknown')
             TRIGGER="${{ github.event_name }}"
           fi
 
-          # Post comment
-          COMMENT_BODY=$(cat <<EOF
-## GAIA Image Build Complete ✅
-
-**SDK Version:** [\`${SDK_SHA:0:7}\`](https://github.com/OpenHands/software-agent-sdk/commit/${SDK_SHA})
-**Image Tag:** \`${IMAGE_TAG}\`
-**Workflow Run:** [#${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
-**Triggered by:** ${TRIGGER}
-EOF
-          )
-
+          # Post comment using jq to properly handle multi-line content
+          jq -n \
+            --arg sdk_short "${SDK_SHA_SHORT}" \
+            --arg sdk_full "${SDK_SHA}" \
+            --arg image "${IMAGE_TAG}" \
+            --arg mcp_image "${MCP_IMAGE_TAG}" \
+            --arg run_id "${{ github.run_id }}" \
+            --arg server_url "${{ github.server_url }}" \
+            --arg repo "${{ github.repository }}" \
+            --arg trigger "${TRIGGER}" \
+            '{body: "## GAIA Image Build Complete ✅\n\n**SDK Version:** [`\($sdk_short)`](https://github.com/OpenHands/software-agent-sdk/commit/\($sdk_full))\n**Base Image:** `\($image)`\n**MCP Image:** `\($mcp_image)` ⚡ _(MCP server pre-cached)_\n**Workflow Run:** [#\($run_id)](\($server_url)/\($repo)/actions/runs/\($run_id))\n**Triggered by:** \($trigger)\n\nThe MCP-enhanced image includes pre-cached `mcp-server-fetch` to eliminate 1-18 minute startup delays."}' | \
           curl -L -X POST \
             -H "Accept: application/vnd.github+json" \
             -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
             -H "X-GitHub-Api-Version: 2022-11-28" \
+            -H "Content-Type: application/json" \
             "${{ github.api_url }}/repos/${{ github.repository }}/issues/81/comments" \
-            -d "$(jq -n --arg body "$COMMENT_BODY" '{body: $body}')"
+            -d @-
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

benchmarks/gaia/Dockerfile.gaia

@@ -0,0 +1,17 @@
+# Dockerfile for GAIA evaluation with MCP server pre-installed
+# Extends the base SDK image to pre-cache mcp-server-fetch and eliminate startup delays
+
+ARG SDK_IMAGE=ghcr.io/openhands/eval-agent-server:f715937-gaia-binary-minimal
+FROM ${SDK_IMAGE}
+
+# Switch to root to install packages
+USER root
+
+# Pre-install MCP server to avoid 1-18 minute startup delays during agent initialization
+# This caches the mcp-server-fetch package so uvx can start it instantly at runtime
+RUN uvx mcp-server-fetch --version 2>&1 || echo "MCP server cached"
+
+# Switch back to openhands user
+USER openhands
+
+# Inherit all other settings from base image

benchmarks/gaia/build_images.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""
+Build a universal agent-server image for GAIA benchmark.
+
+Unlike SWE-bench which requires per-instance images with specific repository environments,
+GAIA uses a single universal image for all instances since they share the same Python+Node.js environment.
+
+Example:
+  uv run benchmarks/gaia/build_images.py \
+    --image ghcr.io/openhands/eval-agent-server --target binary-minimal --push
+"""
+
+import sys
+
+from benchmarks.utils.build_utils import (
+    build_all_images,
+    default_build_output_dir,
+    get_build_parser,
+)
+from openhands.sdk import get_logger
+
+
+logger = get_logger(__name__)
+
+# GAIA base image: Python 3.12 + Node.js 22 (default for agent server)
+GAIA_BASE_IMAGE = "nikolaik/python-nodejs:python3.12-nodejs22"
+
+
+def gaia_tag_fn(base_image: str) -> str:
+    """Return custom tag for GAIA images (all use 'gaia' tag)."""
+    return "gaia"
+
+
+def main(argv: list[str]) -> int:
+    parser = get_build_parser()
+    args = parser.parse_args(argv)
+
+    # GAIA only needs one universal image for all instances
+    base_images = [GAIA_BASE_IMAGE]
+
+    logger.info(f"Building GAIA agent server image from base: {GAIA_BASE_IMAGE}")
+    logger.info(f"Target: {args.target}")
+    logger.info(f"Image: {args.image}")
+    logger.info(f"Push: {args.push}")
+
+    build_dir = default_build_output_dir("gaia", "validation")
+    return build_all_images(
+        base_images=base_images,
+        target=args.target,
+        build_dir=build_dir,
+        image=args.image,
+        push=args.push,
+        max_workers=1,  # Only building one image
+        dry_run=args.dry_run,
+        max_retries=args.max_retries,
+        base_image_to_custom_tag_fn=gaia_tag_fn,  # Tag all with "gaia"
+    )
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))