V1

.github/workflows/ci.yml

@@ -7,6 +7,7 @@ on:
   pull_request:
     branches:
       - main
+      - "v1"
 
 permissions:
   contents: read

.github/workflows/lint.yml

@@ -7,6 +7,7 @@ on:
   pull_request:
     branches:
       - main
+      - "v1"
 
 permissions:
   contents: read
@@ -49,4 +50,3 @@ jobs:
 
     - name: Biome lint
       run: npx biome check . --diagnostic-level=warn --no-errors-on-unmatched
-

.gitignore

@@ -112,6 +112,7 @@ midscene_run/dump
 *.ignore.png
 extension_output
 .cursor
+apps/chrome-extension/web-ext-profile
 packages/android-playground/static/
 packages/ios-playground/static/
 packages/ios/static/
@@ -121,7 +122,7 @@ CLAUDE.md
 .claude
 **/.claude
 **/CLAUDE.md
-AGENTS.md
+
 .cursor/rules/nx-rules.mdc
 .github/instructions/nx.instructions.md
 .gemini-clipboard

CONTRIBUTING.md

@@ -121,7 +121,7 @@ To change the AI-related code of this repository, you need to create a '.env 'fi
 
 ```
 OPENAI_API_KEY="your_token"
-MIDSCENE_MODEL_NAME="gpt-4o-2024-08-06"
+MIDSCENE_MODEL_NAME="qwen3-vl-plus"
 ```
 
 

README.md

@@ -81,7 +81,7 @@ Read more about [Choose a model](https://midscenejs.com/choose-a-model)
 Midscene will automatically plan the steps and execute them. It may be slower and heavily rely on the quality of the AI model.
 
 ```javascript
-await aiAction('click all the records one by one. If one record contains the text "completed", skip it');
+await aiAct('click all the records one by one. If one record contains the text "completed", skip it');
 ```
 
 ### Workflow Style
@@ -131,6 +131,8 @@ Community projects that extend Midscene.js capabilities:
 
 * [midscene-ios](https://github.com/lhuanyu/midscene-ios) - iOS automation support for Midscene
 * [Midscene-Python](https://github.com/Python51888/Midscene-Python) - Python SDK for Midscene automation
+* [midscene-java](https://github.com/Master-Frank/midscene-java) by @Master-Frank - Java SDK that brings Midscene automation features to JVM projects
+* [midscene-java](https://github.com/alstafeev/midscene-java) by @alstafeev - Java SDK for Midscene automation
 
 
 ## 📝 Credits

README.zh.md

@@ -81,7 +81,7 @@ Midscene.js 支持视觉语言模型，例如 `Qwen3-VL`、`Doubao-1.6-vision`
 Midscene 会自动规划步骤并执行。它可能较慢，并且深度依赖于 AI 模型的质量。
 
 ```javascript
-await aiAction('click all the records one by one. If one record contains the text "completed", skip it');
+await aiAct('click all the records one by one. If one record contains the text "completed", skip it');
 ```
 
 ### 工作流风格
@@ -133,6 +133,8 @@ for (const record of recordList) {
 
 * [midscene-ios](https://github.com/lhuanyu/midscene-ios) - iOS 设备自动化工具
 * [Midscene-Python](https://github.com/Python51888/Midscene-Python) - Python 版本的 Midscene SDK
+* [midscene-java](https://github.com/Master-Frank/midscene-java) by @Master-Frank - Java 版本的 Midscene SDK，便于在 JVM 项目中使用自动化能力
+* [midscene-java](https://github.com/alstafeev/midscene-java) by @alstafeev - Java SDK,用于 Midscene 自动化
 
 ## 📝 致谢
 

apps/android-playground/package.json

@@ -27,12 +27,12 @@
     "socket.io-client": "4.8.1"
   },
   "devDependencies": {
-    "@rsbuild/core": "^1.3.22",
-    "@rsbuild/plugin-less": "^1.2.4",
-    "@rsbuild/plugin-node-polyfill": "1.3.0",
-    "@rsbuild/plugin-react": "^1.3.1",
-    "@rsbuild/plugin-svgr": "^1.1.1",
-    "@rsbuild/plugin-type-check": "1.2.3",
+    "@rsbuild/core": "^1.5.17",
+    "@rsbuild/plugin-less": "^1.5.0",
+    "@rsbuild/plugin-node-polyfill": "1.4.2",
+    "@rsbuild/plugin-react": "^1.4.1",
+    "@rsbuild/plugin-svgr": "^1.2.2",
+    "@rsbuild/plugin-type-check": "1.2.4",
     "@types/react": "^18.3.1",
     "@types/react-dom": "^18.3.1",
     "archiver": "^6.0.0",

apps/android-playground/rsbuild.config.ts

@@ -1,5 +1,8 @@
 import path from 'node:path';
-import { createPlaygroundCopyPlugin } from '@midscene/shared';
+import {
+  commonIgnoreWarnings,
+  createPlaygroundCopyPlugin,
+} from '@midscene/shared';
 import { defineConfig } from '@rsbuild/core';
 import { pluginLess } from '@rsbuild/plugin-less';
 import { pluginNodePolyfill } from '@rsbuild/plugin-node-polyfill';
@@ -9,6 +12,11 @@ import { pluginTypeCheck } from '@rsbuild/plugin-type-check';
 import { version as playgroundVersion } from '../../packages/playground/package.json';
 
 export default defineConfig({
+  tools: {
+    rspack: {
+      ignoreWarnings: commonIgnoreWarnings,
+    },
+  },
   environments: {
     web: {
       source: {

apps/android-playground/src/App.tsx

@@ -1,6 +1,10 @@
 import './App.less';
-import { SCRCPY_SERVER_PORT } from '@midscene/shared/constants';
 import {
+  PLAYGROUND_SERVER_PORT,
+  SCRCPY_SERVER_PORT,
+} from '@midscene/shared/constants';
+import {
+  ScreenshotViewer,
   globalThemeConfig,
   safeOverrideAIConfig,
   useEnvConfig,
@@ -17,6 +21,13 @@ import ScrcpyPlayer, {
 
 const { Content } = Layout;
 
+// Helper function to detect if device is remote (IP:Port format)
+const isRemoteDevice = (deviceId: string | null): boolean => {
+  if (!deviceId) return false;
+  // Remote device format: IP:Port (e.g., 192.168.1.10:5555)
+  return /^\d+\.\d+\.\d+\.\d+:\d+$/.test(deviceId);
+};
+
 export default function App() {
   // Device and connection state - now simplified since device is pre-selected
   const [selectedDeviceId, setSelectedDeviceId] = useState<string | null>(null);
@@ -26,6 +37,7 @@ export default function App() {
     `http://localhost:${SCRCPY_SERVER_PORT}`,
   );
   const [isNarrowScreen, setIsNarrowScreen] = useState(false);
+  const [usePollingMode, setUsePollingMode] = useState(false);
 
   // Configuration state
   const { config } = useEnvConfig();
@@ -125,6 +137,11 @@ export default function App() {
     return () => window.removeEventListener('resize', handleResize);
   }, []);
 
+  // Detect if device is remote and switch to polling mode
+  useEffect(() => {
+    setUsePollingMode(isRemoteDevice(selectedDeviceId));
+  }, [selectedDeviceId]);
+
   return (
     <ConfigProvider theme={globalThemeConfig()}>
       {contextHolder}
@@ -157,11 +174,28 @@ export default function App() {
                   selectedDeviceId={selectedDeviceId}
                   scrcpyPlayerRef={scrcpyPlayerRef}
                 />
-                <ScrcpyPlayer
-                  ref={scrcpyPlayerRef}
-                  serverUrl={serverUrl}
-                  autoConnect={connectToDevice}
-                />
+                {!usePollingMode ? (
+                  <ScrcpyPlayer
+                    ref={scrcpyPlayerRef}
+                    serverUrl={serverUrl}
+                    autoConnect={connectToDevice}
+                  />
+                ) : (
+                  <ScreenshotViewer
+                    getScreenshot={() =>
+                      fetch(
+                        `http://localhost:${PLAYGROUND_SERVER_PORT}/screenshot`,
+                      ).then((r) => r.json())
+                    }
+                    getInterfaceInfo={() =>
+                      fetch(
+                        `http://localhost:${PLAYGROUND_SERVER_PORT}/interface-info`,
+                      ).then((r) => r.json())
+                    }
+                    serverOnline={true}
+                    isUserOperating={false}
+                  />
+                )}
               </div>
             </Panel>
           </PanelGroup>

apps/android-playground/src/components/playground-panel/index.tsx

@@ -77,6 +77,7 @@ export default function PlaygroundPanel() {
             enableScrollToBottom: true,
             serverMode: true,
             showEnvConfigReminder: true,
+            deviceType: 'android',
           }}
           branding={{
             title: 'Android Playground',

apps/chrome-extension/package.json

@@ -5,7 +5,8 @@
   "type": "module",
   "scripts": {
     "build": "rsbuild build && npm run pack-extension",
-    "dev": "rsbuild dev --open",
+    "dev": "concurrently -k -n build,run \"rsbuild dev\" \"node scripts/wait-for-build.js && web-ext run --config web-ext-config.cjs --source-dir dist --target chromium\"",
+    "dev:simple": "rsbuild dev --open",
     "preview": "rsbuild preview",
     "pack-extension": "node scripts/pack-extension.js"
   },
@@ -18,7 +19,6 @@
     "@midscene/shared": "workspace:*",
     "@midscene/visualizer": "workspace:*",
     "@midscene/web": "workspace:*",
-
     "@types/file-saver": "2.0.7",
     "antd": "^5.21.6",
     "canvas-confetti": "1.9.3",
@@ -32,20 +32,22 @@
     "zustand": "4.5.2"
   },
   "devDependencies": {
-    "@rsbuild/core": "^1.3.22",
-    "@rsbuild/plugin-less": "^1.2.4",
-    "@rsbuild/plugin-node-polyfill": "1.3.0",
-    "@rsbuild/plugin-react": "^1.3.1",
-    "@rsbuild/plugin-svgr": "^1.1.1",
-    "@rsbuild/plugin-type-check": "1.2.3",
+    "@rsbuild/core": "^1.5.17",
+    "@rsbuild/plugin-less": "^1.5.0",
+    "@rsbuild/plugin-node-polyfill": "1.4.2",
+    "@rsbuild/plugin-react": "^1.4.1",
+    "@rsbuild/plugin-svgr": "^1.2.2",
+    "@rsbuild/plugin-type-check": "1.2.4",
     "@tailwindcss/postcss": "4.1.11",
     "@types/chrome": "0.0.279",
     "@types/react": "^18.3.1",
     "@types/react-dom": "^18.3.1",
     "archiver": "^6.0.0",
+    "concurrently": "^8.2.0",
     "less": "^4.2.0",
+    "openai": "6.3.0",
     "tailwindcss": "4.1.11",
     "typescript": "^5.8.3",
-    "openai": "4.81.0"
+    "web-ext": "9.0.0"
   }
 }

apps/chrome-extension/rsbuild.config.ts

@@ -1,4 +1,5 @@
 import path from 'node:path';
+import { commonIgnoreWarnings } from '@midscene/shared';
 import { defineConfig } from '@rsbuild/core';
 import { pluginLess } from '@rsbuild/plugin-less';
 import { pluginNodePolyfill } from '@rsbuild/plugin-node-polyfill';
@@ -11,8 +12,15 @@ export default defineConfig({
   tools: {
     rspack: {
       watchOptions: {
-        ignored: /\.git/,
+        ignored: [
+          '**/.git/**',
+          '**/web-ext-profile/**',
+          '**/extension_output/**',
+          'dist/**', // Only ignore THIS app's dist folder, not workspace packages
+          '**/node_modules/**',
+        ],
       },
+      ignoreWarnings: commonIgnoreWarnings,
     },
   },
   environments: {
@@ -48,6 +56,7 @@ export default defineConfig({
   },
   dev: {
     writeToDisk: true,
+    lazyCompilation: false, // Disable lazy compilation for Chrome extension compatibility
   },
   output: {
     polyfill: 'entry',

apps/chrome-extension/scripts/wait-for-build.js

@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const manifestPath = path.resolve(__dirname, '../dist/manifest.json');
+const indexHtmlPath = path.resolve(__dirname, '../dist/index.html');
+const indexJsPath = path.resolve(__dirname, '../dist/static/js/index.js');
+const popupJsPath = path.resolve(__dirname, '../dist/static/js/popup.js');
+const maxWaitTime = 60000; // 60 seconds
+const checkInterval = 500; // 500ms
+const stabilityWait = 1000; // Wait 1 second after detection to ensure files are stable
+
+let elapsed = 0;
+
+console.log('Waiting for initial build to complete...');
+
+const checkBuildComplete = () => {
+  const manifestExists = fs.existsSync(manifestPath);
+  const indexExists = fs.existsSync(indexHtmlPath);
+  const indexJsExists = fs.existsSync(indexJsPath);
+  const popupJsExists = fs.existsSync(popupJsPath);
+
+  if (manifestExists && indexExists && indexJsExists && popupJsExists) {
+    // Wait a bit more to ensure all files are written
+    console.log('Build files detected, waiting for stability...');
+    setTimeout(() => {
+      // Double check the files still exist
+      if (
+        fs.existsSync(manifestPath) &&
+        fs.existsSync(indexHtmlPath) &&
+        fs.existsSync(indexJsPath) &&
+        fs.existsSync(popupJsPath)
+      ) {
+        console.log('Build complete! Starting web-ext...');
+        process.exit(0);
+      } else {
+        console.log('Build files disappeared, continuing to wait...');
+        setTimeout(checkBuildComplete, checkInterval);
+      }
+    }, stabilityWait);
+    return;
+  }
+
+  elapsed += checkInterval;
+
+  if (elapsed >= maxWaitTime) {
+    console.error('Timeout waiting for build to complete');
+    process.exit(1);
+  }
+
+  setTimeout(checkBuildComplete, checkInterval);
+};
+
+checkBuildComplete();