system_prompt = """# Tools

You may call one or more functions to assist with the user query.

You are provided with function signatures within <tools></tools> XML tags:
<tools>
{"type": "function", "function": {"name": "computer_use", "description": "Use a mouse and keyboard to interact with a computer, and take screenshots.\\n* This is an interface to a desktop GUI. You do not have access to a terminal or applications menu. You must click on desktop icons to start applications.\\n* Some applications may take time to start or process actions, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you click on Firefox and a window doesn't open, try wait and taking another screenshot.\\n* The screen's resolution is 1000x1000.\\n* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges unless asked.", "parameters": {"properties": {"action": {"description": "The action to perform. The available actions are:\\n* `key`: Performs key down presses on the arguments passed in order, then performs key releases in reverse order.\\n* `type`: Type a string of text on the keyboard.\\n* `mouse_move`: Move the cursor to a specified (x, y) pixel coordinate on the screen.\\n* `left_click`: Click the left mouse button at a specified (x, y) pixel coordinate on the screen.\\n* `left_click_drag`: Click and drag the cursor to a specified (x, y) pixel coordinate on the screen.\\n* `right_click`: Click the right mouse button at a specified (x, y) pixel coordinate on the screen.\\n* `middle_click`: Click the middle mouse button at a specified (x, y) pixel coordinate on the screen.\\n* `double_click`: Double-click the left mouse button at a specified (x, y) pixel coordinate on the screen.\\n* `triple_click`: Triple-click the left mouse button at a specified (x, y) pixel coordinate on the screen (simulated as double-click since it's the closest action).\\n* `scroll`: Performs a scroll of the mouse scroll wheel.\\n* `hscroll`: Performs a horizontal scroll (mapped to regular scroll).\\n* `wait`: Wait specified seconds for the change to happen.\\n* `terminate`: Terminate the current task and report its completion status.\\n* `answer`: Answer a question.\\n* `interact`: Resolve the blocking window by interacting with the user.", "enum": ["key", "type", "mouse_move", "left_click", "left_click_drag", "right_click", "middle_click", "double_click", "triple_click", "scroll", "hscroll", "wait", "terminate", "answer", "interact"], "type": "string"}, "keys": {"description": "Required only by `action=key`.", "type": "array"}, "text": {"description": "Required only by `action=type`, `action=answer` and `action=interact`.", "type": "string"}, "coordinate": {"description": "(x, y): The x (pixels from the left edge) and y (pixels from the top edge) coordinates to move the mouse to. Required only by `action=mouse_move` and `action=left_click_drag`.", "type": "array"}, "pixels": {"description": "The amount of scrolling to perform. Positive values scroll up, negative values scroll down. Required only by `action=scroll` and `action=hscroll`.", "type": "number"}, "time": {"description": "The seconds to wait. Required only by `action=wait`.", "type": "number"}, "status": {"description": "The status of the task. Required only by `action=terminate`.", "type": "string", "enum": ["success", "failure"]}}, "required": ["action"], "type": "object"}}}
</tools>

For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
<tool_call>
{"name": <function-name>, "arguments": <args-json-object>}
</tool_call>

# Response format

Response format for every step:
1) Action: a short imperative describing what to do in the UI.
2) A single <tool_call>...</tool_call> block containing only the JSON: {"name": <function-name>, "arguments": <args-json-object>}.

Rules:
- Output exactly in the order: Action, <tool_call>.
- Be brief: one for Action.
- Do not output anything else outside those two parts.
- If finishing, use action=terminate in the tool call."""

def get_messages(image, instruction, history_output, model_name, system_prompt):
  """
  构造多轮对话消息

  参数:
    image: 当前截图路径
    instruction: 用户指令
    history_output: 历史对话记录 [{"output": "...", "image": "..."}]
    model_name: 模型名称
  """
  history_n = 4  # 保留最近4轮历史
  current_step = len(history_output)
    
  # 构造历史操作摘要
  history_start_idx = max(0, current_step - history_n)
  previous_actions = []
  for i in range(history_start_idx):
    if i < len(history_output):
      history_output_str = history_output[i]['output']
      if 'Action:' in history_output_str and '<tool_call>':
        history_output_str = history_output_str.split('Action:')[1].split('<tool_call>')[0].strip()
      previous_actions.append(f"Step {i + 1}: {history_output_str}")

  previous_actions_str = "\\n".join(previous_actions) if previous_actions else "None"

  instruction_prompt = f"""
   Please generate the next move according to the UI screenshot, instruction and previous actions.
      
   Instruction: {instruction}
      
   Previous actions:
   {previous_actions_str}"""

  # 构造 messages 数组
  messages = [
    {
      "role": "system",
      "content": [{"text": system_prompt}],
    }
  ]

  history_len = min(history_n, len(history_output))
  if history_len > 0:
    # 添加历史对话
    for history_id, history_item in enumerate(history_output[-history_n:], 0):
      if history_id == 0:
        messages.append({
          "role": "user",
          "content": [
            {"text": instruction_prompt},
            {"image": "file://" + history_item['image']}
          ]
        })
      else:
        messages.append({
          "role": "user",
          "content": [{"image": "file://" + history_item['image']}]
        })

      messages.append({
        "role": "assistant",
        "content": [{"text": history_item['output']}],
      })

    # 添加当前截图
    messages.append({
      "role": "user",
      "content": [{"image": "file://" + image}]
    })
  else:
    # 首轮对话
    messages.append({
      "role": "user",
      "content": [
        {"text": instruction_prompt},
        {"image": "file://" + image}
      ]
    })

  return messages

model_input
  [{
    "role": "system",
    "content": [{
      "text": "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{\"type\": \"function\", \"function\": {\"name_for_human\": \"mobile_use\", \"name\": \"mobile_use\", \"description\": \"Use a touchscreen to interact with a mobile device, and take screenshots.\n* This is an interface to a mobile device with touchscreen. You can perform actions like clicking, typing, swiping, etc.\n* Some applications may take time to start or process actions, so you may need to wait and take successive screenshots to see the results of your actions.\n* The screen's resolution is 1000x1000.\n* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges unless asked.\", \"parameters\": {\"properties\": {\"action\": {\"description\": \"The action to perform. The available actions are:\n* `key`: Perform a key event on the mobile device.\n    - This supports adb's `keyevent` syntax.\n    - Examples: \"volume_up\", \"volume_down\", \"power\", \"camera\", \"clear\".\n* `click`: Click the point on the screen with coordinate (x, y).\n* `long_press`: Press the point on the screen with coordinate (x, y) for specified seconds.\n* `swipe`: Swipe from the starting point with coordinate (x, y) to the end point with coordinates2 (x2, y2).\n* `type`: Input the specified text into the activated input box.\n* `system_button`: Press the system button.\n* `open`: Open an app on the device.\n* `wait`: Wait specified seconds for the change to happen.\n* `answer`: Terminate the current task and output the answer.\n* `interact`: Resolve the blocking window by interacting with the user.\n* `terminate`: Terminate the current task and report its completion status.\", \"enum\": [\"key\", \"click\", \"long_press\", \"swipe\", \"type\", \"system_button\", \"open\", \"wait\", \"answer\", \"interact\", \"terminate\"], \"type\": \"string\"}, \"coordinate\": {\"description\": \"(x, y): The x (pixels from the left edge) and y (pixels from the top edge) coordinates to move the mouse to. Required only by `action=click`, `action=long_press`, and `action=swipe`.\", \"type\": \"array\"}, \"coordinate2\": {\"description\": \"(x, y): The x (pixels from the left edge) and y (pixels from the top edge) coordinates to move the mouse to. Required only by `action=swipe`.\", \"type\": \"array\"}, \"text\": {\"description\": \"Required only by `action=key`, `action=type`, `action=open`, `action=answer`,and `action=interact`.\", \"type\": \"string\"}, \"time\": {\"description\": \"The seconds to wait. Required only by `action=long_press` and `action=wait`.\", \"type\": \"number\"}, \"button\": {\"description\": \"Back means returning to the previous interface, Home means returning to the desktop, Menu means opening the application background menu, and Enter means pressing the enter. Required only by `action=system_button`\", \"enum\": [\"Back\", \"Home\", \"Menu\", \"Enter\"], \"type\": \"string\"}, \"status\": {\"description\": \"The status of the task. Required only by `action=terminate`.\", \"type\": \"string\", \"enum\": [\"success\", \"failure\"]}}, \"required\": [\"action\"], \"type\": \"object\"}, \"args_format\": \"Format the arguments as a JSON object.\"}}\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call>\n\n# Response format\n\nResponse format for every step:\n1) Action: a short imperative describing what to do in the UI.\n2) A single <tool_call>...</tool_call> block containing only the JSON: {\"name\": <function-name>, \"arguments\": <args-json-object>}.\n\nRules:\n- Output exactly in the order: Action, <tool_call>.\n- Be brief: one for Action.\n- Do not output anything else outside those two parts.\n- If finishing, use action=terminate in the tool call."
    }]
  }, {
    "role": "user",
    "content": [{
      "text": "\nPlease generate the next move according to the UI screenshot, instruction and previous actions.\n\nInstruction: 帮我在携程搜一下今天济南喜来登酒店的价格\n\nPrevious actions:\nStep 1: 点击携程旅行应用图标以启动携程旅行预订应用程序。\nStep 2: 等待促销启动画面自动过渡到携程主应用界面。"
    }, {
      "image": "http://nlp-mobile-agent.oss-cn-zhangjiakou.aliyuncs.com/computer-use%2Fscreenshot%2Fscreenshot_a84122ac_853a630315784b64988492c9c07b5534.png"
    }]
  }, {
    "role": "assistant",
    "content": [{
      "text": "Action: 点击应用更新通知弹窗右上角的关闭按钮（X图标）以将其关闭。\n<tool_call>\n{\"name\": \"mobile_use\", \"arguments\": {\"action\": \"click\", \"coordinate\": [789, 280]}}\n</tool_call>"
    }]
  }, {
    "role": "user",
    "content": [{
      "image": "http://nlp-mobile-agent.oss-cn-zhangjiakou.aliyuncs.com/computer-use%2Fscreenshot%2Fscreenshot_6010a769_089b9b35b1904913bd5df492563b02b9.png"
    }]
  }, {
    "role": "assistant",
    "content": [{
      "text": "Action: 点击搜索栏中的“济南的酒店”文本区域，以激活搜索输入框并准备修改搜索词。\n<tool_call>\n{\"name\": \"mobile_use\", \"arguments\": {\"action\": \"click\", \"coordinate\": [112, 134]}}\n</tool_call>"
    }]
  }, {
    "role": "user",
    "content": [{
      "image": "http://nlp-mobile-agent.oss-cn-zhangjiakou.aliyuncs.com/computer-use%2Fscreenshot%2Fscreenshot_47446db4_fd4a5022002c4db99f110d5c7261fea2.png"
    }]
  }, {
    "role": "assistant",
    "content": [{
      "text": "Action: 点击显示“厦门”的位置字段，将搜索位置从厦门更改为济南。\n<tool_call>\n{\"name\": \"mobile_use\", \"arguments\": {\"action\": \"click\", \"coordinate\": [156, 347]}}\n</tool_call>"
    }]
  }, {
    "role": "user",
    "content": [{
      "image": "http://nlp-mobile-agent.oss-cn-zhangjiakou.aliyuncs.com/computer-use%2Fscreenshot%2Fscreenshot_3832132c_8c55861c1716467e802a3554402f3580.png"
    }]
  }, {
    "role": "assistant",
    "content": [{
      "text": "Action: 在搜索输入框中键入“济南”，以指定酒店搜索的城市位置。\n<tool_call>\n{\"name\": \"mobile_use\", \"arguments\": {\"action\": \"type\", \"text\": \"济南\"}}\n</tool_call>"
    }]
  }, {
    "role": "user",
    "content": [{
      "image": "http://nlp-mobile-agent.oss-cn-zhangjiakou.aliyuncs.com/computer-use%2Fscreenshot%2Fscreenshot_ff247bac_39c3e20be32c4baf8677a2b6b61bc021.png"
    }]
  }]                                              

import re
import json

def extract_tool_calls(text):
  """
  从模型输出中提取所有 <tool_call> 块

  参数:
    text: 模型返回的文本

  返回:
    actions: 解析后的操作列表
  """
  pattern = re.compile(r'<tool_call>(.*?)</tool_call>', re.DOTALL | re.IGNORECASE)
  blocks = pattern.findall(text)

  actions = []
  for blk in blocks:
    blk = blk.strip()
    try:
      actions.append(json.loads(blk))
    except json.JSONDecodeError as e:
      print(f'解析失败: {e} | 片段: {blk[:80]}...')

  return actions

import math
from PIL import Image

def smart_resize(height, width, factor=32, min_pixels=32*32*4, max_pixels=32*32*1280, max_long_side=8192):
  """
  计算模型内部缩放后的图像尺寸

  参数:
    height: 原始图像高度
    width: 原始图像宽度
    factor: 分辨率因子（固定为 16）
    min_pixels: 最小像素值
    max_pixels: 最大像素值
    max_long_side: 最长边限制

  返回:
    (h_bar, w_bar): 缩放后的高度和宽度
  """
  def round_by_factor(number, factor):
    return round(number / factor) * factor

  def ceil_by_factor(number, factor):
    return math.ceil(number / factor) * factor

  def floor_by_factor(number, factor):
    return math.floor(number / factor) * factor

  if height < 2 or width < 2:
    raise ValueError(f"height:{height} or width:{width} must be larger than factor:{factor}")
  elif max(height, width) / min(height, width) > 200:
    raise ValueError(f"absolute aspect ratio must be smaller than 200, got {height} / {width}")

  # 限制最长边
  if max(height, width) > max_long_side:
    beta = max(height, width) / max_long_side
    height, width = int(height / beta), int(width / beta)

  # 计算缩放后的尺寸
  h_bar = round_by_factor(height, factor)
  w_bar = round_by_factor(width, factor)

  if h_bar * w_bar > max_pixels:
    beta = math.sqrt((height * width) / max_pixels)
    h_bar = floor_by_factor(height / beta, factor)
    w_bar = floor_by_factor(width / beta, factor)
  elif h_bar * w_bar < min_pixels:
    beta = math.sqrt(min_pixels / (height * width))
    h_bar = ceil_by_factor(height * beta, factor)
    w_bar = ceil_by_factor(width * beta, factor)

  return h_bar, w_bar

import pyautogui
import pyperclip
import time
from PIL import Image
import os

class ComputerTools:
  """电脑端 GUI 操作工具类"""

  def __init__(self):
    self.image_info = None

  def load_image_info(self, path):
    """加载图像尺寸信息"""
    width, height = Image.open(path).size
    self.image_info = (width, height)

  def get_screenshot(self, image_path, retry_times=3):
    """获取桌面截图"""
    if os.path.exists(image_path):
      os.remove(image_path)

    for i in range(retry_times):
      screenshot = pyautogui.screenshot()
      screenshot.save(image_path)
      if os.path.exists(image_path):
        self.load_image_info(image_path)
        return True
      else:
        time.sleep(0.1)
    return False

  def reset(self):
    """显示桌面"""
    pyautogui.hotkey('win', 'd')

  def press_key(self, keys):
    """按键操作"""
    if isinstance(keys, list):
      cleaned_keys = []
      for key in keys:
        if isinstance(key, str):
          # 处理键名格式
          if key.startswith("keys=["):
            key = key[6:]
          if key.endswith("]"):
            key = key[:-1]
          if key.startswith("['") or key.startswith('["'):
            key = key[2:] if len(key) > 2 else key
          if key.endswith("']") or key.endswith('"]'):
            key = key[:-2] if len(key) > 2 else key
          key = key.strip()

          # 转换键名
          key_map = {
            "arrowleft": "left",
            "arrowright": "right",
            "arrowup": "up",
            "arrowdown": "down"
          }
          key = key_map.get(key, key)
          cleaned_keys.append(key)
        else:
          cleaned_keys.append(key)
      keys = cleaned_keys
    else:
      keys = [keys]

    if len(keys) > 1:
      pyautogui.hotkey(*keys)
    else:
      pyautogui.press(keys[0])

  def type(self, text):
    """输入文本（使用剪贴板方式支持中文）"""
    pyperclip.copy(text)
    pyautogui.keyDown('ctrl')
    pyautogui.keyDown('v')
    pyautogui.keyUp('v')
    pyautogui.keyUp('ctrl')

  def mouse_move(self, x, y):
    """移动鼠标到指定坐标"""
    pyautogui.moveTo(x, y)
    time.sleep(0.1)
    pyautogui.moveTo(x, y)

  def left_click(self, x, y):
    """左键点击"""
    pyautogui.moveTo(x, y)
    time.sleep(0.1)
    pyautogui.click()

  def left_click_drag(self, x, y):
    """从当前位置拖拽到指定坐标"""
    pyautogui.dragTo(x, y, duration=0.5)
    pyautogui.moveTo(x, y)

  def right_click(self, x, y):
    """右键点击"""
    pyautogui.moveTo(x, y)
    time.sleep(0.1)
    pyautogui.rightClick()

  def middle_click(self, x, y):
    """中键点击"""
    pyautogui.moveTo(x, y)
    time.sleep(0.1)
    pyautogui.middleClick()

  def double_click(self, x, y):
    """双击"""
    pyautogui.moveTo(x, y)
    time.sleep(0.1)
    pyautogui.doubleClick()

  def triple_click(self, x, y):
    """三击"""
    pyautogui.moveTo(x, y)
    time.sleep(0.1)
    pyautogui.tripleClick()

  def scroll(self, pixels):
    """滚轮滚动"""
    pyautogui.scroll(pixels)

import os
import dashscope
import time

def run_gui_automation(instruction, max_step=30):
  """
  运行完整的 GUI 自动化流程

  参数:
    instruction: 用户指令
    max_step: 最大执行步骤数
  """
  # 配置 API
  dashscope.api_key = os.getenv("DASHSCOPE_API_KEY")
  dashscope.base_http_api_url = 'https://dashscope.aliyuncs.com/api/v1'
  model_name = 'gui-plus-2026-02-26'

  # 初始化工具
  computer_tools = ComputerTools()
  computer_tools.reset()  # 显示桌面

  # 创建输出目录
  output_dir = os.path.join(os.path.expanduser("~"), "Desktop", "gui_automation")
  os.makedirs(output_dir, exist_ok=True)

  # 对话历史
  history = []
  stop_flag = False

  print(f"[任务] {instruction}")
  print("=" * 60)

  for step_id in range(max_step):
    if stop_flag:
      break

    print(f"\n[步骤 {step_id + 1}]")

    # 1. 截图
    screen_shot = os.path.join(output_dir, f'screenshot_{step_id}.png')
    computer_tools.get_screenshot(screen_shot)

    # 2. 构造消息
    messages = get_messages(screen_shot, instruction, history, model_name)

    # 3. 调用模型
    response = dashscope.MultiModalConversation.call(
      model=model_name,
      messages=messages,
      vl_high_resolution_images=True,
      stream=False
    )

    output_text = response.output.choices[0].message.content[0]['text']
    print(f"[模型输出]\n{output_text}\n")

    # 4. 解析操作
    action_list = extract_tool_calls(output_text)
    if not action_list:
      print("未提取到有效操作")
      break

    # 5. 执行操作
    for action_id, action in enumerate(action_list):
      action_parameter = action['arguments']
      action_type = action_parameter['action']

      # 获取图像尺寸用于坐标映射
      dummy_image = Image.open(screen_shot)
      resized_height, resized_width = smart_resize(
        dummy_image.height,
        dummy_image.width,
        factor=16,
        min_pixels=3136,
        max_pixels=1003520 * 200
      )

      # 映射坐标（从归一化坐标 1000x1000 映射到实际尺寸）
      for key in ['coordinate', 'coordinate1', 'coordinate2']:
        if key in action_parameter:
          action_parameter[key][0] = int(action_parameter[key][0] / 1000 * resized_width)
          action_parameter[key][1] = int(action_parameter[key][1] / 1000 * resized_height)

      # 执行对应操作
      if action_type in ['click', 'left_click']:
        computer_tools.left_click(
          action_parameter['coordinate'][0],
          action_parameter['coordinate'][1]
        )
        print(f"[OK] 左键点击 ({action_parameter['coordinate'][0]}, {action_parameter['coordinate'][1]})")

      elif action_type == 'mouse_move':
        computer_tools.mouse_move(
          action_parameter['coordinate'][0],
          action_parameter['coordinate'][1]
        )
        print(f"[OK] 移动鼠标到 ({action_parameter['coordinate'][0]}, {action_parameter['coordinate'][1]})")

      elif action_type == 'middle_click':
        computer_tools.middle_click(
          action_parameter['coordinate'][0],
          action_parameter['coordinate'][1]
        )
        print(f"[OK] 中键点击")

      elif action_type in ['right click', 'right_click']:
        computer_tools.right_click(
          action_parameter['coordinate'][0],
          action_parameter['coordinate'][1]
        )
        print(f"[OK] 右键点击")

      elif action_type in ['key', 'hotkey']:
        computer_tools.press_key(action_parameter['keys'])
        print(f"[OK] 按键 {action_parameter['keys']}")

      elif action_type == 'type':
        text = action_parameter['text']
        computer_tools.type(text)
        print(f"[OK] 输入文本: {text}")

      elif action_type == 'drag':
        computer_tools.left_click_drag(
          action_parameter['coordinate'][0],
          action_parameter['coordinate'][1]
        )
        print(f"[OK] 拖拽到 ({action_parameter['coordinate'][0]}, {action_parameter['coordinate'][1]})")

      elif action_type == 'scroll':
        if 'coordinate' in action_parameter:
          computer_tools.mouse_move(
            action_parameter['coordinate'][0],
            action_parameter['coordinate'][1]
          )
        computer_tools.scroll(action_parameter.get("pixels", 1))
        print(f"[OK] 滚动 {action_parameter.get('pixels', 1)} 像素")

      elif action_type in ['computer_double_click', 'double_click']:
        computer_tools.double_click(
          action_parameter['coordinate'][0],
          action_parameter['coordinate'][1]
        )
        print(f"[OK] 双击")

      elif action_type == 'wait':
        time.sleep(action_parameter.get('time', 2))
        print(f"[OK] 等待 {action_parameter.get('time', 2)} 秒")

      elif action_type == 'answer':
        print(f"[OK] 任务完成: {action_parameter.get('text', '')}")
        stop_flag = True
        break

      elif action_type in ['stop', 'terminate', 'done']:
        print(f"[OK] 任务终止: {action_parameter.get('status', 'success')}")
        stop_flag = True
        break

      else:
        print(f"未知操作类型: {action_type}")

    # 6. 保存历史
    history.append({
      'output': output_text,
      'image': screen_shot
    })

    time.sleep(2)  # 操作间隔

  print("\n" + "=" * 60)
  print(f"[完成] 共执行 {len(history)} 步")

# 使用示例
if __name__ == '__main__':
  run_gui_automation(
    instruction='帮我打开chrome,在百度中搜索阿里巴巴',
    max_step=30
  )