From 24c1cff9a4864a9a3dd85e8e7cb8a50ea2109db6 Mon Sep 17 00:00:00 2001 From: Jim Hildensperger Date: Fri, 21 Jan 2022 10:39:08 +0100 Subject: [PATCH 1/4] Add axtap for tapping accessibility elements --- idb/cli/commands/hid.py | 33 +++++++++++++++++++++++++++++---- idb/cli/main.py | 2 ++ idb/common/types.py | 5 +++++ idb/grpc/client.py | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 4 deletions(-) diff --git a/idb/cli/commands/hid.py b/idb/cli/commands/hid.py index 7b20b812b..5989a2390 100644 --- a/idb/cli/commands/hid.py +++ b/idb/cli/commands/hid.py @@ -7,8 +7,33 @@ from argparse import ArgumentParser, Namespace from idb.cli import ClientCommand -from idb.common.types import Client, HIDButtonType +from idb.common.types import Client, HIDButtonType, HIDElementType +class AXTapCommand(ClientCommand): + @property + def description(self) -> str: + return "Tap Accessibility Element On the Screen" + + @property + def name(self) -> str: + return "axtap" + + def add_parser_arguments(self, parser: ArgumentParser) -> None: + parser.add_argument( + "element", + choices=[element.name for element in HIDElementType], + help="Accessibility Type", + type=str + ) + parser.add_argument("label", nargs='?', help="AXLabel", type=str) + parser.add_argument("x", nargs='?', default=1, help="The x-coordinate", type=float) + parser.add_argument("y", nargs='?', default=1, help="The y-coordinate", type=float) + parser.add_argument("--duration", help="Press duration", type=float) + parser.add_argument("--count", default=1, help="Number of taps", type=int) + super().add_parser_arguments(parser) + + async def run_with_client(self, args: Namespace, client: Client) -> None: + await client.axtap(element=HIDElementType[args.element], label=args.label, x=args.x, y=args.y, duration=args.duration, count=args.count) class TapCommand(ClientCommand): @property @@ -20,8 +45,8 @@ def name(self) -> str: return "tap" def add_parser_arguments(self, parser: ArgumentParser) -> None: - parser.add_argument("x", help="The x-coordinate", type=int) - parser.add_argument("y", help="The y-coordinate", type=int) + parser.add_argument("x", help="The x-coordinate", type=float) + parser.add_argument("y", help="The y-coordinate", type=float) parser.add_argument("--duration", help="Press duration", type=float) super().add_parser_arguments(parser) @@ -43,7 +68,7 @@ def add_parser_arguments(self, parser: ArgumentParser) -> None: "button", help="The button name", choices=[button.name for button in HIDButtonType], - type=str, + type=str ) parser.add_argument("--duration", help="Press duration", type=float) super().add_parser_arguments(parser) diff --git a/idb/cli/main.py b/idb/cli/main.py index d9b5e687f..d2ec4cb47 100644 --- a/idb/cli/main.py +++ b/idb/cli/main.py @@ -54,6 +54,7 @@ from idb.cli.commands.focus import FocusCommand from idb.cli.commands.framework import FrameworkInstallCommand from idb.cli.commands.hid import ( + AXTapCommand, ButtonCommand, KeyCommand, KeySequenceCommand, @@ -242,6 +243,7 @@ async def gen_main(cmd_input: Optional[List[str]] = None) -> int: commands=[ AccessibilityInfoAllCommand(), AccessibilityInfoAtPointCommand(), + AXTapCommand(), TapCommand(), ButtonCommand(), TextCommand(), diff --git a/idb/common/types.py b/idb/common/types.py index 7aeb8c81e..5f5637b8c 100644 --- a/idb/common/types.py +++ b/idb/common/types.py @@ -120,6 +120,11 @@ class HIDButtonType(Enum): SIDE_BUTTON = 4 SIRI = 5 +class HIDElementType(Enum): + back = "back button" + button = "button" + text = "text" + textfield = "text field" ConnectionDestination = Union[str, Address] diff --git a/idb/grpc/client.py b/idb/grpc/client.py index bb70fa743..3644483d0 100644 --- a/idb/grpc/client.py +++ b/idb/grpc/client.py @@ -8,6 +8,7 @@ import functools import inspect import logging +import json import os import shutil import sys @@ -65,6 +66,7 @@ FileEntryInfo, FileListing, HIDButtonType, + HIDElementType, HIDEvent, IdbConnectionException, IdbException, @@ -789,6 +791,40 @@ async def list_xctests(self) -> List[InstalledTestInfo]: async def send_events(self, events: Iterable[HIDEvent]) -> None: await self.hid(iterator_to_async_iterator(events)) + @log_and_handle_exceptions + async def axtap(self, element: HIDElementType, label: str, x: float, y: float, duration: Optional[float] = None, count: Optional[int] = 1) -> None: + accessibilityInfo = await self.stub.accessibility_info( + AccessibilityInfoRequest( + point=None, + format=( + AccessibilityInfoRequest.LEGACY + ), + ) + ) + + elementFound = False + + for item in json.loads(accessibilityInfo.json): + try: + axElement = HIDElementType(item["role_description"]) + axLabel = item["AXLabel"] + + elementFound = axElement == element and (label is None or axLabel == label) + + if elementFound: + axframe = item["frame"] + x += axframe["x"] + y += axframe["y"] + break + except: + pass + + if elementFound: + for n in range(count): + await self.send_events(tap_to_events(x, y, duration)) + else: + print("AXElement with AXLabel: ", label, " type: ", element, " not found.") + @log_and_handle_exceptions async def tap(self, x: float, y: float, duration: Optional[float] = None) -> None: await self.send_events(tap_to_events(x, y, duration)) From 6cb90e23547b723b645eb5f842eeca6aed1581bd Mon Sep 17 00:00:00 2001 From: Jim Hildensperger Date: Fri, 21 Jan 2022 10:54:38 +0100 Subject: [PATCH 2/4] Add Pods directory to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 961b6c9fe..23bd45e19 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,9 @@ fbsimctl/cli-tests/supports_metal output/ .clang-format +# CocoaPods +Pods/ + # Files copied from Xcode Fixtures/Source/FBTestRunnerApp/Frameworks From b52faee96d3b4c4e03ada6fe9dfecea6778f6d4d Mon Sep 17 00:00:00 2001 From: Jim Hildensperger Date: Fri, 21 Jan 2022 10:54:57 +0100 Subject: [PATCH 3/4] Add generated protoc-gen-python_grpc to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 23bd45e19..eb12be5c6 100644 --- a/.gitignore +++ b/.gitignore @@ -78,6 +78,7 @@ var/ *.egg-info/ .installed.cfg *.egg +protoc-gen-python_grpc # Installer logs pip-log.txt From 329d6e19e7346d079c8da204419d14d2778f2dfb Mon Sep 17 00:00:00 2001 From: Jim Hildensperger Date: Fri, 21 Jan 2022 13:31:58 +0100 Subject: [PATCH 4/4] Run black to format changes --- idb/cli/commands/hid.py | 27 ++++++++++++++++++++------- idb/common/types.py | 2 ++ idb/common/udid.py | 2 +- idb/grpc/client.py | 20 ++++++++++++++------ 4 files changed, 37 insertions(+), 14 deletions(-) diff --git a/idb/cli/commands/hid.py b/idb/cli/commands/hid.py index 5989a2390..2a0956ec7 100644 --- a/idb/cli/commands/hid.py +++ b/idb/cli/commands/hid.py @@ -9,6 +9,7 @@ from idb.cli import ClientCommand from idb.common.types import Client, HIDButtonType, HIDElementType + class AXTapCommand(ClientCommand): @property def description(self) -> str: @@ -20,20 +21,32 @@ def name(self) -> str: def add_parser_arguments(self, parser: ArgumentParser) -> None: parser.add_argument( - "element", + "element", choices=[element.name for element in HIDElementType], help="Accessibility Type", - type=str + type=str, + ) + parser.add_argument("label", nargs="?", help="AXLabel", type=str) + parser.add_argument( + "x", nargs="?", default=1, help="The x-coordinate", type=float + ) + parser.add_argument( + "y", nargs="?", default=1, help="The y-coordinate", type=float ) - parser.add_argument("label", nargs='?', help="AXLabel", type=str) - parser.add_argument("x", nargs='?', default=1, help="The x-coordinate", type=float) - parser.add_argument("y", nargs='?', default=1, help="The y-coordinate", type=float) parser.add_argument("--duration", help="Press duration", type=float) parser.add_argument("--count", default=1, help="Number of taps", type=int) super().add_parser_arguments(parser) async def run_with_client(self, args: Namespace, client: Client) -> None: - await client.axtap(element=HIDElementType[args.element], label=args.label, x=args.x, y=args.y, duration=args.duration, count=args.count) + await client.axtap( + element=HIDElementType[args.element], + label=args.label, + x=args.x, + y=args.y, + duration=args.duration, + count=args.count, + ) + class TapCommand(ClientCommand): @property @@ -68,7 +81,7 @@ def add_parser_arguments(self, parser: ArgumentParser) -> None: "button", help="The button name", choices=[button.name for button in HIDButtonType], - type=str + type=str, ) parser.add_argument("--duration", help="Press duration", type=float) super().add_parser_arguments(parser) diff --git a/idb/common/types.py b/idb/common/types.py index 5f5637b8c..0886a294e 100644 --- a/idb/common/types.py +++ b/idb/common/types.py @@ -120,12 +120,14 @@ class HIDButtonType(Enum): SIDE_BUTTON = 4 SIRI = 5 + class HIDElementType(Enum): back = "back button" button = "button" text = "text" textfield = "text field" + ConnectionDestination = Union[str, Address] diff --git a/idb/common/udid.py b/idb/common/udid.py index 04dc8aaf9..f16222410 100644 --- a/idb/common/udid.py +++ b/idb/common/udid.py @@ -10,7 +10,7 @@ SIMULATOR_UDID = r"^[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}$" OLD_DEVICE_UDID = r"^[0-9a-f]{40}$" NEW_DEVICE_UDID = r"^[0-9]{8}-[0-9A-F]{16}$" -UDID = fr"({SIMULATOR_UDID}|{OLD_DEVICE_UDID}|{NEW_DEVICE_UDID})" +UDID = rf"({SIMULATOR_UDID}|{OLD_DEVICE_UDID}|{NEW_DEVICE_UDID})" def is_udid(udid: str) -> bool: diff --git a/idb/grpc/client.py b/idb/grpc/client.py index 3644483d0..459fab178 100644 --- a/idb/grpc/client.py +++ b/idb/grpc/client.py @@ -792,16 +792,22 @@ async def send_events(self, events: Iterable[HIDEvent]) -> None: await self.hid(iterator_to_async_iterator(events)) @log_and_handle_exceptions - async def axtap(self, element: HIDElementType, label: str, x: float, y: float, duration: Optional[float] = None, count: Optional[int] = 1) -> None: + async def axtap( + self, + element: HIDElementType, + label: str, + x: float, + y: float, + duration: Optional[float] = None, + count: Optional[int] = 1, + ) -> None: accessibilityInfo = await self.stub.accessibility_info( AccessibilityInfoRequest( point=None, - format=( - AccessibilityInfoRequest.LEGACY - ), + format=(AccessibilityInfoRequest.LEGACY), ) ) - + elementFound = False for item in json.loads(accessibilityInfo.json): @@ -809,7 +815,9 @@ async def axtap(self, element: HIDElementType, label: str, x: float, y: float, d axElement = HIDElementType(item["role_description"]) axLabel = item["AXLabel"] - elementFound = axElement == element and (label is None or axLabel == label) + elementFound = axElement == element and ( + label is None or axLabel == label + ) if elementFound: axframe = item["frame"]