|
| 1 | +import argparse |
| 2 | +import asyncio |
| 3 | +import json |
| 4 | +import plistlib |
| 5 | +import shutil |
| 6 | +import subprocess |
| 7 | +import sys |
| 8 | +from contextlib import asynccontextmanager |
| 9 | +from datetime import datetime |
| 10 | +from pathlib import Path |
| 11 | + |
| 12 | + |
| 13 | +DECODE_ARGS = ("UTF-8", "backslashreplace") |
| 14 | + |
| 15 | + |
| 16 | +# Work around a bug involving sys.exit and TaskGroups |
| 17 | +# (https://github.com/python/cpython/issues/101515). |
| 18 | +def exit(*args): |
| 19 | + raise MySystemExit(*args) |
| 20 | + |
| 21 | + |
| 22 | +class MySystemExit(Exception): |
| 23 | + pass |
| 24 | + |
| 25 | + |
| 26 | +# All subprocesses are executed through this context manager so that no matter |
| 27 | +# what happens, they can always be cancelled from another task, and they will |
| 28 | +# always be cleaned up on exit. |
| 29 | +@asynccontextmanager |
| 30 | +async def async_process(*args, **kwargs): |
| 31 | + process = await asyncio.create_subprocess_exec(*args, **kwargs) |
| 32 | + try: |
| 33 | + yield process |
| 34 | + finally: |
| 35 | + if process.returncode is None: |
| 36 | + # Allow a reasonably long time for Xcode to clean itself up, |
| 37 | + # because we don't want stale emulators left behind. |
| 38 | + timeout = 10 |
| 39 | + process.terminate() |
| 40 | + try: |
| 41 | + await asyncio.wait_for(process.wait(), timeout) |
| 42 | + except TimeoutError: |
| 43 | + print( |
| 44 | + f"Command {args} did not terminate after {timeout} seconds " |
| 45 | + f" - sending SIGKILL" |
| 46 | + ) |
| 47 | + process.kill() |
| 48 | + |
| 49 | + # Even after killing the process we must still wait for it, |
| 50 | + # otherwise we'll get the warning "Exception ignored in __del__". |
| 51 | + await asyncio.wait_for(process.wait(), timeout=1) |
| 52 | + |
| 53 | + |
| 54 | +async def async_check_output(*args, **kwargs): |
| 55 | + async with async_process( |
| 56 | + *args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs |
| 57 | + ) as process: |
| 58 | + stdout, stderr = await process.communicate() |
| 59 | + if process.returncode == 0: |
| 60 | + return stdout.decode(*DECODE_ARGS) |
| 61 | + else: |
| 62 | + raise subprocess.CalledProcessError( |
| 63 | + process.returncode, args, |
| 64 | + stdout.decode(*DECODE_ARGS), stderr.decode(*DECODE_ARGS) |
| 65 | + ) |
| 66 | + |
| 67 | + |
| 68 | +# Return a list of UDIDs associated with booted simulators |
| 69 | +async def list_devices(): |
| 70 | + # List the testing simulators, in JSON format |
| 71 | + raw_json = await async_check_output( |
| 72 | + "xcrun", "simctl", "--set", "testing", "list", "-j" |
| 73 | + ) |
| 74 | + json_data = json.loads(raw_json) |
| 75 | + |
| 76 | + # Filter out the booted iOS simulators |
| 77 | + return [ |
| 78 | + simulator["udid"] |
| 79 | + for runtime, simulators in json_data['devices'].items() |
| 80 | + for simulator in simulators |
| 81 | + if runtime.split(".")[-1].startswith("iOS") |
| 82 | + and simulator['state'] == "Booted" |
| 83 | + ] |
| 84 | + |
| 85 | + |
| 86 | +async def find_device(initial_devices): |
| 87 | + while True: |
| 88 | + new_devices = set(await list_devices()).difference(initial_devices) |
| 89 | + if len(new_devices) == 0: |
| 90 | + await asyncio.sleep(1) |
| 91 | + elif len(new_devices) == 1: |
| 92 | + udid = new_devices.pop() |
| 93 | + print(f"Test simulator UDID: {udid}") |
| 94 | + return udid |
| 95 | + else: |
| 96 | + exit(f"Found more than one new device: {new_devices}") |
| 97 | + |
| 98 | + |
| 99 | +async def log_stream_task(initial_devices): |
| 100 | + # Wait up to 5 minutes for the build to complete and the simulator to boot. |
| 101 | + udid = await asyncio.wait_for(find_device(initial_devices), 5*60) |
| 102 | + |
| 103 | + # Stream the iOS device's logs, filtering out messages that come from the |
| 104 | + # XCTest test suite (catching NSLog messages from the test method), or |
| 105 | + # Python itself (catching stdout/stderr content routed to the system log |
| 106 | + # with config->use_system_logger). |
| 107 | + args = [ |
| 108 | + "xcrun", |
| 109 | + "simctl", |
| 110 | + "--set", |
| 111 | + "testing", |
| 112 | + "spawn", |
| 113 | + udid, |
| 114 | + "log", |
| 115 | + "stream", |
| 116 | + "--style", |
| 117 | + "compact", |
| 118 | + "--predicate", |
| 119 | + ( |
| 120 | + 'senderImagePath ENDSWITH "/iOSTestbedTests.xctest/iOSTestbedTests"' |
| 121 | + ' OR senderImagePath ENDSWITH "/Python.framework/Python"' |
| 122 | + ) |
| 123 | + ] |
| 124 | + |
| 125 | + async with async_process( |
| 126 | + *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, |
| 127 | + ) as process: |
| 128 | + while line := (await process.stdout.readline()).decode(*DECODE_ARGS): |
| 129 | + sys.stdout.write(line) |
| 130 | + |
| 131 | + |
| 132 | +async def xcode_test(location, simulator): |
| 133 | + # Run the test suite on the named simulator |
| 134 | + args = [ |
| 135 | + "xcodebuild", |
| 136 | + "test", |
| 137 | + "-project", |
| 138 | + str(location / "iOSTestbed.xcodeproj"), |
| 139 | + "-scheme", |
| 140 | + "iOSTestbed", |
| 141 | + "-destination", |
| 142 | + f"platform=iOS Simulator,name={simulator}", |
| 143 | + "-resultBundlePath", |
| 144 | + str(location / f"{datetime.now():%Y%m%d-%H%M%S}.xcresult"), |
| 145 | + "-derivedDataPath", |
| 146 | + str(location / "DerivedData",) |
| 147 | + ] |
| 148 | + async with async_process( |
| 149 | + *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, |
| 150 | + ) as process: |
| 151 | + while line := (await process.stdout.readline()).decode(*DECODE_ARGS): |
| 152 | + sys.stdout.write(line) |
| 153 | + |
| 154 | + status = await asyncio.wait_for(process.wait(), timeout=1) |
| 155 | + exit(status) |
| 156 | + |
| 157 | + |
| 158 | +def create_testbed(location: Path, framework: Path, apps: list[Path]) -> None: |
| 159 | + if location.exists(): |
| 160 | + print(f"{location} already exists; aborting without creating project.") |
| 161 | + sys.exit(10) |
| 162 | + |
| 163 | + print("Copying template testbed project...") |
| 164 | + shutil.copytree(Path(__file__).parent, location) |
| 165 | + |
| 166 | + if framework.suffix == ".xcframework": |
| 167 | + print("Installing XCFramework...") |
| 168 | + xc_framework_path = location / "Python.xcframework" |
| 169 | + shutil.rmtree(xc_framework_path) |
| 170 | + shutil.copytree(framework, xc_framework_path) |
| 171 | + else: |
| 172 | + print("Installing simulator Framework...") |
| 173 | + sim_framework_path = ( |
| 174 | + location |
| 175 | + / "Python.xcframework" |
| 176 | + / "ios-arm64_x86_64-simulator" |
| 177 | + ) |
| 178 | + shutil.rmtree(sim_framework_path) |
| 179 | + shutil.copytree(framework, sim_framework_path) |
| 180 | + |
| 181 | + for app in apps: |
| 182 | + print(f"Installing app {app!r}...") |
| 183 | + shutil.copytree(app, location / "iOSTestbed/app/{app.name}") |
| 184 | + |
| 185 | + print(f"Testbed project created in {location}") |
| 186 | + |
| 187 | + |
| 188 | +def update_plist(testbed_path, args): |
| 189 | + # Add the test runner arguments to the testbed's Info.plist file. |
| 190 | + info_plist = testbed_path / "iOSTestbed" / "iOSTestbed-Info.plist" |
| 191 | + with info_plist.open("rb") as f: |
| 192 | + info = plistlib.load(f) |
| 193 | + |
| 194 | + info["TestArgs"] = args |
| 195 | + |
| 196 | + with info_plist.open("wb") as f: |
| 197 | + plistlib.dump(info, f) |
| 198 | + |
| 199 | + |
| 200 | +async def run_testbed(simulator: str, args: list[str]): |
| 201 | + location = Path(__file__).parent |
| 202 | + print("Updating plist...") |
| 203 | + update_plist(location, args) |
| 204 | + |
| 205 | + # Get the list of devices that are booted at the start of the test run. |
| 206 | + # The simulator started by the test suite will be detected as the new |
| 207 | + # entry that appears on the device list. |
| 208 | + initial_devices = await list_devices() |
| 209 | + |
| 210 | + try: |
| 211 | + async with asyncio.TaskGroup() as tg: |
| 212 | + tg.create_task(log_stream_task(initial_devices)) |
| 213 | + tg.create_task(xcode_test(location, simulator)) |
| 214 | + except* MySystemExit as e: |
| 215 | + raise SystemExit(*e.exceptions[0].args) from None |
| 216 | + except* subprocess.CalledProcessError as e: |
| 217 | + # Extract it from the ExceptionGroup so it can be handled by `main`. |
| 218 | + raise e.exceptions[0] |
| 219 | + |
| 220 | + |
| 221 | +def main(): |
| 222 | + parser = argparse.ArgumentParser( |
| 223 | + prog="testbed", |
| 224 | + description=( |
| 225 | + "Manages the process of testing a Python project in the iOS simulator" |
| 226 | + ) |
| 227 | + ) |
| 228 | + |
| 229 | + subcommands = parser.add_subparsers(dest="subcommand") |
| 230 | + |
| 231 | + create = subcommands.add_parser( |
| 232 | + "create", |
| 233 | + description=( |
| 234 | + "Clone the testbed project, copying in an iOS Python framework and" |
| 235 | + "any specified application code." |
| 236 | + ), |
| 237 | + help="Create a new testbed project" |
| 238 | + ) |
| 239 | + create.add_argument( |
| 240 | + "--framework", |
| 241 | + required=True, |
| 242 | + help=( |
| 243 | + "The location of the XCFramework (or simulator-only slice of an XCFramework) " |
| 244 | + "to use when running the testbed" |
| 245 | + ) |
| 246 | + ) |
| 247 | + create.add_argument( |
| 248 | + "--app", |
| 249 | + dest="apps", |
| 250 | + action="append", |
| 251 | + default=[], |
| 252 | + help="The location of any code to include in the testbed project", |
| 253 | + ) |
| 254 | + create.add_argument( |
| 255 | + "location", |
| 256 | + help="The path where the testbed will be created." |
| 257 | + ) |
| 258 | + |
| 259 | + run = subcommands.add_parser( |
| 260 | + "run", |
| 261 | + usage='%(prog)s [-h] [--simulator SIMULATOR] -- <test arg> [<test arg> ...]', |
| 262 | + description=( |
| 263 | + "Run a testbed project. The arguments provided after `--` will be passed to " |
| 264 | + "the running iOS process as if they were arguments to `python -m`." |
| 265 | + ), |
| 266 | + help="Run a testbed project", |
| 267 | + ) |
| 268 | + run.add_argument( |
| 269 | + "--simulator", |
| 270 | + default="iPhone SE (3rd Generation)", |
| 271 | + help="The name of the simulator to use (default: 'iPhone SE (3rd Generation)')", |
| 272 | + ) |
| 273 | + |
| 274 | + try: |
| 275 | + pos = sys.argv.index("--") |
| 276 | + testbed_args = sys.argv[1:pos] |
| 277 | + test_args = sys.argv[pos+1:] |
| 278 | + except ValueError: |
| 279 | + testbed_args = sys.argv[1:] |
| 280 | + test_args = [] |
| 281 | + |
| 282 | + context = parser.parse_args(testbed_args) |
| 283 | + |
| 284 | + if context.subcommand == "create": |
| 285 | + create_testbed( |
| 286 | + location=Path(context.location), |
| 287 | + framework=Path(context.framework), |
| 288 | + apps=[Path(app) for app in context.apps], |
| 289 | + ) |
| 290 | + elif context.subcommand == "run": |
| 291 | + if test_args: |
| 292 | + asyncio.run( |
| 293 | + run_testbed( |
| 294 | + simulator=context.simulator, |
| 295 | + args=test_args |
| 296 | + ) |
| 297 | + ) |
| 298 | + else: |
| 299 | + print(f"Must specify test arguments (e.g., {sys.argv[0]} run -- test)") |
| 300 | + print() |
| 301 | + parser.print_help(sys.stderr) |
| 302 | + sys.exit(2) |
| 303 | + else: |
| 304 | + parser.print_help(sys.stderr) |
| 305 | + sys.exit(1) |
| 306 | + |
| 307 | + |
| 308 | +if __name__ == "__main__": |
| 309 | + main() |
0 commit comments