diff --git a/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml b/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml index 7fa548240f865..fe83a91b2f3d4 100644 --- a/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml +++ b/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml @@ -45,8 +45,6 @@ jobs: - template: use-android-ndk.yml - - template: install-appcenter.yml - - template: use-android-emulator.yml parameters: create: true @@ -68,19 +66,21 @@ jobs: parameters: stop: true + # we run e2e tests on one older device (Pixel 3) and one newer device (Galaxy 23) - script: | set -e -x - cd android_test/android - appcenter test run espresso \ - --app "AI-Frameworks/ORT-Mobile-Android" \ - --devices $(app_center_android_test_devices) \ - --app-path ./app/build/outputs/apk/debug/app-debug.apk \ - --test-series "master" \ - --locale "en_US" \ - --build-dir ./app/build/outputs/apk/androidTest/debug \ - --token $(app_center_api_token) - displayName: Run E2E tests using App Center - workingDirectory: $(Build.BinariesDirectory) + pip install requests + python $(Build.SourcesDirectory)/tools/python/upload_and_run_browserstack_tests.py \ + --test_platform espresso \ + --app_apk_path "debug/app-debug.apk" \ + --test_apk_path "androidTest/debug/app-debug-androidTest.apk" \ + --devices "Samsung Galaxy S23-13.0" "Google Pixel 3-9.0" + displayName: Run E2E tests using Browserstack + workingDirectory: $(Build.BinariesDirectory)/android_test/android/app/build/outputs/apk + timeoutInMinutes: 15 + env: + BROWSERSTACK_ID: $(browserstack_username) + BROWSERSTACK_TOKEN: $(browserstack_access_key) - template: component-governance-component-detection-steps.yml parameters : diff --git a/tools/python/upload_and_run_browserstack_tests.py b/tools/python/upload_and_run_browserstack_tests.py new file mode 100644 index 0000000000000..8751368e1b2fc --- /dev/null +++ b/tools/python/upload_and_run_browserstack_tests.py @@ -0,0 +1,158 @@ +import argparse +import os +import sys +import time +from pathlib import Path + +import requests + +script_description = """ +After building ONNXRuntime for Android or iOS, use this script to upload the app and test files to BrowserStack then +run the tests on the specified devices. + +Find the Android test app in the repo here (as of rel-1.19.2): +java/src/test/android +""" + + +def response_to_json(response): + response.raise_for_status() + response_json = response.json() + print(response_json) + print("-" * 30) + return response_json + + +def upload_apk_parse_json(post_url, apk_path, id, token): + with open(apk_path, "rb") as apk_file: + response = requests.post(post_url, files={"file": apk_file}, auth=(id, token), timeout=180) + return response_to_json(response) + + +def browserstack_build_request(devices, app_url, test_suite_url, test_platform, id, token): + headers = {} + + json_data = { + "devices": devices, + "app": app_url, + "testSuite": test_suite_url, + } + + build_response = requests.post( + f"https://api-cloud.browserstack.com/app-automate/{test_platform}/v2/build", + headers=headers, + json=json_data, + auth=(id, token), + timeout=180, + ) + + return response_to_json(build_response) + + +def build_query_loop(build_id, test_platform, id, token): + """Called after a build is initiated on Browserstack. Repeatedly queries the REST endpoint to check for the + status of the tests in 30 second intervals. + """ + tests_status = "running" + + while tests_status == "running": + time.sleep(30) + + test_response = requests.get( + f"https://api-cloud.browserstack.com/app-automate/{test_platform}/v2/builds/{build_id}", + auth=(id, token), + timeout=30, + ) + + test_response_json = response_to_json(test_response) + tests_status = test_response_json["status"] + + return tests_status + + +if __name__ == "__main__": + # handle cli args + parser = argparse.ArgumentParser(script_description) + + parser.add_argument( + "--test_platform", type=str, help="Testing platform", choices=["espresso", "xcuitest"], required=True + ) + parser.add_argument( + "--app_apk_path", + type=Path, + help=( + "Path to the app APK. " + "Typically, the app APK is in " + "{build_output_dir}/android_test/android/app/build/outputs/apk/debug/app-debug.apk" + ), + required=True, + ) + parser.add_argument( + "--test_apk_path", + type=Path, + help=( + "Path to the test APK. " + "Typically, the test APK is in " + "{build_output_dir}/android_test/android/app/build/outputs/apk/androidTest/debug/app-debug-androidTest.apk" + ), + required=True, + ) + parser.add_argument( + "--devices", + type=str, + nargs="+", + help="List of devices to run the tests on. For more info, " + "see https://www.browserstack.com/docs/app-automate/espresso/specify-devices", + required=True, + ) + + args = parser.parse_args() + + try: + browserstack_id = os.environ["BROWSERSTACK_ID"] + browserstack_token = os.environ["BROWSERSTACK_TOKEN"] + except KeyError: + print("Please set the environment variables BROWSERSTACK_ID and BROWSERSTACK_TOKEN") + print( + "These values will be found at https://app-automate.browserstack.com/dashboard/v2 & clicking 'ACCESS KEY'" + ) + sys.exit(1) + + # Upload the app and test suites + upload_app_json = upload_apk_parse_json( + f"https://api-cloud.browserstack.com/app-automate/{args.test_platform}/v2/app", + args.app_apk_path, + browserstack_id, + browserstack_token, + ) + upload_test_json = upload_apk_parse_json( + f"https://api-cloud.browserstack.com/app-automate/{args.test_platform}/v2/test-suite", + args.test_apk_path, + browserstack_id, + browserstack_token, + ) + + # Initiate build (send request to run the tests) + build_response_json = browserstack_build_request( + args.devices, + upload_app_json["app_url"], + upload_test_json["test_suite_url"], + args.test_platform, + browserstack_id, + browserstack_token, + ) + + # Get build status until the tests are no longer running + tests_status = build_query_loop( + build_response_json["build_id"], args.test_platform, browserstack_id, browserstack_token + ) + + test_suite_details_url = ( + f"https://app-automate.browserstack.com/dashboard/v2/builds/{build_response_json['build_id']}" + ) + + print("=" * 30) + print("Test suite details: ", test_suite_details_url) + print("=" * 30) + if tests_status != "passed": + raise Exception(f"Tests failed. Go to {test_suite_details_url} for more details.")