ci(android-java): add BrowserStack integration tests #57
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # .github/workflows/android-java-browserstack.yml | |
| # Workflow for building and testing android-java on BrowserStack physical devices | |
| # | |
| --- | |
| name: android-java-browserstack | |
| on: | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - 'android-java/**' | |
| - '.github/workflows/android-java-browserstack.yml' | |
| push: | |
| branches: [main] | |
| paths: | |
| - 'android-java/**' | |
| - '.github/workflows/android-java-browserstack.yml' | |
| workflow_dispatch: # Allow manual trigger | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build-and-test: | |
| name: Build and Test on BrowserStack | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '17' | |
| distribution: 'temurin' | |
| - name: Setup Android SDK | |
| uses: android-actions/setup-android@v3 | |
| - name: Setup Gradle | |
| uses: gradle/actions/setup-gradle@v3 | |
| - name: Create .env file | |
| run: | | |
| echo "DITTO_APP_ID=${{ secrets.DITTO_APP_ID }}" > .env | |
| echo "DITTO_PLAYGROUND_TOKEN=${{ secrets.DITTO_PLAYGROUND_TOKEN }}" >> .env | |
| echo "DITTO_AUTH_URL=${{ secrets.DITTO_AUTH_URL }}" >> .env | |
| echo "DITTO_WEBSOCKET_URL=${{ secrets.DITTO_WEBSOCKET_URL }}" >> .env | |
| - name: Cache Gradle dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.gradle/caches | |
| ~/.gradle/wrapper | |
| key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} | |
| restore-keys: | | |
| ${{ runner.os }}-gradle- | |
| - name: Insert test document into Ditto Cloud | |
| run: | | |
| # Use GitHub run ID to create deterministic document ID | |
| DOC_ID="github_test_${GITHUB_RUN_ID}_${GITHUB_RUN_NUMBER}" | |
| TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC") | |
| # Insert document using curl with correct JSON structure for android-java | |
| RESPONSE=$(curl -s -w "\n%{http_code}" -X POST \ | |
| -H 'Content-type: application/json' \ | |
| -H "Authorization: Bearer ${{ secrets.DITTO_API_KEY }}" \ | |
| -d "{ | |
| \"statement\": \"INSERT INTO tasks DOCUMENTS (:newTask) ON ID CONFLICT DO UPDATE\", | |
| \"args\": { | |
| \"newTask\": { | |
| \"_id\": \"${DOC_ID}\", | |
| \"title\": \"GitHub Test Task ${GITHUB_RUN_ID}\", | |
| \"done\": false, | |
| \"deleted\": false | |
| } | |
| } | |
| }" \ | |
| "https://${{ secrets.DITTO_API_URL }}/api/v4/store/execute") | |
| # Extract HTTP status code and response body | |
| HTTP_CODE=$(echo "$RESPONSE" | tail -n1) | |
| BODY=$(echo "$RESPONSE" | head -n-1) | |
| # Check if insertion was successful | |
| if [ "$HTTP_CODE" -eq 200 ] || [ "$HTTP_CODE" -eq 201 ]; then | |
| echo "✓ Successfully inserted test document with ID: ${DOC_ID}" | |
| echo "GITHUB_TEST_DOC_ID=${DOC_ID}" >> $GITHUB_ENV | |
| else | |
| echo "❌ Failed to insert document. HTTP Status: $HTTP_CODE" | |
| echo "Response: $BODY" | |
| exit 1 | |
| fi | |
| - name: Run linter | |
| working-directory: android-java | |
| run: ./gradlew lint | |
| - name: Build APK | |
| working-directory: android-java | |
| run: | | |
| ./gradlew assembleDebug assembleDebugAndroidTest | |
| echo "APK built successfully" | |
| - name: Run Unit Tests | |
| working-directory: android-java | |
| run: ./gradlew test | |
| - name: Upload APKs to BrowserStack | |
| id: upload | |
| run: | | |
| CREDS="${{ secrets.BROWSERSTACK_USERNAME }}:${{ secrets.BROWSERSTACK_ACCESS_KEY }}" | |
| # 1. Upload AUT (app-debug.apk) | |
| APP_UPLOAD_RESPONSE=$(curl -u "$CREDS" \ | |
| -X POST "https://api-cloud.browserstack.com/app-automate/espresso/v2/app" \ | |
| -F "file=@android-java/app/build/outputs/apk/debug/app-debug.apk" \ | |
| -F "custom_id=ditto-android-java-app") | |
| APP_URL=$(echo "$APP_UPLOAD_RESPONSE" | jq -r .app_url) | |
| echo "app_url=$APP_URL" >> "$GITHUB_OUTPUT" | |
| echo "App upload response: $APP_UPLOAD_RESPONSE" | |
| # Validate app upload | |
| if [ "$APP_URL" = "null" ] || [ -z "$APP_URL" ]; then | |
| echo "Error: Failed to upload app APK" | |
| echo "Response: $APP_UPLOAD_RESPONSE" | |
| exit 1 | |
| fi | |
| # 2. Upload Espresso test-suite (app-debug-androidTest.apk) | |
| TEST_UPLOAD_RESPONSE=$(curl -u "$CREDS" \ | |
| -X POST "https://api-cloud.browserstack.com/app-automate/espresso/v2/test-suite" \ | |
| -F "file=@android-java/app/build/outputs/apk/androidTest/debug/app-debug-androidTest.apk" \ | |
| -F "custom_id=ditto-android-java-test") | |
| TEST_URL=$(echo "$TEST_UPLOAD_RESPONSE" | jq -r .test_suite_url) | |
| echo "test_url=$TEST_URL" >> "$GITHUB_OUTPUT" | |
| echo "Test upload response: $TEST_UPLOAD_RESPONSE" | |
| # Validate test upload | |
| if [ "$TEST_URL" = "null" ] || [ -z "$TEST_URL" ]; then | |
| echo "Error: Failed to upload test APK" | |
| echo "Response: $TEST_UPLOAD_RESPONSE" | |
| exit 1 | |
| fi | |
| - name: Execute tests on BrowserStack | |
| id: test | |
| run: | | |
| # Validate inputs before creating test execution request | |
| APP_URL="${{ steps.upload.outputs.app_url }}" | |
| TEST_URL="${{ steps.upload.outputs.test_url }}" | |
| echo "App URL: $APP_URL" | |
| echo "Test URL: $TEST_URL" | |
| if [ -z "$APP_URL" ] || [ "$APP_URL" = "null" ]; then | |
| echo "Error: No valid app URL available" | |
| exit 1 | |
| fi | |
| if [ -z "$TEST_URL" ] || [ "$TEST_URL" = "null" ]; then | |
| echo "Error: No valid test URL available" | |
| exit 1 | |
| fi | |
| # Create test execution request | |
| BUILD_RESPONSE=$(curl -u "${{ secrets.BROWSERSTACK_USERNAME }}:${{ secrets.BROWSERSTACK_ACCESS_KEY }}" \ | |
| -X POST "https://api-cloud.browserstack.com/app-automate/espresso/v2/build" \ | |
| -H "Content-Type: application/json" \ | |
| -d "{ | |
| \"app\": \"$APP_URL\", | |
| \"testSuite\": \"$TEST_URL\", | |
| \"devices\": [ | |
| \"Google Pixel 8-14.0\", | |
| \"Samsung Galaxy S23-13.0\", | |
| \"Google Pixel 6-12.0\", | |
| \"OnePlus 9-11.0\" | |
| ], | |
| \"project\": \"Ditto Android Java\", | |
| \"buildName\": \"Build #${{ github.run_number }}\", | |
| \"buildTag\": \"${{ github.ref_name }}\", | |
| \"deviceLogs\": true, | |
| \"video\": true, | |
| \"networkLogs\": true, | |
| \"autoGrantPermissions\": true, | |
| \"environmentVariables\": { | |
| \"GITHUB_TEST_DOC_ID\": \"${{ env.GITHUB_TEST_DOC_ID }}\" | |
| } | |
| }") | |
| echo "BrowserStack API Response:" | |
| echo "$BUILD_RESPONSE" | |
| BUILD_ID=$(echo "$BUILD_RESPONSE" | jq -r .build_id) | |
| # Check if BUILD_ID is null or empty | |
| if [ "$BUILD_ID" = "null" ] || [ -z "$BUILD_ID" ]; then | |
| echo "Error: Failed to create BrowserStack build" | |
| echo "Response: $BUILD_RESPONSE" | |
| exit 1 | |
| fi | |
| echo "build_id=$BUILD_ID" >> $GITHUB_OUTPUT | |
| echo "Build started with ID: $BUILD_ID" | |
| - name: Wait for BrowserStack tests to complete | |
| run: | | |
| BUILD_ID="${{ steps.test.outputs.build_id }}" | |
| # Validate BUILD_ID before proceeding | |
| if [ "$BUILD_ID" = "null" ] || [ -z "$BUILD_ID" ]; then | |
| echo "Error: No valid BUILD_ID available. Skipping test monitoring." | |
| exit 1 | |
| fi | |
| MAX_WAIT_TIME=1800 # 30 minutes | |
| CHECK_INTERVAL=30 # Check every 30 seconds | |
| ELAPSED=0 | |
| while [ $ELAPSED -lt $MAX_WAIT_TIME ]; do | |
| BUILD_STATUS_RESPONSE=$(curl -s -u "${{ secrets.BROWSERSTACK_USERNAME }}:${{ secrets.BROWSERSTACK_ACCESS_KEY }}" \ | |
| "https://api-cloud.browserstack.com/app-automate/espresso/v2/builds/$BUILD_ID") | |
| BUILD_STATUS=$(echo "$BUILD_STATUS_RESPONSE" | jq -r .status) | |
| # Check for API errors | |
| if [ "$BUILD_STATUS" = "null" ] || [ -z "$BUILD_STATUS" ]; then | |
| echo "Error getting build status. Response: $BUILD_STATUS_RESPONSE" | |
| sleep $CHECK_INTERVAL | |
| ELAPSED=$((ELAPSED + CHECK_INTERVAL)) | |
| continue | |
| fi | |
| echo "Build status: $BUILD_STATUS (elapsed: ${ELAPSED}s)" | |
| echo "Full response: $BUILD_STATUS_RESPONSE" | |
| # Check for completion states - BrowserStack uses different status values | |
| if [ "$BUILD_STATUS" = "done" ] || [ "$BUILD_STATUS" = "failed" ] || [ "$BUILD_STATUS" = "error" ] || [ "$BUILD_STATUS" = "passed" ] || [ "$BUILD_STATUS" = "completed" ]; then | |
| echo "Build completed with status: $BUILD_STATUS" | |
| break | |
| fi | |
| sleep $CHECK_INTERVAL | |
| ELAPSED=$((ELAPSED + CHECK_INTERVAL)) | |
| done | |
| # Get final results | |
| FINAL_RESULT=$(curl -s -u "${{ secrets.BROWSERSTACK_USERNAME }}:${{ secrets.BROWSERSTACK_ACCESS_KEY }}" \ | |
| "https://api-cloud.browserstack.com/app-automate/espresso/v2/builds/$BUILD_ID") | |
| echo "Final build result:" | |
| echo "$FINAL_RESULT" | jq . | |
| # Check if we got valid results | |
| if echo "$FINAL_RESULT" | jq -e .devices > /dev/null 2>&1; then | |
| # Check if the overall build passed | |
| BUILD_STATUS=$(echo "$FINAL_RESULT" | jq -r .status) | |
| if [ "$BUILD_STATUS" != "passed" ]; then | |
| echo "Build failed with status: $BUILD_STATUS" | |
| # Check each device for failures | |
| FAILED_TESTS=$(echo "$FINAL_RESULT" | jq -r '.devices[] | select(.sessions[].status != "passed") | .device') | |
| if [ -n "$FAILED_TESTS" ]; then | |
| echo "Tests failed on devices: $FAILED_TESTS" | |
| fi | |
| exit 1 | |
| else | |
| echo "All tests passed successfully!" | |
| fi | |
| else | |
| echo "Warning: Could not parse final results" | |
| echo "Raw response: $FINAL_RESULT" | |
| fi | |
| - name: Generate test report | |
| if: always() | |
| run: | | |
| BUILD_ID="${{ steps.test.outputs.build_id }}" | |
| # Create test report | |
| echo "# BrowserStack Test Report" > test-report.md | |
| echo "" >> test-report.md | |
| if [ "$BUILD_ID" = "null" ] || [ -z "$BUILD_ID" ]; then | |
| echo "Build ID: N/A (Build creation failed)" >> test-report.md | |
| echo "" >> test-report.md | |
| echo "## Error" >> test-report.md | |
| echo "Failed to create BrowserStack build. Check the 'Execute tests on BrowserStack' step for details." >> test-report.md | |
| else | |
| echo "Build ID: $BUILD_ID" >> test-report.md | |
| echo "View full report: https://app-automate.browserstack.com/dashboard/v2/builds/$BUILD_ID" >> test-report.md | |
| echo "" >> test-report.md | |
| # Get detailed results | |
| RESULTS=$(curl -s -u "${{ secrets.BROWSERSTACK_USERNAME }}:${{ secrets.BROWSERSTACK_ACCESS_KEY }}" \ | |
| "https://api-cloud.browserstack.com/app-automate/espresso/v2/builds/$BUILD_ID") | |
| echo "## Device Results" >> test-report.md | |
| if echo "$RESULTS" | jq -e .devices > /dev/null 2>&1; then | |
| echo "$RESULTS" | jq -r '.devices[] | "- \(.device): \(.status)"' >> test-report.md | |
| else | |
| echo "Unable to retrieve device results" >> test-report.md | |
| fi | |
| echo "" >> test-report.md | |
| echo "## Sync Verification" >> test-report.md | |
| echo "- GitHub Test Document ID: ${GITHUB_TEST_DOC_ID:-Not generated}" >> test-report.md | |
| fi | |
| - name: Upload test artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-results | |
| path: | | |
| android-java/app/build/outputs/apk/ | |
| android-java/app/build/reports/ | |
| test-report.md | |
| - name: Comment PR with results | |
| if: github.event_name == 'pull_request' && always() | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const buildId = '${{ steps.test.outputs.build_id }}'; | |
| const status = '${{ job.status }}'; | |
| const runUrl = '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'; | |
| const testDocId = '${{ env.GITHUB_TEST_DOC_ID }}'; | |
| let body; | |
| if (buildId === 'null' || buildId === '' || !buildId) { | |
| body = `## 📱 BrowserStack Test Results (Android Java) | |
| **Status:** ❌ Failed (Build creation failed) | |
| **Build:** [#${{ github.run_number }}](${runUrl}) | |
| **Issue:** Failed to create BrowserStack build. Check the workflow logs for details. | |
| ### Expected Devices: | |
| - Google Pixel 8 (Android 14) | |
| - Samsung Galaxy S23 (Android 13) | |
| - Google Pixel 6 (Android 12) | |
| - OnePlus 9 (Android 11) | |
| `; | |
| } else { | |
| const bsUrl = `https://app-automate.browserstack.com/dashboard/v2/builds/${buildId}`; | |
| body = `## 📱 BrowserStack Test Results (Android Java) | |
| **Status:** ${status === 'success' ? '✅ Passed' : '❌ Failed'} | |
| **Build:** [#${{ github.run_number }}](${runUrl}) | |
| **BrowserStack:** [View detailed results](${bsUrl}) | |
| **Test Document ID:** ${testDocId || 'Not generated'} | |
| ### Tested Devices: | |
| - Google Pixel 8 (Android 14) | |
| - Samsung Galaxy S23 (Android 13) | |
| - Google Pixel 6 (Android 12) | |
| - OnePlus 9 (Android 11) | |
| ### Test Verification: | |
| - ✅ Lint check completed | |
| - ✅ APK build successful | |
| - ✅ Unit tests passed | |
| - ✅ Test document seeded to Ditto Cloud | |
| - ${status === 'success' ? '✅' : '❌'} Integration test verification on BrowserStack | |
| `; | |
| } | |
| github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: body | |
| }); |