From 1691e885f21d69382c9f4e8381e38b8af84e1a40 Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Tue, 19 Aug 2025 06:00:29 -0500 Subject: [PATCH] Display test results --- .github/workflows/pr_tests.yml | 244 +++++++++++++++++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 .github/workflows/pr_tests.yml diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml new file mode 100644 index 000000000..f6a678a45 --- /dev/null +++ b/.github/workflows/pr_tests.yml @@ -0,0 +1,244 @@ +name: Tests + +on: + pull_request: + branches: [master, develop] + paths-ignore: + - "**.md" + - "docs/**" + - "images/**" + pull_request_target: + branches: [master, develop] + paths-ignore: + - "**.md" + - "docs/**" + - "images/**" + +concurrency: + group: tests-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + actions: read + checks: write + pull-requests: write + +jobs: + native-tests: + name: "πŸ§ͺ Native Tests" + if: github.repository == 'meshtastic/firmware' + uses: ./.github/workflows/test_native.yml + permissions: + contents: read + actions: read + checks: write + + test-summary: + name: "πŸ“Š Test Results" + runs-on: ubuntu-latest + needs: [native-tests] + if: always() + permissions: + contents: read + actions: read + checks: write + pull-requests: write + steps: + - uses: actions/checkout@v5 + with: + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + + - name: Get release version string + run: echo "long=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT + id: version + + - name: Download test artifacts + if: needs.native-tests.result != 'skipped' + uses: actions/download-artifact@v5 + with: + name: platformio-test-report-${{ steps.version.outputs.long }}.zip + merge-multiple: true + + - name: Parse test results and create detailed summary + id: test-results + run: | + echo "## πŸ§ͺ Test Results Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + # Check overall job status first + if [[ "${{ needs.native-tests.result }}" == "success" ]]; then + echo "βœ… **Overall Status**: PASSED" >> $GITHUB_STEP_SUMMARY + elif [[ "${{ needs.native-tests.result }}" == "failure" ]]; then + echo "❌ **Overall Status**: FAILED" >> $GITHUB_STEP_SUMMARY + elif [[ "${{ needs.native-tests.result }}" == "cancelled" ]]; then + echo "⏸️ **Overall Status**: CANCELLED" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Tests were cancelled before completion." >> $GITHUB_STEP_SUMMARY + exit 0 + else + echo "⚠️ **Overall Status**: SKIPPED" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Tests were skipped." >> $GITHUB_STEP_SUMMARY + exit 0 + fi + + echo "" >> $GITHUB_STEP_SUMMARY + + # Parse detailed test results if available + if [ -f "testreport.xml" ]; then + echo "### πŸ” Individual Test Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + python3 << 'EOF' + import xml.etree.ElementTree as ET + import os + + try: + tree = ET.parse('testreport.xml') + root = tree.getroot() + + total_tests = 0 + passed_tests = 0 + failed_tests = 0 + skipped_tests = 0 + + # Parse testsuite elements + for testsuite in root.findall('.//testsuite'): + suite_name = testsuite.get('name', 'Unknown') + suite_tests = int(testsuite.get('tests', '0')) + suite_failures = int(testsuite.get('failures', '0')) + suite_errors = int(testsuite.get('errors', '0')) + suite_skipped = int(testsuite.get('skipped', '0')) + + total_tests += suite_tests + failed_tests += suite_failures + suite_errors + skipped_tests += suite_skipped + passed_tests += suite_tests - suite_failures - suite_errors - suite_skipped + + if suite_tests > 0: + status = "βœ…" if (suite_failures + suite_errors) == 0 else "❌" + print(f"**{status} Test Suite: {suite_name}**") + print(f"- Total: {suite_tests}") + print(f"- Passed: βœ… {suite_tests - suite_failures - suite_errors - suite_skipped}") + print(f"- Failed: ❌ {suite_failures + suite_errors}") + if suite_skipped > 0: + print(f"- Skipped: ⏭️ {suite_skipped}") + print("") + + # Show individual test results for failed suites + if suite_failures + suite_errors > 0: + print("**Failed Tests:**") + for testcase in testsuite.findall('testcase'): + test_name = testcase.get('name', 'Unknown') + failure = testcase.find('failure') + error = testcase.find('error') + + if failure is not None: + msg = failure.get('message', 'Unknown error')[:100] + print(f"- ❌ `{test_name}`: {msg}") + elif error is not None: + msg = error.get('message', 'Unknown error')[:100] + print(f"- ❌ `{test_name}`: ERROR - {msg}") + print("") + else: + # Show passed tests for successful suites + passed_count = 0 + for testcase in testsuite.findall('testcase'): + if testcase.find('failure') is None and testcase.find('error') is None: + if passed_count < 5: # Limit to first 5 to avoid spam + test_name = testcase.get('name', 'Unknown') + print(f"- βœ… `{test_name}`: PASSED") + passed_count += 1 + if passed_count > 5: + print(f"- ... and {passed_count - 5} more tests passed") + print("") + + # Summary statistics + print("### πŸ“Š Test Statistics") + print(f"- **Total Tests**: {total_tests}") + print(f"- **Passed**: βœ… {passed_tests}") + print(f"- **Failed**: ❌ {failed_tests}") + if skipped_tests > 0: + print(f"- **Skipped**: ⏭️ {skipped_tests}") + + if failed_tests > 0: + print(f"\n❌ **{failed_tests} tests failed out of {total_tests} total**") + else: + print(f"\nβœ… **All {total_tests} tests passed!**") + + except Exception as e: + print(f"❌ Error parsing test results: {e}") + EOF + else + echo "⚠️ **No detailed test report available**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Test artifacts may not have been generated properly." >> $GITHUB_STEP_SUMMARY + fi + + echo "" >> $GITHUB_STEP_SUMMARY + echo "---" >> $GITHUB_STEP_SUMMARY + echo "View detailed logs in the [Actions tab](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> $GITHUB_STEP_SUMMARY + + - name: Comment test results on PR + if: github.event_name == 'pull_request' && needs.native-tests.result != 'skipped' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + + // Read the step summary to use as PR comment + let testSummary = "## πŸ§ͺ Test Results Summary\n\n"; + + if ("${{ needs.native-tests.result }}" === "success") { + testSummary += "βœ… **All tests passed!**\n\n"; + } else if ("${{ needs.native-tests.result }}" === "failure") { + testSummary += "❌ **Some tests failed.**\n\n"; + } else { + testSummary += "⚠️ **Tests did not complete normally.**\n\n"; + } + + testSummary += `View detailed results: [Actions Run](${context.payload.repository.html_url}/actions/runs/${context.runId})\n\n`; + testSummary += "---\n"; + testSummary += "*This comment will be automatically updated when new commits are pushed.*"; + + // Find existing comment + const comments = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number + }); + + const botComment = comments.data.find(comment => + comment.user.type === 'Bot' && + comment.body.includes('πŸ§ͺ Test Results Summary') + ); + + if (botComment) { + // Update existing comment + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: testSummary + }); + } else { + // Create new comment + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: testSummary + }); + } + + - name: Set overall status + run: | + if [[ "${{ needs.native-tests.result }}" == "success" ]]; then + echo "All tests passed! βœ…" + exit 0 + else + echo "Some tests failed! ❌" + exit 1 + fi