diff --git a/.github/scripts/generate-qit-wc-matrix.sh b/.github/scripts/generate-qit-wc-matrix.sh new file mode 100755 index 00000000000..bb14b10b939 --- /dev/null +++ b/.github/scripts/generate-qit-wc-matrix.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +# Simplified script for QIT WooCommerce version matrix +# QIT handles version resolution for stable, rc, beta, nightly keywords +# We only need to fetch L-1 version for backward compatibility testing + +set -e + +# Function to get the latest WooCommerce version from WordPress.org API +get_latest_wc_version() { + curl -s https://api.wordpress.org/plugins/info/1.0/woocommerce.json | jq -r '.version' +} + +# Function to get the latest stable version for a specific major version +get_latest_stable_for_major() { + local major_version=$1 + curl -s https://api.wordpress.org/plugins/info/1.0/woocommerce.json | \ + jq -r --arg major "$major_version" '.versions | with_entries(select(.key | startswith($major + ".") and (contains("-") | not))) | keys | sort_by( . | split(".") | map(tonumber) ) | last' +} + +# Function to get the L-1 version (previous major version's latest stable) +get_l1_version() { + local latest_version=$1 + local major_version=$(echo "$latest_version" | cut -d. -f1) + local l1_major=$((major_version - 1)) + get_latest_stable_for_major "$l1_major" +} + +# Get the latest WooCommerce version +echo "Fetching latest WooCommerce version..." >&2 +LATEST_WC_VERSION=$(get_latest_wc_version) +echo "Latest WC version: $LATEST_WC_VERSION" >&2 + +# Get the L-1 version (we need the actual version number for this) +L1_VERSION=$(get_l1_version "$LATEST_WC_VERSION") +echo "L-1 version: $L1_VERSION" >&2 + +# Validate L-1 version +if [[ -z "$L1_VERSION" || "$L1_VERSION" == "null" ]]; then + echo "Error: Could not extract L-1 version" >&2 + exit 1 +fi + +if [[ ! "$L1_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Error: Invalid L-1 version: $L1_VERSION" >&2 + exit 1 +fi + +# Check if RC and beta are available (for metadata only) +# QIT will handle the actual version resolution +RC_AVAILABLE="false" +BETA_AVAILABLE="false" + +LATEST_RC=$(curl -s https://api.wordpress.org/plugins/info/1.0/woocommerce.json | \ + jq -r '.versions | with_entries(select(.key|match("rc";"i"))) | keys | sort_by( . | split("-")[0] | split(".") | map(tonumber) ) | last') + +if [[ -n "$LATEST_RC" && "$LATEST_RC" != "null" ]]; then + RC_BASE="${LATEST_RC%%-*}" + HIGHEST=$(printf '%s\n%s\n' "$RC_BASE" "$LATEST_WC_VERSION" | sort -V | tail -n1) + if [[ "$HIGHEST" == "$RC_BASE" && "$RC_BASE" != "$LATEST_WC_VERSION" ]]; then + RC_AVAILABLE="true" + echo "RC available: $LATEST_RC" >&2 + else + echo "RC not applicable (stable $LATEST_WC_VERSION already released)" >&2 + fi +else + echo "No RC version available" >&2 +fi + +LATEST_BETA=$(curl -s https://api.wordpress.org/plugins/info/1.0/woocommerce.json | \ + jq -r --arg major "$(echo "$LATEST_WC_VERSION" | cut -d. -f1)" \ + '.versions | with_entries(select(.key | startswith($major + ".") and contains("beta"))) | keys | sort_by( . | split("-")[0] | split(".") | map(tonumber) ) | last') + +if [[ -n "$LATEST_BETA" && "$LATEST_BETA" != "null" ]]; then + BETA_AVAILABLE="true" + echo "Beta available: $LATEST_BETA" >&2 +else + echo "No beta version available" >&2 +fi + +# Output JSON with L-1 version and availability flags +# QIT workflows will use keywords (stable, rc, beta) and check availability flags +RESULT=$(jq -n \ + --arg l1_version "$L1_VERSION" \ + --arg rc_available "$RC_AVAILABLE" \ + --arg beta_available "$BETA_AVAILABLE" \ + '{ + l1_version: $l1_version, + rc_available: ($rc_available == "true"), + beta_available: ($beta_available == "true") + }') + +echo "$RESULT" diff --git a/.github/workflows/qit-e2e-pull-request.yml b/.github/workflows/qit-e2e-pull-request.yml new file mode 100644 index 00000000000..a0bdac8de48 --- /dev/null +++ b/.github/workflows/qit-e2e-pull-request.yml @@ -0,0 +1,359 @@ +name: QIT E2E Tests - Pull Request + +on: + pull_request: + paths: + - 'client/**' + - 'includes/**' + - 'src/**' + - 'tests/qit/e2e/**' + - 'tests/qit/qit.yml' + - '.github/workflows/qit-e2e-pull-request.yml' # Trigger on changes to this workflow for testing + workflow_dispatch: + inputs: + woocommerce_version: + description: 'WooCommerce version to test against' + required: false + default: 'stable' + type: choice + options: + - 'stable' + - '8.9.3' + - '7.7.0' + - 'rc' + - 'beta' + php_version: + description: 'PHP version' + required: false + default: '8.3' + type: choice + options: + - '8.3' + - '8.2' + - '8.1' + - '7.4' + wordpress_version: + description: 'WordPress version to test against' + required: false + default: 'stable' + type: choice + options: + - 'stable' + - 'nightly' + - 'rc' + test_tag: + description: 'Test tag to run (e.g., "merchant" or "shopper subscriptions")' + required: false + default: '' + type: string + run_ui_mode: + description: 'Run in UI mode' + required: false + default: false + type: boolean + +jobs: + generate-matrix: + name: "Generate the test matrix dynamically" + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.generate_matrix.outputs.matrix }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: "Generate matrix" + id: generate_matrix + run: | + # Use simplified QIT script - QIT handles version resolution for stable + SCRIPT_RESULT=$( .github/scripts/generate-qit-wc-matrix.sh ) + + # Extract L-1 version from JSON + L1_VERSION=$(echo "$SCRIPT_RESULT" | jq -r '.l1_version') + + echo "Using L-1 version: $L1_VERSION" >&2 + + # Define common values to reduce repetition + PHP_STABLE="8.3" + + # Initialize empty matrix array + MATRIX_ENTRIES=() + + # PR matrix: Test both L-1 and stable versions for comprehensive coverage + # Strategy: + # - Single tag (e.g., "merchant") for base tests - uses --grep-invert "@(subscriptions|blocks)" + # - Multiple tags (e.g., "merchant subscriptions") for combined tests - uses AND logic + + # Add L-1 version with PHP 8.3 (backward compatibility) + MATRIX_ENTRIES+=("{\"woocommerce\":\"$L1_VERSION\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"merchant\",\"test_name\":\"merchant\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"$L1_VERSION\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"shopper\",\"test_name\":\"shopper\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"$L1_VERSION\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"merchant subscriptions\",\"test_name\":\"subscriptions-merchant\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"$L1_VERSION\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"shopper subscriptions\",\"test_name\":\"subscriptions-shopper\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"$L1_VERSION\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"shopper blocks\",\"test_name\":\"blocks-shopper\"}") + + # Add stable version with PHP 8.3 (current WooCommerce release compatibility) + MATRIX_ENTRIES+=("{\"woocommerce\":\"stable\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"merchant\",\"test_name\":\"merchant\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"stable\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"shopper\",\"test_name\":\"shopper\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"stable\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"merchant subscriptions\",\"test_name\":\"subscriptions-merchant\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"stable\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"shopper subscriptions\",\"test_name\":\"subscriptions-shopper\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"stable\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"shopper blocks\",\"test_name\":\"blocks-shopper\"}") + + # Convert array to JSON + MATRIX_INCLUDE=$(printf '%s\n' "${MATRIX_ENTRIES[@]}" | jq -s . -c) + + # Output the full matrix + echo "matrix={\"include\":${MATRIX_INCLUDE}}" >> $GITHUB_OUTPUT + + # Debug: log matrix to stderr so it appears in actions log + echo "Generated matrix with ${#MATRIX_ENTRIES[@]} combinations:" >&2 + echo "$MATRIX_INCLUDE" | jq . >&2 + + build-plugin: + name: "Build plugin artifact" + runs-on: ubuntu-latest + steps: + - name: "Checkout repository" + uses: actions/checkout@v4 + + - name: "Set up repository" + uses: ./.github/actions/setup-repo + + - name: "Build the plugin" + uses: ./.github/actions/build + + - name: "Upload plugin artifact" + uses: actions/upload-artifact@v4 + with: + name: woocommerce-payments-plugin + path: woocommerce-payments.zip + retention-days: 1 + + run-qit-tests: + name: "WC - ${{ matrix.woocommerce }} | PHP - ${{ matrix.php }} | ${{ matrix.test_name }}" + needs: [generate-matrix, build-plugin] + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up repository + uses: ./.github/actions/setup-repo + + - name: Download plugin artifact + uses: actions/download-artifact@v4 + with: + name: woocommerce-payments-plugin + + - name: "Install QIT CLI for running tests" + run: | + # Install dev dependencies to get QIT CLI + composer install --optimize-autoloader + + - name: Authenticate QIT + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + run: ./vendor/bin/qit partner:add --user='${{ secrets.QIT_CI_USER }}' --application_password='${{ secrets.QIT_CI_SECRET }}' + + - name: Set up test options + id: test_options + run: | + OPTIONS="" + + # UI mode is primarily for manual testing + if [[ "${{ inputs.run_ui_mode }}" == "true" ]]; then + OPTIONS="${OPTIONS} --ui" + fi + + # Use input versions if provided, otherwise use matrix + WC_VERSION="${{ inputs.woocommerce_version || matrix.woocommerce }}" + PHP_VERSION="${{ inputs.php_version || matrix.php }}" + TEST_TAG="${{ inputs.test_tag || matrix.test_tag }}" + WP_VERSION="${{ inputs.wordpress_version || matrix.wordpress || 'stable' }}" + + OPTIONS="${OPTIONS} --php_version=${PHP_VERSION} --woo=${WC_VERSION} --wp=${WP_VERSION}" + + # Determine test selection using tags + # Single tag (e.g., "merchant") runs only base tests (excludes subscriptions/blocks) + # Multiple tags (e.g., "merchant subscriptions") use AND logic to run combined tests + if [[ -n "$TEST_TAG" ]]; then + # Remove @ prefix if present (QIT will add it) + CLEAN_TAG="${TEST_TAG#@}" + # For combined tags like "@shopper and @subscriptions", strip all @ symbols + CLEAN_TAG="${CLEAN_TAG//@/}" + + # Check if this is a single tag (base test) or multiple tags (combined test) + WORD_COUNT=$(echo "$CLEAN_TAG" | wc -w | tr -d ' ') + + if [[ "$WORD_COUNT" -eq 1 ]]; then + # Single tag: exclude subscriptions and blocks tests using --grep-invert with regex alternation + # Example: --grep @shopper --grep-invert "@(subscriptions|blocks)" + OPTIONS="${OPTIONS} --pw_test_tag=\"${CLEAN_TAG}\" --pw_options=\"--retries=0 --grep-invert '@(subscriptions|blocks)'\"" + echo "Using test tag: ${CLEAN_TAG} (excluding @subscriptions and @blocks)" + else + # Multiple tags: use AND logic for combined tests + OPTIONS="${OPTIONS} --pw_test_tag=\"${CLEAN_TAG}\" --pw_options=\"--retries=0\"" + echo "Using test tags: ${CLEAN_TAG}" + fi + else + # No specific test selection, just disable retries + OPTIONS="${OPTIONS} --pw_options=\"--retries=0\"" + fi + + echo "options=${OPTIONS}" >> $GITHUB_OUTPUT + echo "Will run with options: ${OPTIONS}" + echo "WordPress: ${WP_VERSION}" + + - name: Mask sensitive values + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + run: | + # Mask tokens in GitHub Actions logs to prevent accidental exposure + echo "::add-mask::${{ secrets.E2E_QIT_JP_SITE_ID }}" + echo "::add-mask::${{ secrets.E2E_QIT_JP_BLOG_TOKEN }}" + echo "::add-mask::${{ secrets.E2E_QIT_JP_USER_TOKEN }}" + + - name: First Run QIT Tests + id: first_run_qit_tests + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + # Use continue-on-error to allow parsing results even if tests fail + continue-on-error: true + run: | + # Always use the full tests/qit/e2e directory + # Test filtering is handled via --pw_test_tag and --grep-invert + TEST_DIR="tests/qit/e2e" + + echo "Running tests from: $TEST_DIR" + + # Run QIT tests - exit code 0 = all passed, non-zero = tests failed or QIT error + ./vendor/bin/qit run:e2e woocommerce-payments "$TEST_DIR" \ + --config tests/qit/qit.yml \ + --source woocommerce-payments.zip \ + --env E2E_JP_SITE_ID="${{ secrets.E2E_QIT_JP_SITE_ID }}" \ + --env E2E_JP_BLOG_TOKEN="${{ secrets.E2E_QIT_JP_BLOG_TOKEN }}" \ + --env E2E_JP_USER_TOKEN="${{ secrets.E2E_QIT_JP_USER_TOKEN }}" \ + --no_upload_report \ + ${{ steps.test_options.outputs.options }} + + QIT_EXIT_CODE=$? + echo "qit_exit_code=$QIT_EXIT_CODE" >> $GITHUB_OUTPUT + + # Check if QIT actually ran tests by looking for results directory + QIT_RESULTS_DIR=$(find /tmp -maxdepth 1 -name "qit-results-*" -type d 2>/dev/null | sort -r | head -1) + if [[ -z "$QIT_RESULTS_DIR" ]]; then + echo "::error::QIT did not produce test results. This indicates an environment or setup failure." + exit 1 + fi + + exit $QIT_EXIT_CODE + + - name: Parse Failed Specs + id: parse_failed_specs + if: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && steps.first_run_qit_tests.outcome == 'failure' }} + run: | + # QIT outputs results to a temporary directory like /tmp/qit-results-XXXXX + # Find the most recent QIT results directory + QIT_RESULTS_DIR=$(find /tmp -maxdepth 1 -name "qit-results-*" -type d 2>/dev/null | sort -r | head -1) + + if [[ -z "$QIT_RESULTS_DIR" ]]; then + echo "No QIT results directory found" + echo "failed_specs_count=0" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Found QIT results directory: $QIT_RESULTS_DIR" + + # Look for CTRF report file + RESULTS_JSON="" + for filename in "ctrf-report.json" "results.json" "test-results.json"; do + FOUND=$(find "$QIT_RESULTS_DIR" -name "$filename" -type f 2>/dev/null | head -1) + if [[ -n "$FOUND" ]]; then + RESULTS_JSON="$FOUND" + break + fi + done + + if [[ -z "$RESULTS_JSON" || ! -f "$RESULTS_JSON" ]]; then + echo "No results JSON found" + echo "failed_specs_count=0" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Parsing results from: $RESULTS_JSON" + + # Extract failed spec files from CTRF format + # CTRF format: .results.tests[] with .status and .filePath fields + FAILED_SPECS_COUNT=$(jq -r '[.results.tests[] | select(.status == "failed") | .filePath] | unique | length' "$RESULTS_JSON" 2>/dev/null || echo "0") + + echo "failed_specs_count=$FAILED_SPECS_COUNT" >> $GITHUB_OUTPUT + echo "results_json=$RESULTS_JSON" >> $GITHUB_OUTPUT + + if [[ ${FAILED_SPECS_COUNT} -gt 0 ]]; then + echo "::notice::${FAILED_SPECS_COUNT} spec file(s) failed in the first run. Will re-run only the failed specs." + fi + + - name: Re-try Failed Specs + if: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && steps.parse_failed_specs.outputs.failed_specs_count > 0 }} + run: | + RESULTS_JSON="${{ steps.parse_failed_specs.outputs.results_json }}" + + # Extract failed spec file paths from CTRF format + mapfile -t FAILED_SPECS < <(jq -r '[.results.tests[] | select(.status == "failed") | .filePath] | unique | .[]' "$RESULTS_JSON") + + if [[ ${#FAILED_SPECS[@]} -eq 0 ]]; then + echo "::notice::No failed specs found. Skipping retry." + exit 0 + fi + + echo "Retrying ${#FAILED_SPECS[@]} failed spec(s):" + for spec in "${FAILED_SPECS[@]}"; do + echo " - $spec" + done + + # Build grep pattern to match only failed spec files + # CTRF filePath format: /qit/tests/e2e/woocommerce-payments/local/specs/... + # Extract filenames and build regex pattern + SPEC_PATTERN="" + for spec in "${FAILED_SPECS[@]}"; do + # Extract just the filename without path (e.g., merchant-disputes-respond.spec.ts) + SPEC_NAME=$(basename "$spec") + if [[ -z "$SPEC_PATTERN" ]]; then + SPEC_PATTERN="$SPEC_NAME" + else + SPEC_PATTERN="${SPEC_PATTERN}|${SPEC_NAME}" + fi + done + + # Always use full test directory for retry + TEST_DIR="tests/qit/e2e" + + # For retry, we only need PHP, WC, and WP versions + WC_VERSION="${{ matrix.woocommerce }}" + PHP_VERSION="${{ matrix.php }}" + WP_VERSION="${{ matrix.wordpress || 'stable' }}" + + echo "Retry grep pattern: ($SPEC_PATTERN)" + + # Re-run QIT with grep pattern matching only failed specs + ./vendor/bin/qit run:e2e woocommerce-payments "$TEST_DIR" \ + --config tests/qit/qit.yml \ + --source woocommerce-payments.zip \ + --env E2E_JP_SITE_ID="${{ secrets.E2E_QIT_JP_SITE_ID }}" \ + --env E2E_JP_BLOG_TOKEN="${{ secrets.E2E_QIT_JP_BLOG_TOKEN }}" \ + --env E2E_JP_USER_TOKEN="${{ secrets.E2E_QIT_JP_USER_TOKEN }}" \ + --no_upload_report \ + --php_version=${PHP_VERSION} \ + --woo=${WC_VERSION} \ + --wp=${WP_VERSION} \ + --pw_options="--retries=0 --grep '(${SPEC_PATTERN})'" + + - name: Upload Playwright test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: qit-results-wc-${{ matrix.woocommerce }}-php-${{ matrix.php }}-${{ matrix.test_name }}-${{ github.run_id }} + path: /tmp/qit-results-*/ + retention-days: 7 + if-no-files-found: warn diff --git a/.github/workflows/qit-e2e.yml b/.github/workflows/qit-e2e.yml index 5336f2ac45b..ec5580e9ede 100644 --- a/.github/workflows/qit-e2e.yml +++ b/.github/workflows/qit-e2e.yml @@ -1,22 +1,13 @@ -name: QIT E2E Tests +name: QIT E2E Tests - Scheduled & Push on: - pull_request: - paths: - - 'client/**' - - 'includes/**' - - 'src/**' - - 'tests/qit/e2e/**' - - 'tests/qit/qit.yml' - - '.github/workflows/qit-e2e.yml' push: branches: - 'develop' - 'trunk' - - 'dev/qit-e2e-*' # Allow testing on QIT E2E development branches schedule: - # Run daily at 2 AM UTC - - cron: '0 2 * * *' + # Run every 6 hours like legacy e2e-test.yml. TODO: Should this be nightly only? + - cron: '0 */6 * * *' workflow_dispatch: inputs: woocommerce_version: @@ -40,6 +31,19 @@ on: - '8.2' - '8.1' - '7.4' + wordpress_version: + description: 'WordPress version to test against' + required: false + default: 'stable' + type: choice + options: + - 'stable' + - 'nightly' + - 'rc' + test_tag: + description: 'Playwright test tag to run (e.g., @shopper, @merchant, @critical)' + required: false + type: string run_ui_mode: description: 'Run tests in UI mode for debugging' required: false @@ -54,24 +58,100 @@ concurrency: cancel-in-progress: true jobs: - qit-e2e-tests: + generate-matrix: + name: "Generate the test matrix dynamically" runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - # Pull requests: Test against stable and L-1 WooCommerce versions - - wc_version: "stable" - php_version: "8.3" - test_group: "basic" - - wc_version: "8.9.3" # L-1 version - php_version: "8.3" - test_group: "basic" - # Full runs: Include business continuity version - - wc_version: "7.7.0" - php_version: "7.4" - test_group: "basic" + outputs: + matrix: ${{ steps.generate_matrix.outputs.matrix }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: "Generate matrix" + id: generate_matrix + run: | + # Use simplified QIT script - QIT handles version resolution for stable/rc/beta + SCRIPT_RESULT=$( .github/scripts/generate-qit-wc-matrix.sh ) + + # Extract L-1 version and availability flags from JSON + L1_VERSION=$(echo "$SCRIPT_RESULT" | jq -r '.l1_version') + RC_AVAILABLE=$(echo "$SCRIPT_RESULT" | jq -r '.rc_available') + BETA_AVAILABLE=$(echo "$SCRIPT_RESULT" | jq -r '.beta_available') + + echo "Using L-1 version: $L1_VERSION" >&2 + echo "RC available: $RC_AVAILABLE" >&2 + echo "Beta available: $BETA_AVAILABLE" >&2 + + # Define common values to reduce repetition + PHP_LEGACY="7.4" + PHP_STABLE="8.3" + PHP_LATEST="8.4" + + # Initialize empty matrix array + MATRIX_ENTRIES=() + + # Scheduled/push matrix: Run comprehensive test suite across multiple versions + echo "Scheduled/push run detected - running full test suite" >&2 + + # WC 7.7.0 with PHP 7.4 only (legacy - business continuity) + MATRIX_ENTRIES+=("{\"woocommerce\":\"7.7.0\",\"php\":\"$PHP_LEGACY\",\"test_tag\":\"merchant\",\"test_name\":\"merchant\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"7.7.0\",\"php\":\"$PHP_LEGACY\",\"test_tag\":\"shopper\",\"test_name\":\"shopper\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"7.7.0\",\"php\":\"$PHP_LEGACY\",\"test_tag\":\"merchant subscriptions\",\"test_name\":\"subscriptions-merchant\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"7.7.0\",\"php\":\"$PHP_LEGACY\",\"test_tag\":\"shopper subscriptions\",\"test_name\":\"subscriptions-shopper\"}") + + # Add L-1 version with PHP 8.3 (QIT uses actual version number) + MATRIX_ENTRIES+=("{\"woocommerce\":\"$L1_VERSION\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"merchant\",\"test_name\":\"merchant\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"$L1_VERSION\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"shopper\",\"test_name\":\"shopper\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"$L1_VERSION\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"merchant subscriptions\",\"test_name\":\"subscriptions-merchant\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"$L1_VERSION\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"shopper subscriptions\",\"test_name\":\"subscriptions-shopper\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"$L1_VERSION\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"shopper blocks\",\"test_name\":\"blocks-shopper\"}") + # Add stable with PHP 8.3 (QIT keyword - resolves to latest stable) + MATRIX_ENTRIES+=("{\"woocommerce\":\"stable\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"merchant\",\"test_name\":\"merchant\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"stable\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"shopper\",\"test_name\":\"shopper\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"stable\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"merchant subscriptions\",\"test_name\":\"subscriptions-merchant\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"stable\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"shopper subscriptions\",\"test_name\":\"subscriptions-shopper\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"stable\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"shopper blocks\",\"test_name\":\"blocks-shopper\"}") + + # Add beta with PHP 8.3 (QIT keyword - only if available) + if [[ "$BETA_AVAILABLE" == "true" ]]; then + MATRIX_ENTRIES+=("{\"woocommerce\":\"beta\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"merchant\",\"test_name\":\"merchant\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"beta\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"shopper\",\"test_name\":\"shopper\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"beta\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"merchant subscriptions\",\"test_name\":\"subscriptions-merchant\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"beta\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"shopper subscriptions\",\"test_name\":\"subscriptions-shopper\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"beta\",\"php\":\"$PHP_STABLE\",\"test_tag\":\"shopper blocks\",\"test_name\":\"blocks-shopper\"}") + echo "Including beta tests (QIT will resolve to latest beta)" >&2 + else + echo "Skipping beta tests - no beta available" >&2 + fi + + # Add rc with PHP 8.4 (QIT keyword - only if available) + if [[ "$RC_AVAILABLE" == "true" ]]; then + MATRIX_ENTRIES+=("{\"woocommerce\":\"rc\",\"php\":\"$PHP_LATEST\",\"test_tag\":\"merchant\",\"test_name\":\"merchant\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"rc\",\"php\":\"$PHP_LATEST\",\"test_tag\":\"shopper\",\"test_name\":\"shopper\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"rc\",\"php\":\"$PHP_LATEST\",\"test_tag\":\"merchant subscriptions\",\"test_name\":\"subscriptions-merchant\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"rc\",\"php\":\"$PHP_LATEST\",\"test_tag\":\"shopper subscriptions\",\"test_name\":\"subscriptions-shopper\"}") + echo "Including RC tests (QIT will resolve to latest RC)" >&2 + else + echo "Skipping RC tests - no RC available" >&2 + fi + + # Add WordPress nightly with WC latest and PHP 8.3 (match e2e-test.yml wp-nightly-tests) + # Test new WordPress features with latest stable WooCommerce + MATRIX_ENTRIES+=("{\"woocommerce\":\"stable\",\"php\":\"$PHP_STABLE\",\"wordpress\":\"nightly\",\"test_tag\":\"merchant\",\"test_name\":\"merchant\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"stable\",\"php\":\"$PHP_STABLE\",\"wordpress\":\"nightly\",\"test_tag\":\"shopper\",\"test_name\":\"shopper\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"stable\",\"php\":\"$PHP_STABLE\",\"wordpress\":\"nightly\",\"test_tag\":\"merchant subscriptions\",\"test_name\":\"subscriptions-merchant\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"stable\",\"php\":\"$PHP_STABLE\",\"wordpress\":\"nightly\",\"test_tag\":\"shopper subscriptions\",\"test_name\":\"subscriptions-shopper\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"stable\",\"php\":\"$PHP_STABLE\",\"wordpress\":\"nightly\",\"test_tag\":\"shopper blocks\",\"test_name\":\"blocks-shopper\"}") + + # Convert array to JSON + MATRIX_INCLUDE=$(printf '%s\n' "${MATRIX_ENTRIES[@]}" | jq -s . -c) + + echo "matrix={\"include\":$MATRIX_INCLUDE}" >> $GITHUB_OUTPUT + + build-plugin: + name: "Build plugin artifact" + runs-on: ubuntu-latest steps: - name: "Checkout repository" uses: actions/checkout@v4 @@ -85,6 +165,32 @@ jobs: id: build_plugin uses: ./.github/actions/build + - name: "Upload plugin artifact" + uses: actions/upload-artifact@v4 + with: + name: woocommerce-payments-plugin + path: woocommerce-payments.zip + retention-days: 1 + + qit-e2e-tests: + runs-on: ubuntu-latest + needs: [generate-matrix, build-plugin] + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }} + name: "${{ matrix.wordpress && format('WP - {0} | ', matrix.wordpress) || '' }}WC - ${{ matrix.woocommerce }} | PHP - ${{ matrix.php }} | ${{ matrix.test_name }}" + + steps: + - name: "Checkout repository" + uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + + - name: "Download plugin artifact" + uses: actions/download-artifact@v4 + with: + name: woocommerce-payments-plugin + - name: "Install QIT CLI for running tests" run: | # Install dev dependencies to get QIT CLI @@ -103,13 +209,43 @@ jobs: fi # Use input versions if provided, otherwise use matrix - WC_VERSION="${{ inputs.woocommerce_version || matrix.wc_version }}" - PHP_VERSION="${{ inputs.php_version || matrix.php_version }}" + WC_VERSION="${{ inputs.woocommerce_version || matrix.woocommerce }}" + PHP_VERSION="${{ inputs.php_version || matrix.php }}" + TEST_TAG="${{ inputs.test_tag || matrix.test_tag }}" + WP_VERSION="${{ inputs.wordpress_version || matrix.wordpress || 'stable' }}" - OPTIONS="${OPTIONS} --php_version=${PHP_VERSION} --woo=${WC_VERSION}" + OPTIONS="${OPTIONS} --php_version=${PHP_VERSION} --woo=${WC_VERSION} --wp=${WP_VERSION}" + + # Determine test selection using tags + # Single tag (e.g., "merchant") needs to exclude subscriptions/blocks tests + # Multiple tags (e.g., "merchant subscriptions") use AND logic to run combined tests + if [[ -n "$TEST_TAG" ]]; then + # Remove @ prefix if present (QIT will add it) + CLEAN_TAG="${TEST_TAG#@}" + # For combined tags like "@shopper and @subscriptions", strip all @ symbols + CLEAN_TAG="${CLEAN_TAG//@/}" + + # Check if this is a single tag (base test) or multiple tags (combined test) + WORD_COUNT=$(echo "$CLEAN_TAG" | wc -w | tr -d ' ') + + if [[ "$WORD_COUNT" -eq 1 ]]; then + # Single tag: exclude subscriptions and blocks tests using --grep-invert with regex alternation + # Example: --grep @shopper --grep-invert "@(subscriptions|blocks)" + OPTIONS="${OPTIONS} --pw_test_tag=\"${CLEAN_TAG}\" --pw_options=\"--retries=0 --grep-invert '@(subscriptions|blocks)'\"" + echo "Using test tag: ${CLEAN_TAG} (excluding @subscriptions and @blocks)" + else + # Multiple tags: use AND logic for combined tests + OPTIONS="${OPTIONS} --pw_test_tag=\"${CLEAN_TAG}\" --pw_options=\"--retries=0\"" + echo "Using test tags: ${CLEAN_TAG}" + fi + else + # No specific test selection, just disable retries + OPTIONS="${OPTIONS} --pw_options=\"--retries=0\"" + fi echo "options=${OPTIONS}" >> $GITHUB_OUTPUT echo "Will run with options: ${OPTIONS}" + echo "WordPress: ${WP_VERSION}" - name: Mask sensitive values if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} @@ -119,13 +255,145 @@ jobs: echo "::add-mask::${{ secrets.E2E_QIT_JP_BLOG_TOKEN }}" echo "::add-mask::${{ secrets.E2E_QIT_JP_USER_TOKEN }}" - - name: Run QIT Tests + - name: First Run QIT Tests + id: first_run_qit_tests if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + # Use continue-on-error to allow parsing results even if tests fail + continue-on-error: true run: | - cd tests/qit - ../../vendor/bin/qit run:e2e woocommerce-payments ./e2e \ - --source ../../woocommerce-payments.zip \ + # Always use the full tests/qit/e2e directory + # Test filtering is handled via --pw_test_tag and --grep-invert + TEST_DIR="tests/qit/e2e" + + echo "Running tests from: $TEST_DIR" + + # Run QIT tests - exit code 0 = all passed, non-zero = tests failed or QIT error + ./vendor/bin/qit run:e2e woocommerce-payments "$TEST_DIR" \ + --config tests/qit/qit.yml \ + --source woocommerce-payments.zip \ --env E2E_JP_SITE_ID="${{ secrets.E2E_QIT_JP_SITE_ID }}" \ --env E2E_JP_BLOG_TOKEN="${{ secrets.E2E_QIT_JP_BLOG_TOKEN }}" \ --env E2E_JP_USER_TOKEN="${{ secrets.E2E_QIT_JP_USER_TOKEN }}" \ + --no_upload_report \ ${{ steps.test_options.outputs.options }} + + QIT_EXIT_CODE=$? + echo "qit_exit_code=$QIT_EXIT_CODE" >> $GITHUB_OUTPUT + + # Check if QIT actually ran tests by looking for results directory + QIT_RESULTS_DIR=$(find /tmp -maxdepth 1 -name "qit-results-*" -type d 2>/dev/null | sort -r | head -1) + if [[ -z "$QIT_RESULTS_DIR" ]]; then + echo "::error::QIT did not produce test results. This indicates an environment or setup failure." + exit 1 + fi + + exit $QIT_EXIT_CODE + + - name: Parse Failed Specs + id: parse_failed_specs + if: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && steps.first_run_qit_tests.outcome == 'failure' }} + run: | + # QIT outputs results to a temporary directory like /tmp/qit-results-XXXXX + # Find the most recent QIT results directory + QIT_RESULTS_DIR=$(find /tmp -maxdepth 1 -name "qit-results-*" -type d 2>/dev/null | sort -r | head -1) + + if [[ -z "$QIT_RESULTS_DIR" ]]; then + echo "No QIT results directory found" + echo "failed_specs_count=0" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Found QIT results directory: $QIT_RESULTS_DIR" + + # Look for CTRF report file + RESULTS_JSON="" + for filename in "ctrf-report.json" "results.json" "test-results.json"; do + FOUND=$(find "$QIT_RESULTS_DIR" -name "$filename" -type f 2>/dev/null | head -1) + if [[ -n "$FOUND" ]]; then + RESULTS_JSON="$FOUND" + break + fi + done + + if [[ -z "$RESULTS_JSON" || ! -f "$RESULTS_JSON" ]]; then + echo "No results JSON found" + echo "failed_specs_count=0" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Parsing results from: $RESULTS_JSON" + + # Extract failed spec files from CTRF format + # CTRF format: .results.tests[] with .status and .filePath fields + FAILED_SPECS_COUNT=$(jq -r '[.results.tests[] | select(.status == "failed") | .filePath] | unique | length' "$RESULTS_JSON" 2>/dev/null || echo "0") + + echo "failed_specs_count=$FAILED_SPECS_COUNT" >> $GITHUB_OUTPUT + echo "results_json=$RESULTS_JSON" >> $GITHUB_OUTPUT + + if [[ ${FAILED_SPECS_COUNT} -gt 0 ]]; then + echo "::notice::${FAILED_SPECS_COUNT} spec file(s) failed in the first run. Will re-run only the failed specs." + fi + + - name: Re-try Failed Specs + if: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && steps.parse_failed_specs.outputs.failed_specs_count > 0 }} + run: | + RESULTS_JSON="${{ steps.parse_failed_specs.outputs.results_json }}" + + # Extract failed spec file paths from CTRF format + mapfile -t FAILED_SPECS < <(jq -r '[.results.tests[] | select(.status == "failed") | .filePath] | unique | .[]' "$RESULTS_JSON") + + if [[ ${#FAILED_SPECS[@]} -eq 0 ]]; then + echo "::notice::No failed specs found. Skipping retry." + exit 0 + fi + + echo "Retrying ${#FAILED_SPECS[@]} failed spec(s):" + for spec in "${FAILED_SPECS[@]}"; do + echo " - $spec" + done + + # Build grep pattern to match only failed spec files + # CTRF filePath format: /qit/tests/e2e/woocommerce-payments/local/specs/... + # Extract filenames and build regex pattern + SPEC_PATTERN="" + for spec in "${FAILED_SPECS[@]}"; do + # Extract just the filename without path (e.g., merchant-disputes-respond.spec.ts) + SPEC_NAME=$(basename "$spec") + if [[ -z "$SPEC_PATTERN" ]]; then + SPEC_PATTERN="$SPEC_NAME" + else + SPEC_PATTERN="${SPEC_PATTERN}|${SPEC_NAME}" + fi + done + + # Always use full test directory for retry + TEST_DIR="tests/qit/e2e" + + # For retry, we only need PHP, WC, and WP versions + WC_VERSION="${{ matrix.woocommerce }}" + PHP_VERSION="${{ matrix.php }}" + WP_VERSION="${{ matrix.wordpress || 'stable' }}" + + echo "Retry grep pattern: ($SPEC_PATTERN)" + + # Re-run QIT with grep pattern matching only failed specs + ./vendor/bin/qit run:e2e woocommerce-payments "$TEST_DIR" \ + --config tests/qit/qit.yml \ + --source woocommerce-payments.zip \ + --env E2E_JP_SITE_ID="${{ secrets.E2E_QIT_JP_SITE_ID }}" \ + --env E2E_JP_BLOG_TOKEN="${{ secrets.E2E_QIT_JP_BLOG_TOKEN }}" \ + --env E2E_JP_USER_TOKEN="${{ secrets.E2E_QIT_JP_USER_TOKEN }}" \ + --no_upload_report \ + --php_version=${PHP_VERSION} \ + --woo=${WC_VERSION} \ + --wp=${WP_VERSION} \ + --pw_options="--retries=0 --grep '(${SPEC_PATTERN})'" + + - name: Upload Playwright test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: qit-results-wc-${{ matrix.woocommerce }}-php-${{ matrix.php }}-${{ matrix.test_name }}-${{ github.run_id }} + path: /tmp/qit-results-*/ + retention-days: 7 + if-no-files-found: warn diff --git a/tests/qit/e2e/specs/basic.spec.ts b/tests/qit/e2e/specs/basic.spec.ts index 708ea2beef8..594f053ba10 100644 --- a/tests/qit/e2e/specs/basic.spec.ts +++ b/tests/qit/e2e/specs/basic.spec.ts @@ -5,6 +5,7 @@ import { test, expect } from '../fixtures/auth'; test.describe( 'A basic set of tests to ensure WP, wp-admin and my-account load', + { tag: '@basic' }, () => { test( 'Load the home page', async ( { page } ) => { await page.goto( '/' );