Merge pull request #994 from airweave-ai/fix/tests/make-public-api-test-transparent-and-add-retry

fix(tests): make public api test transparent and add retry

Author: orhanrauf

.github/workflows/test-public-api.yml

@@ -142,7 +142,75 @@ jobs:
           cd backend
           # Exclude rate_limit tests from CI/CD - they consume quota and should run separately
           # To run rate limit tests locally: pytest tests/e2e/smoke -m rate_limit
-          pytest tests/e2e/smoke -m "not rate_limit" -n 3 -v --tb=short
+
+          MAX_RETRIES=2
+
+          # Run all tests first
+          set +e
+          pytest tests/e2e/smoke -m "not rate_limit" -n 3 -v --tb=short 2>&1 | tee pytest_output.txt
+          EXIT_CODE=${PIPESTATUS[0]}
+          set -e
+
+          if [ $EXIT_CODE -eq 0 ]; then
+            echo "All tests passed!"
+            exit 0
+          fi
+
+          # Count failures from pytest output
+          FAILURES=$(grep -oP '\d+(?= failed)' pytest_output.txt | head -1 || echo "0")
+          echo "Detected $FAILURES test failure(s)"
+
+          # If 3 or more failures, don't retry
+          if [ "$FAILURES" -ge 3 ]; then
+            echo "3 or more tests failed. Not retrying."
+            exit 1
+          fi
+
+          # Retry only the failed tests
+          ATTEMPT=1
+          while [ $ATTEMPT -le $MAX_RETRIES ]; do
+            echo ""
+            echo "=========================================="
+            echo "Retry attempt $ATTEMPT of $MAX_RETRIES (running ONLY failed tests)"
+            echo "=========================================="
+            sleep 5
+
+            set +e
+            pytest tests/e2e/smoke -m "not rate_limit" --lf -n 3 -v --tb=short 2>&1 | tee pytest_retry_output.txt
+            EXIT_CODE=${PIPESTATUS[0]}
+            set -e
+
+            if [ $EXIT_CODE -eq 0 ]; then
+              echo "Failed tests passed on retry!"
+              exit 0
+            fi
+
+            ATTEMPT=$((ATTEMPT + 1))
+          done
+
+          # If we get here, tests failed after retries
+          echo "Tests failed after $MAX_RETRIES retry attempts"
+          exit 1
+
+      - name: Print service logs on failure
+        if: failure()
+        run: |
+          echo "=========================================="
+          echo "FASTAPI SERVER LOGS (last 200 lines)"
+          echo "=========================================="
+          docker logs airweave-backend --tail 200 2>&1 || echo "Could not fetch backend logs"
+
+          echo ""
+          echo "=========================================="
+          echo "TEMPORAL WORKER LOGS (last 800 lines)"
+          echo "=========================================="
+          docker logs airweave-temporal-worker --tail 800 2>&1 || echo "Could not fetch worker logs"
+
+          echo ""
+          echo "=========================================="
+          echo "All running containers:"
+          echo "=========================================="
+          docker ps
 
       - name: Cleanup Docker containers
         if: always()