Skip to content

test: [DO NOT REVIEW] WIP #3901

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 5 commits into
base: master
Choose a base branch
from
Draft

test: [DO NOT REVIEW] WIP #3901

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f10d5fb
test new format
meet2mky Oct 16, 2025
e5cd647
test PDT
meet2mky Oct 16, 2025
8748df7
remove bash
meet2mky Oct 16, 2025
9755094
test timings
meet2mky Oct 17, 2025
b91b307
repro killed vm
meet2mky Oct 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 20 additions & 11 deletions perfmetrics/scripts/install_bash.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,19 @@
# Exit on error, treat unset variables as errors, and propagate pipeline errors.
set -euo pipefail

# Logging Helpers
log_info() {
echo "[$(date +"%H:%M:%S %Z")] INFO: $1"
}

log_error() {
echo "[$(date +"%H:%M:%S %Z")] ERROR: $1"
}

if [[ $# -ne 1 ]]; then
echo "This script requires exactly one argument."
echo "Usage: $0 <bash-version>"
echo "Example: $0 5.3"
log_error "This script requires exactly one argument."
log_info "Usage: $0 <bash-version>"
log_info "Example: $0 5.3"
exit 1
fi

Expand All @@ -32,19 +41,19 @@ INSTALL_DIR="/usr/local/" # Installation directory
# Function to install dependencies like gcc and make if not present
install_dependencies() {
if ! command -v gcc &>/dev/null || ! command -v make &>/dev/null; then
echo "GCC or make not found. Attempting to install build tools..."
log_info "GCC or make not found. Attempting to install build tools..."
if command -v apt-get &>/dev/null; then
sudo apt-get update && sudo apt-get install -y build-essential
elif command -v dnf &>/dev/null; then
sudo dnf install -y gcc make
elif command -v yum &>/dev/null; then
sudo yum install -y gcc make
else
echo "Error: Could not find a known package manager (apt, dnf, yum)."
echo "Please install gcc and make manually before running this script."
log_error "Error: Could not find a known package manager (apt, dnf, yum)."
log_error "Please install gcc and make manually before running this script."
exit 1
fi
echo "Build tools installed successfully."
log_info "Build tools installed successfully."
fi
}

Expand All @@ -68,7 +77,7 @@ install_bash() {
)
}

echo "Installing bash version ${BASH_VERSION} to ${INSTALL_DIR}bin/bash"
log_info "Installing bash version ${BASH_VERSION} to ${INSTALL_DIR}bin/bash"
INSTALLATION_LOG=$(mktemp /tmp/bash_install_log.XXXXXX)

# Installing dependencies before installing Bash
Expand All @@ -78,13 +87,13 @@ install_bash >"$INSTALLATION_LOG" 2>&1
installation_status=$?
set -e
if [[ $installation_status -ne 0 ]]; then
echo "Error: Bash version ${BASH_VERSION} installation failed."
log_error "Bash version ${BASH_VERSION} installation failed."
cat "$INSTALLATION_LOG"
rm -f "$INSTALLATION_LOG"
exit 1
else
echo "Bash ${BASH_VERSION} installed successfully."
echo "Checking bash version at ${INSTALL_DIR}bin/bash:"
log_info "Bash ${BASH_VERSION} installed successfully."
log_info "Checking bash version at ${INSTALL_DIR}bin/bash:"
"${INSTALL_DIR}bin/bash" --version
rm -f "$INSTALLATION_LOG"
fi
25 changes: 17 additions & 8 deletions perfmetrics/scripts/install_go.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,19 @@
# Exit on error, treat unset variables as errors, and propagate pipeline errors.
set -euo pipefail

# Logging Helpers
log_info() {
echo "[$(date +"%H:%M:%S %Z")] INFO: $1"
}

log_error() {
echo "[$(date +"%H:%M:%S %Z")] ERROR: $1"
}

if [[ $# -ne 1 ]]; then
echo "This script requires exactly one argument."
echo "Usage: $0 <go-version>"
echo "Example: $0 1.24.5"
log_error "This script requires exactly one argument."
log_info "Usage: $0 <go-version>"
log_info "Example: $0 1.24.5"
exit 1
fi

Expand All @@ -44,21 +53,21 @@ install_go() {
sudo rm -rf "$temp_dir"
}

echo "Installing Go version ${GO_VERSION} to ${INSTALL_DIR}"
log_info "Installing Go version ${GO_VERSION} to ${INSTALL_DIR}"
INSTALLATION_LOG=$(mktemp /tmp/go_install_log.XXXXXX)
if ! install_go > "$INSTALLATION_LOG" 2>&1; then
echo "Go version ${GO_VERSION} installation failed."
log_error "Go version ${GO_VERSION} installation failed."
cat "$INSTALLATION_LOG"
rm -f "$INSTALLATION_LOG"
exit 1
else
echo "Go version ${GO_VERSION} installed successfully."
log_info "Go version ${GO_VERSION} installed successfully."
# If this script is run in background or different shell then
# export PATH needs to be called from the shell or use absolute go path
# or permanently add this to path variable in bashrc.
export PATH="${INSTALL_DIR}/go/bin:$PATH"
echo "Go version is: "
log_info "Go version is: "
go version
echo "Go is present at: $( (which go) )"
log_info "Go is present at: $( (which go) )"
rm -f "$INSTALLATION_LOG"
fi
23 changes: 16 additions & 7 deletions perfmetrics/scripts/install_latest_gcloud.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,18 @@
# Exit on error, treat unset variables as errors, and propagate pipeline errors.
set -euo pipefail

# Logging Helpers
log_info() {
echo "[$(date +"%H:%M:%S %Z")] INFO: $1"
}

log_error() {
echo "[$(date +"%H:%M:%S %Z")] ERROR: $1"
}

if [[ $# -ne 0 ]]; then
echo "This script requires no argument."
echo "Usage: $0"
log_error "This script requires no argument."
log_info "Usage: $0"
exit 1
fi

Expand All @@ -41,21 +50,21 @@ install_latest_gcloud() {
sudo rm -rf "$temp_dir"
}

echo "Installing latest gcloud version to ${INSTALL_DIR}"
log_info "Installing latest gcloud version to ${INSTALL_DIR}"
INSTALLATION_LOG=$(mktemp /tmp/gcloud_install_log.XXXXXX)
if ! install_latest_gcloud >"$INSTALLATION_LOG" 2>&1; then
echo "latest gcloud installation failed."
log_error "latest gcloud installation failed."
cat "$INSTALLATION_LOG"
rm -f "$INSTALLATION_LOG"
exit 1
else
echo "latest gcloud installed successfully."
log_info "latest gcloud installed successfully."
# If this script is run in background or different shell then
# export PATH needs to be called from the shell or use absolute gcloud path
# or permanently add this path to path variable in bashrc.
export PATH="${INSTALL_DIR}/google-cloud-sdk/bin:$PATH"
echo "gcloud Version is:"
log_info "gcloud Version is:"
gcloud version
echo "Gcloud is present at: $( (which gcloud) )"
log_info "Gcloud is present at: $( (which gcloud) )"
rm -f "$INSTALLATION_LOG"
fi
174 changes: 89 additions & 85 deletions perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,18 +13,29 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# Running test only for when PR contains execute-perf-test,
# execute-integration-tests or execute-checkpoint-test label.
readonly EXECUTE_PERF_TEST_LABEL="execute-perf-test"
readonly EXECUTE_INTEGRATION_TEST_LABEL="execute-integration-tests"
readonly EXECUTE_INTEGRATION_TEST_LABEL_ON_ZB="execute-integration-tests-on-zb"
readonly EXECUTE_PACKAGE_BUILD_TEST_LABEL="execute-package-build-tests"
readonly EXECUTE_CHECKPOINT_TEST_LABEL="execute-checkpoint-test"
readonly BUCKET_LOCATION=us-west4
# Set the timezone for all date commands in the script.
export TZ='America/Los_Angeles'

# Logging Helpers
log_info() {
echo "[$(date +"%H:%M:%S %Z")] INFO: $1"
}

log_error() {
echo "[$(date +"%H:%M:%S %Z")] ERROR: $1"
}

# Constants
readonly EXECUTE_PERF_TEST_LABEL="execute-perf-test" # Label to execute perf tests.
readonly EXECUTE_INTEGRATION_TEST_LABEL="execute-integration-tests" # Label to execute regional e2e integration tests.
readonly EXECUTE_INTEGRATION_TEST_LABEL_ON_ZB="execute-integration-tests-on-zb" # Label to execute zonal e2e integration tests.
readonly EXECUTE_PACKAGE_BUILD_TEST_LABEL="execute-package-build-tests" # Label to execute package build tests.
readonly EXECUTE_CHECKPOINT_TEST_LABEL="execute-checkpoint-test" # Label to execute JAX checkpoint tests.
readonly BUCKET_LOCATION=us-west4 # The Google Cloud Storage bucket location.
readonly GO_VERSION="1.24.5"
readonly REQUIRED_BASH_VERSION_FOR_E2E_SCRIPT="5.3"

curl https://api.github.com/repos/GoogleCloudPlatform/gcsfuse/pulls/$KOKORO_GITHUB_PULL_REQUEST_NUMBER >> pr.json
curl https://api.github.com/repos/GoogleCloudPlatform/gcsfuse/pulls/$KOKORO_GITHUB_PULL_REQUEST_NUMBER >>pr.json
perfTest=$(grep "$EXECUTE_PERF_TEST_LABEL" pr.json)
integrationTests=$(grep "\"$EXECUTE_INTEGRATION_TEST_LABEL\"" pr.json)
integrationTestsOnZB=$(grep "\"$EXECUTE_INTEGRATION_TEST_LABEL_ON_ZB\"" pr.json)
Expand All @@ -36,15 +47,14 @@ integrationTestsStr="$integrationTests"
integrationTestsOnZBStr="$integrationTestsOnZB"
packageBuildTestsStr="$packageBuildTests"
checkpointTestStr="$checkpointTests"
if [[ "$perfTestStr" != *"$EXECUTE_PERF_TEST_LABEL"* && "$integrationTestsStr" != *"$EXECUTE_INTEGRATION_TEST_LABEL"* && "$integrationTestsOnZBStr" != *"$EXECUTE_INTEGRATION_TEST_LABEL_ON_ZB"* && "$packageBuildTestsStr" != *"$EXECUTE_PACKAGE_BUILD_TEST_LABEL"* && "$checkpointTestStr" != *"$EXECUTE_CHECKPOINT_TEST_LABEL"* ]]
then
echo "No need to execute tests"
exit 0
if [[ "$perfTestStr" != *"$EXECUTE_PERF_TEST_LABEL"* && "$integrationTestsStr" != *"$EXECUTE_INTEGRATION_TEST_LABEL"* && "$integrationTestsOnZBStr" != *"$EXECUTE_INTEGRATION_TEST_LABEL_ON_ZB"* && "$packageBuildTestsStr" != *"$EXECUTE_PACKAGE_BUILD_TEST_LABEL"* && "$checkpointTestStr" != *"$EXECUTE_CHECKPOINT_TEST_LABEL"* ]]; then
log_info "No need to execute tests"
exit 0
fi

set -e
sudo apt-get update
echo Installing git
log_info "Installing git"
sudo apt-get install git
cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse"
# Install required go version.
Expand All @@ -54,101 +64,95 @@ export PATH="/usr/local/go/bin:$PATH"

# Fetch PR branch
echo '[remote "origin"]
fetch = +refs/pull/*/head:refs/remotes/origin/pr/*' >> .git/config
fetch = +refs/pull/*/head:refs/remotes/origin/pr/*' >>.git/config
git fetch origin -q

function execute_perf_test() {
mkdir -p gcs
GCSFUSE_FLAGS="--implicit-dirs --prometheus-port=48341"
BUCKET_NAME=presubmit-perf-tests
MOUNT_POINT=gcs
# The VM will itself exit if the gcsfuse mount fails.
go run . $GCSFUSE_FLAGS $BUCKET_NAME $MOUNT_POINT
# Running FIO test
time ./perfmetrics/scripts/presubmit/run_load_test_on_presubmit.sh
sudo umount gcs
mkdir -p gcs
GCSFUSE_FLAGS="--implicit-dirs --prometheus-port=48341"
BUCKET_NAME=presubmit-perf-tests
MOUNT_POINT=gcs
# The VM will itself exit if the gcsfuse mount fails.
go run . $GCSFUSE_FLAGS $BUCKET_NAME $MOUNT_POINT
# Running FIO test
time ./perfmetrics/scripts/presubmit/run_load_test_on_presubmit.sh
sudo umount gcs
}

function install_requirements() {
# Installing requirements
echo installing requirements
echo Installing python3-pip
sudo apt-get -y install python3-pip
echo Installing Bigquery module requirements...
pip install --require-hashes -r ./perfmetrics/scripts/bigquery/requirements.txt --user
echo Installing libraries to run python script
pip install google-cloud
pip install google-cloud-vision
pip install google-api-python-client
pip install prettytable
"${KOKORO_ARTIFACTS_DIR}/github/gcsfuse/perfmetrics/scripts/fio/install_fio.sh" "${KOKORO_ARTIFACTS_DIR}/github"
cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse"
# Installing requirements
log_info "installing requirements"
log_info "Installing python3-pip"
sudo apt-get -y install python3-pip
log_info "Installing Bigquery module requirements..."
pip install --require-hashes -r ./perfmetrics/scripts/bigquery/requirements.txt --user
log_info "Installing libraries to run python script"
pip install google-cloud
pip install google-cloud-vision
pip install google-api-python-client
pip install prettytable
"${KOKORO_ARTIFACTS_DIR}/github/gcsfuse/perfmetrics/scripts/fio/install_fio.sh" "${KOKORO_ARTIFACTS_DIR}/github"
cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse"
}

# execute perf tests.
if [[ "$perfTestStr" == *"$EXECUTE_PERF_TEST_LABEL"* ]];
then
# Executing perf tests for master branch
install_requirements
git checkout master
# Store results
touch result.txt
echo Mounting gcs bucket for master branch and execute tests
execute_perf_test


# Executing perf tests for PR branch
echo checkout PR branch
git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER
echo Mounting gcs bucket from pr branch and execute tests
execute_perf_test

# Show results
echo showing results...
python3 ./perfmetrics/scripts/presubmit/print_results.py
if [[ "$perfTestStr" == *"$EXECUTE_PERF_TEST_LABEL"* ]]; then
# Executing perf tests for master branch
install_requirements
git checkout master
# Store results
touch "result.txt"
log_info "Mounting gcs bucket for master branch and execute tests"
execute_perf_test

# Executing perf tests for PR branch
log_info "checkout PR branch"
git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER
log_info "Mounting gcs bucket from pr branch and execute tests"
execute_perf_test

# Show results
log_info "showing results..."
python3 ./perfmetrics/scripts/presubmit/print_results.py
fi

# Install required bash version for e2e script as kokoro has outdated bash versions.
./perfmetrics/scripts/install_bash.sh "$REQUIRED_BASH_VERSION_FOR_E2E_SCRIPT"

# Execute integration tests on zonal bucket(s).
if test -n "${integrationTestsOnZBStr}" ;
then
echo checkout PR branch
git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER

echo "Running e2e tests on zonal bucket(s) ..."
# $1 argument is refering to value of testInstalledPackage.
/usr/local/bin/bash ./tools/integration_tests/improved_run_e2e_tests.sh --bucket-location=$BUCKET_LOCATION --presubmit --zonal --track-resource-usage
if test -n "${integrationTestsOnZBStr}"; then
log_info "checkout PR branch"
git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER

log_info "Running e2e tests on zonal bucket(s) ..."
# $1 argument is refering to value of testInstalledPackage.
/usr/local/bin/bash ./tools/integration_tests/improved_run_e2e_tests.sh --bucket-location=$BUCKET_LOCATION --presubmit --zonal --track-resource-usage
fi

# Execute integration tests on non-zonal bucket(s).
if test -n "${integrationTestsStr}" ;
then
echo checkout PR branch
git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER

echo "Running e2e tests on non-zonal bucket(s) ..."
# $1 argument is refering to value of testInstalledPackage.
/usr/local/bin/bash ./tools/integration_tests/improved_run_e2e_tests.sh --bucket-location=$BUCKET_LOCATION --presubmit --track-resource-usage
if test -n "${integrationTestsStr}"; then
log_info "checkout PR branch"
git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER

log_info "Running e2e tests on non-zonal bucket(s) ..."
# $1 argument is refering to value of testInstalledPackage.
/usr/local/bin/bash ./tools/integration_tests/improved_run_e2e_tests.sh --bucket-location=$BUCKET_LOCATION --presubmit --track-resource-usage
fi

# Execute package build tests.
if [[ "$packageBuildTestsStr" == *"$EXECUTE_PACKAGE_BUILD_TEST_LABEL"* ]];
then
echo checkout PR branch
git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER
if [[ "$packageBuildTestsStr" == *"$EXECUTE_PACKAGE_BUILD_TEST_LABEL"* ]]; then
log_info "checkout PR branch"
git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER

echo "Running package build tests...."
./perfmetrics/scripts/build_and_install_gcsfuse.sh master
log_info "Running package build tests...."
./perfmetrics/scripts/build_and_install_gcsfuse.sh master
fi

# Execute JAX checkpoints tests.
if [[ "$checkpointTestStr" == *"$EXECUTE_CHECKPOINT_TEST_LABEL"* ]];
then
echo checkout PR branch
git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER
if [[ "$checkpointTestStr" == *"$EXECUTE_CHECKPOINT_TEST_LABEL"* ]]; then
log_info "checkout PR branch"
git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER

echo "Running checkpoint tests...."
./perfmetrics/scripts/ml_tests/checkpoint/Jax/run_checkpoints.sh
log_info "Running checkpoint tests...."
./perfmetrics/scripts/ml_tests/checkpoint/Jax/run_checkpoints.sh
fi
Loading
Loading