Skip to content

Migrate gsutil usage to gcloud storage #3027

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Migrate gsutil usage to gcloud storage #3027

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
12abc08
Migrate gsutil usage to gcloud
googlyrahman Oct 27, 2025
3100296
Manual changes
googlyrahman Oct 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 3 additions & 6 deletions ...machine_learning/asl/open_project/ASL_youtube8m_models/frame/youtube_8m_frame_level.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1057,8 +1057,7 @@
"OUTDIR=gs://$BUCKET/youtube_8m_frame_level/trained_model\n",
"JOBNAME=job_youtube_8m_frame_level$(date -u +%y%m%d_%H%M%S)\n",
"echo $OUTDIR $REGION $JOBNAME\n",
"gsutil -m rm -rf $OUTDIR\n",
"gcloud ml-engine jobs submit training $JOBNAME \\\n",
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "gcloud ml-engine jobs submit training $JOBNAME \\\n",
" --region=$REGION \\\n",
" --module-name=trainer.task \\\n",
" --package-path=$PWD/trainer \\\n",
Expand Down Expand Up @@ -1156,8 +1155,7 @@
"OUTDIR=gs://$BUCKET/youtube_8m_frame_level/hyperparam\n",
"JOBNAME=job_youtube_8m_frame_level$(date -u +%y%m%d_%H%M%S)\n",
"echo $OUTDIR $REGION $JOBNAME\n",
"gsutil -m rm -rf $OUTDIR\n",
"gcloud ml-engine jobs submit training $JOBNAME \\\n",
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "gcloud ml-engine jobs submit training $JOBNAME \\\n",
" --region=$REGION \\\n",
" --module-name=trainer.task \\\n",
" --package-path=$PWD/trainer \\\n",
Expand Down Expand Up @@ -1215,8 +1213,7 @@
"%bash\n",
"MODEL_NAME=\"youtube_8m_frame_level\"\n",
"MODEL_VERSION=\"v1\"\n",
"MODEL_LOCATION=$(gsutil ls gs://$BUCKET/youtube_8m_frame_level/trained_model/export/exporter/ | tail -1)\n",
"echo \"Deleting and deploying $MODEL_NAME $MODEL_VERSION from $MODEL_LOCATION ... this will take a few minutes\"\n",
"MODEL_LOCATION=$(gcloud storage ls gs://$BUCKET/youtube_8m_frame_level/trained_model/export/exporter/ | tail -1)\n", "echo \"Deleting and deploying $MODEL_NAME $MODEL_VERSION from $MODEL_LOCATION ... this will take a few minutes\"\n",
"#gcloud ml-engine versions delete ${MODEL_VERSION} --model ${MODEL_NAME}\n",
"#gcloud ml-engine models delete ${MODEL_NAME}\n",
"gcloud ml-engine models create $MODEL_NAME --regions $REGION\n",
Expand Down
9 changes: 3 additions & 6 deletions ...machine_learning/asl/open_project/ASL_youtube8m_models/video/youtube_8m_video_level.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -749,8 +749,7 @@
"OUTDIR=gs://$BUCKET/youtube_8m_video_level/trained_model\n",
"JOBNAME=job_youtube_8m_video_level$(date -u +%y%m%d_%H%M%S)\n",
"echo $OUTDIR $REGION $JOBNAME\n",
"gsutil -m rm -rf $OUTDIR\n",
"gcloud ml-engine jobs submit training $JOBNAME \\\n",
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "gcloud ml-engine jobs submit training $JOBNAME \\\n",
" --region=$REGION \\\n",
" --module-name=trainer.task \\\n",
" --package-path=$PWD/trainer \\\n",
Expand Down Expand Up @@ -855,8 +854,7 @@
"OUTDIR=gs://$BUCKET/youtube_8m_video_level/hyperparam\n",
"JOBNAME=job_youtube_8m_video_level$(date -u +%y%m%d_%H%M%S)\n",
"echo $OUTDIR $REGION $JOBNAME\n",
"gsutil -m rm -rf $OUTDIR\n",
"gcloud ml-engine jobs submit training $JOBNAME \\\n",
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "gcloud ml-engine jobs submit training $JOBNAME \\\n",
" --region=$REGION \\\n",
" --module-name=trainer.task \\\n",
" --package-path=$PWD/trainer \\\n",
Expand Down Expand Up @@ -914,8 +912,7 @@
"%bash\n",
"MODEL_NAME=\"youtube_8m_video_level\"\n",
"MODEL_VERSION=\"v1\"\n",
"MODEL_LOCATION=$(gsutil ls gs://$BUCKET/youtube_8m_video_level/trained_model/export/exporter/ | tail -1)\n",
"echo \"Deleting and deploying $MODEL_NAME $MODEL_VERSION from $MODEL_LOCATION ... this will take a few minutes\"\n",
"MODEL_LOCATION=$(gcloud storage ls gs://$BUCKET/youtube_8m_video_level/trained_model/export/exporter/ | tail -1)\n", "echo \"Deleting and deploying $MODEL_NAME $MODEL_VERSION from $MODEL_LOCATION ... this will take a few minutes\"\n",
"#gcloud ml-engine versions delete ${MODEL_VERSION} --model ${MODEL_NAME}\n",
"#gcloud ml-engine models delete ${MODEL_NAME}\n",
"gcloud ml-engine models create $MODEL_NAME --regions $REGION\n",
Expand Down
9 changes: 3 additions & 6 deletions ...n_project/ASL_youtube8m_models/video_using_datasets/youtube_8m_video_level_datasets.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3783,8 +3783,7 @@
"OUTDIR=gs://$BUCKET/youtube_8m_video_level_datasets/trained_model\n",
"JOBNAME=job_youtube_8m_video_level_datasets$(date -u +%y%m%d_%H%M%S)\n",
"echo $OUTDIR $REGION $JOBNAME\n",
"gsutil -m rm -rf $OUTDIR\n",
"gcloud ml-engine jobs submit training $JOBNAME \\\n",
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "gcloud ml-engine jobs submit training $JOBNAME \\\n",
" --region=$REGION \\\n",
" --module-name=trainer.task \\\n",
" --package-path=$PWD/trainer \\\n",
Expand Down Expand Up @@ -3882,8 +3881,7 @@
"OUTDIR=gs://$BUCKET/youtube_8m_video_level_datasets/hyperparam\n",
"JOBNAME=job_youtube_8m_video_level_datasets$(date -u +%y%m%d_%H%M%S)\n",
"echo $OUTDIR $REGION $JOBNAME\n",
"gsutil -m rm -rf $OUTDIR\n",
"gcloud ml-engine jobs submit training $JOBNAME \\\n",
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "gcloud ml-engine jobs submit training $JOBNAME \\\n",
" --region=$REGION \\\n",
" --module-name=trainer.task \\\n",
" --package-path=$PWD/trainer \\\n",
Expand Down Expand Up @@ -3941,8 +3939,7 @@
"%bash\n",
"MODEL_NAME=\"youtube_8m_video_level_datasets\"\n",
"MODEL_VERSION=\"v1\"\n",
"MODEL_LOCATION=$(gsutil ls gs://$BUCKET/youtube_8m_video_level_datasets/trained_model/export/exporter/ | tail -1)\n",
"echo \"Deleting and deploying $MODEL_NAME $MODEL_VERSION from $MODEL_LOCATION ... this will take a few minutes\"\n",
"MODEL_LOCATION=$(gcloud storage ls gs://$BUCKET/youtube_8m_video_level_datasets/trained_model/export/exporter/ | tail -1)\n", "echo \"Deleting and deploying $MODEL_NAME $MODEL_VERSION from $MODEL_LOCATION ... this will take a few minutes\"\n",
"#gcloud ml-engine versions delete ${MODEL_VERSION} --model ${MODEL_NAME}\n",
"#gcloud ml-engine models delete ${MODEL_NAME}\n",
"gcloud ml-engine models create $MODEL_NAME --regions $REGION\n",
Expand Down
2 changes: 1 addition & 1 deletion ...n_project/cloud_composer_automated_ml_pipeline_taxifare/airflow/dags/subdag/preprocess.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ def preprocess_tasks(model, parent_dag_name, child_dag_name, default_args, PROJE
# BigQuery training data export to GCS
bash_remove_old_data_op = BashOperator(
task_id="bash_remove_old_data_{}_task".format(model.replace(".","_")),
bash_command="if gsutil ls {0}/taxifare/data/{1} 2> /dev/null; then gsutil -m rm -rf {0}/taxifare/data/{1}/*; else true; fi".format(BUCKET, model.replace(".","_")),
bash_command="if gcloud storage ls {0}/taxifare/data/{1} 2> /dev/null; then gcloud storage rm --recursive --continue-on-error {0}/taxifare/data/{1}/*; else true; fi".format(BUCKET, model.replace(".","_")),
dag=dag
)

Expand Down
4 changes: 2 additions & 2 deletions ...pen_project/cloud_composer_automated_ml_pipeline_taxifare/airflow/dags/subdag/training.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,13 @@ def training_tasks(model, parent_dag_name, child_dag_name, default_args, PROJECT

bash_remove_old_saved_model_op = BashOperator(
task_id="bash_remove_old_saved_model_{}_task".format(model.replace(".","_")),
bash_command="if gsutil ls {0} 2> /dev/null; then gsutil -m rm -rf {0}/*; else true; fi".format(MODEL_LOCATION + model.replace(".","_")),
bash_command="if gcloud storage ls {0} 2> /dev/null; then gcloud storage rm --recursive --continue-on-error {0}/*; else true; fi".format(MODEL_LOCATION + model.replace(".","_")),
dag=dag
)

bash_copy_new_saved_model_op = BashOperator(
task_id="bash_copy_new_saved_model_{}_task".format(model.replace(".","_")),
bash_command="gsutil -m rsync -d -r `gsutil ls {0}/export/exporter/ | tail -1` {1}".format(output_dir, MODEL_LOCATION + model.replace(".","_")),
bash_command="gcloud storage rsync --delete-unmatched-destination-objects --recursive `gcloud storage ls {0}/export/exporter/ | tail -1` {1}".format(output_dir, MODEL_LOCATION + model.replace(".","_")),
dag=dag
)

Expand Down
8 changes: 4 additions & 4 deletions courses/machine_learning/feateng/asl_2.0_feat_eng-old.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,16 +159,16 @@
"fi \n",
" \n",
"## Create GCS bucket if it doesn't exist already...\n",
"exists=$(gsutil ls -d | grep -w gs://${PROJECT}/)\n",
"exists=$(gcloud storage ls | grep -w gs://${PROJECT}/)\n",
"\n",
"if [ -n \"$exists\" ]; then\n",
" echo -e \"Bucket exists, let's not recreate it.\"\n",
" \n",
"else\n",
" echo \"Creating a new GCS bucket.\"\n",
" gsutil mb -l ${REGION} gs://${PROJECT}\n",
" gcloud storage buckets create --location ${REGION} gs://${PROJECT}\n",
" echo \"\\nHere are your current buckets:\"\n",
" gsutil ls\n",
" gcloud storage ls\n",
"fi"
]
},
Expand Down Expand Up @@ -3961,4 +3961,4 @@
},
"nbformat": 4,
"nbformat_minor": 4
}
}
9 changes: 3 additions & 6 deletions courses/machine_learning/feateng/asl_2.0_feat_eng.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,17 +159,14 @@
"fi \n",
" \n",
"## Create GCS bucket if it doesn't exist already...\n",
"exists=$(gsutil ls -d | grep -w gs://${PROJECT}/)\n",
"\n",
"exists=$(gcloud storage ls | grep -w gs://${PROJECT}/)\n", "\n",
"if [ -n \"$exists\" ]; then\n",
" echo -e \"Bucket exists, let's not recreate it.\"\n",
" \n",
"else\n",
" echo \"Creating a new GCS bucket.\"\n",
" gsutil mb -l ${REGION} gs://${PROJECT}\n",
" echo \"\\nHere are your current buckets:\"\n",
" gsutil ls\n",
"fi"
" gcloud storage buckets create --location ${REGION} gs://${PROJECT}\n", " echo \"\\nHere are your current buckets:\"\n",
" gcloud storage ls\n", "fi"
]
},
{
Expand Down
24 changes: 7 additions & 17 deletions courses/machine_learning/feateng/feateng.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,9 +224,7 @@
"outputs": [],
"source": [
"%%bash\n",
"if gsutil ls | grep -q gs://${BUCKET}/taxifare/ch4/taxi_preproc/; then\n",
" gsutil -m rm -rf gs://$BUCKET/taxifare/ch4/taxi_preproc/\n",
"fi"
"if gcloud storage ls | grep -q gs://${BUCKET}/taxifare/ch4/taxi_preproc/; then\n", " gcloud storage rm --recursive --continue-on-error gs://$BUCKET/taxifare/ch4/taxi_preproc/\n", "fi"
]
},
{
Expand Down Expand Up @@ -346,8 +344,7 @@
],
"source": [
"%%bash\n",
"gsutil ls gs://$BUCKET/taxifare/ch4/taxi_preproc/"
]
"gcloud storage ls gs://$BUCKET/taxifare/ch4/taxi_preproc/" ]
},
{
"cell_type": "markdown",
Expand All @@ -370,9 +367,7 @@
"outputs": [],
"source": [
"%%bash\n",
"if gsutil ls | grep -q gs://${BUCKET}/taxifare/ch4/taxi_preproc/; then\n",
" gsutil -m rm -rf gs://$BUCKET/taxifare/ch4/taxi_preproc/\n",
"fi"
"if gcloud storage ls | grep -q gs://${BUCKET}/taxifare/ch4/taxi_preproc/; then\n", " gcloud storage rm --recursive --continue-on-error gs://$BUCKET/taxifare/ch4/taxi_preproc/\n", "fi"
]
},
{
Expand Down Expand Up @@ -433,8 +428,7 @@
],
"source": [
"%%bash\n",
"gsutil ls -l gs://$BUCKET/taxifare/ch4/taxi_preproc/"
]
"gcloud storage ls --long gs://$BUCKET/taxifare/ch4/taxi_preproc/" ]
},
{
"cell_type": "code",
Expand Down Expand Up @@ -462,8 +456,7 @@
"source": [
"%%bash\n",
"#print first 10 lines of first shard of train.csv\n",
"gsutil cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/train.csv-00000-of-*\" | head"
]
"gcloud storage cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/train.csv-00000-of-*\" | head" ]
},
{
"cell_type": "markdown",
Expand All @@ -485,9 +478,7 @@
" rm -rf sample\n",
"fi\n",
"mkdir sample\n",
"gsutil cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/train.csv-00000-of-*\" > sample/train.csv\n",
"gsutil cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/valid.csv-00000-of-*\" > sample/valid.csv"
]
"gcloud storage cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/train.csv-00000-of-*\" > sample/train.csv\n", "gcloud storage cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/valid.csv-00000-of-*\" > sample/valid.csv" ]
},
{
"cell_type": "markdown",
Expand Down Expand Up @@ -673,8 +664,7 @@
"OUTDIR=gs://${BUCKET}/taxifare/ch4/taxi_trained\n",
"JOBNAME=lab4a_$(date -u +%y%m%d_%H%M%S)\n",
"echo $OUTDIR $REGION $JOBNAME\n",
"gsutil -m rm -rf $OUTDIR\n",
"gcloud ai-platform jobs submit training $JOBNAME \\\n",
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "gcloud ai-platform jobs submit training $JOBNAME \\\n",
" --region=$REGION \\\n",
" --module-name=trainer.task \\\n",
" --package-path=${PWD}/taxifare/trainer \\\n",
Expand Down
2 changes: 1 addition & 1 deletion courses/machine_learning/feateng/hyperparam.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@
"OUTDIR=gs://${BUCKET}/taxifare/ch4/taxi_trained\n",
"JOBNAME=lab4a_$(date -u +%y%m%d_%H%M%S)\n",
"echo $OUTDIR $REGION $JOBNAME\n",
"gsutil -m rm -rf $OUTDIR\n",
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n",
"gcloud ml-engine jobs submit training $JOBNAME \\\n",
" --region=$REGION \\\n",
" --module-name=trainer.task \\\n",
Expand Down
8 changes: 4 additions & 4 deletions courses/machine_learning/feateng/tftransform.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,8 @@
"outputs": [],
"source": [
"%%bash\n",
"if ! gsutil ls | grep -q gs://${BUCKET}/; then\n",
" gsutil mb -l ${REGION} gs://${BUCKET}\n",
"if ! gcloud storage ls | grep -q gs://${BUCKET}/; then\n",
" gcloud storage buckets create --location=${REGION} gs://${BUCKET}\n",
"fi"
]
},
Expand Down Expand Up @@ -265,7 +265,7 @@
" print 'Launching Dataflow job {} ... hang on'.format(job_name)\n",
" OUTPUT_DIR = 'gs://{0}/taxifare/preproc_tft/'.format(BUCKET)\n",
" import subprocess\n",
" subprocess.call('gsutil rm -r {}'.format(OUTPUT_DIR).split())\n",
" subprocess.call('gcloud storage rm --recursive {}'.format(OUTPUT_DIR).split())\n",
" EVERY_N = 10000\n",
" \n",
" options = {\n",
Expand Down Expand Up @@ -358,7 +358,7 @@
"source": [
"%%bash\n",
"# ls preproc_tft\n",
"gsutil ls gs://${BUCKET}/taxifare/preproc_tft/"
"gcloud storage ls gs://${BUCKET}/taxifare/preproc_tft/"
]
},
{
Expand Down