Skip to content

Commit 12abc08

Browse files
googlyrahmangooglyrahman
committed
Migrate gsutil usage to gcloud
1 parent a76ffef commit 12abc08

File tree

10 files changed

+31
-55
lines changed

10 files changed

+31
-55
lines changed

‎courses/machine_learning/asl/open_project/ASL_youtube8m_models/frame/youtube_8m_frame_level.ipynb‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1057,8 +1057,7 @@
10571057
"OUTDIR=gs://$BUCKET/youtube_8m_frame_level/trained_model\n",
10581058
"JOBNAME=job_youtube_8m_frame_level$(date -u +%y%m%d_%H%M%S)\n",
10591059
"echo $OUTDIR $REGION $JOBNAME\n",
1060-
"gsutil -m rm -rf $OUTDIR\n",
1061-
"gcloud ml-engine jobs submit training $JOBNAME \\\n",
1060+
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "gcloud ml-engine jobs submit training $JOBNAME \\\n",
10621061
" --region=$REGION \\\n",
10631062
" --module-name=trainer.task \\\n",
10641063
" --package-path=$PWD/trainer \\\n",
@@ -1156,8 +1155,7 @@
11561155
"OUTDIR=gs://$BUCKET/youtube_8m_frame_level/hyperparam\n",
11571156
"JOBNAME=job_youtube_8m_frame_level$(date -u +%y%m%d_%H%M%S)\n",
11581157
"echo $OUTDIR $REGION $JOBNAME\n",
1159-
"gsutil -m rm -rf $OUTDIR\n",
1160-
"gcloud ml-engine jobs submit training $JOBNAME \\\n",
1158+
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "gcloud ml-engine jobs submit training $JOBNAME \\\n",
11611159
" --region=$REGION \\\n",
11621160
" --module-name=trainer.task \\\n",
11631161
" --package-path=$PWD/trainer \\\n",
@@ -1215,8 +1213,7 @@
12151213
"%bash\n",
12161214
"MODEL_NAME=\"youtube_8m_frame_level\"\n",
12171215
"MODEL_VERSION=\"v1\"\n",
1218-
"MODEL_LOCATION=$(gsutil ls gs://$BUCKET/youtube_8m_frame_level/trained_model/export/exporter/ | tail -1)\n",
1219-
"echo \"Deleting and deploying $MODEL_NAME $MODEL_VERSION from $MODEL_LOCATION ... this will take a few minutes\"\n",
1216+
"MODEL_LOCATION=$(gcloud storage ls gs://$BUCKET/youtube_8m_frame_level/trained_model/export/exporter/ | tail -1)\n", "echo \"Deleting and deploying $MODEL_NAME $MODEL_VERSION from $MODEL_LOCATION ... this will take a few minutes\"\n",
12201217
"#gcloud ml-engine versions delete ${MODEL_VERSION} --model ${MODEL_NAME}\n",
12211218
"#gcloud ml-engine models delete ${MODEL_NAME}\n",
12221219
"gcloud ml-engine models create $MODEL_NAME --regions $REGION\n",

‎courses/machine_learning/asl/open_project/ASL_youtube8m_models/video/youtube_8m_video_level.ipynb‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -749,8 +749,7 @@
749749
"OUTDIR=gs://$BUCKET/youtube_8m_video_level/trained_model\n",
750750
"JOBNAME=job_youtube_8m_video_level$(date -u +%y%m%d_%H%M%S)\n",
751751
"echo $OUTDIR $REGION $JOBNAME\n",
752-
"gsutil -m rm -rf $OUTDIR\n",
753-
"gcloud ml-engine jobs submit training $JOBNAME \\\n",
752+
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "gcloud ml-engine jobs submit training $JOBNAME \\\n",
754753
" --region=$REGION \\\n",
755754
" --module-name=trainer.task \\\n",
756755
" --package-path=$PWD/trainer \\\n",
@@ -855,8 +854,7 @@
855854
"OUTDIR=gs://$BUCKET/youtube_8m_video_level/hyperparam\n",
856855
"JOBNAME=job_youtube_8m_video_level$(date -u +%y%m%d_%H%M%S)\n",
857856
"echo $OUTDIR $REGION $JOBNAME\n",
858-
"gsutil -m rm -rf $OUTDIR\n",
859-
"gcloud ml-engine jobs submit training $JOBNAME \\\n",
857+
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "gcloud ml-engine jobs submit training $JOBNAME \\\n",
860858
" --region=$REGION \\\n",
861859
" --module-name=trainer.task \\\n",
862860
" --package-path=$PWD/trainer \\\n",
@@ -914,8 +912,7 @@
914912
"%bash\n",
915913
"MODEL_NAME=\"youtube_8m_video_level\"\n",
916914
"MODEL_VERSION=\"v1\"\n",
917-
"MODEL_LOCATION=$(gsutil ls gs://$BUCKET/youtube_8m_video_level/trained_model/export/exporter/ | tail -1)\n",
918-
"echo \"Deleting and deploying $MODEL_NAME $MODEL_VERSION from $MODEL_LOCATION ... this will take a few minutes\"\n",
915+
"MODEL_LOCATION=$(gcloud storage ls gs://$BUCKET/youtube_8m_video_level/trained_model/export/exporter/ | tail -1)\n", "echo \"Deleting and deploying $MODEL_NAME $MODEL_VERSION from $MODEL_LOCATION ... this will take a few minutes\"\n",
919916
"#gcloud ml-engine versions delete ${MODEL_VERSION} --model ${MODEL_NAME}\n",
920917
"#gcloud ml-engine models delete ${MODEL_NAME}\n",
921918
"gcloud ml-engine models create $MODEL_NAME --regions $REGION\n",

‎courses/machine_learning/asl/open_project/ASL_youtube8m_models/video_using_datasets/youtube_8m_video_level_datasets.ipynb‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3783,8 +3783,7 @@
37833783
"OUTDIR=gs://$BUCKET/youtube_8m_video_level_datasets/trained_model\n",
37843784
"JOBNAME=job_youtube_8m_video_level_datasets$(date -u +%y%m%d_%H%M%S)\n",
37853785
"echo $OUTDIR $REGION $JOBNAME\n",
3786-
"gsutil -m rm -rf $OUTDIR\n",
3787-
"gcloud ml-engine jobs submit training $JOBNAME \\\n",
3786+
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "gcloud ml-engine jobs submit training $JOBNAME \\\n",
37883787
" --region=$REGION \\\n",
37893788
" --module-name=trainer.task \\\n",
37903789
" --package-path=$PWD/trainer \\\n",
@@ -3882,8 +3881,7 @@
38823881
"OUTDIR=gs://$BUCKET/youtube_8m_video_level_datasets/hyperparam\n",
38833882
"JOBNAME=job_youtube_8m_video_level_datasets$(date -u +%y%m%d_%H%M%S)\n",
38843883
"echo $OUTDIR $REGION $JOBNAME\n",
3885-
"gsutil -m rm -rf $OUTDIR\n",
3886-
"gcloud ml-engine jobs submit training $JOBNAME \\\n",
3884+
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "gcloud ml-engine jobs submit training $JOBNAME \\\n",
38873885
" --region=$REGION \\\n",
38883886
" --module-name=trainer.task \\\n",
38893887
" --package-path=$PWD/trainer \\\n",
@@ -3941,8 +3939,7 @@
39413939
"%bash\n",
39423940
"MODEL_NAME=\"youtube_8m_video_level_datasets\"\n",
39433941
"MODEL_VERSION=\"v1\"\n",
3944-
"MODEL_LOCATION=$(gsutil ls gs://$BUCKET/youtube_8m_video_level_datasets/trained_model/export/exporter/ | tail -1)\n",
3945-
"echo \"Deleting and deploying $MODEL_NAME $MODEL_VERSION from $MODEL_LOCATION ... this will take a few minutes\"\n",
3942+
"MODEL_LOCATION=$(gcloud storage ls gs://$BUCKET/youtube_8m_video_level_datasets/trained_model/export/exporter/ | tail -1)\n", "echo \"Deleting and deploying $MODEL_NAME $MODEL_VERSION from $MODEL_LOCATION ... this will take a few minutes\"\n",
39463943
"#gcloud ml-engine versions delete ${MODEL_VERSION} --model ${MODEL_NAME}\n",
39473944
"#gcloud ml-engine models delete ${MODEL_NAME}\n",
39483945
"gcloud ml-engine models create $MODEL_NAME --regions $REGION\n",

‎courses/machine_learning/asl/open_project/cloud_composer_automated_ml_pipeline_taxifare/airflow/dags/subdag/preprocess.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ def preprocess_tasks(model, parent_dag_name, child_dag_name, default_args, PROJE
9898
# BigQuery training data export to GCS
9999
bash_remove_old_data_op = BashOperator(
100100
task_id="bash_remove_old_data_{}_task".format(model.replace(".","_")),
101-
bash_command="if gsutil ls {0}/taxifare/data/{1} 2> /dev/null; then gsutil -m rm -rf {0}/taxifare/data/{1}/*; else true; fi".format(BUCKET, model.replace(".","_")),
101+
bash_command="if gcloud storage ls {0}/taxifare/data/{1} 2> /dev/null; then gcloud storage rm --recursive --continue-on-error {0}/taxifare/data/{1}/*; else true; fi".format(BUCKET, model.replace(".","_")),
102102
dag=dag
103103
)
104104

‎courses/machine_learning/asl/open_project/cloud_composer_automated_ml_pipeline_taxifare/airflow/dags/subdag/training.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,13 +55,13 @@ def training_tasks(model, parent_dag_name, child_dag_name, default_args, PROJECT
5555

5656
bash_remove_old_saved_model_op = BashOperator(
5757
task_id="bash_remove_old_saved_model_{}_task".format(model.replace(".","_")),
58-
bash_command="if gsutil ls {0} 2> /dev/null; then gsutil -m rm -rf {0}/*; else true; fi".format(MODEL_LOCATION + model.replace(".","_")),
58+
bash_command="if gcloud storage ls {0} 2> /dev/null; then gcloud storage rm --recursive --continue-on-error {0}/*; else true; fi".format(MODEL_LOCATION + model.replace(".","_")),
5959
dag=dag
6060
)
6161

6262
bash_copy_new_saved_model_op = BashOperator(
6363
task_id="bash_copy_new_saved_model_{}_task".format(model.replace(".","_")),
64-
bash_command="gsutil -m rsync -d -r `gsutil ls {0}/export/exporter/ | tail -1` {1}".format(output_dir, MODEL_LOCATION + model.replace(".","_")),
64+
bash_command="gcloud storage rsync --delete-unmatched-destination-objects --recursive `gcloud storage ls {0}/export/exporter/ | tail -1` {1}".format(output_dir, MODEL_LOCATION + model.replace(".","_")),
6565
dag=dag
6666
)
6767

‎courses/machine_learning/feateng/asl_2.0_feat_eng-old.ipynb‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -159,17 +159,14 @@
159159
"fi \n",
160160
" \n",
161161
"## Create GCS bucket if it doesn't exist already...\n",
162-
"exists=$(gsutil ls -d | grep -w gs://${PROJECT}/)\n",
163-
"\n",
162+
"# TODO: Flag \"-d\" is not supported in \"gcloud storage ls\". Manual review required.\nexists=$(gsutil ls -d | grep -w gs://${PROJECT}/)\n", "\n",
164163
"if [ -n \"$exists\" ]; then\n",
165164
" echo -e \"Bucket exists, let's not recreate it.\"\n",
166165
" \n",
167166
"else\n",
168167
" echo \"Creating a new GCS bucket.\"\n",
169-
" gsutil mb -l ${REGION} gs://${PROJECT}\n",
170-
" echo \"\\nHere are your current buckets:\"\n",
171-
" gsutil ls\n",
172-
"fi"
168+
" gcloud storage buckets create --location ${REGION} gs://${PROJECT}\n", " echo \"\\nHere are your current buckets:\"\n",
169+
" gcloud storage ls\n", "fi"
173170
]
174171
},
175172
{

‎courses/machine_learning/feateng/asl_2.0_feat_eng.ipynb‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -159,17 +159,15 @@
159159
"fi \n",
160160
" \n",
161161
"## Create GCS bucket if it doesn't exist already...\n",
162-
"exists=$(gsutil ls -d | grep -w gs://${PROJECT}/)\n",
163-
"\n",
162+
# TODO: Flag "-d" is not supported in "gcloud storage ls". Manual review required.
163+
"exists=$(gcloud storage ls | grep -w gs://${PROJECT}/)\n", "\n",
164164
"if [ -n \"$exists\" ]; then\n",
165165
" echo -e \"Bucket exists, let's not recreate it.\"\n",
166166
" \n",
167167
"else\n",
168168
" echo \"Creating a new GCS bucket.\"\n",
169-
" gsutil mb -l ${REGION} gs://${PROJECT}\n",
170-
" echo \"\\nHere are your current buckets:\"\n",
171-
" gsutil ls\n",
172-
"fi"
169+
" gcloud storage buckets create --location ${REGION} gs://${PROJECT}\n", " echo \"\\nHere are your current buckets:\"\n",
170+
" gcloud storage ls\n", "fi"
173171
]
174172
},
175173
{

‎courses/machine_learning/feateng/feateng.ipynb‎

Lines changed: 7 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -224,9 +224,7 @@
224224
"outputs": [],
225225
"source": [
226226
"%%bash\n",
227-
"if gsutil ls | grep -q gs://${BUCKET}/taxifare/ch4/taxi_preproc/; then\n",
228-
" gsutil -m rm -rf gs://$BUCKET/taxifare/ch4/taxi_preproc/\n",
229-
"fi"
227+
"if gcloud storage ls | grep -q gs://${BUCKET}/taxifare/ch4/taxi_preproc/; then\n", " gcloud storage rm --recursive --continue-on-error gs://$BUCKET/taxifare/ch4/taxi_preproc/\n", "fi"
230228
]
231229
},
232230
{
@@ -346,8 +344,7 @@
346344
],
347345
"source": [
348346
"%%bash\n",
349-
"gsutil ls gs://$BUCKET/taxifare/ch4/taxi_preproc/"
350-
]
347+
"gcloud storage ls gs://$BUCKET/taxifare/ch4/taxi_preproc/" ]
351348
},
352349
{
353350
"cell_type": "markdown",
@@ -370,9 +367,7 @@
370367
"outputs": [],
371368
"source": [
372369
"%%bash\n",
373-
"if gsutil ls | grep -q gs://${BUCKET}/taxifare/ch4/taxi_preproc/; then\n",
374-
" gsutil -m rm -rf gs://$BUCKET/taxifare/ch4/taxi_preproc/\n",
375-
"fi"
370+
"if gcloud storage ls | grep -q gs://${BUCKET}/taxifare/ch4/taxi_preproc/; then\n", " gcloud storage rm --recursive --continue-on-error gs://$BUCKET/taxifare/ch4/taxi_preproc/\n", "fi"
376371
]
377372
},
378373
{
@@ -433,8 +428,7 @@
433428
],
434429
"source": [
435430
"%%bash\n",
436-
"gsutil ls -l gs://$BUCKET/taxifare/ch4/taxi_preproc/"
437-
]
431+
"gcloud storage ls --long gs://$BUCKET/taxifare/ch4/taxi_preproc/" ]
438432
},
439433
{
440434
"cell_type": "code",
@@ -462,8 +456,7 @@
462456
"source": [
463457
"%%bash\n",
464458
"#print first 10 lines of first shard of train.csv\n",
465-
"gsutil cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/train.csv-00000-of-*\" | head"
466-
]
459+
"gcloud storage cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/train.csv-00000-of-*\" | head" ]
467460
},
468461
{
469462
"cell_type": "markdown",
@@ -485,9 +478,7 @@
485478
" rm -rf sample\n",
486479
"fi\n",
487480
"mkdir sample\n",
488-
"gsutil cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/train.csv-00000-of-*\" > sample/train.csv\n",
489-
"gsutil cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/valid.csv-00000-of-*\" > sample/valid.csv"
490-
]
481+
"gcloud storage cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/train.csv-00000-of-*\" > sample/train.csv\n", "gcloud storage cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/valid.csv-00000-of-*\" > sample/valid.csv" ]
491482
},
492483
{
493484
"cell_type": "markdown",
@@ -673,8 +664,7 @@
673664
"OUTDIR=gs://${BUCKET}/taxifare/ch4/taxi_trained\n",
674665
"JOBNAME=lab4a_$(date -u +%y%m%d_%H%M%S)\n",
675666
"echo $OUTDIR $REGION $JOBNAME\n",
676-
"gsutil -m rm -rf $OUTDIR\n",
677-
"gcloud ai-platform jobs submit training $JOBNAME \\\n",
667+
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "gcloud ai-platform jobs submit training $JOBNAME \\\n",
678668
" --region=$REGION \\\n",
679669
" --module-name=trainer.task \\\n",
680670
" --package-path=${PWD}/taxifare/trainer \\\n",

‎courses/machine_learning/feateng/hyperparam.ipynb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,7 @@
218218
"OUTDIR=gs://${BUCKET}/taxifare/ch4/taxi_trained\n",
219219
"JOBNAME=lab4a_$(date -u +%y%m%d_%H%M%S)\n",
220220
"echo $OUTDIR $REGION $JOBNAME\n",
221-
"gsutil -m rm -rf $OUTDIR\n",
221+
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n",
222222
"gcloud ml-engine jobs submit training $JOBNAME \\\n",
223223
" --region=$REGION \\\n",
224224
" --module-name=trainer.task \\\n",

‎courses/machine_learning/feateng/tftransform.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -96,8 +96,8 @@
9696
"outputs": [],
9797
"source": [
9898
"%%bash\n",
99-
"if ! gsutil ls | grep -q gs://${BUCKET}/; then\n",
100-
" gsutil mb -l ${REGION} gs://${BUCKET}\n",
99+
"if ! gcloud storage ls | grep -q gs://${BUCKET}/; then\n",
100+
" gcloud storage buckets create --location=${REGION} gs://${BUCKET}\n",
101101
"fi"
102102
]
103103
},
@@ -265,7 +265,7 @@
265265
" print 'Launching Dataflow job {} ... hang on'.format(job_name)\n",
266266
" OUTPUT_DIR = 'gs://{0}/taxifare/preproc_tft/'.format(BUCKET)\n",
267267
" import subprocess\n",
268-
" subprocess.call('gsutil rm -r {}'.format(OUTPUT_DIR).split())\n",
268+
" subprocess.call('gcloud storage rm --recursive {}'.format(OUTPUT_DIR).split())\n",
269269
" EVERY_N = 10000\n",
270270
" \n",
271271
" options = {\n",
@@ -358,7 +358,7 @@
358358
"source": [
359359
"%%bash\n",
360360
"# ls preproc_tft\n",
361-
"gsutil ls gs://${BUCKET}/taxifare/preproc_tft/"
361+
"gcloud storage ls gs://${BUCKET}/taxifare/preproc_tft/"
362362
]
363363
},
364364
{

0 commit comments

Comments
 (0)