Skip to content

Commit dd790f3

Browse files
gstriplinggstripling
authored
Merge pull request #2975 from googlyrahman/refactor-ai-migration-bbd8432477d74c2b8ac6711a1b0b90fb
Migrate gsutil usage to gcloud storage
2 parents d447022 + 4614c36 commit dd790f3

File tree

10 files changed

+33
-62
lines changed

10 files changed

+33
-62
lines changed

‎courses/machine_learning/deepdive/03_model_performance/a_feature_engineering_dnn.ipynb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
"metadata": {},
4646
"outputs": [],
4747
"source": [
48-
"!gsutil cp gs://cloud-training-demos/taxifare/small/*.csv .\n",
48+
"!gcloud storage cp gs://cloud-training-demos/taxifare/small/*.csv .\n",
4949
"!ls -l *.csv"
5050
]
5151
},

‎courses/machine_learning/deepdive/03_model_performance/b_feature_engineering_wd.ipynb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@
4646
"metadata": {},
4747
"outputs": [],
4848
"source": [
49-
"!gsutil cp gs://cloud-training-demos/taxifare/small/*.csv .\n",
49+
"!gcloud storage cp gs://cloud-training-demos/taxifare/small/*.csv .\n",
5050
"!ls -l *.csv"
5151
]
5252
},

‎courses/machine_learning/deepdive/03_model_performance/d_hyperparameter_tuning.ipynb‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -423,8 +423,7 @@
423423
"outputs": [],
424424
"source": [
425425
"OUTDIR=\"gs://{}/taxifare/trained_hp_tune\".format(BUCKET)\n",
426-
"!gsutil -m rm -rf {OUTDIR} # start fresh each time\n",
427-
"!gcloud ai-platform jobs submit training taxifare_$(date -u +%y%m%d_%H%M%S) \\\n",
426+
"!gcloud storage rm --recursive --continue-on-error {OUTDIR} # start fresh each time\n", "!gcloud ai-platform jobs submit training taxifare_$(date -u +%y%m%d_%H%M%S) \\\n",
428427
" --package-path=taxifaremodel \\\n",
429428
" --module-name=taxifaremodel.task \\\n",
430429
" --config=hyperparam.yaml \\\n",
@@ -459,8 +458,7 @@
459458
"outputs": [],
460459
"source": [
461460
"OUTDIR=\"gs://{}/taxifare/trained_large_tuned\".format(BUCKET)\n",
462-
"!gsutil -m rm -rf {OUTDIR} # start fresh each time\n",
463-
"!gcloud ai-platform jobs submit training taxifare_large_$(date -u +%y%m%d_%H%M%S) \\\n",
461+
"!gcloud storage rm --recursive --continue-on-error {OUTDIR} # start fresh each time\n", "!gcloud ai-platform jobs submit training taxifare_large_$(date -u +%y%m%d_%H%M%S) \\\n",
464462
" --package-path=taxifaremodel \\\n",
465463
" --module-name=taxifaremodel.task \\\n",
466464
" --job-dir=gs://{BUCKET}/taxifare \\\n",

‎courses/machine_learning/deepdive/04_advanced_preprocessing/a_dataflow.ipynb‎

Lines changed: 11 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -210,9 +210,7 @@
210210
"outputs": [],
211211
"source": [
212212
"%%bash\n",
213-
"if gsutil ls | grep -q gs://${BUCKET}/taxifare/ch4/taxi_preproc/; then\n",
214-
" gsutil -m rm -rf gs://$BUCKET/taxifare/ch4/taxi_preproc/\n",
215-
"fi"
213+
"if gcloud storage ls | grep -q gs://${BUCKET}/taxifare/ch4/taxi_preproc/; then\n", " gcloud storage rm --recursive --continue-on-error gs://$BUCKET/taxifare/ch4/taxi_preproc/\n", "fi"
216214
]
217215
},
218216
{
@@ -340,9 +338,7 @@
340338
"outputs": [],
341339
"source": [
342340
"%%bash\n",
343-
"if gsutil ls -r gs://${BUCKET} | grep -q gs://${BUCKET}/taxifare/ch4/taxi_preproc/; then\n",
344-
" gsutil -m rm -rf gs://${BUCKET}/taxifare/ch4/taxi_preproc/\n",
345-
"fi"
341+
"if gcloud storage ls --recursive gs://${BUCKET} | grep -q gs://${BUCKET}/taxifare/ch4/taxi_preproc/; then\n", " gcloud storage rm --recursive --continue-on-error gs://${BUCKET}/taxifare/ch4/taxi_preproc/\n", "fi"
346342
]
347343
},
348344
{
@@ -382,8 +378,7 @@
382378
"outputs": [],
383379
"source": [
384380
"%%bash\n",
385-
"gsutil ls -l gs://$BUCKET/taxifare/ch4/taxi_preproc/"
386-
]
381+
"gcloud storage ls --long gs://$BUCKET/taxifare/ch4/taxi_preproc/" ]
387382
},
388383
{
389384
"cell_type": "code",
@@ -392,8 +387,7 @@
392387
"outputs": [],
393388
"source": [
394389
"%%bash\n",
395-
"gsutil cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/TRAIN.csv-00000-of-*\" | head"
396-
]
390+
"gcloud storage cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/TRAIN.csv-00000-of-*\" | head" ]
397391
},
398392
{
399393
"cell_type": "markdown",
@@ -415,9 +409,7 @@
415409
" rm -rf sample\n",
416410
"fi\n",
417411
"mkdir sample\n",
418-
"gsutil cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/TRAIN.csv-00000-of-*\" > sample/train.csv\n",
419-
"gsutil cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/VALID.csv-00000-of-*\" > sample/valid.csv"
420-
]
412+
"gcloud storage cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/TRAIN.csv-00000-of-*\" > sample/train.csv\n", "gcloud storage cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/VALID.csv-00000-of-*\" > sample/valid.csv" ]
421413
},
422414
{
423415
"cell_type": "markdown",
@@ -582,8 +574,7 @@
582574
"OUTDIR=gs://${BUCKET}/taxifare/ch4/taxi_trained\n",
583575
"JOBNAME=lab4a_$(date -u +%y%m%d_%H%M%S)\n",
584576
"echo $OUTDIR $REGION $JOBNAME\n",
585-
"gsutil -m rm -rf $OUTDIR\n",
586-
"gcloud ml-engine jobs submit training $JOBNAME \\\n",
577+
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "gcloud ml-engine jobs submit training $JOBNAME \\\n",
587578
" --region=$REGION \\\n",
588579
" --module-name=taxifaremodel.task \\\n",
589580
" --package-path=${PWD}/taxifaremodel \\\n",
@@ -612,8 +603,7 @@
612603
"outputs": [],
613604
"source": [
614605
"%%bash\n",
615-
"gsutil ls gs://${BUCKET}/taxifare/ch4/taxi_trained/export/exporter | tail -1"
616-
]
606+
"gcloud storage ls gs://${BUCKET}/taxifare/ch4/taxi_trained/export/exporter | tail -1" ]
617607
},
618608
{
619609
"cell_type": "markdown",
@@ -629,8 +619,7 @@
629619
"outputs": [],
630620
"source": [
631621
"%%bash\n",
632-
"model_dir=$(gsutil ls gs://${BUCKET}/taxifare/ch4/taxi_trained/export/exporter | tail -1)\n",
633-
"saved_model_cli show --dir ${model_dir} --all"
622+
"model_dir=$(gcloud storage ls gs://${BUCKET}/taxifare/ch4/taxi_trained/export/exporter | tail -1)\n", "saved_model_cli show --dir ${model_dir} --all"
634623
]
635624
},
636625
{
@@ -647,8 +636,7 @@
647636
"outputs": [],
648637
"source": [
649638
"%%bash\n",
650-
"model_dir=$(gsutil ls gs://${BUCKET}/taxifare/ch4/taxi_trained/export/exporter | tail -1)\n",
651-
"gcloud ml-engine local predict \\\n",
639+
"model_dir=$(gcloud storage ls gs://${BUCKET}/taxifare/ch4/taxi_trained/export/exporter | tail -1)\n", "gcloud ml-engine local predict \\\n",
652640
" --model-dir=${model_dir} \\\n",
653641
" --json-instances=/tmp/test.json"
654642
]
@@ -685,9 +673,7 @@
685673
"outputs": [],
686674
"source": [
687675
"%%bash\n",
688-
"if gsutil ls -r gs://${BUCKET} | grep -q gs://${BUCKET}/taxifare/ch4/taxi_preproc/; then\n",
689-
" gsutil -m rm -rf gs://${BUCKET}/taxifare/ch4/taxi_preproc/\n",
690-
"fi"
676+
"if gcloud storage ls --recursive gs://${BUCKET} | grep -q gs://${BUCKET}/taxifare/ch4/taxi_preproc/; then\n", " gcloud storage rm --recursive --continue-on-error gs://${BUCKET}/taxifare/ch4/taxi_preproc/\n", "fi"
691677
]
692678
},
693679
{
@@ -714,8 +700,7 @@
714700
"JOBNAME=lab4a_$(date -u +%y%m%d_%H%M%S)\n",
715701
"echo $OUTDIR $REGION $JOBNAME\n",
716702
"TIER=STANDARD_1 \n",
717-
"gsutil -m rm -rf $OUTDIR\n",
718-
"gcloud ml-engine jobs submit training $JOBNAME \\\n",
703+
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "gcloud ml-engine jobs submit training $JOBNAME \\\n",
719704
" --region=$REGION \\\n",
720705
" --module-name=taxifaremodel.task \\\n",
721706
" --package-path=${PWD}/taxifaremodel \\\n",

‎courses/machine_learning/deepdive/04_advanced_preprocessing/labs/a_dataflow.ipynb‎

Lines changed: 9 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -210,9 +210,7 @@
210210
"outputs": [],
211211
"source": [
212212
"%%bash\n",
213-
"if gsutil ls | grep -q gs://${BUCKET}/taxifare/ch4/taxi_preproc/; then\n",
214-
" gsutil -m rm -rf gs://$BUCKET/taxifare/ch4/taxi_preproc/\n",
215-
"fi"
213+
"if gcloud storage ls | grep -q gs://${BUCKET}/taxifare/ch4/taxi_preproc/; then\n", " gcloud storage rm --recursive --continue-on-error gs://$BUCKET/taxifare/ch4/taxi_preproc/\n", "fi"
216214
]
217215
},
218216
{
@@ -349,9 +347,7 @@
349347
"outputs": [],
350348
"source": [
351349
"%%bash\n",
352-
"if gsutil ls -r gs://${BUCKET} | grep -q gs://${BUCKET}/taxifare/ch4/taxi_preproc/; then\n",
353-
" gsutil -m rm -rf gs://${BUCKET}/taxifare/ch4/taxi_preproc/\n",
354-
"fi"
350+
"if gcloud storage ls --recursive gs://${BUCKET} | grep -q gs://${BUCKET}/taxifare/ch4/taxi_preproc/; then\n", " gcloud storage rm --recursive --continue-on-error gs://${BUCKET}/taxifare/ch4/taxi_preproc/\n", "fi"
355351
]
356352
},
357353
{
@@ -391,8 +387,7 @@
391387
"outputs": [],
392388
"source": [
393389
"%%bash\n",
394-
"gsutil ls -l gs://$BUCKET/taxifare/ch4/taxi_preproc/"
395-
]
390+
"gcloud storage ls --long gs://$BUCKET/taxifare/ch4/taxi_preproc/" ]
396391
},
397392
{
398393
"cell_type": "code",
@@ -401,8 +396,7 @@
401396
"outputs": [],
402397
"source": [
403398
"%%bash\n",
404-
"gsutil cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/TRAIN.csv-00000-of-*\" | head"
405-
]
399+
"gcloud storage cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/TRAIN.csv-00000-of-*\" | head" ]
406400
},
407401
{
408402
"cell_type": "markdown",
@@ -424,9 +418,7 @@
424418
" rm -rf sample\n",
425419
"fi\n",
426420
"mkdir sample\n",
427-
"gsutil cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/TRAIN.csv-00000-of-*\" > sample/train.csv\n",
428-
"gsutil cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/VALID.csv-00000-of-*\" > sample/valid.csv"
429-
]
421+
"gcloud storage cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/TRAIN.csv-00000-of-*\" > sample/train.csv\n", "gcloud storage cat \"gs://$BUCKET/taxifare/ch4/taxi_preproc/VALID.csv-00000-of-*\" > sample/valid.csv" ]
430422
},
431423
{
432424
"cell_type": "markdown",
@@ -591,8 +583,7 @@
591583
"OUTDIR=gs://${BUCKET}/taxifare/ch4/taxi_trained\n",
592584
"JOBNAME=lab4a_$(date -u +%y%m%d_%H%M%S)\n",
593585
"echo $OUTDIR $REGION $JOBNAME\n",
594-
"gsutil -m rm -rf $OUTDIR\n",
595-
"gcloud ml-engine jobs submit training $JOBNAME \\\n",
586+
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n", "gcloud ml-engine jobs submit training $JOBNAME \\\n",
596587
" --region=$REGION \\\n",
597588
" --module-name=taxifaremodel.task \\\n",
598589
" --package-path=${PWD}/taxifaremodel \\\n",
@@ -621,8 +612,7 @@
621612
"outputs": [],
622613
"source": [
623614
"%%bash\n",
624-
"gsutil ls gs://${BUCKET}/taxifare/ch4/taxi_trained/export/exporter | tail -1"
625-
]
615+
"gcloud storage ls gs://${BUCKET}/taxifare/ch4/taxi_trained/export/exporter | tail -1" ]
626616
},
627617
{
628618
"cell_type": "markdown",
@@ -638,8 +628,7 @@
638628
"outputs": [],
639629
"source": [
640630
"%%bash\n",
641-
"model_dir=$(gsutil ls gs://${BUCKET}/taxifare/ch4/taxi_trained/export/exporter | tail -1)\n",
642-
"saved_model_cli show --dir ${model_dir} --all"
631+
"model_dir=$(gcloud storage ls gs://${BUCKET}/taxifare/ch4/taxi_trained/export/exporter | tail -1)\n", "saved_model_cli show --dir ${model_dir} --all"
643632
]
644633
},
645634
{
@@ -656,8 +645,7 @@
656645
"outputs": [],
657646
"source": [
658647
"%%bash\n",
659-
"model_dir=$(gsutil ls gs://${BUCKET}/taxifare/ch4/taxi_trained/export/exporter | tail -1)\n",
660-
"gcloud ml-engine local predict \\\n",
648+
"model_dir=$(gcloud storage ls gs://${BUCKET}/taxifare/ch4/taxi_trained/export/exporter | tail -1)\n", "gcloud ml-engine local predict \\\n",
661649
" --model-dir=${model_dir} \\\n",
662650
" --json-instances=/tmp/test.json"
663651
]

‎courses/machine_learning/deepdive/04_advanced_preprocessing/labs/taxicab_traffic/deploy.ipynb‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@
5050
"metadata": {},
5151
"outputs": [],
5252
"source": [
53-
"!gsutil cp -r $MODEL_PATH/* gs://$BUCKET/taxifare/model/"
53+
"!gcloud storage cp --recursive $MODEL_PATH/* gs://$BUCKET/taxifare/model/"
5454
]
5555
},
5656
{
@@ -178,7 +178,7 @@
178178
"metadata": {},
179179
"outputs": [],
180180
"source": [
181-
"!gsutil cp dist/taxifare_custom_predict_code-0.1.tar.gz gs://$BUCKET/taxifare/predict_code/"
181+
"!gcloud storage cp dist/taxifare_custom_predict_code-0.1.tar.gz gs://$BUCKET/taxifare/predict_code/"
182182
]
183183
},
184184
{

‎courses/machine_learning/deepdive/04_advanced_preprocessing/labs/taxicab_traffic/train.ipynb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636
"metadata": {},
3737
"outputs": [],
3838
"source": [
39-
"!gsutil cp gs://cloud-training-demos/taxifare/traffic/small/*.csv .\n",
39+
"!gcloud storage cp gs://cloud-training-demos/taxifare/traffic/small/*.csv .\n",
4040
"!ls -l *.csv"
4141
]
4242
},

‎courses/machine_learning/deepdive/04_advanced_preprocessing/taxicab_traffic/deploy.ipynb‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@
5050
"metadata": {},
5151
"outputs": [],
5252
"source": [
53-
"!gsutil cp -r $MODEL_PATH/* gs://$BUCKET/taxifare/model/"
53+
"!gcloud storage cp --recursive $MODEL_PATH/* gs://$BUCKET/taxifare/model/"
5454
]
5555
},
5656
{
@@ -182,7 +182,7 @@
182182
"metadata": {},
183183
"outputs": [],
184184
"source": [
185-
"!gsutil cp dist/taxifare_custom_predict_code-0.1.tar.gz gs://$BUCKET/taxifare/predict_code/"
185+
"!gcloud storage cp dist/taxifare_custom_predict_code-0.1.tar.gz gs://$BUCKET/taxifare/predict_code/"
186186
]
187187
},
188188
{

‎courses/machine_learning/deepdive/04_advanced_preprocessing/taxicab_traffic/train.ipynb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636
"metadata": {},
3737
"outputs": [],
3838
"source": [
39-
"!gsutil cp gs://cloud-training-demos/taxifare/traffic/small/*.csv .\n",
39+
"!gcloud storage cp gs://cloud-training-demos/taxifare/traffic/small/*.csv .\n",
4040
"!ls -l *.csv"
4141
]
4242
},

‎courses/machine_learning/deepdive/09_sequence_keras/labs/sinewaves.ipynb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -178,8 +178,8 @@
178178
"outputs": [],
179179
"source": [
180180
"%%bash\n",
181-
"gsutil -m rm -rf gs://${BUCKET}/sines/*\n",
182-
"gsutil -m cp data/sines/*.csv gs://${BUCKET}/sines"
181+
"gcloud storage rm --recursive --continue-on-error gs://${BUCKET}/sines/*\n",
182+
"gcloud storage cp data/sines/*.csv gs://${BUCKET}/sines"
183183
]
184184
},
185185
{
@@ -192,7 +192,7 @@
192192
"for MODEL in linear dnn cnn rnn rnn2; do\n",
193193
" OUTDIR=gs://${BUCKET}/sinewaves/${MODEL}\n",
194194
" JOBNAME=sines_${MODEL}_$(date -u +%y%m%d_%H%M%S)\n",
195-
" gsutil -m rm -rf $OUTDIR\n",
195+
" gcloud storage rm --recursive --continue-on-error $OUTDIR\n",
196196
" gcloud ml-engine jobs submit training $JOBNAME \\\n",
197197
" --region=$REGION \\\n",
198198
" --module-name=sinemodel.task \\\n",

0 commit comments

Comments
 (0)