Migrate gsutil usage to gcloud storage (#3010)

* Migrate gsutil usage to gcloud

* Manual Changes

Author: googlyrahman

courses/ahybrid/v1.0/common/scripts/create_remote.sh

@@ -14,12 +14,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-gsutil mb $KOPS_STORE
+gcloud storage buckets create $KOPS_STORE
 
 n=0
 until [ $n -ge 5 ]
 do
-    gsutil ls | grep $KOPS_STORE && break
+    gcloud storage ls | grep $KOPS_STORE && break
     n=$[$n+1]
     sleep 3
 done
@@ -61,7 +61,7 @@ fi
 
 echo "copying the kubeconfig file for later use..."
 kops export kubecfg --name $C2_FULLNAME --state=$KOPS_STORE
-gsutil cp $KF $KOPS_STORE
+gcloud storage cp $KF $KOPS_STORE
 
 echo "creating service account and granting role..."
 gcloud iam service-accounts create connect-sa-op

courses/bdml_fundamentals/demos/earthquakevm/README.md

@@ -27,7 +27,7 @@ cd training-data-analyst/courses/bdml_fundamentals
 ```
 * Go to the Storage | Browser in the Cloud Console and create a new bucket
 * Copy the files to the bucket:
-```gsutil cp earthquakes.* gs://[YOURBUCKET]```
+```gcloud storage cp earthquakes.* gs://[YOURBUCKET]```
 * Refresh the Storage | Browser to verify that you have new files in Cloud Storage
 * Edit the bucket permissions and add a new member named ```allUsers``` 
 and give this member Cloud Storage Object Viewer permissions

courses/machine_learning/cloudmle/cloudmle.ipynb

@@ -84,9 +84,9 @@
     "    print(response['serviceAccount'])\")\n",
     "\n",
     "echo \"Authorizing the Cloud ML Service account $SVC_ACCOUNT to access files in $BUCKET\"\n",
-    "gsutil -m defacl ch -u $SVC_ACCOUNT:R gs://$BUCKET\n",
-    "gsutil -m acl ch -u $SVC_ACCOUNT:R -r gs://$BUCKET  # error message (if bucket is empty) can be ignored\n",
-    "gsutil -m acl ch -u $SVC_ACCOUNT:W gs://$BUCKET"
+    "gcloud storage buckets update --add-default-object-acl-grant entity=user-$SVC_ACCOUNT,role=READER gs://$BUCKET\n",
+    "gcloud storage objects update --add-acl-grant entity=user-$SVC_ACCOUNT,role=READER --recursive gs://$BUCKET  # error message (if bucket is empty) can be ignored\n",
+    "gcloud storage buckets update --add-acl-grant entity=user-$SVC_ACCOUNT,role=WRITER gs://$BUCKET",
    ]
   },
   {
@@ -271,8 +271,8 @@
    "source": [
     "%%bash\n",
     "echo $BUCKET\n",
-    "gsutil -m rm -rf gs://${BUCKET}/taxifare/smallinput/\n",
-    "gsutil -m cp ${PWD}/*.csv gs://${BUCKET}/taxifare/smallinput/"
+    "gcloud storage rm --recursive --continue-on-error gs://${BUCKET}/taxifare/smallinput/\n",
+    "gcloud storage cp ${PWD}/*.csv gs://${BUCKET}/taxifare/smallinput/"
    ]
   },
   {
@@ -285,7 +285,7 @@
     "OUTDIR=gs://${BUCKET}/taxifare/smallinput/taxi_trained\n",
     "JOBNAME=lab3a_$(date -u +%y%m%d_%H%M%S)\n",
     "echo $OUTDIR $REGION $JOBNAME\n",
-    "gsutil -m rm -rf $OUTDIR\n",
+    "gcloud storage rm --recursive --continue-on-error $OUTDIR\n",
     "gcloud ai-platform jobs submit training $JOBNAME \\\n",
     "   --region=$REGION \\\n",
     "   --module-name=trainer.task \\\n",
@@ -327,8 +327,8 @@
    "outputs": [],
    "source": [
     "%%bash\n",
-    "gsutil cp -r ${PWD}/taxi_trained gs://${BUCKET}/taxifare/smallinput/ \n",
-    "gsutil ls gs://${BUCKET}/taxifare/smallinput/taxi_trained/export/exporter"
+    "gcloud storage cp --recursive ${PWD}/taxi_trained gs://${BUCKET}/taxifare/smallinput/ \n",
+    "gcloud storage ls gs://${BUCKET}/taxifare/smallinput/taxi_trained/export/exporter"
    ]
   },
   {
@@ -340,7 +340,7 @@
     "%%bash\n",
     "MODEL_NAME=\"taxifare\"\n",
     "MODEL_VERSION=\"v1\"\n",
-    "MODEL_LOCATION=$(gsutil ls gs://${BUCKET}/taxifare/smallinput/taxi_trained/export/exporter | tail -1)\n",
+    "MODEL_LOCATION=$(gcloud storage ls gs://${BUCKET}/taxifare/smallinput/taxi_trained/export/exporter | tail -1)\n",
     "echo \"Run these commands one-by-one (the very first time, you'll create a model and then create a version)\"\n",
     "#gcloud ai-platform versions delete ${MODEL_VERSION} --model ${MODEL_NAME}\n",
     "#gcloud ai-platform models delete ${MODEL_NAME}\n",
@@ -463,7 +463,7 @@
     "JOBNAME=lab3a_$(date -u +%y%m%d_%H%M%S)\n",
     "CRS_BUCKET=cloud-training-demos # use the already exported data\n",
     "echo $OUTDIR $REGION $JOBNAME\n",
-    "gsutil -m rm -rf $OUTDIR\n",
+    "gcloud storage rm --recursive --continue-on-error $OUTDIR\n",
     "gcloud ai-platform jobs submit training $JOBNAME \\\n",
     "   --region=$REGION \\\n",
     "   --module-name=trainer.task \\\n",

courses/unstructured/ML-Tests-Solution.ipynb

@@ -68,7 +68,7 @@
    ],
    "source": [
     "import subprocess\n",
-    "images = subprocess.check_output([\"gsutil\", \"ls\", \"gs://{}/unstructured/photos\".format(BUCKET)])\n",
+    "images = subprocess.check_output([\"gcloud\", \"storage\", \"ls\", \"gs://{}/unstructured/photos\".format(BUCKET)])\n",
     "images = list(filter(None,images.split('\\n')))\n",
     "print(images)"
    ]