From dbdb16b8fc4e19c6e8c673f9d3754157e405c571 Mon Sep 17 00:00:00 2001
From: Daniel Ecer <de-code@users.noreply.github.com>
Date: Tue, 2 Jan 2018 10:53:39 +0000
Subject: [PATCH] optionally use specific temp bucket when running ml-engine
jobs
---
eval.sh | 3 ++-
predict.sh | 2 +-
prepare-shell.sh | 5 +++++
save-model.sh | 2 +-
train.sh | 2 +-
5 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/eval.sh b/eval.sh
index d6f04df..e097904 100755
--- a/eval.sh
+++ b/eval.sh
@@ -41,11 +41,12 @@ if [ $USE_SEPARATE_CHANNELS == true ]; then
fi
if [ $USE_CLOUD == true ]; then
+ JOB_ID="eval_$JOB_ID"
gcloud ml-engine jobs submit training "$JOB_ID" \
--stream-logs \
--module-name sciencebeam_gym.trainer.task \
--package-path sciencebeam_gym \
- --staging-bucket "$BUCKET" \
+ --staging-bucket "$TEMP_BUCKET" \
--region us-central1 \
--runtime-version=1.2 \
-- \
diff --git a/predict.sh b/predict.sh
index 38735cf..351325c 100755
--- a/predict.sh
+++ b/predict.sh
@@ -55,7 +55,7 @@ if [ $USE_CLOUD == true ]; then
--stream-logs \
--module-name sciencebeam_gym.trainer.task \
--package-path sciencebeam_gym \
- --staging-bucket "$BUCKET" \
+ --staging-bucket "$TEMP_BUCKET" \
--region us-central1 \
--runtime-version=1.2 \
-- \
diff --git a/prepare-shell.sh b/prepare-shell.sh
index 5f378b3..f13c3dc 100755
--- a/prepare-shell.sh
+++ b/prepare-shell.sh
@@ -11,6 +11,7 @@ TRAINING_ARGS=""
PROJECT=$(gcloud config list project --format "value(core.project)")
LOCAL_PATH_ROOT="./.models"
BUCKET="gs://${PROJECT}-ml"
+TEMP_BUCKET=
COLOR_MAP_FILENAME="color_map.conf"
USE_SEPARATE_CHANNELS=true
DATASET_SUFFIX=
@@ -61,6 +62,10 @@ if [ -z "$JOB_ID" ]; then
JOB_ID="${DEFAULT_JOB_ID}"
fi
+if [ -z "$TEMP_BUCKET" ]; then
+ TEMP_BUCKET="${BUCKET}"
+fi
+
# cloud paths
GCS_SUB_PROJECT_PATH="${BUCKET}/${SUB_PROJECT_NAME}"
GCS_PATH="${GCS_SUB_PROJECT_PATH}/${MODEL_NAME}/${VERSION_NAME}"
diff --git a/save-model.sh b/save-model.sh
index 03b4337..68c36fc 100755
--- a/save-model.sh
+++ b/save-model.sh
@@ -41,7 +41,7 @@ if [ $USE_CLOUD == true ]; then
--stream-logs \
--module-name sciencebeam_gym.trainer.task \
--package-path sciencebeam_gym \
- --staging-bucket "$BUCKET" \
+ --staging-bucket "$TEMP_BUCKET" \
--region us-central1 \
--runtime-version=1.2 \
-- \
diff --git a/train.sh b/train.sh
index 6d466bf..1059d2f 100755
--- a/train.sh
+++ b/train.sh
@@ -39,7 +39,7 @@ if [ $USE_CLOUD == true ]; then
--stream-logs \
--module-name sciencebeam_gym.trainer.task \
--package-path sciencebeam_gym \
- --staging-bucket "$BUCKET" \
+ --staging-bucket "$TEMP_BUCKET" \
--region us-central1 \
--runtime-version=1.2 \
--scale-tier=BASIC_GPU \
--
GitLab