Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
#!/bin/bash
export JOB_ID_FILE='.job-id'
if [ -f "$JOB_ID_FILE" ]; then
rm "${JOB_ID_FILE}"
fi
source prepare-shell.sh
COMMON_ARGS=(
--output_path "${TRAIN_MODEL_PATH}/"
--train_data_paths "${TRAIN_PREPROC_PATH}/*tfrecord*"
--eval_data_paths "${EVAL_PREPROC_PATH}/*tfrecord*"
--model "${MODEL_NAME}"
--color_map "${CONFIG_PATH}/${COLOR_MAP_FILENAME}"
--class_weights="${CLASS_WEIGHTS_URL}"
--channels="$CHANNEL_NAMES"
--use_separate_channels $USE_SEPARATE_CHANNELS
--batch_size $BATCH_SIZE
--eval_set_size $EVAL_SET_SIZE
--max_steps 0
--base_loss $BASE_LOSS
--seed $RANDOM_SEED
--save_model="${MODEL_EXPORT_PATH}"
${TRAINING_ARGS[@]}
)
if [ ! -z "$QUALITATIVE_FOLDER_NAME" ]; then
COMMON_ARGS=(
${COMMON_ARGS[@]}
--qualitative_data_paths "${PREPROC_PATH}/${QUALITATIVE_FOLDER_NAME}/*tfrecord*"
--qualitative_set_size ${QUALITATIVE_SET_SIZE}
)
fi
if [ $USE_SEPARATE_CHANNELS == true ]; then
COMMON_ARGS=(
${COMMON_ARGS[@]}
--color_map "${CONFIG_PATH}/${COLOR_MAP_FILENAME}"
)
fi
if [ $USE_CLOUD == true ]; then
gcloud ml-engine jobs submit training "$JOB_ID" \
--stream-logs \
--module-name sciencebeam_gym.trainer.task \
--package-path sciencebeam_gym \
--staging-bucket "$BUCKET" \
--region us-central1 \
--runtime-version=1.2 \
-- \
--cloud \
${COMMON_ARGS[@]}
else
gcloud ml-engine local train \
--module-name sciencebeam_gym.trainer.task \
--package-path sciencebeam_gym.trainer \
-- \
${COMMON_ARGS[@]}
fi