-
Notifications
You must be signed in to change notification settings - Fork 0
/
reproduce_mnist_experiments.sh
executable file
·173 lines (163 loc) · 5.37 KB
/
reproduce_mnist_experiments.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
RNG_SEEDS="42 142 242"
DATASETS="mnist emnist fmnist kmnist omni"
DATASETS=($DATASETS)
echo $DATASETS
# general parameters
# reduce the N_SUBSAMPLE to run only on the subsample
export N_SUBSAMPLE=10000
export SAMPLE_RANGE="0:$N_SUBSAMPLE"
export BASE_WORKDIR=$(pwd)
# handle the base directory
if [ -d $BASE_WORKDIR ]; then
echo 'Base directory exists $BASE_WORKDIR'
else
mkdir $BASE_WORKDIR
echo 'Created the base directory at $BASE_WORKDIR'
fi
# run methods
# run the MC Dropout for all the datasets
export USE_METHOD_CONF='baseline_mcdo_mnist'
echo "Computing samples for method $USE_METHOD_CONF..."
for ds in "${DATASETS[@]}"; do
for rand_seed in $RNG_SEEDS; do
echo "$ds" "$rand_seed"
export RNG_SEED=$rand_seed
export RUN_ON_DATASET=$ds
export DS_BATCHSIZE=1000 #$N_SUBSAMPLE
export RUN_ON_SUBSAMPLE=$SAMPLE_RANGE
echo "Running with seed $RNG_SEED on dataset $RUN_ON_DATASET"
python -m source.run_method --config baseline_mcdo_mnist;
# if python -m source.run_method --config baseline_mcdo_mnist; then
# exit
# fi
done
done
# evaluate the ood detection
export ID_DATA=mnist
echo "Computing uncertainties from samples for method $USE_METHOD_CONF..."
for ds in "${DATASETS[@]:1}"; do
for rand_seed in $RNG_SEEDS; do
echo "$met" "$rand_seed"
export RNG_SEED=$rand_seed
export OOD_DATA=$ds
echo "Running with seed $RNG_SEED on dataset OOD_DATA"
python -m source.run_experiment --config ood_nontempered_prerun;
done
done
# run laplace
export USE_METHOD_CONF='baseline_laplace_mnist'
echo "Computing samples for method $USE_METHOD_CONF..."
export LAPLACE_HESSIAN_STRUCTURE=kron
for ds in "${DATASETS[@]}"; do
for rand_seed in $RNG_SEEDS; do
echo "$ds" "$rand_seed"
export RNG_SEED=$rand_seed
export RUN_ON_DATASET=$ds
export DS_BATCHSIZE=$N_SUBSAMPLE
export RUN_ON_SUBSAMPLE=$SAMPLE_RANGE
echo "Running with seed $RNG_SEED on dataset $RUN_ON_DATASET"
python -m source.run_method --config baseline_laplace_mnist;
done
done
# evaluate the ood detection
export ID_DATA=mnist
echo "Computing uncertainties from samples for method $USE_METHOD_CONF..."
for ds in "${DATASETS[@]:1}"; do
for rand_seed in $RNG_SEEDS; do
echo "$met" "$rand_seed"
export RNG_SEED=$rand_seed
export OOD_DATA=$ds
echo "Running with seed $RNG_SEED on dataset OOD_DATA"
python -m source.run_experiment --config ood_nontempered_prerun;
done
done
# run sghmc
export USE_METHOD_CONF='baseline_sghmc_mnist'
echo "Computing samples for method $USE_METHOD_CONF..."
# default parameters in the file are sufficient
for ds in "${DATASETS[@]}"; do
for rand_seed in $RNG_SEEDS; do
echo "$ds" "$rand_seed"
export RNG_SEED=$rand_seed
export RUN_ON_DATASET=$ds
export DS_BATCHSIZE=$N_SUBSAMPLE
export RUN_ON_SUBSAMPLE=$SAMPLE_RANGE
echo "Running with seed $RNG_SEED on dataset $RUN_ON_DATASET"
python -m source.run_method --config baseline_sghmc_mnist;
done
done
# evaluate the ood detection
export ID_DATA=mnist
echo "Computing uncertainties from samples for method $USE_METHOD_CONF..."
for ds in "${DATASETS[@]:1}"; do
for rand_seed in $RNG_SEEDS; do
echo "$met" "$rand_seed"
export RNG_SEED=$rand_seed
export OOD_DATA=$ds
echo "Running with seed $RNG_SEED on dataset OOD_DATA"
python -m source.run_experiment --config ood_nontempered_prerun;
done
done
# run ensembles
export USE_METHOD_CONF='baseline_ensembles_mnist'
echo "Computing samples for method $USE_METHOD_CONF..."
# default parameters in the file are sufficient
for ds in "${DATASETS[@]}"; do
for rand_seed in $RNG_SEEDS; do
echo "$ds" "$rand_seed"
export RNG_SEED=$rand_seed
export RUN_ON_DATASET=$ds
export DS_BATCHSIZE=$N_SUBSAMPLE
export RUN_ON_SUBSAMPLE=$SAMPLE_RANGE
echo "Running with seed $RNG_SEED on dataset $RUN_ON_DATASET"
python -m source.run_method --config baseline_ensembles_mnist;
done
done
# evaluate the ood detection
export ID_DATA=mnist
echo "Computing uncertainties from samples for method $USE_METHOD_CONF..."
for ds in "${DATASETS[@]:1}"; do
for rand_seed in $RNG_SEEDS; do
echo "$met" "$rand_seed"
export RNG_SEED=$rand_seed
export OOD_DATA=$ds
echo "Running with seed $RNG_SEED on dataset OOD_DATA"
python -m source.run_experiment --config ood_nontempered_prerun;
done
done
# run sghmc
export USE_METHOD_CONF='baseline_quam_mnist'
echo "Computing samples for method $USE_METHOD_CONF..."
export ATTACK_N_CLASSES=10
export TOTAL_CLASSES=10
export OPT_EPOCHS=2
#export OPT_GAMMA=1.e-2
#export LR_SCHEDULE
export OPT_C0=6.0
export OPT_ETA=1.9
export OPT_UPDATE_C_EVERY=14
export OPT_LR=5.e-3
export OPT_WEIGHT_DECAY=1.e-3
for ds in "${DATASETS[@]}"; do
for rand_seed in $RNG_SEEDS; do
echo "$ds" "$rand_seed"
export RNG_SEED=$rand_seed
export RUN_ON_DATASET=$ds
export DS_BATCHSIZE=$N_SUBSAMPLE
export RUN_ON_SUBSAMPLE=$SAMPLE_RANGE
echo "Running with seed $RNG_SEED on dataset $RUN_ON_DATASET"
python -m source.run_method --config baseline_quam_mnist;
done
done
# evaluate the ood detection
export ID_DATA=mnist
echo "Computing uncertainties from samples for method $USE_METHOD_CONF..."
for ds in "${DATASETS[@]:1}"; do
for rand_seed in $RNG_SEEDS; do
echo "$met" "$rand_seed"
export RNG_SEED=$rand_seed
export OOD_DATA=$ds
echo "Running with seed $RNG_SEED on dataset OOD_DATA"
python -m source.run_experiment --config ood_tempered_prerun;
done
done