-
Notifications
You must be signed in to change notification settings - Fork 4
/
evaluation_script_final.sh
150 lines (125 loc) · 6.81 KB
/
evaluation_script_final.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
data_path=data/sun360/sun360_processed.h5
model_path=models/sun360
echo "=========================================================================="
echo "=========================== SUN360 results ==============================="
echo "=========================================================================="
echo "===> Evaluating one-view"
echo "====== Average score ======"
python -W ignore eval.py --h5_path $data_path --dataset 0 \
--model_path $model_path/one-view.net --T 1 --M 8 --N 4 \
--start_view 2 --save_path dummy/
echo "===> Evaluating rnd-actions"
echo "====== Average score ======"
python -W ignore eval.py --h5_path $data_path --dataset 0 \
--model_path $model_path/rnd-actions.net --T 4 --M 8 --N 4 \
--start_view 2 --save_path dummy/ --actorType random
echo "===> Evaluating rnd-rewards"
echo "====== Average score ======"
python -W ignore eval.py --h5_path $data_path --dataset 0 \
--model_path $model_path/rnd-rewards.net --T 4 --M 8 --N 4 \
--start_view 2 --save_path dummy/
echo "===> Evaluating ltla"
echo "====== Average score ======"
python -W ignore eval.py --h5_path $data_path --dataset 0 \
--model_path $model_path/ltla.net --T 4 --M 8 --N 4 \
--start_view 2 --save_path dummy/
echo "===> Evaluating asymm-ac"
echo "====== Average score ======"
python -W ignore eval.py --h5_path $data_path --dataset 0 \
--model_path $model_path/asymm-ac.net --T 4 --M 8 --N 4 \
--start_view 2 --save_path dummy/ --baselineType critic \
--critic_full_obs True
echo "===> Evaluating expert-clone"
echo "====== Average score ======"
python -W ignore eval.py --h5_path $data_path --dataset 0 \
--model_path $model_path/expert-clone.net --T 4 --M 8 --N 4 \
--start_view 2 --save_path dummy/
echo "===> Evaluating ours(rew)"
echo "====== Average score ======"
python -W ignore eval.py --h5_path $data_path --dataset 0 \
--model_path $model_path/ours-rew.net --T 4 --M 8 --N 4 \
--start_view 2 --save_path dummy/
echo "===> Evaluating ours(demo)"
echo "====== Average score ======"
python -W ignore eval.py --h5_path $data_path --dataset 0 \
--model_path $model_path/ours-demo.net --T 4 --M 8 --N 4 \
--start_view 2 --save_path dummy/
echo "===> Evaluating ours(rew)+ac"
echo "====== Average score ======"
python -W ignore eval.py --h5_path $data_path --dataset 0 \
--model_path $model_path/ours-rew-ac.net --T 4 --M 8 --N 4 \
--start_view 2 --save_path dummy/ --baselineType critic
echo "===> Evaluating ours(demo)+ac"
echo "====== Average score ======"
python -W ignore eval.py --h5_path $data_path --dataset 0 \
--model_path $model_path/ours-demo-ac.net --T 4 --M 8 --N 4 \
--start_view 2 --save_path dummy/ --baselineType critic
echo "===> Evaluating demo-actions"
echo "====== Average score ======"
python -W ignore eval.py --h5_path $data_path --dataset 0 \
--model_path $model_path/demo-actions.net --T 4 --M 8 --N 4 \
--start_view 2 --save_path dummy/ --actorType demo_sidekick \
--utility_h5_path scores/sun360/ours-demo-scores.h5
data_path=data/modelnet_hard/modelnet30_processed.h5
data_path_unseen=data/modelnet_hard/modelnet10_processed.h5
model_path=models/modelnet_hard
echo "=========================================================================="
echo "======================== ModelNet Hard results ==========================="
echo "=========================================================================="
echo "===> Evaluating one-view"
echo "====== Average score ======"
python -W ignore eval.py --h5_path $data_path --h5_path_unseen $data_path_unseen --dataset 1 \
--model_path $model_path/one-view.net --T 1 --M 9 --N 5 \
--start_view 2 --save_path dummy/
echo "===> Evaluating rnd-actions"
echo "====== Average score ======"
python -W ignore eval.py --h5_path $data_path --h5_path_unseen $data_path_unseen --dataset 1 \
--model_path $model_path/rnd-actions.net --T 4 --M 9 --N 5 \
--start_view 2 --save_path dummy/ --actorType random
echo "===> Evaluating rnd-rewards"
echo "====== Average score ======"
python -W ignore eval.py --h5_path $data_path --h5_path_unseen $data_path_unseen --dataset 1 \
--model_path $model_path/rnd-rewards.net --T 4 --M 9 --N 5 \
--start_view 2 --save_path dummy/
echo "===> Evaluating ltla"
echo "====== Average score ======"
python -W ignore eval.py --h5_path $data_path --h5_path_unseen $data_path_unseen --dataset 1 \
--model_path $model_path/ltla.net --T 4 --M 9 --N 5 \
--start_view 2 --save_path dummy/
echo "===> Evaluating asymm-ac"
echo "====== Average score ======"
python -W ignore eval.py --h5_path $data_path --h5_path_unseen $data_path_unseen --dataset 1 \
--model_path $model_path/asymm-ac.net --T 4 --M 9 --N 5 \
--start_view 2 --save_path dummy/ --baselineType critic \
--critic_full_obs True
echo "===> Evaluating expert-clone"
echo "====== Average score ======"
python -W ignore eval.py --h5_path $data_path --h5_path_unseen $data_path_unseen --dataset 1 \
--model_path $model_path/expert-clone.net --T 4 --M 9 --N 5 \
--start_view 2 --save_path dummy/
echo "===> Evaluating ours(rew)"
echo "====== Average score ======"
python -W ignore eval.py --h5_path $data_path --h5_path_unseen $data_path_unseen --dataset 1 \
--model_path $model_path/ours-rew.net --T 4 --M 9 --N 5 \
--start_view 2 --save_path dummy/
echo "===> Evaluating ours(demo)"
echo "====== Average score ======"
python -W ignore eval.py --h5_path $data_path --h5_path_unseen $data_path_unseen --dataset 1 \
--model_path $model_path/ours-demo.net --T 4 --M 9 --N 5 \
--start_view 2 --save_path dummy/
echo "===> Evaluating ours(rew)+ac"
echo "====== Average score ======"
python -W ignore eval.py --h5_path $data_path --h5_path_unseen $data_path_unseen --dataset 1 \
--model_path $model_path/ours-rew-ac.net --T 4 --M 9 --N 5 \
--start_view 2 --save_path dummy/ --baselineType critic
echo "===> Evaluating ours(demo)+ac"
echo "====== Average score ======"
python -W ignore eval.py --h5_path $data_path --h5_path_unseen $data_path_unseen --dataset 1 \
--model_path $model_path/ours-demo-ac.net --T 4 --M 9 --N 5 \
--start_view 2 --save_path dummy/ --baselineType critic
echo "===> Evaluating demo-actions"
echo "====== Average score ======"
python -W ignore eval.py --h5_path $data_path --h5_path_unseen $data_path_unseen --dataset 1 \
--model_path $model_path/demo-actions.net --T 4 --M 9 --N 5 \
--start_view 2 --save_path dummy/ --actorType demo_sidekick \
--utility_h5_path scores/modelnet_hard/ours-demo-scores.h5