-
Notifications
You must be signed in to change notification settings - Fork 0
/
.gitlab-ci.yml
153 lines (145 loc) · 5.16 KB
/
.gitlab-ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
variables:
BUILD_ENV_REGISTRY: $CI_REGISTRY/maja/maja-build-env
stages:
- prepare
- build
- validation
# ---------------------------- fast pipeline ----------------------------------
prepare-centos7:
stage: prepare
image:
name: gcr.io/kaniko-project/executor:debug-v0.16.0
entrypoint: [""]
tags:
- docker
except:
- schedules
- web
before_script:
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}"
> /kaniko/.docker/config.json
- sed -i "s,^FROM maja-build-env,FROM $BUILD_ENV_REGISTRY/maja-env-centos:7," SuperBuild/CentOS_7/Dockerfile
script:
- echo $CI_COMMIT_SHA > sha1.txt
- /kaniko/executor --cleanup
--cache=true
--cache-repo=$CI_REGISTRY_IMAGE/maja-cots-centos7-cache
--verbosity warn
--dockerfile $CI_PROJECT_DIR/SuperBuild/CentOS_7/Dockerfile
--context $CI_PROJECT_DIR
--destination $CI_REGISTRY_IMAGE/maja-cots-centos7:$CI_COMMIT_REF_NAME
only:
changes:
- SuperBuild/**/*
- CMakeConfig/CommonCmakeOptions.cmake
- CMakeConfig/MAJAConfigurationSetting.cmake
- .gitlab-ci.yml
prepare-ubuntu18:
stage: prepare
image:
name: gcr.io/kaniko-project/executor:debug-v0.16.0
entrypoint: [""]
tags:
- docker
except:
- schedules
- web
before_script:
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}"
> /kaniko/.docker/config.json
- sed -i "s,^FROM maja-build-env,FROM $BUILD_ENV_REGISTRY/maja-env-ubuntu:18.04," SuperBuild/Ubuntu_18.04/Dockerfile
script:
- /kaniko/executor --cleanup
--cache=true
--cache-repo=$CI_REGISTRY_IMAGE/maja-cots-ubuntu18-cache
--verbosity warn
--dockerfile $CI_PROJECT_DIR/SuperBuild/Ubuntu_18.04/Dockerfile
--context $CI_PROJECT_DIR
--destination $CI_REGISTRY_IMAGE/maja-cots-ubuntu18:$CI_COMMIT_REF_NAME
only:
changes:
- SuperBuild/**/*
- CMakeConfig/CommonCmakeOptions.cmake
- CMakeConfig/MAJAConfigurationSetting.cmake
- .gitlab-ci.yml
build-centos7:
stage: build
image: $CI_REGISTRY_IMAGE/maja-cots-centos7:$CI_COMMIT_REF_NAME
timeout: 1h 15m
tags:
- docker
except:
- schedules
- web
variables:
OTB_MAX_RAM_HINT: 20000
ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS: 8
script:
- echo $CI_COMMIT_SHA > sha1.txt
- ctest -V -S CI/maja_build.cmake
- ctest -VV -S CI/maja_pkg.cmake
- time rclone copy s3_otb:maja-data/TVA data/TVA
- time rclone copy s3_otb:maja-data/cropped_context/TV/SENTINEL2_MUSCATE data/TV/SENTINEL2_MUSCATE
- time rclone copy s3_otb:maja-data/TU data/TU
- cd build
- ctest -V -N -R pyTv- > ../tv_content.txt
- ctest -R pyTv-S2-L2INIT-012-SENTINEL2-MUSCATE-NEWCAMS
- ctest -R pyTv-S2-L2NOMINAL-011-F-N-SENTINEL2_MUSCATE-GENERAL
- ctest -R pyTv-S2-L2NOMINAL-011-F-N-SENTINEL2_MUSCATE-LUTDIRCOR
- ctest -R pyTv-S2-L2NOMINAL-011-F-N-SENTINEL2_MUSCATE-MIXRES
- ctest -E pyTv
- cd ..
- ls -lh build/Testing/*-*/
# Filter tv_content.txt to be used in a different job
- sed -i "s,$(pwd)/data,\$MAJA_DATA,g" tv_content.txt
- sed -i "s,$(pwd)/build/Testing/Temporary,\$MAJA_TEST_OUTPUT_ROOT,g" tv_content.txt
- sed -i "s,/opt/cots/maja/[0-9]*\.[0-9]*\.[0-9]*/bin,\$MAJA_BIN,g" tv_content.txt
- sed -i "s,$(pwd)/build/bin,\$MAJA_BIN,g" tv_content.txt
- sed -i "s,$(pwd)/orchestrator/launcher,\$MAJA_LAUNCHER_PYTHON,g" tv_content.txt
artifacts:
expire_in: 20 days
when: on_success
paths:
- sha1.txt
- tv_content.txt
- build-pkg/*.run
# ---------------------------- heavy pipeline ----------------------------------
tv-centos7:
stage: validation
image: $BUILD_ENV_REGISTRY/maja-env-centos:7
timeout: 20h
tags:
- docker
- heavy-task
only:
- schedules
- web
variables:
MAJA_DATA: "data"
MAJA_TEST_OUTPUT_ROOT: "test_output"
MAJA_BIN: "/opt/maja/bin"
MAJA_LAUNCHER_PYTHON: "/opt/maja/lib/python/orchestrator/launcher"
OTB_MAX_RAM_HINT: 40000
ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS: 16
script:
- CI/get_ci_pkg.py
- file artifacts.zip
- unzip artifacts.zip
- mkdir test_output
- mkdir /opt/maja
- chmod +x build-pkg/*.run
- $(ls build-pkg/*.run) --target /opt/maja
- ln -s /opt/maja/bin/python3.7 /opt/maja/bin/python3
- source /opt/maja/bin/.majaenv.sh
- time rclone copy s3_otb:maja-data/TV/SENTINEL2_MUSCATE data/TV/SENTINEL2_MUSCATE
- time rclone copy s3_otb:maja-data/TV/SENTINEL2 data/TV/SENTINEL2
- time rclone copy s3_otb:maja-data/TV/LANDSAT8 data/TV/LANDSAT8
- time rclone copy s3_otb:maja-data/TV/VENUS data/TV/VENUS
- time rclone copy s3_otb:maja-data/TV/VENUS_MUSCATE data/TV/VENUS_MUSCATE
- time rclone copy s3_otb:maja-data/TV/START_MAJA data/TV/START_MAJA
- CI/run_tv.sh tv_content.txt
artifacts:
expire_in: 20 days
when: on_failure
paths:
- logs/*.log