.gitlab-ci: Increase build timeout
[samba.git] / .gitlab-ci-main.yml
1 # see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options
2
3 # Stages explained
4 #
5 # images: Build the images with the bootstrap script
6 # build_first: Build a few things first to find silly errors (fast job)
7 #              (don't pay for 35 machines until something compiles)
8 # build: The main parallel job
9 #              (keep these to 1hour as we are billed per hour)
10 # test_only: Tests using the build from prior stages, these typically
11 #            have an explicit dependency defined to a specific build job,
12 #            which means that start as soon as the build job finished.
13 # test_private: Like test_only, but running on private runners
14 # report: Code coverage reporting
15
16 stages:
17   - images
18   - build_first
19   - build
20   - test_only
21   - test_private
22   - report
23
24 variables:
25   # We want to be resilient to runner failures
26   ARTIFACT_DOWNLOAD_ATTEMPTS: "3"
27   EXECUTOR_JOB_SECTION_ATTEMPTS: "3"
28   GET_SOURCES_ATTEMPTS: "3"
29   RESTORE_CACHE_ATTEMPTS: "3"
30   #
31   GIT_STRATEGY: fetch
32   GIT_DEPTH: "3"
33   #
34   # we run autobuild.py inside a samba CI docker image located on gitlab's registry
35   # overwrite this variable if you want use your own image registry.
36   #
37   # Or better ask for access to the shared development repository, see
38   # https://wiki.samba.org/index.php/Samba_CI_on_gitlab#Getting_Access
39   #
40   SAMBA_CI_CONTAINER_REGISTRY: registry.gitlab.com/samba-team/devel/samba
41   #
42   # Set this to the contents of bootstrap/sha1sum.txt
43   # which is generated by bootstrap/template.py --render
44   #
45   SAMBA_CI_CONTAINER_TAG: 752c448d3186fe93a0c4039b8fbe897bb67a1f33
46   #
47   # We use the ubuntu1804 image as default as
48   # it matches what we have on sn-devel-184.
49   #
50   SAMBA_CI_CONTAINER_IMAGE: ubuntu1804
51   #
52   # The following images are available
53   # Please see the samba-o3 sections at the end of this file!
54   # We should run that for each available image
55   #
56   SAMBA_CI_CONTAINER_IMAGE_ubuntu1604: ubuntu1604
57   SAMBA_CI_CONTAINER_IMAGE_ubuntu1804: ubuntu1804
58   SAMBA_CI_CONTAINER_IMAGE_ubuntu2004: ubuntu2004
59   SAMBA_CI_CONTAINER_IMAGE_debian9: debian9
60   SAMBA_CI_CONTAINER_IMAGE_debian10: debian10
61   SAMBA_CI_CONTAINER_IMAGE_opensuse151: opensuse151
62   SAMBA_CI_CONTAINER_IMAGE_opensuse152: opensuse152
63   SAMBA_CI_CONTAINER_IMAGE_fedora33: fedora33
64   SAMBA_CI_CONTAINER_IMAGE_fedora34: fedora34
65   SAMBA_CI_CONTAINER_IMAGE_centos7: centos7
66   SAMBA_CI_CONTAINER_IMAGE_centos8: centos8
67
68 include:
69   # The image creation details are specified in a separate file
70   # See bootstrap/README.md for details
71   - 'bootstrap/.gitlab-ci.yml'
72
73 .shared_runner_build_image:
74   extends: .shared_runner_build
75   variables:
76     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE}
77   image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-${SAMBA_CI_JOB_IMAGE}:${SAMBA_CI_CONTAINER_TAG}
78
79 .shared_template:
80   extends: .shared_runner_build_image
81   # All Samba jobs are interruptible, this avoids burning CPU when a
82   # newer branch is pushed.
83   interruptible: true
84   timeout: 2h
85
86   variables:
87     AUTOBUILD_JOB_NAME: $CI_JOB_NAME
88   stage: build
89   cache:
90     key: ccache.${CI_JOB_NAME}.${SAMBA_CI_JOB_IMAGE}.${SAMBA_CI_FLAVOR}
91     paths:
92       - ccache
93   before_script:
94     - uname -a
95     - lsb_release -a
96     - cat /etc/os-release
97     - lscpu
98     - cat /proc/cpuinfo
99     - mount
100     - df -h
101     - cat /proc/swaps
102     - free -h
103       # ld will fail if coverage enabled, force link ld to ld.bfd
104     - if [ -n "$SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE" ]; then sudo ln -sf $(which ld.bfd) $(which ld); fi
105       # See bootstrap/.gitlab-ci.yml how to generate a new image
106     - echo "SAMBA_CI_CONTAINER_REGISTRY[${SAMBA_CI_CONTAINER_REGISTRY}]"
107     - echo "SAMBA_CI_CONTAINER_TAG[${SAMBA_CI_CONTAINER_TAG}]"
108     - echo "SAMBA_CI_JOB_IMAGE[${SAMBA_CI_JOB_IMAGE}]"
109     - echo "CI_JOB_IMAGE[${CI_JOB_IMAGE}]"
110     - bootstrap/template.py --sha1sum > /tmp/sha1sum-template.txt
111     - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-template.txt
112     - echo "${SAMBA_CI_CONTAINER_TAG}" > /tmp/sha1sum-tag.txt
113     - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-tag.txt
114     - diff -u bootstrap/sha1sum.txt /sha1sum.txt
115     - echo "${CI_COMMIT_SHA} ${CI_COMMIT_TITLE}" > /tmp/commit.txt
116     - export CCACHE_BASEDIR="${PWD}"
117     - export CCACHE_DIR="${PWD}/ccache" && mkdir -pv "$CCACHE_DIR"
118     - export CC="ccache cc"
119     - export CXX="ccache c++"
120     - ccache -z -M 500M
121     - ccache -s
122   after_script:
123     - mount
124     - df -h
125     - cat /proc/swaps
126     - free -h
127     - CCACHE_BASEDIR="${PWD}" CCACHE_DIR="${PWD}/ccache" ccache -s -c
128   artifacts:
129     expire_in: 1 week
130     paths:
131       - "*.stdout"
132       - "*.stderr"
133       - "*.info"
134       - public
135       - system-info.txt
136   retry:
137     max: 2
138     when:
139       - runner_system_failure
140       - stuck_or_timeout_failure
141       - api_failure
142       - runner_unsupported
143       - stale_schedule
144       - job_execution_timeout
145       - archived_failure
146       - scheduler_failure
147       - data_integrity_failure
148
149   script:
150     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
151     # autobuild name, which means we can define a default template that runs most autobuild jobs
152     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE  --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
153
154 # Ensure when adding a new job below that you also add it to
155 # the dependencies for 'pages' below for the code coverage page
156 # generation.
157
158 others:
159   extends: .shared_template
160   script:
161     - script/autobuild.py ldb      $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/ldb
162     - script/autobuild.py pidl     $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/pidl
163     - script/autobuild.py replace  $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/replace
164     - script/autobuild.py talloc   $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/talloc
165     - script/autobuild.py tdb      $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/tdb
166     - script/autobuild.py tevent   $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/tevent
167     - script/autobuild.py samba-xc $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/samba-xc
168     - script/autobuild.py docs-xml $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/docs-xml
169
170 .shared_template_build_only:
171   extends: .shared_template
172   timeout: 2h
173   artifacts:
174     expire_in: 1 week
175     paths:
176       - "*.stdout"
177       - "*.stderr"
178       - "*.info"
179       - system-info.txt
180       - samba-testbase.tar.gz
181   script:
182     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
183     # autobuild name, which means we can define a default template that runs most autobuild jobs
184     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE  --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
185     # On success we need to pack everything into an artifacts file
186     # which needs to be in the git checkout.
187     # As tar doesn't handle hardlink of read-only files,
188     # we remember the acls and add write permissions
189     # before creating the archive. The consumer will apply
190     # the acls again.
191     - cp -a /sha1sum.txt /tmp/samba-testbase/image-sha1sum.txt
192     - cp -a /tmp/commit.txt /tmp/samba-testbase/commit.txt
193     - ln -s /tmp/samba-testbase/${AUTOBUILD_JOB_NAME}/ /tmp/samba-testbase/build_subdir_link
194     - pushd /tmp && getfacl -R samba-testbase > samba-testbase.acl.dump && popd
195     - chmod -R +w /tmp/samba-testbase
196     - mv /tmp/samba-testbase.acl.dump /tmp/samba-testbase/
197     - tar cfz samba-testbase.tar.gz /tmp/samba-testbase
198     - ls -la samba-testbase.tar.gz
199     - sha1sum samba-testbase.tar.gz
200
201 .shared_template_test_only:
202   extends:
203     - .shared_template
204     - .shared_runner_test
205   stage: test_only
206   script:
207     # We unpack the artifacts file created by the .shared_template_build_only
208     # run we depend on
209     - ls -la samba-testbase.tar.gz
210     - sha1sum samba-testbase.tar.gz
211     - tar xfz samba-testbase.tar.gz -C /
212     - diff -u /tmp/samba-testbase/image-sha1sum.txt /sha1sum.txt
213     - diff -u /tmp/samba-testbase/commit.txt /tmp/commit.txt
214     - mv /tmp/samba-testbase/samba-testbase.acl.dump /tmp/samba-testbase.acl.dump
215     - pushd /tmp && setfacl --restore=/tmp/samba-testbase.acl.dump && popd
216     - ls -la /tmp/samba-testbase/
217     - ls -la /tmp/samba-testbase/build_subdir_link
218     - ls -la /tmp/samba-testbase/build_subdir_link/
219     - if [ -n "$SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE" ]; then find /tmp/samba-testbase/build_subdir_link/ -type d -printf "'%p'\n" | xargs chmod u+w; fi
220     - ls -la /tmp/samba-testbase/build_subdir_link/
221     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
222     # autobuild name, which means we can define a default template that runs most autobuild jobs
223     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --skip-dependencies --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
224
225 samba-def-build:
226   extends: .shared_template_build_only
227   stage: build_first
228
229 .needs_samba-def-build:
230   extends: .shared_template_test_only
231   needs:
232     - job: samba-def-build
233       artifacts: true
234
235 samba-mit-build:
236   extends: .shared_template_build_only
237   variables:
238     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora34}
239   stage: build_first
240
241 .needs_samba-mit-build:
242   extends: .shared_template_test_only
243   variables:
244     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora34}
245   needs:
246     - job: samba-mit-build
247       artifacts: true
248
249 samba-h5l-build:
250   extends: .shared_template_build_only
251
252 .needs_samba-h5l-build:
253   extends: .shared_template_test_only
254   needs:
255     - job: samba-h5l-build
256       artifacts: true
257
258 samba-nt4-build:
259   extends: .shared_template_build_only
260
261 .needs_samba-nt4-build:
262   extends: .shared_template_test_only
263   needs:
264     - job: samba-nt4-build
265       artifacts: true
266
267 samba-no-opath-build:
268   extends: .shared_template_build_only
269
270 .needs_samba-no-opath-build:
271   extends: .shared_template_test_only
272   needs:
273     - job: samba-no-opath-build
274       artifacts: true
275
276 samba:
277   extends: .shared_template
278
279 samba-mitkrb5:
280   extends: .shared_template
281   variables:
282     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora34}
283
284 samba-minimal-smbd:
285   extends: .shared_template
286
287 samba-nopython:
288   extends: .shared_template
289
290 samba-admem:
291   extends: .needs_samba-def-build
292
293 samba-ad-dc-2:
294   extends: .needs_samba-def-build
295
296 samba-ad-dc-3:
297   extends: .needs_samba-def-build
298
299 samba-ad-dc-4a:
300   extends: .needs_samba-def-build
301
302 samba-ad-dc-4b:
303   extends: .needs_samba-def-build
304
305 samba-ad-dc-5:
306   extends: .needs_samba-def-build
307
308 samba-ad-dc-6:
309   extends: .needs_samba-def-build
310
311 samba-ad-back1:
312   extends: .needs_samba-def-build
313
314 samba-ad-back2:
315   extends: .needs_samba-def-build
316
317 samba-schemaupgrade:
318   extends: .needs_samba-def-build
319
320 samba-libs:
321   extends: .shared_template
322
323 samba-fuzz:
324   extends: .shared_template
325   variables:
326     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu2004}
327
328 ctdb:
329   extends: .shared_template
330
331 samba-ctdb:
332   extends: .shared_template
333
334 samba-ad-dc-ntvfs:
335   extends: .needs_samba-def-build
336
337 samba-admem-mit:
338   extends: .needs_samba-mit-build
339
340 samba-addc-mit-4a:
341   extends: .needs_samba-mit-build
342
343 samba-addc-mit-4b:
344   extends: .needs_samba-mit-build
345
346 # This task is run first to ensure we compile before we start the
347 # main run as it is the fastest full compile of Samba.
348 samba-fips:
349   extends: .shared_template
350   variables:
351     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora34}
352
353 .private_test_only:
354   extends: .private_runner_test
355   stage: test_private
356   only:
357     variables:
358       # These jobs are only run if the gitlab repo has private runners available.
359       # To enable private jobs, you must add the following var and value to
360       # your gitlab repo by navigating to:
361       # settings -> CI/CD -> Environment variables
362       - $SUPPORT_PRIVATE_TEST == "yes"
363
364 .needs_samba-def-build-private:
365   extends:
366     - .needs_samba-def-build
367     - .private_test_only
368
369 .needs_samba-mit-build-private:
370   extends:
371     - .needs_samba-mit-build
372     - .private_test_only
373
374 .needs_samba-h5l-build-private:
375   extends:
376     - .needs_samba-h5l-build
377     - .private_test_only
378
379 .needs_samba-nt4-build-private:
380   extends:
381     - .needs_samba-nt4-build
382     - .private_test_only
383
384 .needs_samba-no-opath-build-private:
385   extends:
386     - .needs_samba-no-opath-build
387     - .private_test_only
388
389 samba-fileserver:
390   extends: .needs_samba-h5l-build-private
391
392 # This is a full build without the AD DC so we test the build with MIT
393 # Kerberos from the default system (Ubuntu 18.04 at this stage).
394 # Runtime behaviour checked via the ktest (static ccache and keytab)
395 # environment
396 samba-ktest-mit:
397  extends: .shared_template
398
399 samba-ad-dc-1:
400   extends: .needs_samba-def-build-private
401
402 samba-nt4:
403   extends: .needs_samba-nt4-build-private
404
405 samba-addc-mit-1:
406   extends: .needs_samba-mit-build-private
407
408 samba-no-opath1:
409   extends: .needs_samba-no-opath-build-private
410
411 samba-no-opath2:
412   extends: .needs_samba-no-opath-build-private
413
414 # 'pages' is a special job which can publish artifacts in `public` dir to gitlab pages
415 pages:
416   extends: .shared_runner_build_image
417   stage: report
418   dependencies:  # tell gitlab to download artifacts for these jobs
419     - others
420     - samba
421     - samba-mitkrb5
422     - samba-admem
423     - samba-ad-dc-2
424     - samba-ad-dc-3
425     - samba-ad-dc-4a
426     - samba-ad-dc-4b
427     - samba-ad-dc-5
428     - samba-ad-dc-6
429     - samba-libs
430     - samba-minimal-smbd
431     - samba-nopython
432     - samba-fuzz
433     # - ctdb  # TODO
434     - samba-ctdb
435     - samba-ad-dc-ntvfs
436     - samba-admem-mit
437     - samba-addc-mit-4a
438     - samba-addc-mit-4b
439     - samba-ad-back1
440     - samba-ad-back2
441     - samba-fileserver
442     - samba-ad-dc-1
443     - samba-nt4
444     - samba-schemaupgrade
445     - samba-addc-mit-1
446     - samba-fips
447     - samba-no-opath1
448     - samba-no-opath2
449     - ubuntu1804-samba-o3
450   script:
451     - ls -la *.info
452     - ./configure.developer
453     - make -j
454     - ls -la *.info
455     - lcov $(ls *.info | xargs -I{} echo -n "-a {} ") -o all.info
456     - ls -la *.info
457     - genhtml all.info --ignore-errors source --output-directory public --prefix=$(pwd) --title "coverage report for $CI_COMMIT_REF_NAME $CI_COMMIT_SHORT_SHA"
458   artifacts:
459     expire_in: 30 days
460     paths:
461       - public
462   only:
463     variables:
464       - $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == "--enable-coverage"
465
466 # Coverity Scan
467 coverity:
468   extends: .shared_runner_build_image
469   variables:
470     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora34}
471   stage: build
472   script:
473     - wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_SCAN_TOKEN&project=$COVERITY_SCAN_PROJECT_NAME" -O /tmp/coverity_tool.tgz
474     - tar xf /tmp/coverity_tool.tgz
475     - ./configure.developer --with-cluster-support --with-system-mitkrb5 --with-experimental-mit-ad-dc
476     - cov-analysis-linux64-*/bin/cov-build --dir cov-int make -j$(nproc)
477     - tar czf cov-int.tar.gz cov-int
478     - curl
479       --form token=$COVERITY_SCAN_TOKEN
480       --form email=$COVERITY_SCAN_EMAIL
481       --form file=@cov-int.tar.gz
482       --form version="`git describe --tags`"
483       --form description="CI build"
484       https://scan.coverity.com/builds?project=$COVERITY_SCAN_PROJECT_NAME
485   only:
486     refs:
487       - master
488       - schedules
489     variables:
490       - $COVERITY_SCAN_TOKEN != null
491       - $COVERITY_SCAN_PROJECT_NAME != null
492       - $COVERITY_SCAN_EMAIL != null
493   artifacts:
494     expire_in: 1 week
495     when: on_failure
496     paths:
497       - cov-int/*.txt
498
499 #
500 # We build samba-o3 on all supported distributions
501 #
502
503 # This job, which matches the main CI, needs to still do coverage so
504 # we show the coverage on the "none" environment tests
505 #
506 # We want --enable-coverage specified here otherwise we will have a
507 # different set of build options on the coverage build and can fail
508 # when -O3 gets combined with --enable-coverage in the scheduled
509 # builds.
510
511 ubuntu1804-samba-o3:
512   extends: .shared_template
513   variables:
514     AUTOBUILD_JOB_NAME: samba-o3
515     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu1804}
516     SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE: "--enable-coverage"
517   only:
518     variables:
519       # do not run o3 builds (which run a lot of VMs) if told not to
520       # (this uses the same variable as autobuild.py)
521       - $AUTOBUILD_SKIP_SAMBA_O3 == "0"
522
523 # All other jobs do not want code coverage.
524 .samba-o3-template:
525   extends: .shared_template
526   variables:
527     AUTOBUILD_JOB_NAME: samba-o3
528   rules:
529     # do not run o3 builds (which run a lot of VMs) if told not to
530     # (this uses the same variable as autobuild.py)
531     - if: $AUTOBUILD_SKIP_SAMBA_O3 == "1"
532       when: never
533     # do not run o3 for coverage since they are using different images
534     - if: $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == ""
535
536 ubuntu2004-samba-o3:
537   extends: .samba-o3-template
538   variables:
539     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu2004}
540
541 debian10-samba-o3:
542   extends: .samba-o3-template
543   variables:
544     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_debian10}
545
546 opensuse151-samba-o3:
547   extends: .samba-o3-template
548   variables:
549     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_opensuse151}
550
551 opensuse152-samba-o3:
552   extends: .samba-o3-template
553   variables:
554     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_opensuse152}
555
556 centos7-samba-o3:
557   extends: .samba-o3-template
558   variables:
559     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_centos7}
560     # Git on CentOS doesn't support shallow git cloning
561     GIT_DEPTH: ""
562     # We need a newer GnuTLS version on CentOS7
563     PKG_CONFIG_PATH: "/usr/lib64/compat-gnutls34/pkgconfig:/usr/lib64/compat-nettle32/pkgconfig"
564
565 centos8-samba-o3:
566   extends: .samba-o3-template
567   variables:
568     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_centos8}
569
570 fedora33-samba-o3:
571   extends: .samba-o3-template
572   variables:
573     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora33}
574
575 fedora34-samba-o3:
576   extends: .samba-o3-template
577   variables:
578     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora34}
579
580 #
581 # Keep the samba-o3 sections at the end ...
582 #