.gitlab-ci-main.yml

   1 # see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options
   2
   3 # Stages explained
   4 #
   5 # images: Build the images with the bootstrap script
   6 # build_first: Build a few things first to find silly errors (fast job)
   7 #              (don't pay for 35 machines until something compiles)
   8 # build: The main parallel job
   9 #              (keep these to 1hour as we are billed per hour)
  10 # test_only: Tests using the build from prior stages, these typically
  11 #            have an explicit dependency defined to a specific build job,
  12 #            which means that start as soon as the build job finished.
  13 # test_private: Like test_only, but running on private runners
  14 # report: Code coverage reporting
  15
  16 stages:
  17   - images
  18   - build_first
  19   - build
  20   - test_only
  21   - test_private
  22   - report
  23
  24 variables:
  25   # We want to be resilient to runner failures
  26   ARTIFACT_DOWNLOAD_ATTEMPTS: "3"
  27   EXECUTOR_JOB_SECTION_ATTEMPTS: "3"
  28   GET_SOURCES_ATTEMPTS: "3"
  29   RESTORE_CACHE_ATTEMPTS: "3"
  30   #
  31   GIT_STRATEGY: fetch
  32   GIT_DEPTH: "3"
  33   # "--enable-coverage" or ""
  34   SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE: ""
  35   #
  36   # we run autobuild.py inside a samba CI docker image located on gitlab's registry
  37   # overwrite this variable if you want use your own image registry.
  38   #
  39   # Or better ask for access to the shared development repository, see
  40   # https://wiki.samba.org/index.php/Samba_CI_on_gitlab#Getting_Access
  41   #
  42   SAMBA_CI_CONTAINER_REGISTRY: registry.gitlab.com/samba-team/devel/samba
  43   #
  44   # Set this to the contents of bootstrap/sha1sum.txt
  45   # which is generated by bootstrap/template.py --render
  46   #
  47   SAMBA_CI_CONTAINER_TAG: f25e1fa0a828f428ade0e4f37d8a646ff1e207d4
  48   #
  49   # We use the ubuntu1804 image as default as
  50   # it matches what we have on sn-devel-184.
  51   #
  52   SAMBA_CI_CONTAINER_IMAGE: ubuntu1804
  53   #
  54   # The following images are available
  55   # Please see the samba-o3 sections at the end of this file!
  56   # We should run that for each available image
  57   #
  58   SAMBA_CI_CONTAINER_IMAGE_ubuntu1604: ubuntu1604
  59   SAMBA_CI_CONTAINER_IMAGE_ubuntu1804: ubuntu1804
  60   SAMBA_CI_CONTAINER_IMAGE_ubuntu2004: ubuntu2004
  61   SAMBA_CI_CONTAINER_IMAGE_debian9: debian9
  62   SAMBA_CI_CONTAINER_IMAGE_debian10: debian10
  63   SAMBA_CI_CONTAINER_IMAGE_opensuse151: opensuse151
  64   SAMBA_CI_CONTAINER_IMAGE_opensuse152: opensuse152
  65   SAMBA_CI_CONTAINER_IMAGE_fedora32: fedora32
  66   SAMBA_CI_CONTAINER_IMAGE_fedora33: fedora33
  67   SAMBA_CI_CONTAINER_IMAGE_centos7: centos7
  68   SAMBA_CI_CONTAINER_IMAGE_centos8: centos8
  69
  70 include:
  71   # The image creation details are specified in a separate file
  72   # See bootstrap/README.md for details
  73   - 'bootstrap/.gitlab-ci.yml'
  74
  75 .shared_runner_build_image:
  76   extends: .shared_runner_build
  77   variables:
  78     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE}
  79   image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-${SAMBA_CI_JOB_IMAGE}:${SAMBA_CI_CONTAINER_TAG}
  80
  81 .shared_template:
  82   extends: .shared_runner_build_image
  83   # All Samba jobs are interruptible, this avoids burning CPU when a
  84   # newer branch is pushed.
  85   interruptible: true
  86   timeout: 2h
  87
  88   variables:
  89     AUTOBUILD_JOB_NAME: $CI_JOB_NAME
  90   stage: build
  91   cache:
  92     key: ccache.${CI_JOB_NAME}.${SAMBA_CI_JOB_IMAGE}
  93     paths:
  94       - ccache
  95   before_script:
  96     - uname -a
  97     - lsb_release -a
  98     - cat /etc/os-release
  99     - lscpu
 100     - cat /proc/cpuinfo
 101     - mount
 102     - df -h
 103     - cat /proc/swaps
 104     - free -h
 105       # ld will fail if coverage enabled, force link ld to ld.bfd
 106     - if [ -n "$SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE" ]; then sudo ln -sf $(which ld.bfd) $(which ld); fi
 107       # See bootstrap/.gitlab-ci.yml how to generate a new image
 108     - echo "SAMBA_CI_CONTAINER_REGISTRY[${SAMBA_CI_CONTAINER_REGISTRY}]"
 109     - echo "SAMBA_CI_CONTAINER_TAG[${SAMBA_CI_CONTAINER_TAG}]"
 110     - echo "SAMBA_CI_JOB_IMAGE[${SAMBA_CI_JOB_IMAGE}]"
 111     - echo "CI_JOB_IMAGE[${CI_JOB_IMAGE}]"
 112     - bootstrap/template.py --sha1sum > /tmp/sha1sum-template.txt
 113     - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-template.txt
 114     - echo "${SAMBA_CI_CONTAINER_TAG}" > /tmp/sha1sum-tag.txt
 115     - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-tag.txt
 116     - diff -u bootstrap/sha1sum.txt /sha1sum.txt
 117     - echo "${CI_COMMIT_SHA} ${CI_COMMIT_TITLE}" > /tmp/commit.txt
 118     - export CCACHE_BASEDIR="${PWD}"
 119     - export CCACHE_DIR="${PWD}/ccache" && mkdir -pv "$CCACHE_DIR"
 120     - export CC="ccache cc"
 121     - export CXX="ccache c++"
 122     - ccache -z -M 500M
 123     - ccache -s
 124   after_script:
 125     - mount
 126     - df -h
 127     - cat /proc/swaps
 128     - free -h
 129     - CCACHE_BASEDIR="${PWD}" CCACHE_DIR="${PWD}/ccache" ccache -s -c
 130   artifacts:
 131     expire_in: 1 week
 132     paths:
 133       - "*.stdout"
 134       - "*.stderr"
 135       - "*.info"
 136       - system-info.txt
 137   retry:
 138     max: 2
 139     when:
 140       - runner_system_failure
 141       - stuck_or_timeout_failure
 142       - api_failure
 143       - runner_unsupported
 144       - stale_schedule
 145       - job_execution_timeout
 146       - archived_failure
 147       - scheduler_failure
 148       - data_integrity_failure
 149
 150   script:
 151     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 152     # autobuild name, which means we can define a default template that runs most autobuild jobs
 153     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE  --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 154
 155 # Ensure when adding a new job below that you also add it to
 156 # the dependencies for 'pages' below for the code coverage page
 157 # generation.
 158
 159 others:
 160   extends: .shared_template
 161   script:
 162     - script/autobuild.py ldb      $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/ldb
 163     - script/autobuild.py pidl     $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/pidl
 164     - script/autobuild.py replace  $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/replace
 165     - script/autobuild.py talloc   $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/talloc
 166     - script/autobuild.py tdb      $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/tdb
 167     - script/autobuild.py tevent   $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/tevent
 168     - script/autobuild.py samba-xc $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/samba-xc
 169     - script/autobuild.py docs-xml $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/docs-xml
 170
 171 .shared_template_build_only:
 172   extends: .shared_template
 173   timeout: 45m
 174   artifacts:
 175     expire_in: 1 week
 176     paths:
 177       - "*.stdout"
 178       - "*.stderr"
 179       - "*.info"
 180       - system-info.txt
 181       - samba-testbase.tar.gz
 182   script:
 183     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 184     # autobuild name, which means we can define a default template that runs most autobuild jobs
 185     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE  --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 186     # On success we need to pack everything into an artifacts file
 187     # which needs to be in the git checkout.
 188     # As tar doesn't handle hardlink of read-only files,
 189     # we remember the acls and add write permissions
 190     # before creating the archive. The consumer will apply
 191     # the acls again.
 192     - cp -a /sha1sum.txt /tmp/samba-testbase/image-sha1sum.txt
 193     - cp -a /tmp/commit.txt /tmp/samba-testbase/commit.txt
 194     - pushd /tmp && getfacl -R samba-testbase > samba-testbase.acl.dump && popd
 195     - chmod -R +w /tmp/samba-testbase
 196     - mv /tmp/samba-testbase.acl.dump /tmp/samba-testbase/
 197     - tar cfz samba-testbase.tar.gz /tmp/samba-testbase
 198     - ls -la samba-testbase.tar.gz
 199     - sha1sum samba-testbase.tar.gz
 200
 201 .shared_template_test_only:
 202   extends:
 203     - .shared_template
 204     - .shared_runner_test
 205   stage: test_only
 206   script:
 207     # We unpack the artifacts file created by the .shared_template_build_only
 208     # run we depend on
 209     - ls -la samba-testbase.tar.gz
 210     - sha1sum samba-testbase.tar.gz
 211     - tar xfz samba-testbase.tar.gz -C /
 212     - diff -u /tmp/samba-testbase/image-sha1sum.txt /sha1sum.txt
 213     - diff -u /tmp/samba-testbase/commit.txt /tmp/commit.txt
 214     - mv /tmp/samba-testbase/samba-testbase.acl.dump /tmp/samba-testbase.acl.dump
 215     - pushd /tmp && setfacl --restore=/tmp/samba-testbase.acl.dump && popd
 216     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 217     # autobuild name, which means we can define a default template that runs most autobuild jobs
 218     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --skip-dependencies --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 219
 220 samba-def-build:
 221   extends: .shared_template_build_only
 222   stage: build_first
 223
 224 .needs_samba-def-build:
 225   extends: .shared_template_test_only
 226   needs:
 227     - job: samba-def-build
 228       artifacts: true
 229
 230 samba-mit-build:
 231   extends: .shared_template_build_only
 232   stage: build_first
 233
 234 .needs_samba-mit-build:
 235   extends: .shared_template_test_only
 236   needs:
 237     - job: samba-mit-build
 238       artifacts: true
 239
 240 samba-h5l-build:
 241   extends: .shared_template_build_only
 242
 243 .needs_samba-h5l-build:
 244   extends: .shared_template_test_only
 245   needs:
 246     - job: samba-h5l-build
 247       artifacts: true
 248
 249 samba-nt4-build:
 250   extends: .shared_template_build_only
 251
 252 .needs_samba-nt4-build:
 253   extends: .shared_template_test_only
 254   needs:
 255     - job: samba-nt4-build
 256       artifacts: true
 257
 258 samba-no-opath-build:
 259   extends: .shared_template_build_only
 260
 261 .needs_samba-no-opath-build:
 262   extends: .shared_template_test_only
 263   needs:
 264     - job: samba-no-opath-build
 265       artifacts: true
 266
 267 samba:
 268   extends: .shared_template
 269
 270 samba-mitkrb5:
 271   extends: .shared_template
 272
 273 samba-minimal-smbd:
 274   extends: .shared_template
 275
 276 samba-nopython:
 277   extends: .shared_template
 278
 279 samba-admem:
 280   extends: .needs_samba-def-build
 281
 282 samba-ad-dc-2:
 283   extends: .needs_samba-def-build
 284
 285 samba-ad-dc-3:
 286   extends: .needs_samba-def-build
 287
 288 samba-ad-dc-4:
 289   extends: .needs_samba-def-build
 290
 291 samba-ad-dc-5:
 292   extends: .needs_samba-def-build
 293
 294 samba-ad-dc-6:
 295   extends: .needs_samba-def-build
 296
 297 samba-ad-back1:
 298   extends: .needs_samba-def-build
 299
 300 samba-ad-back2:
 301   extends: .needs_samba-def-build
 302
 303 samba-schemaupgrade:
 304   extends: .needs_samba-def-build
 305
 306 samba-libs:
 307   extends: .shared_template
 308
 309 samba-fuzz:
 310   extends: .shared_template
 311   variables:
 312     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu1604}
 313
 314 ctdb:
 315   extends: .shared_template
 316
 317 samba-ctdb:
 318   extends: .shared_template
 319
 320 samba-ad-dc-ntvfs:
 321   extends: .needs_samba-def-build
 322
 323 samba-admem-mit:
 324   extends: .needs_samba-mit-build
 325
 326 samba-ad-dc-4-mitkrb5:
 327   extends: .needs_samba-mit-build
 328
 329 # This task is run first to ensure we compile before we start the
 330 # main run as it is the fastest full compile of Samba.
 331 samba-fips:
 332   extends: .shared_template
 333   variables:
 334     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora33}
 335
 336 .private_test_only:
 337   extends: .private_runner_test
 338   stage: test_private
 339   only:
 340     variables:
 341       # These jobs are only run if the gitlab repo has private runners available.
 342       # To enable private jobs, you must add the following var and value to
 343       # your gitlab repo by navigating to:
 344       # settings -> CI/CD -> Environment variables
 345       - $SUPPORT_PRIVATE_TEST == "yes"
 346
 347 .needs_samba-def-build-private:
 348   extends:
 349     - .needs_samba-def-build
 350     - .private_test_only
 351
 352 .needs_samba-mit-build-private:
 353   extends:
 354     - .needs_samba-mit-build
 355     - .private_test_only
 356
 357 .needs_samba-h5l-build-private:
 358   extends:
 359     - .needs_samba-h5l-build
 360     - .private_test_only
 361
 362 .needs_samba-nt4-build-private:
 363   extends:
 364     - .needs_samba-nt4-build
 365     - .private_test_only
 366
 367 .needs_samba-no-opath-build-private:
 368   extends:
 369     - .needs_samba-no-opath-build
 370     - .private_test_only
 371
 372 samba-fileserver:
 373   extends: .needs_samba-h5l-build-private
 374
 375 samba-ad-dc-1:
 376   extends: .needs_samba-def-build-private
 377
 378 samba-nt4:
 379   extends: .needs_samba-nt4-build-private
 380
 381 samba-ad-dc-1-mitkrb5:
 382   extends: .needs_samba-mit-build-private
 383
 384 samba-no-opath1:
 385   extends: .needs_samba-no-opath-build-private
 386
 387 samba-no-opath2:
 388   extends: .needs_samba-no-opath-build-private
 389
 390 # 'pages' is a special job which can publish artifacts in `public` dir to gitlab pages
 391 pages:
 392   extends: .shared_runner_build_image
 393   stage: report
 394   dependencies:  # tell gitlab to download artifacts for these jobs
 395     - others
 396     - samba
 397     - samba-mitkrb5
 398     - samba-admem
 399     - samba-ad-dc-2
 400     - samba-ad-dc-3
 401     - samba-ad-dc-4
 402     - samba-ad-dc-5
 403     - samba-ad-dc-6
 404     - samba-libs
 405     - samba-minimal-smbd
 406     - samba-nopython
 407     - samba-fuzz
 408     # - ctdb  # TODO
 409     - samba-ctdb
 410     - samba-ad-dc-ntvfs
 411     - samba-admem-mit
 412     - samba-ad-dc-4-mitkrb5
 413     - samba-ad-back1
 414     - samba-ad-back2
 415     - samba-fileserver
 416     - samba-ad-dc-1
 417     - samba-nt4
 418     - samba-schemaupgrade
 419     - samba-ad-dc-1-mitkrb5
 420     - samba-fips
 421     - samba-no-opath1
 422     - samba-no-opath2
 423   script:
 424     - ./configure.developer
 425     - make -j
 426     - lcov $(ls *.info | xargs -I{} echo -n "-a {} ") -o all.info
 427     - genhtml all.info --output-directory public --prefix=$(pwd) --title "coverage report for $CI_COMMIT_REF_NAME $CI_COMMIT_SHORT_SHA"
 428   artifacts:
 429     expire_in: 30 days
 430     paths:
 431       - public
 432   only:
 433     variables:
 434       - $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == "--enable-coverage"
 435
 436 # Coverity Scan
 437 coverity:
 438   extends: .shared_runner_build_image
 439   variables:
 440     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora33}
 441   stage: build
 442   script:
 443     - wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_SCAN_TOKEN&project=$COVERITY_SCAN_PROJECT_NAME" -O /tmp/coverity_tool.tgz
 444     - tar xf /tmp/coverity_tool.tgz
 445     - ./configure.developer --with-cluster-support --with-system-mitkrb5 --with-experimental-mit-ad-dc
 446     - cov-analysis-linux64-*/bin/cov-build --dir cov-int make -j$(nproc)
 447     - tar czf cov-int.tar.gz cov-int
 448     - curl
 449       --form token=$COVERITY_SCAN_TOKEN
 450       --form email=$COVERITY_SCAN_EMAIL
 451       --form file=@cov-int.tar.gz
 452       --form version="`git describe --tags`"
 453       --form description="CI build"
 454       https://scan.coverity.com/builds?project=$COVERITY_SCAN_PROJECT_NAME
 455   only:
 456     refs:
 457       - master
 458       - schedules
 459     variables:
 460       - $COVERITY_SCAN_TOKEN != null
 461       - $COVERITY_SCAN_PROJECT_NAME != null
 462       - $COVERITY_SCAN_EMAIL != null
 463   artifacts:
 464     expire_in: 1 week
 465     when: on_failure
 466     paths:
 467       - cov-int/*.txt
 468
 469 #
 470 # We build samba-o3 on all supported distributions
 471 #
 472
 473 .samba-o3-template:
 474   extends: .shared_template
 475   variables:
 476     AUTOBUILD_JOB_NAME: samba-o3
 477   only:
 478     variables:
 479       # do not run o3 for coverage since they are using different images
 480       - $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == ""
 481
 482 ubuntu1804-samba-o3:
 483   extends: .samba-o3-template
 484   variables:
 485     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu1804}
 486
 487 ubuntu2004-samba-o3:
 488   extends: .samba-o3-template
 489   variables:
 490     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu2004}
 491
 492 debian10-samba-o3:
 493   extends: .samba-o3-template
 494   variables:
 495     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_debian10}
 496
 497 opensuse151-samba-o3:
 498   extends: .samba-o3-template
 499   variables:
 500     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_opensuse151}
 501
 502 opensuse152-samba-o3:
 503   extends: .samba-o3-template
 504   variables:
 505     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_opensuse152}
 506
 507 centos7-samba-o3:
 508   extends: .samba-o3-template
 509   variables:
 510     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_centos7}
 511     # Git on CentOS doesn't support shallow git cloning
 512     GIT_DEPTH: ""
 513     # We need a newer GnuTLS version on CentOS7
 514     PKG_CONFIG_PATH: "/usr/lib64/compat-gnutls34/pkgconfig:/usr/lib64/compat-nettle32/pkgconfig"
 515
 516 centos8-samba-o3:
 517   extends: .samba-o3-template
 518   variables:
 519     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_centos8}
 520
 521 fedora32-samba-o3:
 522   extends: .samba-o3-template
 523   variables:
 524     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora32}
 525
 526 fedora33-samba-o3:
 527   extends: .samba-o3-template
 528   variables:
 529     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora33}
 530
 531 #
 532 # Keep the samba-o3 sections at the end ...
 533 #