.gitlab-ci.yml

   1 # see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options
   2
   3 # Stages explained
   4 #
   5 # images: Build the images with the bootstrap script
   6 # build_first: Build a few things first to find silly errors (fast job)
   7 #              (don't pay for 35 machines until something compiles)
   8 # build: The main parallel job
   9 #              (keep these to 1hour as we are billed per hour)
  10 # test_only: Tests using the build from prior stages, these typically
  11 #            have an explicit dependency defined to a specific build job,
  12 #            which means that start as soon as the build job finished.
  13 # test_private: Like test_only, but running on private runners
  14 # report: Code coverage reporting
  15
  16 stages:
  17   - images
  18   - build_first
  19   - build
  20   - test_only
  21   - test_private
  22   - report
  23
  24 variables:
  25   # We want to be resilient to runner failures
  26   ARTIFACT_DOWNLOAD_ATTEMPTS: "3"
  27   EXECUTOR_JOB_SECTION_ATTEMPTS: "3"
  28   GET_SOURCES_ATTEMPTS: "3"
  29   RESTORE_CACHE_ATTEMPTS: "3"
  30   #
  31   GIT_STRATEGY: fetch
  32   GIT_DEPTH: "3"
  33   # "--enable-coverage" or ""
  34   SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE: ""
  35   #
  36   # we run autobuild.py inside a samba CI docker image located on gitlab's registry
  37   # overwrite this variable if you want use your own image registry.
  38   #
  39   # Or better ask for access to the shared development repository, see
  40   # https://wiki.samba.org/index.php/Samba_CI_on_gitlab#Getting_Access
  41   #
  42   SAMBA_CI_CONTAINER_REGISTRY: registry.gitlab.com/samba-team/devel/samba
  43   #
  44   # Set this to the contents of bootstrap/sha1sum.txt
  45   # which is generated by bootstrap/template.py --render
  46   #
  47   SAMBA_CI_CONTAINER_TAG: c4c00eb35cae36d8d6e752ee01fd943432993721
  48   #
  49   # We use the ubuntu1804 image as default as
  50   # it matches what we have on sn-devel-184.
  51   #
  52   SAMBA_CI_CONTAINER_IMAGE: ubuntu1804
  53   #
  54   # The following images are available
  55   # Please see the samba-o3 sections at the end of this file!
  56   # We should run that for each available image
  57   #
  58   SAMBA_CI_CONTAINER_IMAGE_ubuntu1604: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-ubuntu1604:${SAMBA_CI_CONTAINER_TAG}
  59   SAMBA_CI_CONTAINER_IMAGE_ubuntu1804: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-ubuntu1804:${SAMBA_CI_CONTAINER_TAG}
  60   SAMBA_CI_CONTAINER_IMAGE_ubuntu2004: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-ubuntu2004:${SAMBA_CI_CONTAINER_TAG}
  61   SAMBA_CI_CONTAINER_IMAGE_debian9: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-debian9:${SAMBA_CI_CONTAINER_TAG}
  62   SAMBA_CI_CONTAINER_IMAGE_debian10: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-debian10:${SAMBA_CI_CONTAINER_TAG}
  63   SAMBA_CI_CONTAINER_IMAGE_opensuse151: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-opensuse151:${SAMBA_CI_CONTAINER_TAG}
  64   SAMBA_CI_CONTAINER_IMAGE_opensuse152: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-opensuse152:${SAMBA_CI_CONTAINER_TAG}
  65   SAMBA_CI_CONTAINER_IMAGE_fedora32: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-fedora32:${SAMBA_CI_CONTAINER_TAG}
  66   SAMBA_CI_CONTAINER_IMAGE_fedora33: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-fedora33:${SAMBA_CI_CONTAINER_TAG}
  67   SAMBA_CI_CONTAINER_IMAGE_centos7: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-centos7:${SAMBA_CI_CONTAINER_TAG}
  68   SAMBA_CI_CONTAINER_IMAGE_centos8: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-centos8:${SAMBA_CI_CONTAINER_TAG}
  69
  70 include:
  71   # The image creation details are specified in a separate file
  72   # See bootstrap/README.md for details
  73   - 'bootstrap/.gitlab-ci.yml'
  74
  75 .shared_template:
  76   # All Samba jobs are interruptible, this avoids burning CPU when a
  77   # newer branch is pushed.
  78   interruptible: true
  79   timeout: 2h
  80
  81   variables:
  82     AUTOBUILD_JOB_NAME: $CI_JOB_NAME
  83   image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-${SAMBA_CI_CONTAINER_IMAGE}:${SAMBA_CI_CONTAINER_TAG}
  84   stage: build
  85   tags:
  86     - docker
  87     - shared
  88   cache:
  89     key: ccache.${CI_JOB_NAME}
  90     paths:
  91       - ccache
  92   before_script:
  93     - uname -a
  94     - lsb_release -a
  95     - cat /etc/os-release
  96     - lscpu
  97     - cat /proc/cpuinfo
  98     - mount
  99     - df -h
 100     - cat /proc/swaps
 101     - free -h
 102       # ld will fail if coverage enabled, force link ld to ld.bfd
 103     - if [ -n "$SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE" ]; then sudo ln -sf $(which ld.bfd) $(which ld); fi
 104       # See bootstrap/.gitlab-ci.yml how to generate a new image
 105     - echo "SAMBA_CI_CONTAINER_REGISTRY[${SAMBA_CI_CONTAINER_REGISTRY}]"
 106     - echo "SAMBA_CI_CONTAINER_TAG[${SAMBA_CI_CONTAINER_TAG}]"
 107     - bootstrap/template.py --sha1sum > /tmp/sha1sum-template.txt
 108     - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-template.txt
 109     - echo "${SAMBA_CI_CONTAINER_TAG}" > /tmp/sha1sum-tag.txt
 110     - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-tag.txt
 111     - diff -u bootstrap/sha1sum.txt /sha1sum.txt
 112     - echo "${CI_COMMIT_SHA} ${CI_COMMIT_TITLE}" > /tmp/commit.txt
 113     - export CCACHE_BASEDIR="${PWD}"
 114     - export CCACHE_DIR="${PWD}/ccache" && mkdir -pv "$CCACHE_DIR"
 115     - export CC="ccache cc"
 116     - export CXX="ccache c++"
 117     - ccache -z -M 500M
 118     - ccache -s
 119   after_script:
 120     - mount
 121     - df -h
 122     - cat /proc/swaps
 123     - free -h
 124     - CCACHE_BASEDIR="${PWD}" CCACHE_DIR="${PWD}/ccache" ccache -s -c
 125   artifacts:
 126     expire_in: 1 week
 127     paths:
 128       - "*.stdout"
 129       - "*.stderr"
 130       - "*.info"
 131       - system-info.txt
 132   retry:
 133     max: 2
 134     when:
 135       - runner_system_failure
 136       - stuck_or_timeout_failure
 137       - api_failure
 138       - runner_unsupported
 139       - stale_schedule
 140       - job_execution_timeout
 141       - archived_failure
 142       - scheduler_failure
 143       - data_integrity_failure
 144
 145   script:
 146     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 147     # autobuild name, which means we can define a default template that runs most autobuild jobs
 148     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE  --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 149
 150 # Ensure when adding a new job below that you also add it to
 151 # the dependencies for 'pages' below for the code coverage page
 152 # generation.
 153
 154 others:
 155   extends: .shared_template
 156   script:
 157     - script/autobuild.py ldb      $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/ldb
 158     - script/autobuild.py pidl     $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/pidl
 159     - script/autobuild.py replace  $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/replace
 160     - script/autobuild.py talloc   $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/talloc
 161     - script/autobuild.py tdb      $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/tdb
 162     - script/autobuild.py tevent   $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/tevent
 163     - script/autobuild.py samba-xc $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/samba-xc
 164     - script/autobuild.py docs-xml $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/docs-xml
 165
 166 .shared_template_build_only:
 167   extends: .shared_template
 168   timeout: 45m
 169   artifacts:
 170     expire_in: 1 week
 171     paths:
 172       - "*.stdout"
 173       - "*.stderr"
 174       - "*.info"
 175       - system-info.txt
 176       - samba-testbase.tar.gz
 177   script:
 178     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 179     # autobuild name, which means we can define a default template that runs most autobuild jobs
 180     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE  --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 181     # On success we need to pack everything into an artifacts file
 182     # which needs to be in the git checkout.
 183     # As tar doesn't handle hardlink of read-only files,
 184     # we remember the acls and add write permissions
 185     # before creating the archive. The consumer will apply
 186     # the acls again.
 187     - cp -a /sha1sum.txt /tmp/samba-testbase/image-sha1sum.txt
 188     - cp -a /tmp/commit.txt /tmp/samba-testbase/commit.txt
 189     - pushd /tmp && getfacl -R samba-testbase > samba-testbase.acl.dump && popd
 190     - chmod -R +w /tmp/samba-testbase
 191     - mv /tmp/samba-testbase.acl.dump /tmp/samba-testbase/
 192     - tar cfz samba-testbase.tar.gz /tmp/samba-testbase
 193     - ls -la samba-testbase.tar.gz
 194     - sha1sum samba-testbase.tar.gz
 195
 196 .shared_template_test_only:
 197   extends: .shared_template
 198   stage: test_only
 199   script:
 200     # We unpack the artifacts file created by the .shared_template_build_only
 201     # run we depend on
 202     - ls -la samba-testbase.tar.gz
 203     - sha1sum samba-testbase.tar.gz
 204     - tar xfz samba-testbase.tar.gz -C /
 205     - diff -u /tmp/samba-testbase/image-sha1sum.txt /sha1sum.txt
 206     - diff -u /tmp/samba-testbase/commit.txt /tmp/commit.txt
 207     - mv /tmp/samba-testbase/samba-testbase.acl.dump /tmp/samba-testbase.acl.dump
 208     - pushd /tmp && setfacl --restore=/tmp/samba-testbase.acl.dump && popd
 209     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 210     # autobuild name, which means we can define a default template that runs most autobuild jobs
 211     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --skip-dependencies --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 212
 213 samba-def-build:
 214   extends: .shared_template_build_only
 215   stage: build_first
 216
 217 .needs_samba-def-build:
 218   extends: .shared_template_test_only
 219   needs:
 220     - job: samba-def-build
 221       artifacts: true
 222
 223 samba-mit-build:
 224   extends: .shared_template_build_only
 225   stage: build_first
 226
 227 .needs_samba-mit-build:
 228   extends: .shared_template_test_only
 229   needs:
 230     - job: samba-mit-build
 231       artifacts: true
 232
 233 samba-h5l-build:
 234   extends: .shared_template_build_only
 235
 236 .needs_samba-h5l-build:
 237   extends: .shared_template_test_only
 238   needs:
 239     - job: samba-h5l-build
 240       artifacts: true
 241
 242 samba-nt4-build:
 243   extends: .shared_template_build_only
 244
 245 .needs_samba-nt4-build:
 246   extends: .shared_template_test_only
 247   needs:
 248     - job: samba-nt4-build
 249       artifacts: true
 250
 251 samba-no-opath-build:
 252   extends: .shared_template_build_only
 253
 254 .needs_samba-no-opath-build:
 255   extends: .shared_template_test_only
 256   needs:
 257     - job: samba-no-opath-build
 258       artifacts: true
 259
 260 samba:
 261   extends: .shared_template
 262
 263 samba-mitkrb5:
 264   extends: .shared_template
 265
 266 samba-minimal-smbd:
 267   extends: .shared_template
 268
 269 samba-admem:
 270   extends: .needs_samba-def-build
 271
 272 samba-ad-dc-2:
 273   extends: .needs_samba-def-build
 274
 275 samba-ad-dc-3:
 276   extends: .needs_samba-def-build
 277
 278 samba-ad-dc-4:
 279   extends: .needs_samba-def-build
 280
 281 samba-ad-dc-5:
 282   extends: .needs_samba-def-build
 283
 284 samba-ad-dc-6:
 285   extends: .needs_samba-def-build
 286
 287 samba-libs:
 288   extends: .shared_template
 289
 290 samba-fuzz:
 291   extends: .shared_template
 292   image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-ubuntu1604:${SAMBA_CI_CONTAINER_TAG}
 293
 294 ctdb:
 295   extends: .shared_template
 296
 297 samba-ctdb:
 298   extends: .shared_template
 299
 300 samba-ad-dc-ntvfs:
 301   extends: .needs_samba-def-build
 302
 303 samba-admem-mit:
 304   extends: .needs_samba-mit-build
 305
 306 samba-ad-dc-4-mitkrb5:
 307   extends: .needs_samba-mit-build
 308
 309 # This task is run first to ensure we compile before we start the
 310 # main run as it is the fastest full compile of Samba.
 311 samba-fips:
 312   extends: .shared_template
 313   image: $SAMBA_CI_CONTAINER_IMAGE_fedora33
 314
 315 .private_runner:
 316   stage: test_private
 317   tags:
 318     - docker
 319     - samba-ci-private
 320   only:
 321     variables:
 322       # These jobs are only run if the gitlab repo has private runners available.
 323       # To enable private jobs, you must add the following var and value to
 324       # your gitlab repo by navigating to:
 325       # settings -> CI/CD -> Environment variables
 326       - $SUPPORT_PRIVATE_TEST == "yes"
 327
 328 .needs_samba-def-build-private:
 329   extends:
 330     - .needs_samba-def-build
 331     - .private_runner
 332
 333 .needs_samba-mit-build-private:
 334   extends:
 335     - .needs_samba-mit-build
 336     - .private_runner
 337
 338 .needs_samba-h5l-build-private:
 339   extends:
 340     - .needs_samba-h5l-build
 341     - .private_runner
 342
 343 .needs_samba-nt4-build-private:
 344   extends:
 345     - .needs_samba-nt4-build
 346     - .private_runner
 347
 348 .needs_samba-no-opath-build-private:
 349   extends:
 350     - .needs_samba-no-opath-build
 351     - .private_runner
 352
 353 samba-ad-back1:
 354   extends: .needs_samba-def-build-private
 355
 356 samba-ad-back2:
 357   extends: .needs_samba-def-build-private
 358
 359 samba-fileserver:
 360   extends: .needs_samba-h5l-build-private
 361
 362 samba-ad-dc-1:
 363   extends: .needs_samba-def-build-private
 364
 365 samba-nt4:
 366   extends: .needs_samba-nt4-build-private
 367
 368 samba-schemaupgrade:
 369   extends: .needs_samba-def-build-private
 370
 371 samba-ad-dc-1-mitkrb5:
 372   extends: .needs_samba-mit-build-private
 373
 374 samba-no-opath:
 375   extends: .needs_samba-no-opath-build-private
 376
 377 # 'pages' is a special job which can publish artifacts in `public` dir to gitlab pages
 378 pages:
 379   image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-${SAMBA_CI_CONTAINER_IMAGE}:${SAMBA_CI_CONTAINER_TAG}
 380   stage: report
 381   tags:
 382     - docker
 383     - shared
 384   dependencies:  # tell gitlab to download artifacts for these jobs
 385     - others
 386     - samba
 387     - samba-mitkrb5
 388     - samba-admem
 389     - samba-ad-dc-2
 390     - samba-ad-dc-3
 391     - samba-ad-dc-4
 392     - samba-ad-dc-5
 393     - samba-ad-dc-6
 394     - samba-libs
 395     - samba-minimal-smbd
 396     - samba-fuzz
 397     # - ctdb  # TODO
 398     - samba-ctdb
 399     - samba-ad-dc-ntvfs
 400     - samba-admem-mit
 401     - samba-ad-dc-4-mitkrb5
 402     - samba-ad-back1
 403     - samba-ad-back2
 404     - samba-fileserver
 405     - samba-ad-dc-1
 406     - samba-nt4
 407     - samba-schemaupgrade
 408     - samba-ad-dc-1-mitkrb5
 409     - samba-fips
 410   script:
 411     - ./configure.developer
 412     - make -j
 413     - lcov $(ls *.info | xargs -I{} echo -n "-a {} ") -o all.info
 414     - genhtml all.info --output-directory public --prefix=$(pwd) --title "coverage report for $CI_COMMIT_REF_NAME $CI_COMMIT_SHORT_SHA"
 415   artifacts:
 416     expire_in: 30 days
 417     paths:
 418       - public
 419   only:
 420     variables:
 421       - $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == "--enable-coverage"
 422
 423 # Coverity Scan
 424 coverity:
 425   stage: build
 426   image: $SAMBA_CI_CONTAINER_IMAGE_fedora33
 427   tags:
 428     - docker
 429     - shared
 430   script:
 431     - wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_SCAN_TOKEN&project=$COVERITY_SCAN_PROJECT_NAME" -O /tmp/coverity_tool.tgz
 432     - tar xf /tmp/coverity_tool.tgz
 433     - ./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc
 434     - cov-analysis-linux64-*/bin/cov-build --dir cov-int make -j$(nproc)
 435     - tar czf cov-int.tar.gz cov-int
 436     - curl
 437       --form token=$COVERITY_SCAN_TOKEN
 438       --form email=$COVERITY_SCAN_EMAIL
 439       --form file=@cov-int.tar.gz
 440       --form version="`git describe --tags`"
 441       --form description="CI build"
 442       https://scan.coverity.com/builds?project=$COVERITY_SCAN_PROJECT_NAME
 443   only:
 444     refs:
 445       - master
 446       - schedules
 447     variables:
 448       - $COVERITY_SCAN_TOKEN != null
 449       - $COVERITY_SCAN_PROJECT_NAME != null
 450       - $COVERITY_SCAN_EMAIL != null
 451   artifacts:
 452     expire_in: 1 week
 453     when: on_failure
 454     paths:
 455       - cov-int/*.txt
 456
 457 #
 458 # We build samba-o3 on all supported distributions
 459 #
 460
 461 .samba-o3-template:
 462   extends: .shared_template
 463   variables:
 464     AUTOBUILD_JOB_NAME: samba-o3
 465   only:
 466     variables:
 467       # do not run o3 for coverage since they are using different images
 468       - $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == ""
 469
 470 ubuntu1804-samba-o3:
 471   extends: .samba-o3-template
 472   image: $SAMBA_CI_CONTAINER_IMAGE_ubuntu1804
 473
 474 ubuntu2004-samba-o3:
 475   extends: .samba-o3-template
 476   image: $SAMBA_CI_CONTAINER_IMAGE_ubuntu2004
 477
 478 debian10-samba-o3:
 479   extends: .samba-o3-template
 480   image: $SAMBA_CI_CONTAINER_IMAGE_debian10
 481
 482 opensuse151-samba-o3:
 483   extends: .samba-o3-template
 484   image: $SAMBA_CI_CONTAINER_IMAGE_opensuse151
 485
 486 opensuse152-samba-o3:
 487   extends: .samba-o3-template
 488   image: $SAMBA_CI_CONTAINER_IMAGE_opensuse152
 489
 490 centos7-samba-o3:
 491   extends: .samba-o3-template
 492   image: $SAMBA_CI_CONTAINER_IMAGE_centos7
 493   variables:
 494     # Git on CentOS doesn't support shallow git cloning
 495     GIT_DEPTH: ""
 496     # We need a newer GnuTLS version on CentOS7
 497     PKG_CONFIG_PATH: "/usr/lib64/compat-gnutls34/pkgconfig:/usr/lib64/compat-nettle32/pkgconfig"
 498
 499 centos8-samba-o3:
 500   extends: .samba-o3-template
 501   image: $SAMBA_CI_CONTAINER_IMAGE_centos8
 502
 503 fedora32-samba-o3:
 504   extends: .samba-o3-template
 505   image: $SAMBA_CI_CONTAINER_IMAGE_fedora32
 506
 507 fedora33-samba-o3:
 508   extends: .samba-o3-template
 509   image: $SAMBA_CI_CONTAINER_IMAGE_fedora33
 510
 511 #
 512 # Keep the samba-o3 sections at the end ...
 513 #