.gitlab-ci-main.yml

   1 # see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options
   2
   3 # Stages explained
   4 #
   5 # images: Build the images with the bootstrap script
   6 # build_first: Build a few things first to find silly errors (fast job)
   7 #              (don't pay for 35 machines until something compiles)
   8 # build: The main parallel job
   9 #              (keep these to 1hour as we are billed per hour)
  10 # test_only: Tests using the build from prior stages, these typically
  11 #            have an explicit dependency defined to a specific build job,
  12 #            which means that start as soon as the build job finished.
  13 # test_private: Like test_only, but running on private runners
  14 # report: Code coverage reporting
  15
  16 stages:
  17   - images
  18   - build_first
  19   - build
  20   - test_only
  21   - test_private
  22   - report
  23
  24 variables:
  25   # We want to be resilient to runner failures
  26   ARTIFACT_DOWNLOAD_ATTEMPTS: "3"
  27   EXECUTOR_JOB_SECTION_ATTEMPTS: "3"
  28   GET_SOURCES_ATTEMPTS: "3"
  29   RESTORE_CACHE_ATTEMPTS: "3"
  30   #
  31   GIT_STRATEGY: fetch
  32   GIT_DEPTH: "3"
  33   # "--enable-coverage" or ""
  34   SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE: ""
  35   #
  36   # we run autobuild.py inside a samba CI docker image located on gitlab's registry
  37   # overwrite this variable if you want use your own image registry.
  38   #
  39   # Or better ask for access to the shared development repository, see
  40   # https://wiki.samba.org/index.php/Samba_CI_on_gitlab#Getting_Access
  41   #
  42   SAMBA_CI_CONTAINER_REGISTRY: registry.gitlab.com/samba-team/devel/samba
  43   #
  44   # Set this to the contents of bootstrap/sha1sum.txt
  45   # which is generated by bootstrap/template.py --render
  46   #
  47   SAMBA_CI_CONTAINER_TAG: f25e1fa0a828f428ade0e4f37d8a646ff1e207d4
  48   #
  49   # We use the ubuntu1804 image as default as
  50   # it matches what we have on sn-devel-184.
  51   #
  52   SAMBA_CI_CONTAINER_IMAGE: ubuntu1804
  53   #
  54   # The following images are available
  55   # Please see the samba-o3 sections at the end of this file!
  56   # We should run that for each available image
  57   #
  58   SAMBA_CI_CONTAINER_IMAGE_ubuntu1604: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-ubuntu1604:${SAMBA_CI_CONTAINER_TAG}
  59   SAMBA_CI_CONTAINER_IMAGE_ubuntu1804: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-ubuntu1804:${SAMBA_CI_CONTAINER_TAG}
  60   SAMBA_CI_CONTAINER_IMAGE_ubuntu2004: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-ubuntu2004:${SAMBA_CI_CONTAINER_TAG}
  61   SAMBA_CI_CONTAINER_IMAGE_debian9: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-debian9:${SAMBA_CI_CONTAINER_TAG}
  62   SAMBA_CI_CONTAINER_IMAGE_debian10: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-debian10:${SAMBA_CI_CONTAINER_TAG}
  63   SAMBA_CI_CONTAINER_IMAGE_opensuse151: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-opensuse151:${SAMBA_CI_CONTAINER_TAG}
  64   SAMBA_CI_CONTAINER_IMAGE_opensuse152: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-opensuse152:${SAMBA_CI_CONTAINER_TAG}
  65   SAMBA_CI_CONTAINER_IMAGE_fedora32: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-fedora32:${SAMBA_CI_CONTAINER_TAG}
  66   SAMBA_CI_CONTAINER_IMAGE_fedora33: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-fedora33:${SAMBA_CI_CONTAINER_TAG}
  67   SAMBA_CI_CONTAINER_IMAGE_centos7: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-centos7:${SAMBA_CI_CONTAINER_TAG}
  68   SAMBA_CI_CONTAINER_IMAGE_centos8: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-centos8:${SAMBA_CI_CONTAINER_TAG}
  69
  70 include:
  71   # The image creation details are specified in a separate file
  72   # See bootstrap/README.md for details
  73   - 'bootstrap/.gitlab-ci.yml'
  74
  75 .shared_template:
  76   extends: .shared_runner_build
  77   # All Samba jobs are interruptible, this avoids burning CPU when a
  78   # newer branch is pushed.
  79   interruptible: true
  80   timeout: 2h
  81
  82   variables:
  83     AUTOBUILD_JOB_NAME: $CI_JOB_NAME
  84   image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-${SAMBA_CI_CONTAINER_IMAGE}:${SAMBA_CI_CONTAINER_TAG}
  85   stage: build
  86   cache:
  87     key: ccache.${CI_JOB_NAME}
  88     paths:
  89       - ccache
  90   before_script:
  91     - uname -a
  92     - lsb_release -a
  93     - cat /etc/os-release
  94     - lscpu
  95     - cat /proc/cpuinfo
  96     - mount
  97     - df -h
  98     - cat /proc/swaps
  99     - free -h
 100       # ld will fail if coverage enabled, force link ld to ld.bfd
 101     - if [ -n "$SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE" ]; then sudo ln -sf $(which ld.bfd) $(which ld); fi
 102       # See bootstrap/.gitlab-ci.yml how to generate a new image
 103     - echo "SAMBA_CI_CONTAINER_REGISTRY[${SAMBA_CI_CONTAINER_REGISTRY}]"
 104     - echo "SAMBA_CI_CONTAINER_TAG[${SAMBA_CI_CONTAINER_TAG}]"
 105     - bootstrap/template.py --sha1sum > /tmp/sha1sum-template.txt
 106     - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-template.txt
 107     - echo "${SAMBA_CI_CONTAINER_TAG}" > /tmp/sha1sum-tag.txt
 108     - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-tag.txt
 109     - diff -u bootstrap/sha1sum.txt /sha1sum.txt
 110     - echo "${CI_COMMIT_SHA} ${CI_COMMIT_TITLE}" > /tmp/commit.txt
 111     - export CCACHE_BASEDIR="${PWD}"
 112     - export CCACHE_DIR="${PWD}/ccache" && mkdir -pv "$CCACHE_DIR"
 113     - export CC="ccache cc"
 114     - export CXX="ccache c++"
 115     - ccache -z -M 500M
 116     - ccache -s
 117   after_script:
 118     - mount
 119     - df -h
 120     - cat /proc/swaps
 121     - free -h
 122     - CCACHE_BASEDIR="${PWD}" CCACHE_DIR="${PWD}/ccache" ccache -s -c
 123   artifacts:
 124     expire_in: 1 week
 125     paths:
 126       - "*.stdout"
 127       - "*.stderr"
 128       - "*.info"
 129       - system-info.txt
 130   retry:
 131     max: 2
 132     when:
 133       - runner_system_failure
 134       - stuck_or_timeout_failure
 135       - api_failure
 136       - runner_unsupported
 137       - stale_schedule
 138       - job_execution_timeout
 139       - archived_failure
 140       - scheduler_failure
 141       - data_integrity_failure
 142
 143   script:
 144     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 145     # autobuild name, which means we can define a default template that runs most autobuild jobs
 146     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE  --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 147
 148 # Ensure when adding a new job below that you also add it to
 149 # the dependencies for 'pages' below for the code coverage page
 150 # generation.
 151
 152 others:
 153   extends: .shared_template
 154   script:
 155     - script/autobuild.py ldb      $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/ldb
 156     - script/autobuild.py pidl     $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/pidl
 157     - script/autobuild.py replace  $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/replace
 158     - script/autobuild.py talloc   $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/talloc
 159     - script/autobuild.py tdb      $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/tdb
 160     - script/autobuild.py tevent   $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/tevent
 161     - script/autobuild.py samba-xc $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/samba-xc
 162     - script/autobuild.py docs-xml $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/docs-xml
 163
 164 .shared_template_build_only:
 165   extends: .shared_template
 166   timeout: 45m
 167   artifacts:
 168     expire_in: 1 week
 169     paths:
 170       - "*.stdout"
 171       - "*.stderr"
 172       - "*.info"
 173       - system-info.txt
 174       - samba-testbase.tar.gz
 175   script:
 176     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 177     # autobuild name, which means we can define a default template that runs most autobuild jobs
 178     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE  --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 179     # On success we need to pack everything into an artifacts file
 180     # which needs to be in the git checkout.
 181     # As tar doesn't handle hardlink of read-only files,
 182     # we remember the acls and add write permissions
 183     # before creating the archive. The consumer will apply
 184     # the acls again.
 185     - cp -a /sha1sum.txt /tmp/samba-testbase/image-sha1sum.txt
 186     - cp -a /tmp/commit.txt /tmp/samba-testbase/commit.txt
 187     - pushd /tmp && getfacl -R samba-testbase > samba-testbase.acl.dump && popd
 188     - chmod -R +w /tmp/samba-testbase
 189     - mv /tmp/samba-testbase.acl.dump /tmp/samba-testbase/
 190     - tar cfz samba-testbase.tar.gz /tmp/samba-testbase
 191     - ls -la samba-testbase.tar.gz
 192     - sha1sum samba-testbase.tar.gz
 193
 194 .shared_template_test_only:
 195   extends:
 196     - .shared_template
 197     - .shared_runner_test
 198   stage: test_only
 199   script:
 200     # We unpack the artifacts file created by the .shared_template_build_only
 201     # run we depend on
 202     - ls -la samba-testbase.tar.gz
 203     - sha1sum samba-testbase.tar.gz
 204     - tar xfz samba-testbase.tar.gz -C /
 205     - diff -u /tmp/samba-testbase/image-sha1sum.txt /sha1sum.txt
 206     - diff -u /tmp/samba-testbase/commit.txt /tmp/commit.txt
 207     - mv /tmp/samba-testbase/samba-testbase.acl.dump /tmp/samba-testbase.acl.dump
 208     - pushd /tmp && setfacl --restore=/tmp/samba-testbase.acl.dump && popd
 209     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 210     # autobuild name, which means we can define a default template that runs most autobuild jobs
 211     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --skip-dependencies --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 212
 213 samba-def-build:
 214   extends: .shared_template_build_only
 215   stage: build_first
 216
 217 .needs_samba-def-build:
 218   extends: .shared_template_test_only
 219   needs:
 220     - job: samba-def-build
 221       artifacts: true
 222
 223 samba-mit-build:
 224   extends: .shared_template_build_only
 225   stage: build_first
 226
 227 .needs_samba-mit-build:
 228   extends: .shared_template_test_only
 229   needs:
 230     - job: samba-mit-build
 231       artifacts: true
 232
 233 samba-h5l-build:
 234   extends: .shared_template_build_only
 235
 236 .needs_samba-h5l-build:
 237   extends: .shared_template_test_only
 238   needs:
 239     - job: samba-h5l-build
 240       artifacts: true
 241
 242 samba-nt4-build:
 243   extends: .shared_template_build_only
 244
 245 .needs_samba-nt4-build:
 246   extends: .shared_template_test_only
 247   needs:
 248     - job: samba-nt4-build
 249       artifacts: true
 250
 251 samba-no-opath-build:
 252   extends: .shared_template_build_only
 253
 254 .needs_samba-no-opath-build:
 255   extends: .shared_template_test_only
 256   needs:
 257     - job: samba-no-opath-build
 258       artifacts: true
 259
 260 samba:
 261   extends: .shared_template
 262
 263 samba-mitkrb5:
 264   extends: .shared_template
 265
 266 samba-minimal-smbd:
 267   extends: .shared_template
 268
 269 samba-admem:
 270   extends: .needs_samba-def-build
 271
 272 samba-ad-dc-2:
 273   extends: .needs_samba-def-build
 274
 275 samba-ad-dc-3:
 276   extends: .needs_samba-def-build
 277
 278 samba-ad-dc-4:
 279   extends: .needs_samba-def-build
 280
 281 samba-ad-dc-5:
 282   extends: .needs_samba-def-build
 283
 284 samba-ad-dc-6:
 285   extends: .needs_samba-def-build
 286
 287 samba-ad-back1:
 288   extends: .needs_samba-def-build
 289
 290 samba-ad-back2:
 291   extends: .needs_samba-def-build
 292
 293 samba-schemaupgrade:
 294   extends: .needs_samba-def-build
 295
 296 samba-libs:
 297   extends: .shared_template
 298
 299 samba-fuzz:
 300   extends: .shared_template
 301   image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-ubuntu1604:${SAMBA_CI_CONTAINER_TAG}
 302
 303 ctdb:
 304   extends: .shared_template
 305
 306 samba-ctdb:
 307   extends: .shared_template
 308
 309 samba-ad-dc-ntvfs:
 310   extends: .needs_samba-def-build
 311
 312 samba-admem-mit:
 313   extends: .needs_samba-mit-build
 314
 315 samba-ad-dc-4-mitkrb5:
 316   extends: .needs_samba-mit-build
 317
 318 # This task is run first to ensure we compile before we start the
 319 # main run as it is the fastest full compile of Samba.
 320 samba-fips:
 321   extends: .shared_template
 322   image: $SAMBA_CI_CONTAINER_IMAGE_fedora33
 323
 324 .private_test_only:
 325   extends: .private_runner_test
 326   stage: test_private
 327   only:
 328     variables:
 329       # These jobs are only run if the gitlab repo has private runners available.
 330       # To enable private jobs, you must add the following var and value to
 331       # your gitlab repo by navigating to:
 332       # settings -> CI/CD -> Environment variables
 333       - $SUPPORT_PRIVATE_TEST == "yes"
 334
 335 .needs_samba-def-build-private:
 336   extends:
 337     - .needs_samba-def-build
 338     - .private_test_only
 339
 340 .needs_samba-mit-build-private:
 341   extends:
 342     - .needs_samba-mit-build
 343     - .private_test_only
 344
 345 .needs_samba-h5l-build-private:
 346   extends:
 347     - .needs_samba-h5l-build
 348     - .private_test_only
 349
 350 .needs_samba-nt4-build-private:
 351   extends:
 352     - .needs_samba-nt4-build
 353     - .private_test_only
 354
 355 .needs_samba-no-opath-build-private:
 356   extends:
 357     - .needs_samba-no-opath-build
 358     - .private_test_only
 359
 360 samba-fileserver:
 361   extends: .needs_samba-h5l-build-private
 362
 363 samba-ad-dc-1:
 364   extends: .needs_samba-def-build-private
 365
 366 samba-nt4:
 367   extends: .needs_samba-nt4-build-private
 368
 369 samba-ad-dc-1-mitkrb5:
 370   extends: .needs_samba-mit-build-private
 371
 372 samba-no-opath:
 373   extends: .needs_samba-no-opath-build-private
 374
 375 # 'pages' is a special job which can publish artifacts in `public` dir to gitlab pages
 376 pages:
 377   extends: .shared_runner_build
 378   image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-${SAMBA_CI_CONTAINER_IMAGE}:${SAMBA_CI_CONTAINER_TAG}
 379   stage: report
 380   dependencies:  # tell gitlab to download artifacts for these jobs
 381     - others
 382     - samba
 383     - samba-mitkrb5
 384     - samba-admem
 385     - samba-ad-dc-2
 386     - samba-ad-dc-3
 387     - samba-ad-dc-4
 388     - samba-ad-dc-5
 389     - samba-ad-dc-6
 390     - samba-libs
 391     - samba-minimal-smbd
 392     - samba-fuzz
 393     # - ctdb  # TODO
 394     - samba-ctdb
 395     - samba-ad-dc-ntvfs
 396     - samba-admem-mit
 397     - samba-ad-dc-4-mitkrb5
 398     - samba-ad-back1
 399     - samba-ad-back2
 400     - samba-fileserver
 401     - samba-ad-dc-1
 402     - samba-nt4
 403     - samba-schemaupgrade
 404     - samba-ad-dc-1-mitkrb5
 405     - samba-fips
 406   script:
 407     - ./configure.developer
 408     - make -j
 409     - lcov $(ls *.info | xargs -I{} echo -n "-a {} ") -o all.info
 410     - genhtml all.info --output-directory public --prefix=$(pwd) --title "coverage report for $CI_COMMIT_REF_NAME $CI_COMMIT_SHORT_SHA"
 411   artifacts:
 412     expire_in: 30 days
 413     paths:
 414       - public
 415   only:
 416     variables:
 417       - $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == "--enable-coverage"
 418
 419 # Coverity Scan
 420 coverity:
 421   extends: .shared_runner_build
 422   stage: build
 423   image: $SAMBA_CI_CONTAINER_IMAGE_fedora33
 424   script:
 425     - wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_SCAN_TOKEN&project=$COVERITY_SCAN_PROJECT_NAME" -O /tmp/coverity_tool.tgz
 426     - tar xf /tmp/coverity_tool.tgz
 427     - ./configure.developer --with-cluster-support --with-system-mitkrb5 --with-experimental-mit-ad-dc
 428     - cov-analysis-linux64-*/bin/cov-build --dir cov-int make -j$(nproc)
 429     - tar czf cov-int.tar.gz cov-int
 430     - curl
 431       --form token=$COVERITY_SCAN_TOKEN
 432       --form email=$COVERITY_SCAN_EMAIL
 433       --form file=@cov-int.tar.gz
 434       --form version="`git describe --tags`"
 435       --form description="CI build"
 436       https://scan.coverity.com/builds?project=$COVERITY_SCAN_PROJECT_NAME
 437   only:
 438     refs:
 439       - master
 440       - schedules
 441     variables:
 442       - $COVERITY_SCAN_TOKEN != null
 443       - $COVERITY_SCAN_PROJECT_NAME != null
 444       - $COVERITY_SCAN_EMAIL != null
 445   artifacts:
 446     expire_in: 1 week
 447     when: on_failure
 448     paths:
 449       - cov-int/*.txt
 450
 451 #
 452 # We build samba-o3 on all supported distributions
 453 #
 454
 455 .samba-o3-template:
 456   extends: .shared_template
 457   variables:
 458     AUTOBUILD_JOB_NAME: samba-o3
 459   only:
 460     variables:
 461       # do not run o3 for coverage since they are using different images
 462       - $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == ""
 463
 464 ubuntu1804-samba-o3:
 465   extends: .samba-o3-template
 466   image: $SAMBA_CI_CONTAINER_IMAGE_ubuntu1804
 467
 468 ubuntu2004-samba-o3:
 469   extends: .samba-o3-template
 470   image: $SAMBA_CI_CONTAINER_IMAGE_ubuntu2004
 471
 472 debian10-samba-o3:
 473   extends: .samba-o3-template
 474   image: $SAMBA_CI_CONTAINER_IMAGE_debian10
 475
 476 opensuse151-samba-o3:
 477   extends: .samba-o3-template
 478   image: $SAMBA_CI_CONTAINER_IMAGE_opensuse151
 479
 480 opensuse152-samba-o3:
 481   extends: .samba-o3-template
 482   image: $SAMBA_CI_CONTAINER_IMAGE_opensuse152
 483
 484 centos7-samba-o3:
 485   extends: .samba-o3-template
 486   image: $SAMBA_CI_CONTAINER_IMAGE_centos7
 487   variables:
 488     # Git on CentOS doesn't support shallow git cloning
 489     GIT_DEPTH: ""
 490     # We need a newer GnuTLS version on CentOS7
 491     PKG_CONFIG_PATH: "/usr/lib64/compat-gnutls34/pkgconfig:/usr/lib64/compat-nettle32/pkgconfig"
 492
 493 centos8-samba-o3:
 494   extends: .samba-o3-template
 495   image: $SAMBA_CI_CONTAINER_IMAGE_centos8
 496
 497 fedora32-samba-o3:
 498   extends: .samba-o3-template
 499   image: $SAMBA_CI_CONTAINER_IMAGE_fedora32
 500
 501 fedora33-samba-o3:
 502   extends: .samba-o3-template
 503   image: $SAMBA_CI_CONTAINER_IMAGE_fedora33
 504
 505 #
 506 # Keep the samba-o3 sections at the end ...
 507 #