.gitlab-ci-main.yml

   1 # see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options
   2
   3 # Stages explained
   4 #
   5 # images: Build the images with the bootstrap script
   6 # build_first: Build a few things first to find silly errors (fast job)
   7 #              (don't pay for 35 machines until something compiles)
   8 # build: The main parallel job
   9 #              (keep these to 1hour as we are billed per hour)
  10 # test_only: Tests using the build from prior stages, these typically
  11 #            have an explicit dependency defined to a specific build job,
  12 #            which means that start as soon as the build job finished.
  13 # test_private: Like test_only, but running on private runners
  14 # report: Code coverage reporting
  15
  16 stages:
  17   - images
  18   - build_first
  19   - build
  20   - test_only
  21   - test_private
  22   - report
  23
  24 variables:
  25   # We want to be resilient to runner failures
  26   ARTIFACT_DOWNLOAD_ATTEMPTS: "3"
  27   EXECUTOR_JOB_SECTION_ATTEMPTS: "3"
  28   GET_SOURCES_ATTEMPTS: "3"
  29   RESTORE_CACHE_ATTEMPTS: "3"
  30   #
  31   GIT_STRATEGY: fetch
  32   GIT_DEPTH: "3"
  33   # "--enable-coverage" or ""
  34   SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE: ""
  35   #
  36   # we run autobuild.py inside a samba CI docker image located on gitlab's registry
  37   # overwrite this variable if you want use your own image registry.
  38   #
  39   # Or better ask for access to the shared development repository, see
  40   # https://wiki.samba.org/index.php/Samba_CI_on_gitlab#Getting_Access
  41   #
  42   SAMBA_CI_CONTAINER_REGISTRY: registry.gitlab.com/samba-team/devel/samba
  43   #
  44   # Set this to the contents of bootstrap/sha1sum.txt
  45   # which is generated by bootstrap/template.py --render
  46   #
  47   SAMBA_CI_CONTAINER_TAG: f25e1fa0a828f428ade0e4f37d8a646ff1e207d4
  48   #
  49   # We use the ubuntu1804 image as default as
  50   # it matches what we have on sn-devel-184.
  51   #
  52   SAMBA_CI_CONTAINER_IMAGE: ubuntu1804
  53   #
  54   # The following images are available
  55   # Please see the samba-o3 sections at the end of this file!
  56   # We should run that for each available image
  57   #
  58   SAMBA_CI_CONTAINER_IMAGE_ubuntu1604: ubuntu1604
  59   SAMBA_CI_CONTAINER_IMAGE_ubuntu1804: ubuntu1804
  60   SAMBA_CI_CONTAINER_IMAGE_ubuntu2004: ubuntu2004
  61   SAMBA_CI_CONTAINER_IMAGE_debian9: debian9
  62   SAMBA_CI_CONTAINER_IMAGE_debian10: debian10
  63   SAMBA_CI_CONTAINER_IMAGE_opensuse151: opensuse151
  64   SAMBA_CI_CONTAINER_IMAGE_opensuse152: opensuse152
  65   SAMBA_CI_CONTAINER_IMAGE_fedora32: fedora32
  66   SAMBA_CI_CONTAINER_IMAGE_fedora33: fedora33
  67   SAMBA_CI_CONTAINER_IMAGE_centos7: centos7
  68   SAMBA_CI_CONTAINER_IMAGE_centos8: centos8
  69
  70 include:
  71   # The image creation details are specified in a separate file
  72   # See bootstrap/README.md for details
  73   - 'bootstrap/.gitlab-ci.yml'
  74
  75 .shared_runner_build_image:
  76   extends: .shared_runner_build
  77   variables:
  78     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE}
  79   image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-${SAMBA_CI_JOB_IMAGE}:${SAMBA_CI_CONTAINER_TAG}
  80
  81 .shared_template:
  82   extends: .shared_runner_build_image
  83   # All Samba jobs are interruptible, this avoids burning CPU when a
  84   # newer branch is pushed.
  85   interruptible: true
  86   timeout: 2h
  87
  88   variables:
  89     AUTOBUILD_JOB_NAME: $CI_JOB_NAME
  90   stage: build
  91   cache:
  92     key: ccache.${CI_JOB_NAME}.${SAMBA_CI_JOB_IMAGE}
  93     paths:
  94       - ccache
  95   before_script:
  96     - uname -a
  97     - lsb_release -a
  98     - cat /etc/os-release
  99     - lscpu
 100     - cat /proc/cpuinfo
 101     - mount
 102     - df -h
 103     - cat /proc/swaps
 104     - free -h
 105       # ld will fail if coverage enabled, force link ld to ld.bfd
 106     - if [ -n "$SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE" ]; then sudo ln -sf $(which ld.bfd) $(which ld); fi
 107       # See bootstrap/.gitlab-ci.yml how to generate a new image
 108     - echo "SAMBA_CI_CONTAINER_REGISTRY[${SAMBA_CI_CONTAINER_REGISTRY}]"
 109     - echo "SAMBA_CI_CONTAINER_TAG[${SAMBA_CI_CONTAINER_TAG}]"
 110     - echo "SAMBA_CI_JOB_IMAGE[${SAMBA_CI_JOB_IMAGE}]"
 111     - echo "CI_JOB_IMAGE[${CI_JOB_IMAGE}]"
 112     - bootstrap/template.py --sha1sum > /tmp/sha1sum-template.txt
 113     - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-template.txt
 114     - echo "${SAMBA_CI_CONTAINER_TAG}" > /tmp/sha1sum-tag.txt
 115     - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-tag.txt
 116     - diff -u bootstrap/sha1sum.txt /sha1sum.txt
 117     - echo "${CI_COMMIT_SHA} ${CI_COMMIT_TITLE}" > /tmp/commit.txt
 118     - export CCACHE_BASEDIR="${PWD}"
 119     - export CCACHE_DIR="${PWD}/ccache" && mkdir -pv "$CCACHE_DIR"
 120     - export CC="ccache cc"
 121     - export CXX="ccache c++"
 122     - ccache -z -M 500M
 123     - ccache -s
 124   after_script:
 125     - mount
 126     - df -h
 127     - cat /proc/swaps
 128     - free -h
 129     - CCACHE_BASEDIR="${PWD}" CCACHE_DIR="${PWD}/ccache" ccache -s -c
 130   artifacts:
 131     expire_in: 1 week
 132     paths:
 133       - "*.stdout"
 134       - "*.stderr"
 135       - "*.info"
 136       - system-info.txt
 137   retry:
 138     max: 2
 139     when:
 140       - runner_system_failure
 141       - stuck_or_timeout_failure
 142       - api_failure
 143       - runner_unsupported
 144       - stale_schedule
 145       - job_execution_timeout
 146       - archived_failure
 147       - scheduler_failure
 148       - data_integrity_failure
 149
 150   script:
 151     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 152     # autobuild name, which means we can define a default template that runs most autobuild jobs
 153     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE  --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 154
 155 # Ensure when adding a new job below that you also add it to
 156 # the dependencies for 'pages' below for the code coverage page
 157 # generation.
 158
 159 others:
 160   extends: .shared_template
 161   script:
 162     - script/autobuild.py ldb      $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/ldb
 163     - script/autobuild.py pidl     $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/pidl
 164     - script/autobuild.py replace  $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/replace
 165     - script/autobuild.py talloc   $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/talloc
 166     - script/autobuild.py tdb      $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/tdb
 167     - script/autobuild.py tevent   $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/tevent
 168     - script/autobuild.py samba-xc $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/samba-xc
 169     - script/autobuild.py docs-xml $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/docs-xml
 170
 171 .shared_template_build_only:
 172   extends: .shared_template
 173   timeout: 45m
 174   artifacts:
 175     expire_in: 1 week
 176     paths:
 177       - "*.stdout"
 178       - "*.stderr"
 179       - "*.info"
 180       - system-info.txt
 181       - samba-testbase.tar.gz
 182   script:
 183     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 184     # autobuild name, which means we can define a default template that runs most autobuild jobs
 185     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE  --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 186     # On success we need to pack everything into an artifacts file
 187     # which needs to be in the git checkout.
 188     # As tar doesn't handle hardlink of read-only files,
 189     # we remember the acls and add write permissions
 190     # before creating the archive. The consumer will apply
 191     # the acls again.
 192     - cp -a /sha1sum.txt /tmp/samba-testbase/image-sha1sum.txt
 193     - cp -a /tmp/commit.txt /tmp/samba-testbase/commit.txt
 194     - pushd /tmp && getfacl -R samba-testbase > samba-testbase.acl.dump && popd
 195     - chmod -R +w /tmp/samba-testbase
 196     - mv /tmp/samba-testbase.acl.dump /tmp/samba-testbase/
 197     - tar cfz samba-testbase.tar.gz /tmp/samba-testbase
 198     - ls -la samba-testbase.tar.gz
 199     - sha1sum samba-testbase.tar.gz
 200
 201 .shared_template_test_only:
 202   extends:
 203     - .shared_template
 204     - .shared_runner_test
 205   stage: test_only
 206   script:
 207     # We unpack the artifacts file created by the .shared_template_build_only
 208     # run we depend on
 209     - ls -la samba-testbase.tar.gz
 210     - sha1sum samba-testbase.tar.gz
 211     - tar xfz samba-testbase.tar.gz -C /
 212     - diff -u /tmp/samba-testbase/image-sha1sum.txt /sha1sum.txt
 213     - diff -u /tmp/samba-testbase/commit.txt /tmp/commit.txt
 214     - mv /tmp/samba-testbase/samba-testbase.acl.dump /tmp/samba-testbase.acl.dump
 215     - pushd /tmp && setfacl --restore=/tmp/samba-testbase.acl.dump && popd
 216     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 217     # autobuild name, which means we can define a default template that runs most autobuild jobs
 218     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --skip-dependencies --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 219
 220 samba-def-build:
 221   extends: .shared_template_build_only
 222   stage: build_first
 223
 224 .needs_samba-def-build:
 225   extends: .shared_template_test_only
 226   needs:
 227     - job: samba-def-build
 228       artifacts: true
 229
 230 samba-mit-build:
 231   extends: .shared_template_build_only
 232   stage: build_first
 233
 234 .needs_samba-mit-build:
 235   extends: .shared_template_test_only
 236   needs:
 237     - job: samba-mit-build
 238       artifacts: true
 239
 240 samba-h5l-build:
 241   extends: .shared_template_build_only
 242
 243 .needs_samba-h5l-build:
 244   extends: .shared_template_test_only
 245   needs:
 246     - job: samba-h5l-build
 247       artifacts: true
 248
 249 samba-nt4-build:
 250   extends: .shared_template_build_only
 251
 252 .needs_samba-nt4-build:
 253   extends: .shared_template_test_only
 254   needs:
 255     - job: samba-nt4-build
 256       artifacts: true
 257
 258 samba-no-opath-build:
 259   extends: .shared_template_build_only
 260
 261 .needs_samba-no-opath-build:
 262   extends: .shared_template_test_only
 263   needs:
 264     - job: samba-no-opath-build
 265       artifacts: true
 266
 267 samba:
 268   extends: .shared_template
 269
 270 samba-mitkrb5:
 271   extends: .shared_template
 272
 273 samba-minimal-smbd:
 274   extends: .shared_template
 275
 276 samba-admem:
 277   extends: .needs_samba-def-build
 278
 279 samba-ad-dc-2:
 280   extends: .needs_samba-def-build
 281
 282 samba-ad-dc-3:
 283   extends: .needs_samba-def-build
 284
 285 samba-ad-dc-4:
 286   extends: .needs_samba-def-build
 287
 288 samba-ad-dc-5:
 289   extends: .needs_samba-def-build
 290
 291 samba-ad-dc-6:
 292   extends: .needs_samba-def-build
 293
 294 samba-ad-back1:
 295   extends: .needs_samba-def-build
 296
 297 samba-ad-back2:
 298   extends: .needs_samba-def-build
 299
 300 samba-schemaupgrade:
 301   extends: .needs_samba-def-build
 302
 303 samba-libs:
 304   extends: .shared_template
 305
 306 samba-fuzz:
 307   extends: .shared_template
 308   variables:
 309     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu1604}
 310
 311 ctdb:
 312   extends: .shared_template
 313
 314 samba-ctdb:
 315   extends: .shared_template
 316
 317 samba-ad-dc-ntvfs:
 318   extends: .needs_samba-def-build
 319
 320 samba-admem-mit:
 321   extends: .needs_samba-mit-build
 322
 323 samba-ad-dc-4-mitkrb5:
 324   extends: .needs_samba-mit-build
 325
 326 # This task is run first to ensure we compile before we start the
 327 # main run as it is the fastest full compile of Samba.
 328 samba-fips:
 329   extends: .shared_template
 330   variables:
 331     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora33}
 332
 333 .private_test_only:
 334   extends: .private_runner_test
 335   stage: test_private
 336   only:
 337     variables:
 338       # These jobs are only run if the gitlab repo has private runners available.
 339       # To enable private jobs, you must add the following var and value to
 340       # your gitlab repo by navigating to:
 341       # settings -> CI/CD -> Environment variables
 342       - $SUPPORT_PRIVATE_TEST == "yes"
 343
 344 .needs_samba-def-build-private:
 345   extends:
 346     - .needs_samba-def-build
 347     - .private_test_only
 348
 349 .needs_samba-mit-build-private:
 350   extends:
 351     - .needs_samba-mit-build
 352     - .private_test_only
 353
 354 .needs_samba-h5l-build-private:
 355   extends:
 356     - .needs_samba-h5l-build
 357     - .private_test_only
 358
 359 .needs_samba-nt4-build-private:
 360   extends:
 361     - .needs_samba-nt4-build
 362     - .private_test_only
 363
 364 .needs_samba-no-opath-build-private:
 365   extends:
 366     - .needs_samba-no-opath-build
 367     - .private_test_only
 368
 369 samba-fileserver:
 370   extends: .needs_samba-h5l-build-private
 371
 372 samba-ad-dc-1:
 373   extends: .needs_samba-def-build-private
 374
 375 samba-nt4:
 376   extends: .needs_samba-nt4-build-private
 377
 378 samba-ad-dc-1-mitkrb5:
 379   extends: .needs_samba-mit-build-private
 380
 381 samba-no-opath:
 382   extends: .needs_samba-no-opath-build-private
 383
 384 # 'pages' is a special job which can publish artifacts in `public` dir to gitlab pages
 385 pages:
 386   extends: .shared_runner_build_image
 387   stage: report
 388   dependencies:  # tell gitlab to download artifacts for these jobs
 389     - others
 390     - samba
 391     - samba-mitkrb5
 392     - samba-admem
 393     - samba-ad-dc-2
 394     - samba-ad-dc-3
 395     - samba-ad-dc-4
 396     - samba-ad-dc-5
 397     - samba-ad-dc-6
 398     - samba-libs
 399     - samba-minimal-smbd
 400     - samba-fuzz
 401     # - ctdb  # TODO
 402     - samba-ctdb
 403     - samba-ad-dc-ntvfs
 404     - samba-admem-mit
 405     - samba-ad-dc-4-mitkrb5
 406     - samba-ad-back1
 407     - samba-ad-back2
 408     - samba-fileserver
 409     - samba-ad-dc-1
 410     - samba-nt4
 411     - samba-schemaupgrade
 412     - samba-ad-dc-1-mitkrb5
 413     - samba-fips
 414   script:
 415     - ./configure.developer
 416     - make -j
 417     - lcov $(ls *.info | xargs -I{} echo -n "-a {} ") -o all.info
 418     - genhtml all.info --output-directory public --prefix=$(pwd) --title "coverage report for $CI_COMMIT_REF_NAME $CI_COMMIT_SHORT_SHA"
 419   artifacts:
 420     expire_in: 30 days
 421     paths:
 422       - public
 423   only:
 424     variables:
 425       - $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == "--enable-coverage"
 426
 427 # Coverity Scan
 428 coverity:
 429   extends: .shared_runner_build_image
 430   variables:
 431     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora33}
 432   stage: build
 433   script:
 434     - wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_SCAN_TOKEN&project=$COVERITY_SCAN_PROJECT_NAME" -O /tmp/coverity_tool.tgz
 435     - tar xf /tmp/coverity_tool.tgz
 436     - ./configure.developer --with-cluster-support --with-system-mitkrb5 --with-experimental-mit-ad-dc
 437     - cov-analysis-linux64-*/bin/cov-build --dir cov-int make -j$(nproc)
 438     - tar czf cov-int.tar.gz cov-int
 439     - curl
 440       --form token=$COVERITY_SCAN_TOKEN
 441       --form email=$COVERITY_SCAN_EMAIL
 442       --form file=@cov-int.tar.gz
 443       --form version="`git describe --tags`"
 444       --form description="CI build"
 445       https://scan.coverity.com/builds?project=$COVERITY_SCAN_PROJECT_NAME
 446   only:
 447     refs:
 448       - master
 449       - schedules
 450     variables:
 451       - $COVERITY_SCAN_TOKEN != null
 452       - $COVERITY_SCAN_PROJECT_NAME != null
 453       - $COVERITY_SCAN_EMAIL != null
 454   artifacts:
 455     expire_in: 1 week
 456     when: on_failure
 457     paths:
 458       - cov-int/*.txt
 459
 460 #
 461 # We build samba-o3 on all supported distributions
 462 #
 463
 464 .samba-o3-template:
 465   extends: .shared_template
 466   variables:
 467     AUTOBUILD_JOB_NAME: samba-o3
 468   only:
 469     variables:
 470       # do not run o3 for coverage since they are using different images
 471       - $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == ""
 472
 473 ubuntu1804-samba-o3:
 474   extends: .samba-o3-template
 475   variables:
 476     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu1804}
 477
 478 ubuntu2004-samba-o3:
 479   extends: .samba-o3-template
 480   variables:
 481     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu2004}
 482
 483 debian10-samba-o3:
 484   extends: .samba-o3-template
 485   variables:
 486     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_debian10}
 487
 488 opensuse151-samba-o3:
 489   extends: .samba-o3-template
 490   variables:
 491     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_opensuse151}
 492
 493 opensuse152-samba-o3:
 494   extends: .samba-o3-template
 495   variables:
 496     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_opensuse152}
 497
 498 centos7-samba-o3:
 499   extends: .samba-o3-template
 500   variables:
 501     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_centos7}
 502     # Git on CentOS doesn't support shallow git cloning
 503     GIT_DEPTH: ""
 504     # We need a newer GnuTLS version on CentOS7
 505     PKG_CONFIG_PATH: "/usr/lib64/compat-gnutls34/pkgconfig:/usr/lib64/compat-nettle32/pkgconfig"
 506
 507 centos8-samba-o3:
 508   extends: .samba-o3-template
 509   variables:
 510     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_centos8}
 511
 512 fedora32-samba-o3:
 513   extends: .samba-o3-template
 514   variables:
 515     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora32}
 516
 517 fedora33-samba-o3:
 518   extends: .samba-o3-template
 519   variables:
 520     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora33}
 521
 522 #
 523 # Keep the samba-o3 sections at the end ...
 524 #