.gitlab-ci-main.yml

   1 # see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options
   2
   3 # Stages explained
   4 #
   5 # images: Build the images with the bootstrap script
   6 # build_first: Build a few things first to find silly errors (fast job)
   7 #              (don't pay for 35 machines until something compiles)
   8 # build: The main parallel job
   9 #              (keep these to 1hour as we are billed per hour)
  10 # test_only: Tests using the build from prior stages, these typically
  11 #            have an explicit dependency defined to a specific build job,
  12 #            which means that start as soon as the build job finished.
  13 # test_private: Like test_only, but running on private runners
  14 # report: Code coverage reporting
  15
  16 stages:
  17   - images
  18   - build_first
  19   - build
  20   - test_only
  21   - test_private
  22   - report
  23
  24 variables:
  25   # We want to be resilient to runner failures
  26   ARTIFACT_DOWNLOAD_ATTEMPTS: "3"
  27   EXECUTOR_JOB_SECTION_ATTEMPTS: "3"
  28   GET_SOURCES_ATTEMPTS: "3"
  29   RESTORE_CACHE_ATTEMPTS: "3"
  30   #
  31   GIT_STRATEGY: fetch
  32   GIT_DEPTH: "3"
  33   #
  34   # we run autobuild.py inside a samba CI docker image located on gitlab's registry
  35   # overwrite this variable if you want use your own image registry.
  36   #
  37   # Or better ask for access to the shared development repository, see
  38   # https://wiki.samba.org/index.php/Samba_CI_on_gitlab#Getting_Access
  39   #
  40   SAMBA_CI_CONTAINER_REGISTRY: registry.gitlab.com/samba-team/devel/samba
  41   #
  42   # Set this to the contents of bootstrap/sha1sum.txt
  43   # which is generated by bootstrap/template.py --render
  44   #
  45   SAMBA_CI_CONTAINER_TAG: f25e1fa0a828f428ade0e4f37d8a646ff1e207d4
  46   #
  47   # We use the ubuntu1804 image as default as
  48   # it matches what we have on sn-devel-184.
  49   #
  50   SAMBA_CI_CONTAINER_IMAGE: ubuntu1804
  51   #
  52   # The following images are available
  53   # Please see the samba-o3 sections at the end of this file!
  54   # We should run that for each available image
  55   #
  56   SAMBA_CI_CONTAINER_IMAGE_ubuntu1604: ubuntu1604
  57   SAMBA_CI_CONTAINER_IMAGE_ubuntu1804: ubuntu1804
  58   SAMBA_CI_CONTAINER_IMAGE_ubuntu2004: ubuntu2004
  59   SAMBA_CI_CONTAINER_IMAGE_debian9: debian9
  60   SAMBA_CI_CONTAINER_IMAGE_debian10: debian10
  61   SAMBA_CI_CONTAINER_IMAGE_opensuse151: opensuse151
  62   SAMBA_CI_CONTAINER_IMAGE_opensuse152: opensuse152
  63   SAMBA_CI_CONTAINER_IMAGE_fedora32: fedora32
  64   SAMBA_CI_CONTAINER_IMAGE_fedora33: fedora33
  65   SAMBA_CI_CONTAINER_IMAGE_centos7: centos7
  66   SAMBA_CI_CONTAINER_IMAGE_centos8: centos8
  67
  68 include:
  69   # The image creation details are specified in a separate file
  70   # See bootstrap/README.md for details
  71   - 'bootstrap/.gitlab-ci.yml'
  72
  73 .shared_runner_build_image:
  74   extends: .shared_runner_build
  75   variables:
  76     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE}
  77   image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-${SAMBA_CI_JOB_IMAGE}:${SAMBA_CI_CONTAINER_TAG}
  78
  79 .shared_template:
  80   extends: .shared_runner_build_image
  81   # All Samba jobs are interruptible, this avoids burning CPU when a
  82   # newer branch is pushed.
  83   interruptible: true
  84   timeout: 2h
  85
  86   variables:
  87     AUTOBUILD_JOB_NAME: $CI_JOB_NAME
  88   stage: build
  89   cache:
  90     key: ccache.${CI_JOB_NAME}.${SAMBA_CI_JOB_IMAGE}.${SAMBA_CI_FLAVOR}
  91     paths:
  92       - ccache
  93   before_script:
  94     - uname -a
  95     - lsb_release -a
  96     - cat /etc/os-release
  97     - lscpu
  98     - cat /proc/cpuinfo
  99     - mount
 100     - df -h
 101     - cat /proc/swaps
 102     - free -h
 103       # ld will fail if coverage enabled, force link ld to ld.bfd
 104     - if [ -n "$SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE" ]; then sudo ln -sf $(which ld.bfd) $(which ld); fi
 105       # See bootstrap/.gitlab-ci.yml how to generate a new image
 106     - echo "SAMBA_CI_CONTAINER_REGISTRY[${SAMBA_CI_CONTAINER_REGISTRY}]"
 107     - echo "SAMBA_CI_CONTAINER_TAG[${SAMBA_CI_CONTAINER_TAG}]"
 108     - echo "SAMBA_CI_JOB_IMAGE[${SAMBA_CI_JOB_IMAGE}]"
 109     - echo "CI_JOB_IMAGE[${CI_JOB_IMAGE}]"
 110     - bootstrap/template.py --sha1sum > /tmp/sha1sum-template.txt
 111     - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-template.txt
 112     - echo "${SAMBA_CI_CONTAINER_TAG}" > /tmp/sha1sum-tag.txt
 113     - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-tag.txt
 114     - diff -u bootstrap/sha1sum.txt /sha1sum.txt
 115     - echo "${CI_COMMIT_SHA} ${CI_COMMIT_TITLE}" > /tmp/commit.txt
 116     - export CCACHE_BASEDIR="${PWD}"
 117     - export CCACHE_DIR="${PWD}/ccache" && mkdir -pv "$CCACHE_DIR"
 118     - export CC="ccache cc"
 119     - export CXX="ccache c++"
 120     - ccache -z -M 500M
 121     - ccache -s
 122   after_script:
 123     - mount
 124     - df -h
 125     - cat /proc/swaps
 126     - free -h
 127     - CCACHE_BASEDIR="${PWD}" CCACHE_DIR="${PWD}/ccache" ccache -s -c
 128   artifacts:
 129     expire_in: 1 week
 130     paths:
 131       - "*.stdout"
 132       - "*.stderr"
 133       - "*.info"
 134       - system-info.txt
 135   retry:
 136     max: 2
 137     when:
 138       - runner_system_failure
 139       - stuck_or_timeout_failure
 140       - api_failure
 141       - runner_unsupported
 142       - stale_schedule
 143       - job_execution_timeout
 144       - archived_failure
 145       - scheduler_failure
 146       - data_integrity_failure
 147
 148   script:
 149     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 150     # autobuild name, which means we can define a default template that runs most autobuild jobs
 151     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE  --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 152
 153 # Ensure when adding a new job below that you also add it to
 154 # the dependencies for 'pages' below for the code coverage page
 155 # generation.
 156
 157 others:
 158   extends: .shared_template
 159   script:
 160     - script/autobuild.py ldb      $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/ldb
 161     - script/autobuild.py pidl     $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/pidl
 162     - script/autobuild.py replace  $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/replace
 163     - script/autobuild.py talloc   $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/talloc
 164     - script/autobuild.py tdb      $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/tdb
 165     - script/autobuild.py tevent   $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/tevent
 166     - script/autobuild.py samba-xc $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/samba-xc
 167     - script/autobuild.py docs-xml $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/docs-xml
 168
 169 .shared_template_build_only:
 170   extends: .shared_template
 171   timeout: 45m
 172   artifacts:
 173     expire_in: 1 week
 174     paths:
 175       - "*.stdout"
 176       - "*.stderr"
 177       - "*.info"
 178       - system-info.txt
 179       - samba-testbase.tar.gz
 180   script:
 181     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 182     # autobuild name, which means we can define a default template that runs most autobuild jobs
 183     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE  --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 184     # On success we need to pack everything into an artifacts file
 185     # which needs to be in the git checkout.
 186     # As tar doesn't handle hardlink of read-only files,
 187     # we remember the acls and add write permissions
 188     # before creating the archive. The consumer will apply
 189     # the acls again.
 190     - cp -a /sha1sum.txt /tmp/samba-testbase/image-sha1sum.txt
 191     - cp -a /tmp/commit.txt /tmp/samba-testbase/commit.txt
 192     - ln -s /tmp/samba-testbase/${AUTOBUILD_JOB_NAME}/ /tmp/samba-testbase/build_subdir_link
 193     - pushd /tmp && getfacl -R samba-testbase > samba-testbase.acl.dump && popd
 194     - chmod -R +w /tmp/samba-testbase
 195     - mv /tmp/samba-testbase.acl.dump /tmp/samba-testbase/
 196     - tar cfz samba-testbase.tar.gz /tmp/samba-testbase
 197     - ls -la samba-testbase.tar.gz
 198     - sha1sum samba-testbase.tar.gz
 199
 200 .shared_template_test_only:
 201   extends:
 202     - .shared_template
 203     - .shared_runner_test
 204   stage: test_only
 205   script:
 206     # We unpack the artifacts file created by the .shared_template_build_only
 207     # run we depend on
 208     - ls -la samba-testbase.tar.gz
 209     - sha1sum samba-testbase.tar.gz
 210     - tar xfz samba-testbase.tar.gz -C /
 211     - diff -u /tmp/samba-testbase/image-sha1sum.txt /sha1sum.txt
 212     - diff -u /tmp/samba-testbase/commit.txt /tmp/commit.txt
 213     - mv /tmp/samba-testbase/samba-testbase.acl.dump /tmp/samba-testbase.acl.dump
 214     - pushd /tmp && setfacl --restore=/tmp/samba-testbase.acl.dump && popd
 215     - ls -la /tmp/samba-testbase/
 216     - ls -la /tmp/samba-testbase/build_subdir_link
 217     - ls -la /tmp/samba-testbase/build_subdir_link/
 218     - if [ -n "$SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE" ]; then find /tmp/samba-testbase/build_subdir_link/ -type d -printf "'%p'\n" | xargs chmod u+w; fi
 219     - ls -la /tmp/samba-testbase/build_subdir_link/
 220     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 221     # autobuild name, which means we can define a default template that runs most autobuild jobs
 222     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --skip-dependencies --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 223
 224 samba-def-build:
 225   extends: .shared_template_build_only
 226   stage: build_first
 227
 228 .needs_samba-def-build:
 229   extends: .shared_template_test_only
 230   needs:
 231     - job: samba-def-build
 232       artifacts: true
 233
 234 samba-mit-build:
 235   extends: .shared_template_build_only
 236   stage: build_first
 237
 238 .needs_samba-mit-build:
 239   extends: .shared_template_test_only
 240   needs:
 241     - job: samba-mit-build
 242       artifacts: true
 243
 244 samba-h5l-build:
 245   extends: .shared_template_build_only
 246
 247 .needs_samba-h5l-build:
 248   extends: .shared_template_test_only
 249   needs:
 250     - job: samba-h5l-build
 251       artifacts: true
 252
 253 samba-nt4-build:
 254   extends: .shared_template_build_only
 255
 256 .needs_samba-nt4-build:
 257   extends: .shared_template_test_only
 258   needs:
 259     - job: samba-nt4-build
 260       artifacts: true
 261
 262 samba-no-opath-build:
 263   extends: .shared_template_build_only
 264
 265 .needs_samba-no-opath-build:
 266   extends: .shared_template_test_only
 267   needs:
 268     - job: samba-no-opath-build
 269       artifacts: true
 270
 271 samba:
 272   extends: .shared_template
 273
 274 samba-mitkrb5:
 275   extends: .shared_template
 276
 277 samba-minimal-smbd:
 278   extends: .shared_template
 279
 280 samba-nopython:
 281   extends: .shared_template
 282
 283 samba-admem:
 284   extends: .needs_samba-def-build
 285
 286 samba-ad-dc-2:
 287   extends: .needs_samba-def-build
 288
 289 samba-ad-dc-3:
 290   extends: .needs_samba-def-build
 291
 292 samba-ad-dc-4a:
 293   extends: .needs_samba-def-build
 294
 295 samba-ad-dc-4b:
 296   extends: .needs_samba-def-build
 297
 298 samba-ad-dc-5:
 299   extends: .needs_samba-def-build
 300
 301 samba-ad-dc-6:
 302   extends: .needs_samba-def-build
 303
 304 samba-ad-back1:
 305   extends: .needs_samba-def-build
 306
 307 samba-ad-back2:
 308   extends: .needs_samba-def-build
 309
 310 samba-schemaupgrade:
 311   extends: .needs_samba-def-build
 312
 313 samba-libs:
 314   extends: .shared_template
 315
 316 samba-fuzz:
 317   extends: .shared_template
 318   variables:
 319     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu1604}
 320
 321 ctdb:
 322   extends: .shared_template
 323
 324 samba-ctdb:
 325   extends: .shared_template
 326
 327 samba-ad-dc-ntvfs:
 328   extends: .needs_samba-def-build
 329
 330 samba-admem-mit:
 331   extends: .needs_samba-mit-build
 332
 333 samba-ad-dc-4a-mitkrb5:
 334   extends: .needs_samba-mit-build
 335
 336 samba-ad-dc-4b-mitkrb5:
 337   extends: .needs_samba-mit-build
 338
 339 # This task is run first to ensure we compile before we start the
 340 # main run as it is the fastest full compile of Samba.
 341 samba-fips:
 342   extends: .shared_template
 343   variables:
 344     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora33}
 345
 346 .private_test_only:
 347   extends: .private_runner_test
 348   stage: test_private
 349   only:
 350     variables:
 351       # These jobs are only run if the gitlab repo has private runners available.
 352       # To enable private jobs, you must add the following var and value to
 353       # your gitlab repo by navigating to:
 354       # settings -> CI/CD -> Environment variables
 355       - $SUPPORT_PRIVATE_TEST == "yes"
 356
 357 .needs_samba-def-build-private:
 358   extends:
 359     - .needs_samba-def-build
 360     - .private_test_only
 361
 362 .needs_samba-mit-build-private:
 363   extends:
 364     - .needs_samba-mit-build
 365     - .private_test_only
 366
 367 .needs_samba-h5l-build-private:
 368   extends:
 369     - .needs_samba-h5l-build
 370     - .private_test_only
 371
 372 .needs_samba-nt4-build-private:
 373   extends:
 374     - .needs_samba-nt4-build
 375     - .private_test_only
 376
 377 .needs_samba-no-opath-build-private:
 378   extends:
 379     - .needs_samba-no-opath-build
 380     - .private_test_only
 381
 382 samba-fileserver:
 383   extends: .needs_samba-h5l-build-private
 384
 385 samba-ad-dc-1:
 386   extends: .needs_samba-def-build-private
 387
 388 samba-nt4:
 389   extends: .needs_samba-nt4-build-private
 390
 391 samba-ad-dc-1-mitkrb5:
 392   extends: .needs_samba-mit-build-private
 393
 394 samba-no-opath1:
 395   extends: .needs_samba-no-opath-build-private
 396
 397 samba-no-opath2:
 398   extends: .needs_samba-no-opath-build-private
 399
 400 # 'pages' is a special job which can publish artifacts in `public` dir to gitlab pages
 401 pages:
 402   extends: .shared_runner_build_image
 403   stage: report
 404   dependencies:  # tell gitlab to download artifacts for these jobs
 405     - others
 406     - samba
 407     - samba-mitkrb5
 408     - samba-admem
 409     - samba-ad-dc-2
 410     - samba-ad-dc-3
 411     - samba-ad-dc-4a
 412     - samba-ad-dc-4b
 413     - samba-ad-dc-5
 414     - samba-ad-dc-6
 415     - samba-libs
 416     - samba-minimal-smbd
 417     - samba-nopython
 418     - samba-fuzz
 419     # - ctdb  # TODO
 420     - samba-ctdb
 421     - samba-ad-dc-ntvfs
 422     - samba-admem-mit
 423     - samba-ad-dc-4a-mitkrb5
 424     - samba-ad-dc-4b-mitkrb5
 425     - samba-ad-back1
 426     - samba-ad-back2
 427     - samba-fileserver
 428     - samba-ad-dc-1
 429     - samba-nt4
 430     - samba-schemaupgrade
 431     - samba-ad-dc-1-mitkrb5
 432     - samba-fips
 433     - samba-no-opath1
 434     - samba-no-opath2
 435   script:
 436     - ls -la *.info
 437     - ./configure.developer
 438     - make -j
 439     - ls -la *.info
 440     - lcov $(ls *.info | xargs -I{} echo -n "-a {} ") -o all.info
 441     - ls -la *.info
 442     - genhtml all.info --output-directory public --prefix=$(pwd) --title "coverage report for $CI_COMMIT_REF_NAME $CI_COMMIT_SHORT_SHA"
 443   artifacts:
 444     expire_in: 30 days
 445     paths:
 446       - public
 447   only:
 448     variables:
 449       - $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == "--enable-coverage"
 450
 451 # Coverity Scan
 452 coverity:
 453   extends: .shared_runner_build_image
 454   variables:
 455     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora33}
 456   stage: build
 457   script:
 458     - wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_SCAN_TOKEN&project=$COVERITY_SCAN_PROJECT_NAME" -O /tmp/coverity_tool.tgz
 459     - tar xf /tmp/coverity_tool.tgz
 460     - ./configure.developer --with-cluster-support --with-system-mitkrb5 --with-experimental-mit-ad-dc
 461     - cov-analysis-linux64-*/bin/cov-build --dir cov-int make -j$(nproc)
 462     - tar czf cov-int.tar.gz cov-int
 463     - curl
 464       --form token=$COVERITY_SCAN_TOKEN
 465       --form email=$COVERITY_SCAN_EMAIL
 466       --form file=@cov-int.tar.gz
 467       --form version="`git describe --tags`"
 468       --form description="CI build"
 469       https://scan.coverity.com/builds?project=$COVERITY_SCAN_PROJECT_NAME
 470   only:
 471     refs:
 472       - master
 473       - schedules
 474     variables:
 475       - $COVERITY_SCAN_TOKEN != null
 476       - $COVERITY_SCAN_PROJECT_NAME != null
 477       - $COVERITY_SCAN_EMAIL != null
 478   artifacts:
 479     expire_in: 1 week
 480     when: on_failure
 481     paths:
 482       - cov-int/*.txt
 483
 484 #
 485 # We build samba-o3 on all supported distributions
 486 #
 487
 488 .samba-o3-template:
 489   extends: .shared_template
 490   variables:
 491     AUTOBUILD_JOB_NAME: samba-o3
 492   only:
 493     variables:
 494       # do not run o3 for coverage since they are using different images
 495       - $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == ""
 496
 497 ubuntu1804-samba-o3:
 498   extends: .samba-o3-template
 499   variables:
 500     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu1804}
 501
 502 ubuntu2004-samba-o3:
 503   extends: .samba-o3-template
 504   variables:
 505     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu2004}
 506
 507 debian10-samba-o3:
 508   extends: .samba-o3-template
 509   variables:
 510     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_debian10}
 511
 512 opensuse151-samba-o3:
 513   extends: .samba-o3-template
 514   variables:
 515     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_opensuse151}
 516
 517 opensuse152-samba-o3:
 518   extends: .samba-o3-template
 519   variables:
 520     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_opensuse152}
 521
 522 centos7-samba-o3:
 523   extends: .samba-o3-template
 524   variables:
 525     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_centos7}
 526     # Git on CentOS doesn't support shallow git cloning
 527     GIT_DEPTH: ""
 528     # We need a newer GnuTLS version on CentOS7
 529     PKG_CONFIG_PATH: "/usr/lib64/compat-gnutls34/pkgconfig:/usr/lib64/compat-nettle32/pkgconfig"
 530
 531 centos8-samba-o3:
 532   extends: .samba-o3-template
 533   variables:
 534     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_centos8}
 535
 536 fedora32-samba-o3:
 537   extends: .samba-o3-template
 538   variables:
 539     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora32}
 540
 541 fedora33-samba-o3:
 542   extends: .samba-o3-template
 543   variables:
 544     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora33}
 545
 546 #
 547 # Keep the samba-o3 sections at the end ...
 548 #