.gitlab-ci.yml

   1 # see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options
   2
   3 # Stages explained
   4 #
   5 # images: Build the images with the bootstrap script
   6 # build_first: Build one thing first to find silly errors (fast job)
   7 #              (don't pay for 35 machines until something compiles)
   8 # build: The main parallel job
   9 #              (keep these to 1hour as we are billed per hour)
  10 # report: Code coverage reporting
  11
  12 stages:
  13   - images
  14   - build_first
  15   - build
  16   - report
  17
  18 variables:
  19   GIT_STRATEGY: fetch
  20   GIT_DEPTH: "3"
  21   # "--enable-coverage" or ""
  22   SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE: ""
  23   #
  24   # we run autobuild.py inside a samba CI docker image located on gitlab's registry
  25   # overwrite this variable if you want use your own image registry.
  26   #
  27   # Or better ask for access to the shared development repository, see
  28   # https://wiki.samba.org/index.php/Samba_CI_on_gitlab#Getting_Access
  29   #
  30   SAMBA_CI_CONTAINER_REGISTRY: registry.gitlab.com/samba-team/devel/samba
  31   #
  32   # Set this to the contents of bootstrap/sha1sum.txt
  33   # which is generated by bootstrap/template.py --render
  34   #
  35   SAMBA_CI_CONTAINER_TAG: c4c00eb35cae36d8d6e752ee01fd943432993721
  36   #
  37   # We use the ubuntu1804 image as default as
  38   # it matches what we have on sn-devel-184.
  39   #
  40   SAMBA_CI_CONTAINER_IMAGE: ubuntu1804
  41   #
  42   # The following images are available
  43   # Please see the samba-o3 sections at the end of this file!
  44   # We should run that for each available image
  45   #
  46   SAMBA_CI_CONTAINER_IMAGE_ubuntu1604: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-ubuntu1604:${SAMBA_CI_CONTAINER_TAG}
  47   SAMBA_CI_CONTAINER_IMAGE_ubuntu1804: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-ubuntu1804:${SAMBA_CI_CONTAINER_TAG}
  48   SAMBA_CI_CONTAINER_IMAGE_ubuntu2004: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-ubuntu2004:${SAMBA_CI_CONTAINER_TAG}
  49   SAMBA_CI_CONTAINER_IMAGE_debian9: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-debian9:${SAMBA_CI_CONTAINER_TAG}
  50   SAMBA_CI_CONTAINER_IMAGE_debian10: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-debian10:${SAMBA_CI_CONTAINER_TAG}
  51   SAMBA_CI_CONTAINER_IMAGE_opensuse151: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-opensuse151:${SAMBA_CI_CONTAINER_TAG}
  52   SAMBA_CI_CONTAINER_IMAGE_opensuse152: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-opensuse152:${SAMBA_CI_CONTAINER_TAG}
  53   SAMBA_CI_CONTAINER_IMAGE_fedora32: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-fedora32:${SAMBA_CI_CONTAINER_TAG}
  54   SAMBA_CI_CONTAINER_IMAGE_fedora33: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-fedora33:${SAMBA_CI_CONTAINER_TAG}
  55   SAMBA_CI_CONTAINER_IMAGE_centos7: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-centos7:${SAMBA_CI_CONTAINER_TAG}
  56   SAMBA_CI_CONTAINER_IMAGE_centos8: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-centos8:${SAMBA_CI_CONTAINER_TAG}
  57
  58 include:
  59   # The image creation details are specified in a separate file
  60   # See bootstrap/README.md for details
  61   - 'bootstrap/.gitlab-ci.yml'
  62
  63 .shared_template:
  64   # All Samba jobs are interruptible, this avoids burning CPU when a
  65   # newer branch is pushed.
  66   interruptible: true
  67
  68   variables:
  69     AUTOBUILD_JOB_NAME: $CI_JOB_NAME
  70   image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-${SAMBA_CI_CONTAINER_IMAGE}:${SAMBA_CI_CONTAINER_TAG}
  71   stage: build
  72   tags:
  73     - docker
  74     - shared
  75   cache:
  76     key: ccache.${CI_JOB_NAME}
  77     paths:
  78       - ccache
  79   before_script:
  80     - uname -a
  81     - lsb_release -a
  82     - cat /etc/os-release
  83     - lscpu
  84     - cat /proc/cpuinfo
  85     - mount
  86     - df -h
  87     - cat /proc/swaps
  88     - free -h
  89       # ld will fail if coverage enabled, force link ld to ld.bfd
  90     - if [ -n "$SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE" ]; then sudo ln -sf $(which ld.bfd) $(which ld); fi
  91       # See bootstrap/.gitlab-ci.yml how to generate a new image
  92     - echo "SAMBA_CI_CONTAINER_REGISTRY[${SAMBA_CI_CONTAINER_REGISTRY}]"
  93     - echo "SAMBA_CI_CONTAINER_TAG[${SAMBA_CI_CONTAINER_TAG}]"
  94     - bootstrap/template.py --sha1sum > /tmp/sha1sum-template.txt
  95     - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-template.txt
  96     - echo "${SAMBA_CI_CONTAINER_TAG}" > /tmp/sha1sum-tag.txt
  97     - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-tag.txt
  98     - diff -u bootstrap/sha1sum.txt /sha1sum.txt
  99     - echo "${CI_COMMIT_SHA} ${CI_COMMIT_TITLE}" > /tmp/commit.txt
 100     - export CCACHE_BASEDIR="${PWD}"
 101     - export CCACHE_DIR="${PWD}/ccache" && mkdir -pv "$CCACHE_DIR"
 102     - export CC="ccache cc"
 103     - export CXX="ccache c++"
 104     - ccache -z -M 500M
 105     - ccache -s
 106   after_script:
 107     - mount
 108     - df -h
 109     - cat /proc/swaps
 110     - free -h
 111     - CCACHE_BASEDIR="${PWD}" CCACHE_DIR="${PWD}/ccache" ccache -s -c
 112   artifacts:
 113     expire_in: 1 week
 114     paths:
 115       - "*.stdout"
 116       - "*.stderr"
 117       - "*.info"
 118       - system-info.txt
 119   retry:
 120     max: 2
 121     when:
 122       - runner_system_failure
 123       - stuck_or_timeout_failure
 124   script:
 125     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 126     # autobuild name, which means we can define a default template that runs most autobuild jobs
 127     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE  --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 128
 129 # Ensure when adding a new job below that you also add it to
 130 # the dependencies for 'pages' below for the code coverage page
 131 # generation.
 132
 133 others:
 134   extends: .shared_template
 135   script:
 136     - script/autobuild.py ldb      $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/ldb
 137     - script/autobuild.py pidl     $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/pidl
 138     - script/autobuild.py replace  $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/replace
 139     - script/autobuild.py talloc   $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/talloc
 140     - script/autobuild.py tdb      $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/tdb
 141     - script/autobuild.py tevent   $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/tevent
 142     - script/autobuild.py samba-xc $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/samba-xc
 143     - script/autobuild.py docs-xml $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/docs-xml
 144
 145 .shared_template_build_only:
 146   extends: .shared_template
 147   stage: build_first
 148   artifacts:
 149     expire_in: 1 week
 150     paths:
 151       - "*.stdout"
 152       - "*.stderr"
 153       - "*.info"
 154       - system-info.txt
 155       - samba-testbase.tar.gz
 156   script:
 157     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 158     # autobuild name, which means we can define a default template that runs most autobuild jobs
 159     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE  --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 160     # On success we need to pack everything into an artifacts file
 161     # which needs to be in the git checkout.
 162     # As tar doesn't handle hardlink of read-only files,
 163     # we remember the acls and add write permissions
 164     # before creating the archive. The consumer will apply
 165     # the acls again.
 166     - cp -a /sha1sum.txt /tmp/samba-testbase/image-sha1sum.txt
 167     - cp -a /tmp/commit.txt /tmp/samba-testbase/commit.txt
 168     - pushd /tmp && getfacl -R samba-testbase > samba-testbase.acl.dump && popd
 169     - chmod -R +w /tmp/samba-testbase
 170     - mv /tmp/samba-testbase.acl.dump /tmp/samba-testbase/
 171     - tar cfz samba-testbase.tar.gz /tmp/samba-testbase
 172     - ls -la samba-testbase.tar.gz
 173     - sha1sum samba-testbase.tar.gz
 174
 175 .shared_template_test_only:
 176   extends: .shared_template
 177   stage: build
 178   script:
 179     # We unpack the artifacts file created by the .shared_template_build_only
 180     # run we depend on
 181     - ls -la samba-testbase.tar.gz
 182     - sha1sum samba-testbase.tar.gz
 183     - tar xfz samba-testbase.tar.gz -C /
 184     - diff -u /tmp/samba-testbase/image-sha1sum.txt /sha1sum.txt
 185     - diff -u /tmp/samba-testbase/commit.txt /tmp/commit.txt
 186     - mv /tmp/samba-testbase/samba-testbase.acl.dump /tmp/samba-testbase.acl.dump
 187     - pushd /tmp && setfacl --restore=/tmp/samba-testbase.acl.dump && popd
 188     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 189     # autobuild name, which means we can define a default template that runs most autobuild jobs
 190     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --skip-dependencies --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 191
 192 samba-def-build:
 193   extends: .shared_template_build_only
 194
 195 .needs_samba-def-build:
 196   extends: .shared_template_test_only
 197   needs:
 198     - job: samba-def-build
 199       artifacts: true
 200
 201 samba-mit-build:
 202   extends: .shared_template_build_only
 203
 204 .needs_samba-mit-build:
 205   extends: .shared_template_test_only
 206   needs:
 207     - job: samba-mit-build
 208       artifacts: true
 209
 210 samba:
 211   extends: .shared_template
 212
 213 samba-mitkrb5:
 214   extends: .shared_template
 215
 216 samba-minimal-smbd:
 217   extends: .shared_template
 218
 219 samba-admem:
 220   extends: .needs_samba-def-build
 221
 222 samba-ad-dc-2:
 223   extends: .needs_samba-def-build
 224
 225 samba-ad-dc-3:
 226   extends: .needs_samba-def-build
 227
 228 samba-ad-dc-4:
 229   extends: .needs_samba-def-build
 230
 231 samba-ad-dc-5:
 232   extends: .needs_samba-def-build
 233
 234 samba-ad-dc-6:
 235   extends: .needs_samba-def-build
 236
 237 samba-libs:
 238   extends: .shared_template
 239
 240 samba-fuzz:
 241   extends: .shared_template
 242   image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-ubuntu1604:${SAMBA_CI_CONTAINER_TAG}
 243
 244 ctdb:
 245   extends: .shared_template
 246
 247 samba-ctdb:
 248   extends: .shared_template
 249
 250 samba-ad-dc-ntvfs:
 251   extends: .needs_samba-def-build
 252
 253 samba-admem-mit:
 254   extends: .needs_samba-mit-build
 255
 256 samba-ad-dc-4-mitkrb5:
 257   extends: .needs_samba-mit-build
 258
 259 # This task is run first to ensure we compile before we start the
 260 # main run as it is the fastest full compile of Samba.
 261 samba-fips:
 262   extends: .shared_template
 263   image: $SAMBA_CI_CONTAINER_IMAGE_fedora33
 264
 265 .private_template:
 266   extends: .shared_template
 267   tags:
 268     - docker
 269     - samba-ci-private
 270   only:
 271     variables:
 272       # These jobs are only run if the gitlab repo has private runners available.
 273       # To enable private jobs, you must add the following var and value to
 274       # your gitlab repo by navigating to:
 275       # settings -> CI/CD -> Environment variables
 276       - $SUPPORT_PRIVATE_TEST == "yes"
 277
 278 samba-ad-dc-backup:
 279   extends: .private_template
 280
 281 samba-fileserver:
 282   extends: .private_template
 283
 284 samba-ad-dc-1:
 285   extends: .private_template
 286
 287 samba-nt4:
 288   extends: .private_template
 289
 290 samba-schemaupgrade:
 291   extends: .private_template
 292
 293 samba-ad-dc-1-mitkrb5:
 294   extends: .private_template
 295
 296 samba-no-opath:
 297   extends: .private_template
 298
 299 # 'pages' is a special job which can publish artifacts in `public` dir to gitlab pages
 300 pages:
 301   image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-${SAMBA_CI_CONTAINER_IMAGE}:${SAMBA_CI_CONTAINER_TAG}
 302   stage: report
 303   tags:
 304     - docker
 305     - shared
 306   dependencies:  # tell gitlab to download artifacts for these jobs
 307     - others
 308     - samba
 309     - samba-mitkrb5
 310     - samba-admem
 311     - samba-ad-dc-2
 312     - samba-ad-dc-3
 313     - samba-ad-dc-4
 314     - samba-ad-dc-5
 315     - samba-ad-dc-6
 316     - samba-libs
 317     - samba-minimal-smbd
 318     - samba-fuzz
 319     # - ctdb  # TODO
 320     - samba-ctdb
 321     - samba-ad-dc-ntvfs
 322     - samba-admem-mit
 323     - samba-ad-dc-4-mitkrb5
 324     - samba-ad-dc-backup
 325     - samba-fileserver
 326     - samba-ad-dc-1
 327     - samba-nt4
 328     - samba-schemaupgrade
 329     - samba-ad-dc-1-mitkrb5
 330     - samba-fips
 331   script:
 332     - ./configure.developer
 333     - make -j
 334     - lcov $(ls *.info | xargs -I{} echo -n "-a {} ") -o all.info
 335     - genhtml all.info --output-directory public --prefix=$(pwd) --title "coverage report for $CI_COMMIT_REF_NAME $CI_COMMIT_SHORT_SHA"
 336   artifacts:
 337     expire_in: 30 days
 338     paths:
 339       - public
 340   only:
 341     variables:
 342       - $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == "--enable-coverage"
 343
 344 # Coverity Scan
 345 coverity:
 346   stage: build
 347   image: $SAMBA_CI_CONTAINER_IMAGE_fedora33
 348   tags:
 349     - docker
 350     - shared
 351   script:
 352     - wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_SCAN_TOKEN&project=$COVERITY_SCAN_PROJECT_NAME" -O /tmp/coverity_tool.tgz
 353     - tar xf /tmp/coverity_tool.tgz
 354     - ./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc
 355     - cov-analysis-linux64-*/bin/cov-build --dir cov-int make -j$(nproc)
 356     - tar czf cov-int.tar.gz cov-int
 357     - curl
 358       --form token=$COVERITY_SCAN_TOKEN
 359       --form email=$COVERITY_SCAN_EMAIL
 360       --form file=@cov-int.tar.gz
 361       --form version="`git describe --tags`"
 362       --form description="CI build"
 363       https://scan.coverity.com/builds?project=$COVERITY_SCAN_PROJECT_NAME
 364   only:
 365     refs:
 366       - master
 367       - schedules
 368     variables:
 369       - $COVERITY_SCAN_TOKEN != null
 370       - $COVERITY_SCAN_PROJECT_NAME != null
 371       - $COVERITY_SCAN_EMAIL != null
 372   artifacts:
 373     expire_in: 1 week
 374     when: on_failure
 375     paths:
 376       - cov-int/*.txt
 377
 378 #
 379 # We build samba-o3 on all supported distributions
 380 #
 381
 382 .samba-o3-template:
 383   extends: .shared_template
 384   variables:
 385     AUTOBUILD_JOB_NAME: samba-o3
 386   only:
 387     variables:
 388       # do not run o3 for coverage since they are using different images
 389       - $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == ""
 390
 391 ubuntu1804-samba-o3:
 392   extends: .samba-o3-template
 393   image: $SAMBA_CI_CONTAINER_IMAGE_ubuntu1804
 394
 395 ubuntu2004-samba-o3:
 396   extends: .samba-o3-template
 397   image: $SAMBA_CI_CONTAINER_IMAGE_ubuntu2004
 398
 399 debian10-samba-o3:
 400   extends: .samba-o3-template
 401   image: $SAMBA_CI_CONTAINER_IMAGE_debian10
 402
 403 opensuse151-samba-o3:
 404   extends: .samba-o3-template
 405   image: $SAMBA_CI_CONTAINER_IMAGE_opensuse151
 406
 407 opensuse152-samba-o3:
 408   extends: .samba-o3-template
 409   image: $SAMBA_CI_CONTAINER_IMAGE_opensuse152
 410
 411 centos7-samba-o3:
 412   extends: .samba-o3-template
 413   image: $SAMBA_CI_CONTAINER_IMAGE_centos7
 414   variables:
 415     # Git on CentOS doesn't support shallow git cloning
 416     GIT_DEPTH: ""
 417     # We need a newer GnuTLS version on CentOS7
 418     PKG_CONFIG_PATH: "/usr/lib64/compat-gnutls34/pkgconfig:/usr/lib64/compat-nettle32/pkgconfig"
 419
 420 centos8-samba-o3:
 421   extends: .samba-o3-template
 422   image: $SAMBA_CI_CONTAINER_IMAGE_centos8
 423
 424 fedora32-samba-o3:
 425   extends: .samba-o3-template
 426   image: $SAMBA_CI_CONTAINER_IMAGE_fedora32
 427
 428 fedora33-samba-o3:
 429   extends: .samba-o3-template
 430   image: $SAMBA_CI_CONTAINER_IMAGE_fedora33
 431
 432 #
 433 # Keep the samba-o3 sections at the end ...
 434 #