.gitlab-ci.yml

   1 # see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options
   2
   3 # Stages explained
   4 #
   5 # images: Build the images with the bootstrap script
   6 # build_first: Build one thing first to find silly errors (fast job)
   7 #              (don't pay for 35 machines until something compiles)
   8 # build: The main parallel job
   9 #              (keep these to 1hour as we are billed per hour)
  10 # report: Code coverage reporting
  11
  12 stages:
  13   - images
  14   - build_first
  15   - build
  16   - report
  17
  18 variables:
  19   GIT_STRATEGY: fetch
  20   GIT_DEPTH: "3"
  21   # "--enable-coverage" or ""
  22   SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE: ""
  23   #
  24   # we run autobuild.py inside a samba CI docker image located on gitlab's registry
  25   # overwrite this variable if you want use your own image registry.
  26   #
  27   # Or better ask for access to the shared development repository, see
  28   # https://wiki.samba.org/index.php/Samba_CI_on_gitlab#Getting_Access
  29   #
  30   SAMBA_CI_CONTAINER_REGISTRY: registry.gitlab.com/samba-team/devel/samba
  31   #
  32   # Set this to the contents of bootstrap/sha1sum.txt
  33   # which is generated by bootstrap/template.py --render
  34   #
  35   SAMBA_CI_CONTAINER_TAG: c4c00eb35cae36d8d6e752ee01fd943432993721
  36   #
  37   # We use the ubuntu1804 image as default as
  38   # it matches what we have on sn-devel-184.
  39   #
  40   SAMBA_CI_CONTAINER_IMAGE: ubuntu1804
  41   #
  42   # The following images are available
  43   # Please see the samba-o3 sections at the end of this file!
  44   # We should run that for each available image
  45   #
  46   SAMBA_CI_CONTAINER_IMAGE_ubuntu1604: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-ubuntu1604:${SAMBA_CI_CONTAINER_TAG}
  47   SAMBA_CI_CONTAINER_IMAGE_ubuntu1804: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-ubuntu1804:${SAMBA_CI_CONTAINER_TAG}
  48   SAMBA_CI_CONTAINER_IMAGE_ubuntu2004: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-ubuntu2004:${SAMBA_CI_CONTAINER_TAG}
  49   SAMBA_CI_CONTAINER_IMAGE_debian9: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-debian9:${SAMBA_CI_CONTAINER_TAG}
  50   SAMBA_CI_CONTAINER_IMAGE_debian10: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-debian10:${SAMBA_CI_CONTAINER_TAG}
  51   SAMBA_CI_CONTAINER_IMAGE_opensuse151: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-opensuse151:${SAMBA_CI_CONTAINER_TAG}
  52   SAMBA_CI_CONTAINER_IMAGE_opensuse152: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-opensuse152:${SAMBA_CI_CONTAINER_TAG}
  53   SAMBA_CI_CONTAINER_IMAGE_fedora32: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-fedora32:${SAMBA_CI_CONTAINER_TAG}
  54   SAMBA_CI_CONTAINER_IMAGE_fedora33: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-fedora33:${SAMBA_CI_CONTAINER_TAG}
  55   SAMBA_CI_CONTAINER_IMAGE_centos7: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-centos7:${SAMBA_CI_CONTAINER_TAG}
  56   SAMBA_CI_CONTAINER_IMAGE_centos8: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-centos8:${SAMBA_CI_CONTAINER_TAG}
  57
  58 include:
  59   # The image creation details are specified in a separate file
  60   # See bootstrap/README.md for details
  61   - 'bootstrap/.gitlab-ci.yml'
  62
  63 .shared_template:
  64   # All Samba jobs are interruptible, this avoids burning CPU when a
  65   # newer branch is pushed.
  66   interruptible: true
  67   timeout: 2h
  68
  69   variables:
  70     AUTOBUILD_JOB_NAME: $CI_JOB_NAME
  71   image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-${SAMBA_CI_CONTAINER_IMAGE}:${SAMBA_CI_CONTAINER_TAG}
  72   stage: build
  73   tags:
  74     - docker
  75     - shared
  76   cache:
  77     key: ccache.${CI_JOB_NAME}
  78     paths:
  79       - ccache
  80   before_script:
  81     - uname -a
  82     - lsb_release -a
  83     - cat /etc/os-release
  84     - lscpu
  85     - cat /proc/cpuinfo
  86     - mount
  87     - df -h
  88     - cat /proc/swaps
  89     - free -h
  90       # ld will fail if coverage enabled, force link ld to ld.bfd
  91     - if [ -n "$SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE" ]; then sudo ln -sf $(which ld.bfd) $(which ld); fi
  92       # See bootstrap/.gitlab-ci.yml how to generate a new image
  93     - echo "SAMBA_CI_CONTAINER_REGISTRY[${SAMBA_CI_CONTAINER_REGISTRY}]"
  94     - echo "SAMBA_CI_CONTAINER_TAG[${SAMBA_CI_CONTAINER_TAG}]"
  95     - bootstrap/template.py --sha1sum > /tmp/sha1sum-template.txt
  96     - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-template.txt
  97     - echo "${SAMBA_CI_CONTAINER_TAG}" > /tmp/sha1sum-tag.txt
  98     - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-tag.txt
  99     - diff -u bootstrap/sha1sum.txt /sha1sum.txt
 100     - echo "${CI_COMMIT_SHA} ${CI_COMMIT_TITLE}" > /tmp/commit.txt
 101     - export CCACHE_BASEDIR="${PWD}"
 102     - export CCACHE_DIR="${PWD}/ccache" && mkdir -pv "$CCACHE_DIR"
 103     - export CC="ccache cc"
 104     - export CXX="ccache c++"
 105     - ccache -z -M 500M
 106     - ccache -s
 107   after_script:
 108     - mount
 109     - df -h
 110     - cat /proc/swaps
 111     - free -h
 112     - CCACHE_BASEDIR="${PWD}" CCACHE_DIR="${PWD}/ccache" ccache -s -c
 113   artifacts:
 114     expire_in: 1 week
 115     paths:
 116       - "*.stdout"
 117       - "*.stderr"
 118       - "*.info"
 119       - system-info.txt
 120   retry:
 121     max: 2
 122     when:
 123       - runner_system_failure
 124       - stuck_or_timeout_failure
 125   script:
 126     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 127     # autobuild name, which means we can define a default template that runs most autobuild jobs
 128     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE  --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 129
 130 # Ensure when adding a new job below that you also add it to
 131 # the dependencies for 'pages' below for the code coverage page
 132 # generation.
 133
 134 others:
 135   extends: .shared_template
 136   script:
 137     - script/autobuild.py ldb      $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/ldb
 138     - script/autobuild.py pidl     $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/pidl
 139     - script/autobuild.py replace  $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/replace
 140     - script/autobuild.py talloc   $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/talloc
 141     - script/autobuild.py tdb      $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/tdb
 142     - script/autobuild.py tevent   $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/tevent
 143     - script/autobuild.py samba-xc $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/samba-xc
 144     - script/autobuild.py docs-xml $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase/docs-xml
 145
 146 .shared_template_build_only:
 147   extends: .shared_template
 148   stage: build_first
 149   timeout: 45m
 150   artifacts:
 151     expire_in: 1 week
 152     paths:
 153       - "*.stdout"
 154       - "*.stderr"
 155       - "*.info"
 156       - system-info.txt
 157       - samba-testbase.tar.gz
 158   script:
 159     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 160     # autobuild name, which means we can define a default template that runs most autobuild jobs
 161     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE  --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 162     # On success we need to pack everything into an artifacts file
 163     # which needs to be in the git checkout.
 164     # As tar doesn't handle hardlink of read-only files,
 165     # we remember the acls and add write permissions
 166     # before creating the archive. The consumer will apply
 167     # the acls again.
 168     - cp -a /sha1sum.txt /tmp/samba-testbase/image-sha1sum.txt
 169     - cp -a /tmp/commit.txt /tmp/samba-testbase/commit.txt
 170     - pushd /tmp && getfacl -R samba-testbase > samba-testbase.acl.dump && popd
 171     - chmod -R +w /tmp/samba-testbase
 172     - mv /tmp/samba-testbase.acl.dump /tmp/samba-testbase/
 173     - tar cfz samba-testbase.tar.gz /tmp/samba-testbase
 174     - ls -la samba-testbase.tar.gz
 175     - sha1sum samba-testbase.tar.gz
 176
 177 .shared_template_test_only:
 178   extends: .shared_template
 179   stage: build
 180   script:
 181     # We unpack the artifacts file created by the .shared_template_build_only
 182     # run we depend on
 183     - ls -la samba-testbase.tar.gz
 184     - sha1sum samba-testbase.tar.gz
 185     - tar xfz samba-testbase.tar.gz -C /
 186     - diff -u /tmp/samba-testbase/image-sha1sum.txt /sha1sum.txt
 187     - diff -u /tmp/samba-testbase/commit.txt /tmp/commit.txt
 188     - mv /tmp/samba-testbase/samba-testbase.acl.dump /tmp/samba-testbase.acl.dump
 189     - pushd /tmp && setfacl --restore=/tmp/samba-testbase.acl.dump && popd
 190     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
 191     # autobuild name, which means we can define a default template that runs most autobuild jobs
 192     - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --skip-dependencies --verbose --nocleanup --keeplogs --tail --full-testbase /tmp/samba-testbase
 193
 194 samba-def-build:
 195   extends: .shared_template_build_only
 196
 197 .needs_samba-def-build:
 198   extends: .shared_template_test_only
 199   needs:
 200     - job: samba-def-build
 201       artifacts: true
 202
 203 samba-mit-build:
 204   extends: .shared_template_build_only
 205
 206 .needs_samba-mit-build:
 207   extends: .shared_template_test_only
 208   needs:
 209     - job: samba-mit-build
 210       artifacts: true
 211
 212 samba:
 213   extends: .shared_template
 214
 215 samba-mitkrb5:
 216   extends: .shared_template
 217
 218 samba-minimal-smbd:
 219   extends: .shared_template
 220
 221 samba-admem:
 222   extends: .needs_samba-def-build
 223
 224 samba-ad-dc-2:
 225   extends: .needs_samba-def-build
 226
 227 samba-ad-dc-3:
 228   extends: .needs_samba-def-build
 229
 230 samba-ad-dc-4:
 231   extends: .needs_samba-def-build
 232
 233 samba-ad-dc-5:
 234   extends: .needs_samba-def-build
 235
 236 samba-ad-dc-6:
 237   extends: .needs_samba-def-build
 238
 239 samba-libs:
 240   extends: .shared_template
 241
 242 samba-fuzz:
 243   extends: .shared_template
 244   image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-ubuntu1604:${SAMBA_CI_CONTAINER_TAG}
 245
 246 ctdb:
 247   extends: .shared_template
 248
 249 samba-ctdb:
 250   extends: .shared_template
 251
 252 samba-ad-dc-ntvfs:
 253   extends: .needs_samba-def-build
 254
 255 samba-admem-mit:
 256   extends: .needs_samba-mit-build
 257
 258 samba-ad-dc-4-mitkrb5:
 259   extends: .needs_samba-mit-build
 260
 261 # This task is run first to ensure we compile before we start the
 262 # main run as it is the fastest full compile of Samba.
 263 samba-fips:
 264   extends: .shared_template
 265   image: $SAMBA_CI_CONTAINER_IMAGE_fedora33
 266
 267 .private_template:
 268   extends: .shared_template
 269   tags:
 270     - docker
 271     - samba-ci-private
 272   only:
 273     variables:
 274       # These jobs are only run if the gitlab repo has private runners available.
 275       # To enable private jobs, you must add the following var and value to
 276       # your gitlab repo by navigating to:
 277       # settings -> CI/CD -> Environment variables
 278       - $SUPPORT_PRIVATE_TEST == "yes"
 279
 280 samba-ad-dc-backup:
 281   extends: .private_template
 282
 283 samba-fileserver:
 284   extends: .private_template
 285
 286 samba-ad-dc-1:
 287   extends: .private_template
 288
 289 samba-nt4:
 290   extends: .private_template
 291
 292 samba-schemaupgrade:
 293   extends: .private_template
 294
 295 samba-ad-dc-1-mitkrb5:
 296   extends: .private_template
 297
 298 samba-no-opath:
 299   extends: .private_template
 300
 301 # 'pages' is a special job which can publish artifacts in `public` dir to gitlab pages
 302 pages:
 303   image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-${SAMBA_CI_CONTAINER_IMAGE}:${SAMBA_CI_CONTAINER_TAG}
 304   stage: report
 305   tags:
 306     - docker
 307     - shared
 308   dependencies:  # tell gitlab to download artifacts for these jobs
 309     - others
 310     - samba
 311     - samba-mitkrb5
 312     - samba-admem
 313     - samba-ad-dc-2
 314     - samba-ad-dc-3
 315     - samba-ad-dc-4
 316     - samba-ad-dc-5
 317     - samba-ad-dc-6
 318     - samba-libs
 319     - samba-minimal-smbd
 320     - samba-fuzz
 321     # - ctdb  # TODO
 322     - samba-ctdb
 323     - samba-ad-dc-ntvfs
 324     - samba-admem-mit
 325     - samba-ad-dc-4-mitkrb5
 326     - samba-ad-dc-backup
 327     - samba-fileserver
 328     - samba-ad-dc-1
 329     - samba-nt4
 330     - samba-schemaupgrade
 331     - samba-ad-dc-1-mitkrb5
 332     - samba-fips
 333   script:
 334     - ./configure.developer
 335     - make -j
 336     - lcov $(ls *.info | xargs -I{} echo -n "-a {} ") -o all.info
 337     - genhtml all.info --output-directory public --prefix=$(pwd) --title "coverage report for $CI_COMMIT_REF_NAME $CI_COMMIT_SHORT_SHA"
 338   artifacts:
 339     expire_in: 30 days
 340     paths:
 341       - public
 342   only:
 343     variables:
 344       - $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == "--enable-coverage"
 345
 346 # Coverity Scan
 347 coverity:
 348   stage: build
 349   image: $SAMBA_CI_CONTAINER_IMAGE_fedora33
 350   tags:
 351     - docker
 352     - shared
 353   script:
 354     - wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_SCAN_TOKEN&project=$COVERITY_SCAN_PROJECT_NAME" -O /tmp/coverity_tool.tgz
 355     - tar xf /tmp/coverity_tool.tgz
 356     - ./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc
 357     - cov-analysis-linux64-*/bin/cov-build --dir cov-int make -j$(nproc)
 358     - tar czf cov-int.tar.gz cov-int
 359     - curl
 360       --form token=$COVERITY_SCAN_TOKEN
 361       --form email=$COVERITY_SCAN_EMAIL
 362       --form file=@cov-int.tar.gz
 363       --form version="`git describe --tags`"
 364       --form description="CI build"
 365       https://scan.coverity.com/builds?project=$COVERITY_SCAN_PROJECT_NAME
 366   only:
 367     refs:
 368       - master
 369       - schedules
 370     variables:
 371       - $COVERITY_SCAN_TOKEN != null
 372       - $COVERITY_SCAN_PROJECT_NAME != null
 373       - $COVERITY_SCAN_EMAIL != null
 374   artifacts:
 375     expire_in: 1 week
 376     when: on_failure
 377     paths:
 378       - cov-int/*.txt
 379
 380 #
 381 # We build samba-o3 on all supported distributions
 382 #
 383
 384 .samba-o3-template:
 385   extends: .shared_template
 386   variables:
 387     AUTOBUILD_JOB_NAME: samba-o3
 388   only:
 389     variables:
 390       # do not run o3 for coverage since they are using different images
 391       - $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == ""
 392
 393 ubuntu1804-samba-o3:
 394   extends: .samba-o3-template
 395   image: $SAMBA_CI_CONTAINER_IMAGE_ubuntu1804
 396
 397 ubuntu2004-samba-o3:
 398   extends: .samba-o3-template
 399   image: $SAMBA_CI_CONTAINER_IMAGE_ubuntu2004
 400
 401 debian10-samba-o3:
 402   extends: .samba-o3-template
 403   image: $SAMBA_CI_CONTAINER_IMAGE_debian10
 404
 405 opensuse151-samba-o3:
 406   extends: .samba-o3-template
 407   image: $SAMBA_CI_CONTAINER_IMAGE_opensuse151
 408
 409 opensuse152-samba-o3:
 410   extends: .samba-o3-template
 411   image: $SAMBA_CI_CONTAINER_IMAGE_opensuse152
 412
 413 centos7-samba-o3:
 414   extends: .samba-o3-template
 415   image: $SAMBA_CI_CONTAINER_IMAGE_centos7
 416   variables:
 417     # Git on CentOS doesn't support shallow git cloning
 418     GIT_DEPTH: ""
 419     # We need a newer GnuTLS version on CentOS7
 420     PKG_CONFIG_PATH: "/usr/lib64/compat-gnutls34/pkgconfig:/usr/lib64/compat-nettle32/pkgconfig"
 421
 422 centos8-samba-o3:
 423   extends: .samba-o3-template
 424   image: $SAMBA_CI_CONTAINER_IMAGE_centos8
 425
 426 fedora32-samba-o3:
 427   extends: .samba-o3-template
 428   image: $SAMBA_CI_CONTAINER_IMAGE_fedora32
 429
 430 fedora33-samba-o3:
 431   extends: .samba-o3-template
 432   image: $SAMBA_CI_CONTAINER_IMAGE_fedora33
 433
 434 #
 435 # Keep the samba-o3 sections at the end ...
 436 #