s3:dbwrap_watch: let dbwrap_watched_watch_skip_alerting() also clear the selected...
[samba.git] / .gitlab-ci-main.yml
index b0f81e674bbafdbfd6c00c925515026fca6457a9..c3174f5f1b5e2e051150c9175681e76c34d7132a 100644 (file)
@@ -30,6 +30,11 @@ variables:
   #
   GIT_STRATEGY: fetch
   GIT_DEPTH: "3"
+  #
+  # Use GZip by default, it is fast and is good enough.  Other options include --xz
+
+  SAMBA_TESTBASE_TAR_OPTIONS: -z
+
   #
   # we run autobuild.py inside a samba CI docker image located on gitlab's registry
   # overwrite this variable if you want use your own image registry.
@@ -42,7 +47,7 @@ variables:
   # Set this to the contents of bootstrap/sha1sum.txt
   # which is generated by bootstrap/template.py --render
   #
-  SAMBA_CI_CONTAINER_TAG: f25e1fa0a828f428ade0e4f37d8a646ff1e207d4
+  SAMBA_CI_CONTAINER_TAG: 1e06877f1afbb3dbd4283e00310d63216a274333
   #
   # We use the ubuntu1804 image as default as
   # it matches what we have on sn-devel-184.
@@ -53,17 +58,14 @@ variables:
   # Please see the samba-o3 sections at the end of this file!
   # We should run that for each available image
   #
-  SAMBA_CI_CONTAINER_IMAGE_ubuntu1604: ubuntu1604
   SAMBA_CI_CONTAINER_IMAGE_ubuntu1804: ubuntu1804
   SAMBA_CI_CONTAINER_IMAGE_ubuntu2004: ubuntu2004
-  SAMBA_CI_CONTAINER_IMAGE_debian9: debian9
-  SAMBA_CI_CONTAINER_IMAGE_debian10: debian10
-  SAMBA_CI_CONTAINER_IMAGE_opensuse151: opensuse151
-  SAMBA_CI_CONTAINER_IMAGE_opensuse152: opensuse152
-  SAMBA_CI_CONTAINER_IMAGE_fedora32: fedora32
-  SAMBA_CI_CONTAINER_IMAGE_fedora33: fedora33
+  SAMBA_CI_CONTAINER_IMAGE_debian11: debian11
+  SAMBA_CI_CONTAINER_IMAGE_opensuse153: opensuse153
+  SAMBA_CI_CONTAINER_IMAGE_fedora36: fedora36
+  SAMBA_CI_CONTAINER_IMAGE_f36mit120: f36mit120
   SAMBA_CI_CONTAINER_IMAGE_centos7: centos7
-  SAMBA_CI_CONTAINER_IMAGE_centos8: centos8
+  SAMBA_CI_CONTAINER_IMAGE_centos8s: centos8s
 
 include:
   # The image creation details are specified in a separate file
@@ -83,6 +85,13 @@ include:
   interruptible: true
   timeout: 2h
 
+  # Otherwise we run twice, once on push and once on MR
+  # https://forum.gitlab.com/t/new-rules-syntax-and-detached-pipelines/37292
+  rules:
+    - if: $CI_MERGE_REQUEST_ID
+      when: never
+    - when: on_success
+
   variables:
     AUTOBUILD_JOB_NAME: $CI_JOB_NAME
   stage: build
@@ -90,6 +99,16 @@ include:
     key: ccache.${CI_JOB_NAME}.${SAMBA_CI_JOB_IMAGE}.${SAMBA_CI_FLAVOR}
     paths:
       - ccache
+
+  # This is overridden in many cases, but ensures none of the other
+  # main jobs start until and unless this build finishes.  However
+  # this also ensures we do not download artifacts from any build
+  # unless we specifically depend on it, saving bandwidth
+
+  needs:
+    - job: samba-def-build
+      artifacts: false
+
   before_script:
     - uname -a
     - lsb_release -a
@@ -119,6 +138,8 @@ include:
     - export CXX="ccache c++"
     - ccache -z -M 500M
     - ccache -s
+      # We are already running .gitlab-ci directives from this repo, remove additional checks that break our CI
+    - git config --global --add safe.directory `pwd`
   after_script:
     - mount
     - df -h
@@ -141,7 +162,6 @@ include:
       - api_failure
       - runner_unsupported
       - stale_schedule
-      - job_execution_timeout
       - archived_failure
       - scheduler_failure
       - data_integrity_failure
@@ -169,7 +189,8 @@ others:
 
 .shared_template_build_only:
   extends: .shared_template
-  timeout: 45m
+  timeout: 2h
+  needs:
   artifacts:
     expire_in: 1 week
     paths:
@@ -177,7 +198,7 @@ others:
       - "*.stderr"
       - "*.info"
       - system-info.txt
-      - samba-testbase.tar.gz
+      - samba-testbase.tar
   script:
     # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the
     # autobuild name, which means we can define a default template that runs most autobuild jobs
@@ -194,9 +215,9 @@ others:
     - pushd /tmp && getfacl -R samba-testbase > samba-testbase.acl.dump && popd
     - chmod -R +w /tmp/samba-testbase
     - mv /tmp/samba-testbase.acl.dump /tmp/samba-testbase/
-    - tar cfz samba-testbase.tar.gz /tmp/samba-testbase
-    - ls -la samba-testbase.tar.gz
-    - sha1sum samba-testbase.tar.gz
+    - tar $SAMBA_TESTBASE_TAR_OPTIONS -cf samba-testbase.tar /tmp/samba-testbase
+    - ls -la samba-testbase.tar
+    - sha1sum samba-testbase.tar
 
 .shared_template_test_only:
   extends:
@@ -204,11 +225,15 @@ others:
     - .shared_runner_test
   stage: test_only
   script:
+    # Print the Kerberos version to check we ended up with the right one
+    # in the runner. We do not have configure output to recognize it
+    # otherwise.
+    - if [ -x "$(command -v krb5-config)" ]; then krb5-config --version; fi
     # We unpack the artifacts file created by the .shared_template_build_only
     # run we depend on
-    - ls -la samba-testbase.tar.gz
-    - sha1sum samba-testbase.tar.gz
-    - tar xfz samba-testbase.tar.gz -C /
+    - ls -la samba-testbase.tar
+    - sha1sum samba-testbase.tar
+    - tar $SAMBA_TESTBASE_TAR_OPTIONS -xf samba-testbase.tar -C /
     - diff -u /tmp/samba-testbase/image-sha1sum.txt /sha1sum.txt
     - diff -u /tmp/samba-testbase/commit.txt /tmp/commit.txt
     - mv /tmp/samba-testbase/samba-testbase.acl.dump /tmp/samba-testbase.acl.dump
@@ -231,16 +256,37 @@ samba-def-build:
   needs:
     - job: samba-def-build
       artifacts: true
+    - job: samba-shellcheck
 
 samba-mit-build:
   extends: .shared_template_build_only
+  variables:
+    SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora36}
+  stage: build_first
+
+samba-mit120-build:
+  extends: .shared_template_build_only
+  variables:
+    AUTOBUILD_JOB_NAME: samba-mit-build
+    SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_f36mit120}
   stage: build_first
 
 .needs_samba-mit-build:
   extends: .shared_template_test_only
+  variables:
+    SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora36}
   needs:
     - job: samba-mit-build
       artifacts: true
+    - job: samba-shellcheck
+
+.needs_samba-mit120-build:
+  extends: .shared_template_test_only
+  variables:
+    SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_f36mit120}
+  needs:
+    - job: samba-mit120-build
+      artifacts: true
 
 samba-h5l-build:
   extends: .shared_template_build_only
@@ -251,6 +297,15 @@ samba-h5l-build:
     - job: samba-h5l-build
       artifacts: true
 
+samba-without-smb1-build:
+  extends: .shared_template_build_only
+
+.needs_samba-without-smb1-build:
+  extends: .shared_template_test_only
+  needs:
+    - job: samba-without-smb1-build
+      artifacts: true
+
 samba-nt4-build:
   extends: .shared_template_build_only
 
@@ -274,6 +329,8 @@ samba:
 
 samba-mitkrb5:
   extends: .shared_template
+  variables:
+    SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora36}
 
 samba-minimal-smbd:
   extends: .shared_template
@@ -317,7 +374,7 @@ samba-libs:
 samba-fuzz:
   extends: .shared_template
   variables:
-    SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu1604}
+    SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu2004}
 
 ctdb:
   extends: .shared_template
@@ -331,10 +388,10 @@ samba-ad-dc-ntvfs:
 samba-admem-mit:
   extends: .needs_samba-mit-build
 
-samba-ad-dc-4a-mitkrb5:
+samba-addc-mit-4a:
   extends: .needs_samba-mit-build
 
-samba-ad-dc-4b-mitkrb5:
+samba-addc-mit-4b:
   extends: .needs_samba-mit-build
 
 # This task is run first to ensure we compile before we start the
@@ -342,18 +399,28 @@ samba-ad-dc-4b-mitkrb5:
 samba-fips:
   extends: .shared_template
   variables:
-    SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora33}
+    SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora36}
+
+samba-shellcheck:
+  extends: .shared_template
+  needs:
+  variables:
+    SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora36}
+  stage: build_first
 
 .private_test_only:
   extends: .private_runner_test
   stage: test_private
-  only:
-    variables:
+  rules:
+      # See above, to avoid a duplicate CI on the MR (these rules override the others)
+    - if: $CI_MERGE_REQUEST_ID
+      when: never
+
       # These jobs are only run if the gitlab repo has private runners available.
       # To enable private jobs, you must add the following var and value to
       # your gitlab repo by navigating to:
       # settings -> CI/CD -> Environment variables
-      - $SUPPORT_PRIVATE_TEST == "yes"
+    - if: $SUPPORT_PRIVATE_TEST == "yes"
 
 .needs_samba-def-build-private:
   extends:
@@ -365,11 +432,21 @@ samba-fips:
     - .needs_samba-mit-build
     - .private_test_only
 
+.needs_samba-mit120-build-private:
+  extends:
+    - .needs_samba-mit120-build
+    - .private_test_only
+
 .needs_samba-h5l-build-private:
   extends:
     - .needs_samba-h5l-build
     - .private_test_only
 
+.needs_samba-without-smb1-build-private:
+  extends:
+    - .needs_samba-without-smb1-build
+    - .private_test_only
+
 .needs_samba-nt4-build-private:
   extends:
     - .needs_samba-nt4-build
@@ -383,15 +460,30 @@ samba-fips:
 samba-fileserver:
   extends: .needs_samba-h5l-build-private
 
+samba-fileserver-without-smb1:
+  extends: .needs_samba-without-smb1-build-private
+
+# This is a full build without the AD DC so we test the build with MIT
+# Kerberos from the default system (Ubuntu 18.04 at this stage).
+# Runtime behaviour checked via the ktest (static ccache and keytab)
+# environment
+samba-ktest-mit:
+ extends: .shared_template
+
 samba-ad-dc-1:
   extends: .needs_samba-def-build-private
 
 samba-nt4:
   extends: .needs_samba-nt4-build-private
 
-samba-ad-dc-1-mitkrb5:
+samba-addc-mit-1:
   extends: .needs_samba-mit-build-private
 
+samba-addc-mit120:
+  extends: .needs_samba-mit120-build-private
+  variables:
+    AUTOBUILD_JOB_NAME: samba-addc-mit-1
+
 samba-no-opath1:
   extends: .needs_samba-no-opath-build-private
 
@@ -421,15 +513,16 @@ pages:
     - samba-ctdb
     - samba-ad-dc-ntvfs
     - samba-admem-mit
-    - samba-ad-dc-4a-mitkrb5
-    - samba-ad-dc-4b-mitkrb5
+    - samba-addc-mit-4a
+    - samba-addc-mit-4b
     - samba-ad-back1
     - samba-ad-back2
     - samba-fileserver
+    - samba-fileserver-without-smb1
     - samba-ad-dc-1
     - samba-nt4
     - samba-schemaupgrade
-    - samba-ad-dc-1-mitkrb5
+    - samba-addc-mit-1
     - samba-fips
     - samba-no-opath1
     - samba-no-opath2
@@ -441,7 +534,7 @@ pages:
     - ls -la *.info
     - lcov $(ls *.info | xargs -I{} echo -n "-a {} ") -o all.info
     - ls -la *.info
-    - genhtml all.info --output-directory public --prefix=$(pwd) --title "coverage report for $CI_COMMIT_REF_NAME $CI_COMMIT_SHORT_SHA"
+    - genhtml all.info --ignore-errors source --output-directory public --prefix=$(pwd) --title "coverage report for $CI_COMMIT_REF_NAME $CI_COMMIT_SHORT_SHA"
   artifacts:
     expire_in: 30 days
     paths:
@@ -454,7 +547,7 @@ pages:
 coverity:
   extends: .shared_runner_build_image
   variables:
-    SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora33}
+    SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_opensuse153}
   stage: build
   script:
     - wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_SCAN_TOKEN&project=$COVERITY_SCAN_PROJECT_NAME" -O /tmp/coverity_tool.tgz
@@ -501,36 +594,45 @@ ubuntu1804-samba-o3:
     AUTOBUILD_JOB_NAME: samba-o3
     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu1804}
     SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE: "--enable-coverage"
+  rules:
+    # See above, to avoid a duplicate CI on the MR (these rules override the others)
+    - if: $CI_MERGE_REQUEST_ID
+      when: never
+    # do not run o3 builds (which run a lot of VMs) if told not to
+    # (this uses the same variable as autobuild.py)
+    - if: $AUTOBUILD_SKIP_SAMBA_O3 == "1"
+      when: never
 
 # All other jobs do not want code coverage.
 .samba-o3-template:
   extends: .shared_template
   variables:
     AUTOBUILD_JOB_NAME: samba-o3
-  only:
-    variables:
-      # do not run o3 for coverage since they are using different images
-      - $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == ""
+  rules:
+    # See above, to avoid a duplicate CI on the MR (these rules override the others)
+    - if: $CI_MERGE_REQUEST_ID
+      when: never
+    # do not run o3 builds (which run a lot of VMs) if told not to
+    # (this uses the same variable as autobuild.py)
+    - if: $AUTOBUILD_SKIP_SAMBA_O3 == "1"
+      when: never
+    # do not run o3 for coverage since they are using different images
+    - if: $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == ""
 
 ubuntu2004-samba-o3:
   extends: .samba-o3-template
   variables:
     SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu2004}
 
-debian10-samba-o3:
-  extends: .samba-o3-template
-  variables:
-    SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_debian10}
-
-opensuse151-samba-o3:
+debian11-samba-o3:
   extends: .samba-o3-template
   variables:
-    SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_opensuse151}
+    SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_debian11}
 
-opensuse152-samba-o3:
+opensuse153-samba-o3:
   extends: .samba-o3-template
   variables:
-    SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_opensuse152}
+    SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_opensuse153}
 
 centos7-samba-o3:
   extends: .samba-o3-template
@@ -539,22 +641,17 @@ centos7-samba-o3:
     # Git on CentOS doesn't support shallow git cloning
     GIT_DEPTH: ""
     # We need a newer GnuTLS version on CentOS7
-    PKG_CONFIG_PATH: "/usr/lib64/compat-gnutls34/pkgconfig:/usr/lib64/compat-nettle32/pkgconfig"
-
-centos8-samba-o3:
-  extends: .samba-o3-template
-  variables:
-    SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_centos8}
+    PKG_CONFIG_PATH: "/usr/lib64/compat-gnutls37/pkgconfig:/usr/lib64/compat-nettle32/pkgconfig"
 
-fedora32-samba-o3:
+centos8s-samba-o3:
   extends: .samba-o3-template
   variables:
-    SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora32}
+    SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_centos8s}
 
-fedora33-samba-o3:
+fedora36-samba-o3:
   extends: .samba-o3-template
   variables:
-    SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora33}
+    SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora36}
 
 #
 # Keep the samba-o3 sections at the end ...