bootstrap: Bring back a Ubuntu 16.04 build but just for the samba-fuzz task

This is needed to restore oss-fuzz support, as this uses the Ubuntu 16.04 package list
because all the docker images provided start with a Ubuntu 16.04 base.

REF: https://github.com/google/oss-fuzz/issues/3505
REF: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=21189

Signed-off-by: Andrew Bartlett <abartlet@samba.org>
Reviewed-by: Gary Lockyer <gary@catalyst.net.nz>

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6114ccced99ce70871ee9a71a30aa0cc0a910b91..05f410bde3da584df0399537c9b7a66932f85c21 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -22,7 +22,7 @@ variables:
   # Set this to the contents of bootstrap/sha1sum.txt
   # which is generated by bootstrap/template.py --render
   #
-  SAMBA_CI_CONTAINER_TAG: 2b0275df23424240774afcd61fae8abed8663996
+  SAMBA_CI_CONTAINER_TAG: 6bb2eeaf8203467d9a93a722071b0f081027410e
   #
   # We use the ubuntu1804 image as default as
   # it matches what we have on sn-devel-184.
@@ -154,6 +154,7 @@ samba-static:
 
 samba-fuzz:
   extends: .shared_template
+  image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-ubuntu1604:${SAMBA_CI_CONTAINER_TAG}
 
 ctdb:
   extends: .shared_template

diff --git a/bootstrap/.gitlab-ci.yml b/bootstrap/.gitlab-ci.yml
index 4e52da09dcc5dd7ba7d3457d83a216ef46216300..ecd9f4d4223668b66126c083b6baf0142080b35a 100644 (file)
--- a/bootstrap/.gitlab-ci.yml
+++ b/bootstrap/.gitlab-ci.yml
@@ -9,6 +9,7 @@ services:
     - gce
   variables:
     SAMBA_CI_IS_BROKEN_IMAGE: "no"
+    SAMBA_CI_TEST_JOB: "samba-o3"
   before_script:
     # Ensure we are generating correct the container
     - uname -a
@@ -40,9 +41,9 @@ services:
     docker run --volume $(pwd):${samba_repo_root} --workdir ${samba_repo_root} ${ci_image_name} \
         bootstrap/template.py --sha1sum > /tmp/sha1sum-template.txt
     diff -u bootstrap/sha1sum.txt /tmp/sha1sum-template.txt
-    # run smoke test with samba-o3
+    # run smoke test with samba-o3 or samba-fuzz
     docker run --volume $(pwd):${samba_repo_root} --workdir ${samba_repo_root} ${ci_image_name} \
-        /bin/bash -c "sudo chown -R samba:samba ./** && export PKG_CONFIG_PATH=/usr/lib64/compat-gnutls34/pkgconfig:/usr/lib64/compat-nettle32/pkgconfig && script/autobuild.py samba-o3 --verbose --nocleanup --keeplogs --tail --testbase /tmp/samba-testbase"
+        /bin/bash -c "sudo chown -R samba:samba ./** && export PKG_CONFIG_PATH=/usr/lib64/compat-gnutls34/pkgconfig:/usr/lib64/compat-nettle32/pkgconfig && script/autobuild.py ${SAMBA_CI_TEST_JOB} --verbose --nocleanup --keeplogs --tail --testbase /tmp/samba-testbase"
     docker tag ${ci_image_name} ${ci_image_path}:${SAMBA_CI_CONTAINER_TAG}
     docker tag ${ci_image_name} ${ci_image_path}:${timestamp_tag}
     # We build all images, but only upload is it's not marked as broken
@@ -83,6 +84,12 @@ services:
       #
       - $SAMBA_CI_REBUILD_BROKEN_IMAGES == "yes"
 
+# This is ONLY for oss-fuzz, so we test a fuzz build not a real one
+ubuntu1604:
+  extends: .build_image_template
+  variables:
+    SAMBA_CI_TEST_JOB: "samba-fuzz"
+
 ubuntu1804:
   extends: .build_image_template
 

diff --git a/bootstrap/config.py b/bootstrap/config.py
index 19a2cf08b9d3d465e568c7511b3cf552a72a8f2a..ff9bb150672185963b2f5290960a990d1a9e4480 100644 (file)
--- a/bootstrap/config.py
+++ b/bootstrap/config.py
@@ -382,6 +382,17 @@ DEB_DISTS = {
             'liburing-dev': '',   # not available
         }
     },
+    'ubuntu1604': {
+        'docker_image': 'ubuntu:16.04',
+        'vagrant_box': 'ubuntu/xenial64',
+        'replace': {
+            'python-gpg': 'python-gpgme',
+            'python3-gpg': 'python3-gpgme',
+            'glusterfs-common': '',
+            'libcephfs-dev': '',
+            'liburing-dev': '',   # not available
+        }
+    },
     'ubuntu1804': {
         'docker_image': 'ubuntu:18.04',
         'vagrant_box': 'ubuntu/bionic64',

diff --git a/bootstrap/generated-dists/Vagrantfile b/bootstrap/generated-dists/Vagrantfile
index 091c65488cb0237cef8f387fdc3998411e1cba5f..47c58d5a87b4321009145862fa365c34ba5f57e7 100644 (file)
--- a/bootstrap/generated-dists/Vagrantfile
+++ b/bootstrap/generated-dists/Vagrantfile
@@ -66,6 +66,13 @@ Vagrant.configure("2") do |config|
         v.vm.provision :shell, path: "opensuse151/locale.sh"
     end
 
+    config.vm.define "ubuntu1604" do |v|
+        v.vm.box = "ubuntu/xenial64"
+        v.vm.hostname = "ubuntu1604"
+        v.vm.provision :shell, path: "ubuntu1604/bootstrap.sh"
+        v.vm.provision :shell, path: "ubuntu1604/locale.sh"
+    end
+
     config.vm.define "ubuntu1804" do |v|
         v.vm.box = "ubuntu/bionic64"
         v.vm.hostname = "ubuntu1804"

diff --git a/bootstrap/generated-dists/ubuntu1604/Dockerfile b/bootstrap/generated-dists/ubuntu1604/Dockerfile
new file mode 100644 (file)
index 0000000..93001fc
--- /dev/null
+++ b/bootstrap/generated-dists/ubuntu1604/Dockerfile
@@ -0,0 +1,27 @@
+#
+# This file is generated by 'bootstrap/template.py --render'
+# See also bootstrap/config.py
+#
+
+FROM ubuntu:16.04
+
+# pass in with --build-arg while build
+ARG SHA1SUM
+RUN [ -n $SHA1SUM ] && echo $SHA1SUM > /sha1sum.txt
+
+ADD *.sh /tmp/
+# need root permission, do it before USER samba
+RUN /tmp/bootstrap.sh && /tmp/locale.sh
+
+# if ld.gold exists, force link it to ld
+RUN set -x; LD=$(which ld); LD_GOLD=$(which ld.gold); test -x $LD_GOLD && ln -sf $LD_GOLD $LD && test -x $LD && echo "$LD is now $LD_GOLD"
+
+# make test can not work with root, so we have to create a new user
+RUN useradd -m -U -s /bin/bash samba && \
+    mkdir -p /etc/sudoers.d && \
+    echo "samba ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/samba
+
+USER samba
+WORKDIR /home/samba
+# samba tests rely on this
+ENV USER=samba LC_ALL=en_US.utf8 LANG=en_US.utf8
\ No newline at end of file

diff --git a/bootstrap/generated-dists/ubuntu1604/bootstrap.sh b/bootstrap/generated-dists/ubuntu1604/bootstrap.sh
new file mode 100755 (executable)
index 0000000..a8f4776
--- /dev/null
+++ b/bootstrap/generated-dists/ubuntu1604/bootstrap.sh
@@ -0,0 +1,104 @@
+#!/bin/bash
+
+#
+# This file is generated by 'bootstrap/template.py --render'
+# See also bootstrap/config.py
+#
+
+set -xueo pipefail
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get -y update
+
+apt-get -y install \
+    acl \
+    apt-utils \
+    attr \
+    autoconf \
+    bind9utils \
+    binutils \
+    bison \
+    build-essential \
+    chrpath \
+    curl \
+    debhelper \
+    dnsutils \
+    docbook-xml \
+    docbook-xsl \
+    flex \
+    gcc \
+    gdb \
+    git \
+    gzip \
+    heimdal-multidev \
+    hostname \
+    htop \
+    krb5-config \
+    krb5-kdc \
+    krb5-user \
+    language-pack-en \
+    lcov \
+    libacl1-dev \
+    libarchive-dev \
+    libattr1-dev \
+    libavahi-common-dev \
+    libblkid-dev \
+    libbsd-dev \
+    libcap-dev \
+    libcups2-dev \
+    libdbus-1-dev \
+    libglib2.0-dev \
+    libgnutls28-dev \
+    libgpgme11-dev \
+    libicu-dev \
+    libjansson-dev \
+    libjs-jquery \
+    libjson-perl \
+    libkrb5-dev \
+    libldap2-dev \
+    liblmdb-dev \
+    libncurses5-dev \
+    libpam0g-dev \
+    libparse-yapp-perl \
+    libpcap-dev \
+    libpopt-dev \
+    libreadline-dev \
+    libsystemd-dev \
+    libtasn1-bin \
+    libtasn1-dev \
+    libunwind-dev \
+    lmdb-utils \
+    locales \
+    lsb-release \
+    make \
+    mawk \
+    mingw-w64 \
+    patch \
+    perl \
+    perl-modules \
+    pkg-config \
+    procps \
+    psmisc \
+    python3 \
+    python3-dbg \
+    python3-dev \
+    python3-dnspython \
+    python3-gpgme \
+    python3-iso8601 \
+    python3-markdown \
+    python3-matplotlib \
+    python3-pexpect \
+    rng-tools \
+    rsync \
+    sed \
+    sudo \
+    tar \
+    tree \
+    uuid-dev \
+    xfslibs-dev \
+    xsltproc \
+    zlib1g-dev
+
+apt-get -y autoremove
+apt-get -y autoclean
+apt-get -y clean
\ No newline at end of file

diff --git a/bootstrap/generated-dists/ubuntu1604/locale.sh b/bootstrap/generated-dists/ubuntu1604/locale.sh
new file mode 100755 (executable)
index 0000000..cc64e18
--- /dev/null
+++ b/bootstrap/generated-dists/ubuntu1604/locale.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+#
+# This file is generated by 'bootstrap/template.py --render'
+# See also bootstrap/config.py
+#
+
+set -xueo pipefail
+
+# refer to /usr/share/i18n/locales
+INPUTFILE=en_US
+# refer to /usr/share/i18n/charmaps
+CHARMAP=UTF-8
+# locale to generate in /usr/lib/locale
+# glibc/localedef will normalize UTF-8 to utf8, follow the naming style
+LOCALE=$INPUTFILE.utf8
+
+# if locale is already correct, exit
+( locale | grep LC_ALL | grep -i $LOCALE ) && exit 0
+
+# if locale not available, generate locale into /usr/lib/locale
+if ! ( locale --all-locales | grep -i $LOCALE )
+then
+    # no-archive means create its own dir
+    localedef --inputfile $INPUTFILE --charmap $CHARMAP --no-archive $LOCALE
+fi
+
+# update locale conf and global env file
+# set both LC_ALL and LANG for safe
+
+# update conf for Debian family
+FILE=/etc/default/locale
+if [ -f $FILE ]
+then
+    echo LC_ALL="$LOCALE" > $FILE
+    echo LANG="$LOCALE" >> $FILE
+fi
+
+# update conf for RedHat family
+FILE=/etc/locale.conf
+if [ -f $FILE ]
+then
+    # LC_ALL is not valid in this file, set LANG only
+    echo LANG="$LOCALE" > $FILE
+fi
+
+# update global env file
+FILE=/etc/environment
+if [ -f $FILE ]
+then
+    # append LC_ALL if not exist
+    grep LC_ALL $FILE || echo LC_ALL="$LOCALE" >> $FILE
+    # append LANG if not exist
+    grep LANG $FILE || echo LANG="$LOCALE" >> $FILE
+fi
\ No newline at end of file

diff --git a/bootstrap/generated-dists/ubuntu1604/packages.yml b/bootstrap/generated-dists/ubuntu1604/packages.yml
new file mode 100644 (file)
index 0000000..c3cd9af
--- /dev/null
+++ b/bootstrap/generated-dists/ubuntu1604/packages.yml
@@ -0,0 +1,89 @@
+---
+packages:
+  - acl
+  - apt-utils
+  - attr
+  - autoconf
+  - bind9utils
+  - binutils
+  - bison
+  - build-essential
+  - chrpath
+  - curl
+  - debhelper
+  - dnsutils
+  - docbook-xml
+  - docbook-xsl
+  - flex
+  - gcc
+  - gdb
+  - git
+  - gzip
+  - heimdal-multidev
+  - hostname
+  - htop
+  - krb5-config
+  - krb5-kdc
+  - krb5-user
+  - language-pack-en
+  - lcov
+  - libacl1-dev
+  - libarchive-dev
+  - libattr1-dev
+  - libavahi-common-dev
+  - libblkid-dev
+  - libbsd-dev
+  - libcap-dev
+  - libcups2-dev
+  - libdbus-1-dev
+  - libglib2.0-dev
+  - libgnutls28-dev
+  - libgpgme11-dev
+  - libicu-dev
+  - libjansson-dev
+  - libjs-jquery
+  - libjson-perl
+  - libkrb5-dev
+  - libldap2-dev
+  - liblmdb-dev
+  - libncurses5-dev
+  - libpam0g-dev
+  - libparse-yapp-perl
+  - libpcap-dev
+  - libpopt-dev
+  - libreadline-dev
+  - libsystemd-dev
+  - libtasn1-bin
+  - libtasn1-dev
+  - libunwind-dev
+  - lmdb-utils
+  - locales
+  - lsb-release
+  - make
+  - mawk
+  - mingw-w64
+  - patch
+  - perl
+  - perl-modules
+  - pkg-config
+  - procps
+  - psmisc
+  - python3
+  - python3-dbg
+  - python3-dev
+  - python3-dnspython
+  - python3-gpgme
+  - python3-iso8601
+  - python3-markdown
+  - python3-matplotlib
+  - python3-pexpect
+  - rng-tools
+  - rsync
+  - sed
+  - sudo
+  - tar
+  - tree
+  - uuid-dev
+  - xfslibs-dev
+  - xsltproc
+  - zlib1g-dev
\ No newline at end of file

diff --git a/bootstrap/sha1sum.txt b/bootstrap/sha1sum.txt
index 7344075d11de3634b9cb564991c4ece6c247f8a9..1c9d01d5e7d3dccedbb05fbc561b5770c6429cfb 100644 (file)
--- a/bootstrap/sha1sum.txt
+++ b/bootstrap/sha1sum.txt
@@ -1 +1 @@
-2b0275df23424240774afcd61fae8abed8663996
+6bb2eeaf8203467d9a93a722071b0f081027410e