3 # Copyright (C) Catalyst.Net Ltd 2019
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 3 of the License, or
8 # (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program. If not, see <http://www.gnu.org/licenses/>.
19 Manage dependencies and bootstrap environments for Samba.
21 Config file for packages and templates.
23 Author: Joe Guo <joeg@catalyst.net.nz>
26 from os.path import abspath, dirname, join
27 HERE = abspath(dirname(__file__))
28 # output dir for rendered files
29 OUT = join(HERE, 'generated-dists')
32 # pkgs with same name in all packaging systems
51 'psmisc', # for pstree in test
55 'sudo', # docker images has no sudo by default
61 # define pkgs for all packaging systems in parallel
62 # make it easier to find missing ones
63 # use latest ubuntu and fedora as defaults
66 # NAME1-dev, NAME2-devel
67 ('lmdb-utils', 'lmdb'),
68 ('mingw-w64', 'mingw64-gcc'),
69 ('nettle-dev', 'nettle-devel'),
70 ('zlib1g-dev', 'zlib-devel'),
71 ('libbsd-dev', 'libbsd-devel'),
72 ('libaio-dev', 'libaio-devel'),
73 ('libarchive-dev', 'libarchive-devel'),
74 ('libblkid-dev', 'libblkid-devel'),
75 ('libcap-dev', 'libcap-devel'),
76 ('libacl1-dev', 'libacl-devel'),
77 ('libattr1-dev', 'libattr-devel'),
79 # libNAME1-dev, NAME2-devel
80 ('libpopt-dev', 'popt-devel'),
81 ('libreadline-dev', 'readline-devel'),
82 ('libjansson-dev', 'jansson-devel'),
83 ('liblmdb-dev', 'lmdb-devel'),
84 ('libncurses5-dev', 'ncurses-devel'),
85 # NOTE: Debian 7+ or Ubuntu 16.04+
86 ('libsystemd-dev', 'systemd-devel'),
87 ('libkrb5-dev', 'krb5-devel'),
88 ('libldap2-dev', 'openldap-devel'),
89 ('libcups2-dev', 'cups-devel'),
90 ('libpam0g-dev', 'pam-devel'),
91 ('libgpgme11-dev', 'gpgme-devel'),
92 # NOTE: Debian 8+ and Ubuntu 14.04+
93 ('libgnutls28-dev', 'gnutls-devel'),
94 ('libtasn1-bin', 'libtasn1-tools'),
95 ('libtasn1-dev', 'libtasn1-devel'),
97 ('uuid-dev', 'libuuid-devel'),
99 ('libavahi-common-dev', 'avahi-devel'),
100 ('libdbus-1-dev', 'dbus-devel'),
101 ('libpcap-dev', 'libpcap-devel'),
102 ('libunwind-dev', 'libunwind-devel'), # for back trace
103 ('libglib2.0-dev', 'glib2-devel'),
104 ('libicu-dev', 'libicu-devel'),
107 # for debian, locales provide locale support with language packs
108 # ubuntu split language packs to language-pack-xx
109 # for centos, glibc-common provide locale support with language packs
110 # fedora split language packs to glibc-langpack-xx
111 ('locales', 'glibc-common'), # required for locale
112 ('language-pack-en', 'glibc-langpack-en'), # we need en_US.UTF-8
113 ('bind9utils', 'bind-utils'),
115 ('xsltproc', 'libxslt'),
118 ('krb5-kdc', 'krb5-server'),
119 ('apt-utils', 'yum-utils'),
120 ('pkg-config', 'pkgconfig'),
121 ('procps', 'procps-ng'), # required for the free cmd in tests
122 ('lsb-release', 'lsb-release'), # we need lsb_relase to show info
123 ('', 'rpcgen'), # required for test
124 # refer: https://fedoraproject.org/wiki/Changes/SunRPCRemoval
125 ('', 'libtirpc-devel'), # for <rpc/rpc.h> header on fedora
126 ('', 'libnsl2-devel'), # for <rpcsvc/yp_prot.h> header on fedora
127 ('', 'rpcsvc-proto-devel'), # for <rpcsvc/rquota.h> header
130 ('python3', 'python3'),
131 ('python3-dev', 'python3-devel'),
133 ('python3-iso8601', ''),
134 ('python3-gpg', 'python3-gpg'), # defaults to ubuntu/fedora latest
135 ('python3-crypto', 'python3-crypto'),
136 ('python3-markdown', 'python3-markdown'),
137 ('python3-matplotlib', ''),
138 ('python3-dnspython', 'python3-dns'),
139 ('python3-pexpect', ''), # for wintest only
141 ('', 'libsemanage-python'),
142 ('', 'policycoreutils-python'),
145 ('libparse-yapp-perl', 'perl-Parse-Yapp'),
146 ('libjson-perl', 'perl-JSON-Parse'),
147 ('perl-modules', ''),
148 ('', 'perl-Archive-Tar'),
149 ('', 'perl-ExtUtils-MakeMaker'),
150 ('', 'perl-Test-Base'),
151 ('', 'perl-generators'),
152 ('', 'perl-interpreter'),
155 ('xfslibs-dev', 'xfsprogs-devel'), # for xfs quota support
156 ('', 'glusterfs-api-devel'),
157 ('glusterfs-common', 'glusterfs-devel'),
158 ('libcephfs-dev', 'libcephfs-devel'),
161 # @ means group for rpm, use fedora as rpm default
162 ('build-essential', '@development-tools'),
164 # rpm has no pkg for docbook-xml
165 ('docbook-xml', 'docbook-dtds'),
166 ('docbook-xsl', 'docbook-style-xsl'),
167 ('', 'keyutils-libs-devel'),
172 DEB_PKGS = COMMON + [pkg for pkg, _ in PKGS if pkg]
173 RPM_PKGS = COMMON + [pkg for _, pkg in PKGS if pkg]
175 GENERATED_MARKER = r"""
177 # This file is generated by 'bootstrap/template.py --render'
178 # See also bootstrap/config.py
188 export DEBIAN_FRONTEND=noninteractive
194 apt-get -y autoremove
206 yum install -y epel-release
207 yum install -y yum-plugin-copr
208 yum copr enable -y sergiomb/SambaAD
216 if [ ! -f /usr/bin/python3 ]; then
217 ln -sf /usr/bin/python3.6 /usr/bin/python3
221 CENTOS8_YUM_BOOTSTRAP = r"""
227 yum install -y dnf-plugins-core
228 yum install -y epel-release
229 yum config-manager --set-enabled PowerTools -y
233 --setopt=install_weak_deps=False \
247 --setopt=install_weak_deps=False \
253 ZYPPER_BOOTSTRAP = r"""
258 zypper --non-interactive refresh
259 zypper --non-interactive update
260 zypper --non-interactive install \
265 zypper --non-interactive clean
267 if [ -f /usr/lib/mit/bin/krb5-config ]; then
268 ln -sf /usr/lib/mit/bin/krb5-config /usr/bin/krb5-config
272 # A generic shell script to setup locale
278 # refer to /usr/share/i18n/locales
280 # refer to /usr/share/i18n/charmaps
282 # locale to generate in /usr/lib/locale
283 # glibc/localedef will normalize UTF-8 to utf8, follow the naming style
284 LOCALE=$INPUTFILE.utf8
286 # if locale is already correct, exit
287 ( locale | grep LC_ALL | grep -i $LOCALE ) && exit 0
289 # if locale not available, generate locale into /usr/lib/locale
290 if ! ( locale --all-locales | grep -i $LOCALE )
292 # no-archive means create its own dir
293 localedef --inputfile $INPUTFILE --charmap $CHARMAP --no-archive $LOCALE
296 # update locale conf and global env file
297 # set both LC_ALL and LANG for safe
299 # update conf for Debian family
300 FILE=/etc/default/locale
303 echo LC_ALL="$LOCALE" > $FILE
304 echo LANG="$LOCALE" >> $FILE
307 # update conf for RedHat family
308 FILE=/etc/locale.conf
311 # LC_ALL is not valid in this file, set LANG only
312 echo LANG="$LOCALE" > $FILE
315 # update global env file
316 FILE=/etc/environment
319 # append LC_ALL if not exist
320 grep LC_ALL $FILE || echo LC_ALL="$LOCALE" >> $FILE
321 # append LANG if not exist
322 grep LANG $FILE || echo LANG="$LOCALE" >> $FILE
331 # pass in with --build-arg while build
333 RUN [ -n $SHA1SUM ] && echo $SHA1SUM > /sha1sum.txt
336 # need root permission, do it before USER samba
337 RUN /tmp/bootstrap.sh && /tmp/locale.sh
339 # if ld.gold exists, force link it to ld
340 RUN set -x; LD=$(which ld); LD_GOLD=$(which ld.gold); test -x $LD_GOLD && ln -sf $LD_GOLD $LD && test -x $LD && echo "$LD is now $LD_GOLD"
342 # make test can not work with root, so we have to create a new user
343 RUN useradd -m -U -s /bin/bash samba && \
344 mkdir -p /etc/sudoers.d && \
345 echo "samba ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/samba
349 # samba tests rely on this
350 ENV USER=samba LC_ALL=en_US.utf8 LANG=en_US.utf8
353 # Vagrantfile snippet for each dist
354 VAGRANTFILE_SNIPPET = r"""
355 config.vm.define "{name}" do |v|
356 v.vm.box = "{vagrant_box}"
357 v.vm.hostname = "{name}"
358 v.vm.provision :shell, path: "{name}/bootstrap.sh"
359 v.vm.provision :shell, path: "{name}/locale.sh"
363 # global Vagrantfile with snippets for all dists
364 VAGRANTFILE_GLOBAL = r"""
367 Vagrant.configure("2") do |config|
368 config.ssh.insert_key = false
370 {vagrantfile_snippets}
378 'docker_image': 'debian:7',
379 'vagrant_box': 'debian/wheezy64',
381 'libgnutls28-dev': 'libgnutls-dev',
382 'libsystemd-dev': '', # not available, remove
383 'lmdb-utils': '', # not available, remove
384 'liblmdb-dev': '', # not available, remove
385 'python-gpg': 'python-gpgme',
386 'python3-gpg': '', # no python3 gpg pkg available, remove
387 'language-pack-en': '', # included in locales
391 'docker_image': 'debian:8',
392 'vagrant_box': 'debian/jessie64',
394 'python-gpg': 'python-gpgme',
395 'python3-gpg': 'python3-gpgme',
396 'language-pack-en': '', # included in locales
400 'docker_image': 'debian:9',
401 'vagrant_box': 'debian/stretch64',
403 'language-pack-en': '', # included in locales
407 'docker_image': 'ubuntu:14.04',
408 'vagrant_box': 'ubuntu/trusty64',
410 'libsystemd-dev': '', # remove
411 'libgnutls28-dev': 'libgnutls-dev',
412 'python-gpg': 'python-gpgme',
413 'python3-gpg': 'python3-gpgme',
414 'lmdb-utils': 'lmdb-utils/trusty-backports',
415 'liblmdb-dev': 'liblmdb-dev/trusty-backports',
416 'libunwind-dev': 'libunwind8-dev',
417 'glusterfs-common': '',
422 'docker_image': 'ubuntu:16.04',
423 'vagrant_box': 'ubuntu/xenial64',
425 'python-gpg': 'python-gpgme',
426 'python3-gpg': 'python3-gpgme',
427 'glusterfs-common': '',
432 'docker_image': 'ubuntu:18.04',
433 'vagrant_box': 'ubuntu/bionic64',
440 'docker_image': 'centos:6',
441 'vagrant_box': 'centos/6',
442 'bootstrap': YUM_BOOTSTRAP,
444 'lsb-release': 'redhat-lsb',
445 'python3': 'python36',
446 'python3-devel': 'python36-devel',
447 'python2-gpg': 'pygpgme',
448 'python3-gpg': '', # no python3-gpg yet
449 '@development-tools': '"@Development Tools"', # add quotes
450 'glibc-langpack-en': '', # included in glibc-common
451 'glibc-locale-source': '', # included in glibc-common
452 'procps-ng': 'procps', # centos6 still use old name
453 # update perl core modules on centos
454 # fix: Can't locate Archive/Tar.pm in @INC
456 'rpcsvc-proto-devel': '',
457 'glusterfs-api-devel': '',
458 'glusterfs-devel': '',
459 'libcephfs-devel': '',
463 'docker_image': 'centos:7',
464 'vagrant_box': 'centos/7',
465 'bootstrap': YUM_BOOTSTRAP,
467 'lsb-release': 'redhat-lsb',
468 'python3': 'python36',
469 'python3-crypto': 'python36-crypto',
470 'python3-devel': 'python36-devel',
471 'python3-dns': 'python36-dns',
472 'python3-gpg': 'python36-gpg',
473 'python3-iso8601' : 'python36-iso8601',
474 'python3-markdown': 'python36-markdown',
475 # although python36-devel is available
476 # after epel-release installed
477 # however, all other python3 pkgs are still python36-ish
478 'python2-gpg': 'pygpgme',
479 'python3-gpg': '', # no python3-gpg yet
480 '@development-tools': '"@Development Tools"', # add quotes
481 'glibc-langpack-en': '', # included in glibc-common
482 'glibc-locale-source': '', # included in glibc-common
483 # update perl core modules on centos
484 # fix: Can't locate Archive/Tar.pm in @INC
486 'rpcsvc-proto-devel': '',
487 'glusterfs-api-devel': '',
488 'glusterfs-devel': '',
489 'libcephfs-devel': '',
490 'gnutls-devel': 'compat-gnutls34-devel',
494 'docker_image': 'centos:8',
495 'vagrant_box': 'centos/8',
496 'bootstrap': CENTOS8_YUM_BOOTSTRAP,
498 'lsb-release': 'redhat-lsb',
499 '@development-tools': '"@Development Tools"', # add quotes
500 'libsemanage-python': 'python3-libsemanage',
501 'lcov': '', # does not exist
502 'perl-JSON-Parse': '', # does not exist?
503 'perl-Test-Base': 'perl-Test-Simple',
504 'policycoreutils-python': 'python3-policycoreutils',
505 'python3-crypto': '',
506 'quota-devel': '', # FIXME: Add me back, once available!
510 'docker_image': 'fedora:28',
511 'vagrant_box': 'fedora/28-cloud-base',
512 'bootstrap': DNF_BOOTSTRAP,
514 'lsb-release': 'redhat-lsb',
518 'docker_image': 'fedora:29',
519 'vagrant_box': 'fedora/29-cloud-base',
520 'bootstrap': DNF_BOOTSTRAP,
522 'lsb-release': 'redhat-lsb',
526 'docker_image': 'fedora:30',
527 'vagrant_box': 'fedora/30-cloud-base',
528 'bootstrap': DNF_BOOTSTRAP,
530 'lsb-release': 'redhat-lsb',
534 'docker_image': 'opensuse/leap:15.0',
535 'vagrant_box': 'opensuse/openSUSE-15.0-x86_64',
536 'bootstrap': ZYPPER_BOOTSTRAP,
538 '@development-tools': '',
539 'dbus-devel': 'dbus-1-devel',
540 'docbook-style-xsl': 'docbook-xsl-stylesheets',
541 'glibc-common': 'glibc-locale',
542 'glibc-locale-source': 'glibc-i18ndata',
543 'glibc-langpack-en': '',
544 'jansson-devel': 'libjansson-devel',
545 'keyutils-libs-devel': 'keyutils-devel',
546 'krb5-workstation': 'krb5-client',
547 'libnsl2-devel': 'libnsl-devel',
548 'libsemanage-python': 'python2-semanage',
549 'nettle-devel': 'libnettle-devel',
550 'openldap-devel': 'openldap2-devel',
551 'perl-Archive-Tar': 'perl-Archive-Tar-Wrapper',
552 'perl-JSON-Parse': 'perl-JSON-XS',
553 'perl-generators': '',
554 'perl-interpreter': '',
555 'procps-ng': 'procps',
556 'python-dns': 'python2-dnspython',
557 'python3-crypto': 'python3-pycrypto',
558 'python3-dns': 'python3-dnspython',
559 'python3-markdown': 'python3-Markdown',
561 'glusterfs-api-devel': '',
562 'libtasn1-tools': '', # asn1Parser is part of libtasn1
563 'mingw64-gcc': '', # doesn't exist
567 'docker_image': 'opensuse/leap:15.1',
568 'vagrant_box': 'opensuse/openSUSE-15.1-x86_64',
569 'bootstrap': ZYPPER_BOOTSTRAP,
571 '@development-tools': '',
572 'dbus-devel': 'dbus-1-devel',
573 'docbook-style-xsl': 'docbook-xsl-stylesheets',
574 'glibc-common': 'glibc-locale',
575 'glibc-locale-source': 'glibc-i18ndata',
576 'glibc-langpack-en': '',
577 'jansson-devel': 'libjansson-devel',
578 'keyutils-libs-devel': 'keyutils-devel',
579 'krb5-workstation': 'krb5-client',
580 'libnsl2-devel': 'libnsl-devel',
581 'libsemanage-python': 'python2-semanage',
582 'nettle-devel': 'libnettle-devel',
583 'openldap-devel': 'openldap2-devel',
584 'perl-Archive-Tar': 'perl-Archive-Tar-Wrapper',
585 'perl-JSON-Parse': 'perl-JSON-XS',
586 'perl-generators': '',
587 'perl-interpreter': '',
588 'procps-ng': 'procps',
589 'python-dns': 'python2-dnspython',
590 'python3-crypto': 'python3-pycrypto',
591 'python3-dns': 'python3-dnspython',
592 'python3-markdown': 'python3-Markdown',
594 'glusterfs-api-devel': '',
595 'libtasn1-tools': '', # asn1Parser is part of libtasn1
596 'mingw64-gcc': '', # doesn't exist
605 'bootstrap': APT_BOOTSTRAP, # family default
613 'bootstrap': YUM_BOOTSTRAP, # family default
624 def expand_family_dists(family):
626 for name, config in family['dists'].items():
627 config = config.copy()
628 config['name'] = name
629 config['home'] = join(OUT, name)
630 config['family'] = family['name']
631 config['GENERATED_MARKER'] = GENERATED_MARKER
633 # replace dist specific pkgs
634 replace = config.get('replace', {})
636 for pkg in family['pkgs']:
637 pkg = replace.get(pkg, pkg) # replace if exists or get self
642 lines = [' - {}'.format(pkg) for pkg in pkgs]
643 config['packages.yml'] = YML_HEADER.lstrip() + os.linesep.join(lines)
645 sep = ' \\' + os.linesep + ' '
646 config['pkgs'] = sep.join(pkgs)
648 # get dist bootstrap template or fall back to family default
649 bootstrap_template = config.get('bootstrap', family['bootstrap'])
650 config['bootstrap.sh'] = bootstrap_template.format(**config).strip()
651 config['locale.sh'] = LOCALE_SETUP.format(**config).strip()
653 config['Dockerfile'] = DOCKERFILE.format(**config).strip()
654 # keep the indent, no strip
655 config['vagrantfile_snippet'] = VAGRANTFILE_SNIPPET.format(**config)
661 # expanded config for dists
662 DEB_DISTS_EXP = expand_family_dists(DEB_FAMILY)
663 RPM_DISTS_EXP = expand_family_dists(RPM_FAMILY)
665 # assemble all together
667 DISTS.update(DEB_DISTS_EXP)
668 DISTS.update(RPM_DISTS_EXP)
671 def render_vagrantfile(dists):
673 Render all snippets for each dist into global Vagrantfile.
675 Vagrant supports multiple vms in one Vagrantfile.
676 This make it easier to manage the fleet, e.g:
678 start all: vagrant up
679 start one: vagrant up ubuntu1804
681 All other commands apply to above syntax, e.g.: status, destroy, provision
683 # sort dists by name and put all vagrantfile snippets together
685 dists[dist]['vagrantfile_snippet']
686 for dist in sorted(dists.keys())]
688 return VAGRANTFILE_GLOBAL.format(
689 vagrantfile_snippets=''.join(snippets),
690 GENERATED_MARKER=GENERATED_MARKER
694 VAGRANTFILE = render_vagrantfile(DISTS)
697 # data we need to expose
698 __all__ = ['DISTS', 'VAGRANTFILE', 'OUT']