3 # Copyright (C) Catalyst.Net Ltd 2019
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 3 of the License, or
8 # (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program. If not, see <http://www.gnu.org/licenses/>.
19 Manage dependencies and bootstrap environments for Samba.
21 Config file for packages and templates.
23 Author: Joe Guo <joeg@catalyst.net.nz>
26 from os.path import abspath, dirname, join
27 HERE = abspath(dirname(__file__))
28 # output dir for rendered files
29 OUT = join(HERE, 'dists')
32 # pkgs with same name in all packaging systems
46 'psmisc', # for pstree in test
49 'sudo', # docker images has no sudo by default
54 # define pkgs for all packaging systems in parallel
55 # make it easier to find missing ones
56 # use latest ubuntu and fedora as defaults
59 # NAME1-dev, NAME2-devel
60 ('lmdb-utils', 'lmdb-devel'),
61 ('nettle-dev', 'nettle-devel'),
62 ('zlib1g-dev', 'zlib-devel'),
63 ('libbsd-dev', 'libbsd-devel'),
64 ('libaio-dev', 'libaio-devel'),
65 ('libarchive-dev', 'libarchive-devel'),
66 ('libblkid-dev', 'libblkid-devel'),
67 ('libcap-dev', 'libcap-devel'),
68 ('libacl1-dev', 'libacl-devel'),
69 ('libattr1-dev', 'libattr-devel'),
71 # libNAME1-dev, NAME2-devel
72 ('libpopt-dev', 'popt-devel'),
73 ('libreadline-dev', 'readline-devel'),
74 ('libjansson-dev', 'jansson-devel'),
75 ('liblmdb-dev', 'lmdb-devel'),
76 ('libncurses5-dev', 'ncurses-devel'),
77 # NOTE: Debian 7+ or Ubuntu 16.04+
78 ('libsystemd-dev', 'systemd-devel'),
79 ('libkrb5-dev', 'krb5-devel'),
80 ('libldap2-dev', 'openldap-devel'),
81 ('libcups2-dev', 'cups-devel'),
82 ('libpam0g-dev', 'pam-devel'),
83 ('libgpgme11-dev', 'gpgme-devel'),
84 # NOTE: Debian 8+ and Ubuntu 14.04+
85 ('libgnutls28-dev', 'gnutls-devel'),
87 ('libtasn1-dev', 'libtasn1-devel'),
89 ('uuid-dev', 'libuuid-devel'),
91 ('libavahi-common-dev', 'avahi-devel'),
92 ('libdbus-1-dev', 'dbus-devel'),
93 ('libpcap-dev', 'libpcap-devel'),
94 ('libunwind-dev', 'libunwind-devel'), # for back trace
97 # for debian, locales provide locale support with language packs
98 # ubuntu split language packs to language-pack-xx
99 # for centos, glibc-common provide locale support with language packs
100 # fedora split language packs to glibc-langpack-xx
101 ('locales', 'glibc-common'), # required for locale
102 ('language-pack-en', 'glibc-langpack-en'), # we need en_US.UTF-8
103 ('bind9utils', 'bind-utils'),
105 ('xsltproc', 'libxslt'),
109 ('apt-utils', 'yum-utils'),
110 ('pkg-config', 'pkgconfig'),
111 ('procps', 'procps-ng'), # required for the free cmd in tests
112 ('lsb-release', 'redhat-lsb'), # we need lsb_relase to show info
113 ('', 'rpcgen'), # required for test
114 # refer: https://fedoraproject.org/wiki/Changes/SunRPCRemoval
115 ('', 'libtirpc-devel'), # for <rpc/rpc.h> header on fedora
116 ('', 'libnsl2-devel'), # for <rpcsvc/yp_prot.h> header on fedora
120 ('python-dev', 'python-devel'),
122 ('python-iso8601', ''),
123 ('python-gpg', 'python2-gpg'), # defaults to ubuntu/fedora latest
124 ('python-crypto', 'python-crypto'),
125 ('python-markdown', 'python-markdown'),
126 ('python-dnspython', 'python-dns'),
127 ('python-pexpect', ''), # for wintest only
129 ('python3-dev', 'python3-devel'),
131 ('python3-iso8601', ''),
132 ('python3-gpg', 'python3-gpg'), # defaults to ubuntu/fedora latest
133 ('python3-crypto', 'python3-crypto'),
134 ('python3-markdown', 'python3-markdown'),
135 ('python3-matplotlib', ''),
136 ('python3-dnspython', 'python3-dns'),
137 ('python3-pexpect', ''), # for wintest only
139 ('', 'libsemanage-python'),
140 ('', 'policycoreutils-python'),
143 ('libparse-yapp-perl', 'perl-Parse-Yapp'),
144 ('libjson-perl', 'perl-JSON-Parse'),
145 ('perl-modules', ''),
146 ('', 'perl-Archive-Tar'),
147 ('', 'perl-ExtUtils-MakeMaker'),
148 ('', 'perl-Test-Base'),
149 ('', 'perl-generators'),
150 ('', 'perl-interpreter'),
153 # @ means group for rpm, use fedora as rpm default
154 ('build-essential', '@development-tools'),
156 # rpm has no pkg for docbook-xml
157 ('docbook-xml', 'docbook-dtds'),
158 ('docbook-xsl', 'docbook-style-xsl'),
160 ('', 'keyutils-libs-devel'),
165 DEB_PKGS = COMMON + [pkg for pkg, _ in PKGS if pkg]
166 RPM_PKGS = COMMON + [pkg for _, pkg in PKGS if pkg]
173 export DEBIAN_FRONTEND=noninteractive
179 apt-get -y autoremove
190 yum -y -q install epel-release
193 yum -y -q --verbose install \
206 dnf -y -q --verbose install \
213 # A generic shell script to setup locale
218 # refer to /usr/share/i18n/locales
220 # refer to /usr/share/i18n/charmaps
222 # locale to generate in /usr/lib/locale
223 # glibc/localedef will normalize UTF-8 to utf8, follow the naming style
224 LOCALE=$INPUTFILE.utf8
226 # if locale is already correct, exit
227 ( locale | grep LC_ALL | grep -i $LOCALE ) && exit 0
229 # if locale not available, generate locale into /usr/lib/locale
230 if ! ( locale --all-locales | grep -i $LOCALE )
232 # no-archive means create its own dir
233 localedef --inputfile $INPUTFILE --charmap $CHARMAP --no-archive $LOCALE
236 # update locale conf and global env file
237 # set both LC_ALL and LANG for safe
239 # update conf for Debian family
240 FILE=/etc/default/locale
243 echo LC_ALL="$LOCALE" > $FILE
244 echo LANG="$LOCALE" >> $FILE
247 # update conf for RedHat family
248 FILE=/etc/locale.conf
251 # LC_ALL is not valid in this file, set LANG only
252 echo LANG="$LOCALE" > $FILE
255 # update global env file
256 FILE=/etc/environment
259 # append LC_ALL if not exist
260 grep LC_ALL $FILE || echo LC_ALL="$LOCALE" >> $FILE
261 # append LANG if not exist
262 grep LANG $FILE || echo LANG="$LOCALE" >> $FILE
270 # pass in with --build-arg while build
272 RUN [ -n $SHA1SUM ] && echo $SHA1SUM > /sha1sum.txt
275 # need root permission, do it before USER samba
276 RUN /tmp/bootstrap.sh && /tmp/locale.sh
278 # make test can not work with root, so we have to create a new user
279 RUN useradd -m -s /bin/bash samba && \
280 mkdir -p /etc/sudoers.d && \
281 echo "samba ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/samba
285 # samba tests rely on this
286 ENV USER=samba LC_ALL=en_US.utf8 LANG=en_US.utf8
289 # Vagrantfile snippet for each dist
290 VAGRANTFILE_SNIPPET = r"""
291 config.vm.define "{name}" do |v|
292 v.vm.box = "{vagrant_box}"
293 v.vm.hostname = "{name}"
294 v.vm.provision :shell, path: "{name}/bootstrap.sh"
295 v.vm.provision :shell, path: "{name}/locale.sh"
299 # global Vagrantfile with snippets for all dists
300 VAGRANTFILE_GLOBAL = r"""
301 Vagrant.configure("2") do |config|
302 config.ssh.insert_key = false
304 {vagrantfile_snippets}
312 'docker_image': 'debian:7',
313 'vagrant_box': 'debian/wheezy64',
315 'libgnutls28-dev': 'libgnutls-dev',
316 'libsystemd-dev': '', # not available, remove
317 'lmdb-utils': '', # not available, remove
318 'liblmdb-dev': '', # not available, remove
319 'python-gpg': 'python-gpgme',
320 'python3-gpg': '', # no python3 gpg pkg available, remove
321 'language-pack-en': '', # included in locales
325 'docker_image': 'debian:8',
326 'vagrant_box': 'debian/jessie64',
328 'python-gpg': 'python-gpgme',
329 'python3-gpg': 'python3-gpgme',
330 'language-pack-en': '', # included in locales
334 'docker_image': 'debian:9',
335 'vagrant_box': 'debian/stretch64',
337 'language-pack-en': '', # included in locales
341 'docker_image': 'ubuntu:14.04',
342 'vagrant_box': 'ubuntu/trusty64',
344 'libsystemd-dev': '', # remove
345 'libgnutls28-dev': 'libgnutls-dev',
346 'python-gpg': 'python-gpgme',
347 'python3-gpg': 'python3-gpgme',
348 'lmdb-utils': 'lmdb-utils/trusty-backports',
349 'liblmdb-dev': 'liblmdb-dev/trusty-backports',
350 'libunwind-dev': 'libunwind8-dev',
354 'docker_image': 'ubuntu:16.04',
355 'vagrant_box': 'ubuntu/xenial64',
357 'python-gpg': 'python-gpgme',
358 'python3-gpg': 'python3-gpgme',
362 'docker_image': 'ubuntu:18.04',
363 'vagrant_box': 'ubuntu/bionic64',
370 'docker_image': 'centos:6',
371 'vagrant_box': 'centos/6',
372 'bootstrap': YUM_BOOTSTRAP,
374 'python3-devel': 'python34-devel',
375 'python2-gpg': 'pygpgme',
376 'python3-gpg': '', # no python3-gpg yet
377 '@development-tools': '"@Development Tools"', # add quotes
378 'glibc-langpack-en': '', # included in glibc-common
379 'glibc-locale-source': '', # included in glibc-common
380 'procps-ng': 'procps', # centos6 still use old name
381 # update perl core modules on centos
382 # fix: Can't locate Archive/Tar.pm in @INC
387 'docker_image': 'centos:7',
388 'vagrant_box': 'centos/7',
389 'bootstrap': YUM_BOOTSTRAP,
391 'python3-devel': 'python34-devel',
392 # although python36-devel is available
393 # after epel-release installed
394 # however, all other python3 pkgs are still python34-ish
395 'python2-gpg': 'pygpgme',
396 'python3-gpg': '', # no python3-gpg yet
397 '@development-tools': '"@Development Tools"', # add quotes
398 'glibc-langpack-en': '', # included in glibc-common
399 'glibc-locale-source': '', # included in glibc-common
400 # update perl core modules on centos
401 # fix: Can't locate Archive/Tar.pm in @INC
406 'docker_image': 'fedora:28',
407 'vagrant_box': 'fedora/28-cloud-base',
408 'bootstrap': DNF_BOOTSTRAP,
411 'docker_image': 'fedora:29',
412 'vagrant_box': 'fedora/29-cloud-base',
413 'bootstrap': DNF_BOOTSTRAP,
421 'bootstrap': APT_BOOTSTRAP, # family default
429 'bootstrap': YUM_BOOTSTRAP, # family default
440 def expand_family_dists(family):
442 for name, config in family['dists'].items():
443 config = config.copy()
444 config['name'] = name
445 config['home'] = join(OUT, name)
446 config['family'] = family['name']
448 # replace dist specific pkgs
449 replace = config.get('replace', {})
451 for pkg in family['pkgs']:
452 pkg = replace.get(pkg, pkg) # replace if exists or get self
457 lines = [' - {}'.format(pkg) for pkg in pkgs]
458 config['packages.yml'] = YML_HEADER.lstrip() + os.linesep.join(lines)
460 sep = ' \\' + os.linesep + ' '
461 config['pkgs'] = sep.join(pkgs)
463 # get dist bootstrap template or fall back to family default
464 bootstrap_template = config.get('bootstrap', family['bootstrap'])
465 config['bootstrap.sh'] = bootstrap_template.format(**config).strip()
466 config['locale.sh'] = LOCALE_SETUP.format(**config).strip()
468 config['Dockerfile'] = DOCKERFILE.format(**config).strip()
469 # keep the indent, no strip
470 config['vagrantfile_snippet'] = VAGRANTFILE_SNIPPET.format(**config)
476 # expanded config for dists
477 DEB_DISTS_EXP = expand_family_dists(DEB_FAMILY)
478 RPM_DISTS_EXP = expand_family_dists(RPM_FAMILY)
480 # assemble all together
482 DISTS.update(DEB_DISTS_EXP)
483 DISTS.update(RPM_DISTS_EXP)
486 def render_vagrantfile(dists):
488 Render all snippets for each dist into global Vagrantfile.
490 Vagrant supports multiple vms in one Vagrantfile.
491 This make it easier to manage the fleet, e.g:
493 start all: vagrant up
494 start one: vagrant up ubuntu1804
496 All other commands apply to above syntax, e.g.: status, destroy, provision
498 # sort dists by name and put all vagrantfile snippets together
500 dists[dist]['vagrantfile_snippet']
501 for dist in sorted(dists.keys())]
503 return VAGRANTFILE_GLOBAL.format(vagrantfile_snippets=''.join(snippets))
506 VAGRANTFILE = render_vagrantfile(DISTS)
509 # data we need to expose
510 __all__ = ['DISTS', 'VAGRANTFILE', 'OUT']