f2a29497cf273a35d47f28f2f1ac862ac631d34c
[sfrench/samba-autobuild/.git] / script / autobuild.py
1 #!/usr/bin/env python3
2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # released under GNU GPL v3 or later
5
6 from __future__ import print_function
7 from subprocess import call, check_call, Popen, PIPE
8 import os
9 import tarfile
10 import sys
11 import time
12 from optparse import OptionParser
13 import smtplib
14 import email
15 from email.mime.text import MIMEText
16 from email.mime.base import MIMEBase
17 from email.mime.application import MIMEApplication
18 from email.mime.multipart import MIMEMultipart
19 from distutils.sysconfig import get_python_lib
20 import platform
21
22 try:
23     from waflib.Build import CACHE_SUFFIX
24 except ImportError:
25     sys.path.insert(0, "./third_party/waf")
26     from waflib.Build import CACHE_SUFFIX
27
28
29 os.environ["PYTHONUNBUFFERED"] = "1"
30
31 # This speeds up testing remarkably.
32 os.environ['TDB_NO_FSYNC'] = '1'
33
34 cleanup_list = []
35
36 builddirs = {
37     "ctdb": "ctdb",
38     "samba": ".",
39     "samba-nt4": ".",
40     "samba-fileserver": ".",
41     "samba-xc": ".",
42     "samba-o3": ".",
43     "samba-ctdb": ".",
44     "samba-libs": ".",
45     "samba-static": ".",
46     "samba-none-env": ".",
47     "samba-ad-dc": ".",
48     "samba-ad-dc-ntvfs": ".",
49     "samba-ad-dc-2": ".",
50     "samba-ad-dc-backup": ".",
51     "samba-systemkrb5": ".",
52     "samba-nopython": ".",
53     "samba-nopython-py2": ".",
54     "ldb": "lib/ldb",
55     "tdb": "lib/tdb",
56     "talloc": "lib/talloc",
57     "replace": "lib/replace",
58     "tevent": "lib/tevent",
59     "pidl": "pidl"
60 }
61
62 defaulttasks = builddirs.keys()
63
64 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
65     defaulttasks.remove("samba-o3")
66
67 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
68 samba_configure_params = " --picky-developer ${PREFIX} --with-profiling-data"
69
70 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
71 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
72 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
73 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
74 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
75 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
76 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
77
78 tasks = {
79     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
80                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
81                ("make", "make all", "text/plain"),
82                ("install", "make install", "text/plain"),
83                ("test", "make autotest", "text/plain"),
84                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
85                ("clean", "make clean", "text/plain")],
86
87     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
88     "samba": [
89                 ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
90                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
91                 ("make", "make -j", "text/plain"),
92                 ("test", "make test FAIL_IMMEDIATELY=1 "
93                  "TESTS='--exclude-env=none "
94                  "--exclude-env=nt4_dc "
95                  "--exclude-env=nt4_member "
96                  "--exclude-env=ad_dc "
97                  "--exclude-env=ad_dc_ntvfs "
98                  "--exclude-env=ad_dc_no_nss "
99                  "--exclude-env=fl2003dc "
100                  "--exclude-env=fl2008r2dc "
101                  "--exclude-env=ad_member "
102                  "--exclude-env=ad_member_idmap_rid "
103                  "--exclude-env=ad_member_idmap_ad "
104                  "--exclude-env=chgdcpass "
105                  "--exclude-env=vampire_2000_dc "
106                  "--exclude-env=fl2000dc "
107                  "--exclude-env=fileserver "
108                  "--exclude-env=backupfromdc "
109                  "--exclude-env=restoredc "
110                  "--exclude-env=renamedc "
111                  "--exclude-env=offlinebackupdc "
112                  "--exclude-env=labdc "
113                  "'",
114                  "text/plain"),
115                 ("install", "make install", "text/plain"),
116                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
117                 ("clean", "make clean", "text/plain")],
118
119     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
120                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
121                     ("make", "make -j", "text/plain"),
122                     ("test", "make test FAIL_IMMEDIATELY=1 "
123                      "TESTS='--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
124                     ("install", "make install", "text/plain"),
125                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
126                     ("clean", "make clean", "text/plain")],
127
128     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
129                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
130                            ("make", "make -j", "text/plain"),
131                            ("test", "make test FAIL_IMMEDIATELY=1 "
132                             "TESTS='--include-env=fileserver'", "text/plain"),
133                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
134
135     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
136                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
137                       ("make", "make -j", "text/plain"),
138                       ("test", "make test FAIL_IMMEDIATELY=1 "
139                        "TESTS='--include-env=ad_dc "
140                        "--include-env=fl2003dc "
141                        "--include-env=fl2008r2dc "
142                        "--include-env=ad_member "
143                        "--include-env=ad_member_idmap_rid "
144                        "--include-env=ad_member_idmap_ad'", "text/plain"),
145                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
146
147     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
148                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
149                         ("make", "make -j", "text/plain"),
150                         ("test", "make test FAIL_IMMEDIATELY=1 "
151                          "TESTS='--include-env=chgdcpass "
152                          "--include-env=vampire_2000_dc "
153                          "--include-env=fl2000dc "
154                          "--include-env=ad_dc_no_nss "
155                          "'",
156                          "text/plain"),
157                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
158
159     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
160     # This is currently the longest task, so we don't randomly delay it.
161     "samba-ad-dc-ntvfs": [
162                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
163                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
164                       ("make", "make -j", "text/plain"),
165                       ("test", "make test FAIL_IMMEDIATELY=1 "
166                        "TESTS='--include-env=ad_dc_ntvfs'",
167                        "text/plain"),
168                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
169
170     # run the backup/restore testenvs separately as they're fairly standalone
171     # (and CI seems to max out at ~8 different DCs running at once)
172     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
173                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
174                         ("make", "make -j", "text/plain"),
175                         ("test", "make test FAIL_IMMEDIATELY=1 "
176                          "TESTS='--include-env=backupfromdc "
177                          "--include-env=restoredc "
178                          "--include-env=renamedc "
179                          "--include-env=offlinebackupdc "
180                          "--include-env=labdc "
181                          "'",
182                          "text/plain"),
183                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
184
185     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
186                           ("make", "make -j", "text/plain"),
187                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
188
189     # Test cross-compile infrastructure
190     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
191                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
192                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
193                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
194                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
195                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
196                    ("compare-results", "script/compare_cc_results.py "
197                     "./bin/c4che/default{} "
198                     "./bin-xe/c4che/default{} "
199                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
200
201     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
202     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
203                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
204                    ("make", "make -j", "text/plain"),
205                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
206                     "TESTS='--include-env=ad_dc'", "text/plain"),
207                    ("install", "make install", "text/plain"),
208                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
209                    ("clean", "make clean", "text/plain")],
210
211     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
212
213                      # make sure we have tdb around:
214                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
215                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
216                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
217
218
219                      # build samba with cluster support (also building ctdb):
220                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
221                      ("samba-make", "make", "text/plain"),
222                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
223                      ("samba-install", "make install", "text/plain"),
224                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
225
226                      # clean up:
227                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
228                      ("clean", "make clean", "text/plain"),
229                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
230
231     "samba-libs": [
232                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
233                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
234                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
235                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
236
237                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
238                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
239                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
240
241                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
242                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
243                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
244
245                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
246                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
247                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
248
249                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
250                       ("nondevel-make", "make -j", "text/plain"),
251                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
252                       ("nondevel-install", "make install", "text/plain"),
253                       ("nondevel-dist", "make dist", "text/plain"),
254
255                       # retry with all modules shared
256                       ("allshared-distclean", "make distclean", "text/plain"),
257                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
258                       ("allshared-make", "make -j", "text/plain")],
259
260     "samba-none-env": [
261                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
262                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
263                       ("make", "make -j", "text/plain"),
264                       ("test", "make test "
265                        "FAIL_IMMEDIATELY=1 "
266                        "TESTS='--include-env=none'",
267                        "text/plain")],
268
269     "samba-static": [
270                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
271                       # build with all modules static
272                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
273                       ("allstatic-make", "make -j", "text/plain"),
274                       ("allstatic-test", "make test "
275                        "FAIL_IMMEDIATELY=1 "
276                        "TESTS='samba3.smb2.create.*nt4_dc'",
277                        "text/plain"),
278
279                       # retry without any required modules
280                       ("none-distclean", "make distclean", "text/plain"),
281                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
282                       ("none-make", "make -j", "text/plain"),
283
284                       # retry with nonshared smbd and smbtorture
285                       ("nonshared-distclean", "make distclean", "text/plain"),
286                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
287                       ("nonshared-make", "make -j", "text/plain")],
288
289     "samba-systemkrb5": [
290                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
291                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
292                       ("make", "make -j", "text/plain"),
293                       # we currently cannot run a full make test, a limited list of tests could be run
294                       # via "make test TESTS=sometests"
295                       ("test", "make test FAIL_IMMEDIATELY=1 "
296                        "TESTS='--include-env=ktest'", "text/plain"),
297                       ("install", "make install", "text/plain"),
298                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
299                       ("clean", "make clean", "text/plain")
300                       ],
301
302     # Test Samba without python still builds.  When this test fails
303     # due to more use of Python, the expectations is that the newly
304     # failing part of the code should be disabled when
305     # --disable-python is set (rather than major work being done to
306     # support this environment).  The target here is for vendors
307     # shipping a minimal smbd.
308     "samba-nopython": [
309                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
310                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
311                       ("make", "make -j", "text/plain"),
312                       ("install", "make install", "text/plain"),
313                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
314                       ("clean", "make clean", "text/plain"),
315
316                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
317                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
318                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
319
320                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
321                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
322                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
323
324                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
325                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
326                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
327
328                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
329                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
330                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
331
332                       # retry against installed library packages
333                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
334                       ("libs-make", "make -j", "text/plain"),
335                       ("libs-install", "make install", "text/plain"),
336                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
337                       ("libs-clean", "make clean", "text/plain")
338                       ],
339
340     # check we can do the same thing using python2
341     "samba-nopython-py2": [
342                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
343                       ("configure", "PYTHON=python2 ./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
344                       ("make", "PYTHON=python2 make -j", "text/plain"),
345                       ("install", "PYTHON=python2 make install", "text/plain"),
346                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
347                       ("clean", "PYTHON=python2 make clean", "text/plain"),
348
349                       ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
350                       ("talloc-make", "cd lib/talloc && PYTHON=python2 make", "text/plain"),
351                       ("talloc-install", "cd lib/talloc && PYTHON=python2 make install", "text/plain"),
352
353                       ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
354                       ("tdb-make", "cd lib/tdb && PYTHON=python2 make", "text/plain"),
355                       ("tdb-install", "cd lib/tdb && PYTHON=python2 make install", "text/plain"),
356
357                       ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
358                       ("tevent-make", "cd lib/tevent && PYTHON=python2 make", "text/plain"),
359                       ("tevent-install", "cd lib/tevent && PYTHON=python2 make install", "text/plain"),
360
361                       ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
362                       ("ldb-make", "cd lib/ldb && PYTHON=python2 make", "text/plain"),
363                       ("ldb-install", "cd lib/ldb && PYTHON=python2 make install", "text/plain"),
364
365                       # retry against installed library packages
366                       ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
367                       ("libs-make", "PYTHON=python2 make -j", "text/plain"),
368                       ("libs-install", "PYTHON=python2 make install", "text/plain"),
369                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
370                       ("libs-clean", "PYTHON=python2 make clean", "text/plain")
371                       ],
372
373     "ldb": [
374               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
375               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
376               ("make", "make", "text/plain"),
377               ("install", "make install", "text/plain"),
378               ("test", "make test", "text/plain"),
379               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX}", "text/plain"),
380               ("make-no-lmdb", "make", "text/plain"),
381               ("install-no-lmdb", "make install", "text/plain"),
382               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
383               ("distcheck", "make distcheck", "text/plain"),
384               ("clean", "make clean", "text/plain")],
385
386     "tdb": [
387               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
388               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
389               ("make", "make", "text/plain"),
390               ("install", "make install", "text/plain"),
391               ("test", "make test", "text/plain"),
392               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
393               ("distcheck", "make distcheck", "text/plain"),
394               ("clean", "make clean", "text/plain")],
395
396     "talloc": [
397                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
398                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
399                  ("make", "make", "text/plain"),
400                  ("install", "make install", "text/plain"),
401                  ("test", "make test", "text/plain"),
402                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
403                  ("distcheck", "make distcheck", "text/plain"),
404                  ("clean", "make clean", "text/plain")],
405
406     "replace": [
407                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
408                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
409                   ("make", "make", "text/plain"),
410                   ("install", "make install", "text/plain"),
411                   ("test", "make test", "text/plain"),
412                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
413                   ("distcheck", "make distcheck", "text/plain"),
414                   ("clean", "make clean", "text/plain")],
415
416     "tevent": [
417                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
418                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
419                  ("make", "make", "text/plain"),
420                  ("install", "make install", "text/plain"),
421                  ("test", "make test", "text/plain"),
422                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
423                  ("distcheck", "make distcheck", "text/plain"),
424                  ("clean", "make clean", "text/plain")],
425
426     "pidl": [
427         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
428         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
429         ("touch", "touch *.yp", "text/plain"),
430         ("make", "make", "text/plain"),
431         ("test", "make test", "text/plain"),
432         ("install", "make install", "text/plain"),
433         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
434         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
435         ("clean", "make clean", "text/plain")],
436
437
438     # these are useful for debugging autobuild
439     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
440     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
441
442
443 }
444
445
446 def do_print(msg):
447     print("%s" % msg)
448     sys.stdout.flush()
449     sys.stderr.flush()
450
451
452 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
453     if show is None:
454         show = options.verbose
455     if show:
456         do_print("Running: '%s' in '%s'" % (cmd, dir))
457     if output:
458         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
459     elif checkfail:
460         return check_call(cmd, shell=True, cwd=dir)
461     else:
462         return call(cmd, shell=True, cwd=dir)
463
464
465 class builder(object):
466     '''handle build of one directory'''
467
468     def __init__(self, name, sequence, cp=True):
469         self.name = name
470         if name in builddirs:
471             self.dir = builddirs[name]
472         else:
473             self.dir = "."
474
475         self.tag = self.name.replace('/', '_')
476         self.sequence = sequence
477         self.next = 0
478         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
479         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
480         if options.verbose:
481             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
482             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
483         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
484         self.stdout = open(self.stdout_path, 'w')
485         self.stderr = open(self.stderr_path, 'w')
486         self.stdin  = open("/dev/null", 'r')
487         self.sdir = "%s/%s" % (testbase, self.tag)
488         self.prefix = "%s/%s" % (test_prefix, self.tag)
489         run_cmd("rm -rf %s" % self.sdir)
490         run_cmd("rm -rf %s" % self.prefix)
491         if cp:
492             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
493         else:
494             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
495         self.start_next()
496
497     def start_next(self):
498         if self.next == len(self.sequence):
499             if not options.nocleanup:
500                 run_cmd("rm -rf %s" % self.sdir)
501                 run_cmd("rm -rf %s" % self.prefix)
502             do_print('%s: Completed OK' % self.name)
503             self.done = True
504             return
505         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
506         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
507         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
508         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
509         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
510 #        if self.output_mime_type == "text/x-subunit":
511 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
512         cwd = os.getcwd()
513         os.chdir("%s/%s" % (self.sdir, self.dir))
514         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
515         self.proc = Popen(self.cmd, shell=True,
516                           close_fds=True,
517                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
518         os.chdir(cwd)
519         self.next += 1
520
521
522 class buildlist(object):
523     '''handle build of multiple directories'''
524
525     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
526         global tasks
527         self.tlist = []
528         self.tail_proc = None
529         self.retry = None
530         if tasknames == []:
531             if options.restrict_tests:
532                 tasknames = ["samba-test-only"]
533             else:
534                 tasknames = defaulttasks
535         else:
536             # If we are only running one test,
537             # do not sleep randomly to wait for it to start
538             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
539
540         for n in tasknames:
541             b = builder(n, tasks[n], cp=n is not "pidl")
542             self.tlist.append(b)
543         if options.retry:
544             rebase_remote = "rebaseon"
545             retry_task = [("retry",
546                             '''set -e
547                             git remote add -t %s %s %s
548                             git fetch %s
549                             while :; do
550                               sleep 60
551                               git describe %s/%s > old_remote_branch.desc
552                               git fetch %s
553                               git describe %s/%s > remote_branch.desc
554                               diff old_remote_branch.desc remote_branch.desc
555                             done
556                            ''' % (
557                                 rebase_branch, rebase_remote, rebase_url,
558                                rebase_remote,
559                                rebase_remote, rebase_branch,
560                                rebase_remote,
561                                rebase_remote, rebase_branch
562                             ),
563                             "test/plain")]
564
565             self.retry = builder('retry', retry_task, cp=False)
566             self.need_retry = False
567
568     def kill_kids(self):
569         if self.tail_proc is not None:
570             self.tail_proc.terminate()
571             self.tail_proc.wait()
572             self.tail_proc = None
573         if self.retry is not None:
574             self.retry.proc.terminate()
575             self.retry.proc.wait()
576             self.retry = None
577         for b in self.tlist:
578             if b.proc is not None:
579                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
580                 b.proc.terminate()
581                 b.proc.wait()
582                 b.proc = None
583
584     def wait_one(self):
585         while True:
586             none_running = True
587             for b in self.tlist:
588                 if b.proc is None:
589                     continue
590                 none_running = False
591                 b.status = b.proc.poll()
592                 if b.status is None:
593                     continue
594                 b.proc = None
595                 return b
596             if options.retry:
597                 ret = self.retry.proc.poll()
598                 if ret is not None:
599                     self.need_retry = True
600                     self.retry = None
601                     return None
602             if none_running:
603                 return None
604             time.sleep(0.1)
605
606     def run(self):
607         while True:
608             b = self.wait_one()
609             if options.retry and self.need_retry:
610                 self.kill_kids()
611                 do_print("retry needed")
612                 return (0, None, None, None, "retry")
613             if b is None:
614                 break
615             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
616                 self.kill_kids()
617                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
618             b.start_next()
619         self.kill_kids()
620         return (0, None, None, None, "All OK")
621
622     def write_system_info(self):
623         filename = 'system-info.txt'
624         f = open(filename, 'w')
625         for cmd in ['uname -a',
626                     'lsb_release -a',
627                     'free',
628                     'mount',
629                     'cat /proc/cpuinfo',
630                     'cc --version',
631                     'df -m .',
632                     'df -m %s' % testbase]:
633             out = run_cmd(cmd, output=True, checkfail=False)
634             print('### %s' % cmd, file=f)
635             print(out.decode('utf8', 'backslashreplace'), file=f)
636             print(file=f)
637         f.close()
638         return filename
639
640     def tarlogs(self, fname):
641         tar = tarfile.open(fname, "w:gz")
642         for b in self.tlist:
643             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
644             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
645         if os.path.exists("autobuild.log"):
646             tar.add("autobuild.log")
647         sys_info = self.write_system_info()
648         tar.add(sys_info)
649         tar.close()
650
651     def remove_logs(self):
652         for b in self.tlist:
653             os.unlink(b.stdout_path)
654             os.unlink(b.stderr_path)
655
656     def start_tail(self):
657         cmd = ["tail", "-f"]
658         for b in self.tlist:
659             cmd.append(b.stdout_path)
660             cmd.append(b.stderr_path)
661         self.tail_proc = Popen(cmd, close_fds=True)
662
663
664 def cleanup():
665     if options.nocleanup:
666         return
667     run_cmd("stat %s || true" % test_tmpdir, show=True)
668     run_cmd("stat %s" % testbase, show=True)
669     do_print("Cleaning up %r" % cleanup_list)
670     for d in cleanup_list:
671         run_cmd("rm -rf %s" % d)
672
673
674 def find_git_root():
675     '''get to the top of the git repo'''
676     p = os.getcwd()
677     while p != '/':
678         if os.path.isdir(os.path.join(p, ".git")):
679             return p
680         p = os.path.abspath(os.path.join(p, '..'))
681     return None
682
683
684 def daemonize(logfile):
685     pid = os.fork()
686     if pid == 0:  # Parent
687         os.setsid()
688         pid = os.fork()
689         if pid != 0:  # Actual daemon
690             os._exit(0)
691     else:  # Grandparent
692         os._exit(0)
693
694     import resource      # Resource usage information.
695     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
696     if maxfd == resource.RLIM_INFINITY:
697         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
698     for fd in range(0, maxfd):
699         try:
700             os.close(fd)
701         except OSError:
702             pass
703     os.open(logfile, os.O_RDWR | os.O_CREAT)
704     os.dup2(0, 1)
705     os.dup2(0, 2)
706
707
708 def write_pidfile(fname):
709     '''write a pid file, cleanup on exit'''
710     f = open(fname, mode='w')
711     f.write("%u\n" % os.getpid())
712     f.close()
713
714
715 def rebase_tree(rebase_url, rebase_branch="master"):
716     rebase_remote = "rebaseon"
717     do_print("Rebasing on %s" % rebase_url)
718     run_cmd("git describe HEAD", show=True, dir=test_master)
719     run_cmd("git remote add -t %s %s %s" %
720             (rebase_branch, rebase_remote, rebase_url),
721             show=True, dir=test_master)
722     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
723     if options.fix_whitespace:
724         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
725                 (rebase_remote, rebase_branch),
726                 show=True, dir=test_master)
727     else:
728         run_cmd("git rebase --force-rebase %s/%s" %
729                 (rebase_remote, rebase_branch),
730                 show=True, dir=test_master)
731     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
732                    (rebase_remote, rebase_branch),
733                    dir=test_master, output=True)
734     if diff == '':
735         do_print("No differences between HEAD and %s/%s - exiting" %
736                  (rebase_remote, rebase_branch))
737         sys.exit(0)
738     run_cmd("git describe %s/%s" %
739             (rebase_remote, rebase_branch),
740             show=True, dir=test_master)
741     run_cmd("git describe HEAD", show=True, dir=test_master)
742     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
743             (rebase_remote, rebase_branch),
744             show=True, dir=test_master)
745
746
747 def push_to(push_url, push_branch="master"):
748     push_remote = "pushto"
749     do_print("Pushing to %s" % push_url)
750     if options.mark:
751         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
752         run_cmd("git commit --amend -c HEAD", dir=test_master)
753         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
754         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
755     run_cmd("git remote add -t %s %s %s" %
756             (push_branch, push_remote, push_url),
757             show=True, dir=test_master)
758     run_cmd("git push %s +HEAD:%s" %
759             (push_remote, push_branch),
760             show=True, dir=test_master)
761
762
763 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
764
765 gitroot = find_git_root()
766 if gitroot is None:
767     raise Exception("Failed to find git root")
768
769 parser = OptionParser()
770 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
771 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
772 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
773 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
774                   default=def_testbase)
775 parser.add_option("", "--passcmd", help="command to run on success", default=None)
776 parser.add_option("", "--verbose", help="show all commands as they are run",
777                   default=False, action="store_true")
778 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
779                   default=None, type='str')
780 parser.add_option("", "--pushto", help="push to a git url on success",
781                   default=None, type='str')
782 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
783                   default=False, action="store_true")
784 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
785                   default=False, action="store_true")
786 parser.add_option("", "--retry", help="automatically retry if master changes",
787                   default=False, action="store_true")
788 parser.add_option("", "--email", help="send email to the given address on failure",
789                   type='str', default=None)
790 parser.add_option("", "--email-from", help="send email from the given address",
791                   type='str', default="autobuild@samba.org")
792 parser.add_option("", "--email-server", help="send email via the given server",
793                   type='str', default='localhost')
794 parser.add_option("", "--always-email", help="always send email, even on success",
795                   action="store_true")
796 parser.add_option("", "--daemon", help="daemonize after initial setup",
797                   action="store_true")
798 parser.add_option("", "--branch", help="the branch to work on (default=master)",
799                   default="master", type='str')
800 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
801                   default=gitroot, type='str')
802 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
803                   default=False, action="store_true")
804 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
805                   default='')
806
807
808 def send_email(subject, text, log_tar):
809     if options.email is None:
810         do_print("not sending email because the recipient is not set")
811         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
812                  (subject, text))
813         return
814     outer = MIMEMultipart()
815     outer['Subject'] = subject
816     outer['To'] = options.email
817     outer['From'] = options.email_from
818     outer['Date'] = email.utils.formatdate(localtime=True)
819     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
820     outer.attach(MIMEText(text, 'plain'))
821     if options.attach_logs:
822         fp = open(log_tar, 'rb')
823         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
824         fp.close()
825         # Set the filename parameter
826         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
827         outer.attach(msg)
828     content = outer.as_string()
829     s = smtplib.SMTP(options.email_server)
830     s.sendmail(options.email_from, [options.email], content)
831     s.set_debuglevel(1)
832     s.quit()
833
834
835 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
836                   elapsed_time, log_base=None, add_log_tail=True):
837     '''send an email to options.email about the failure'''
838     elapsed_minutes = elapsed_time / 60.0
839     if log_base is None:
840         log_base = gitroot
841     text = '''
842 Dear Developer,
843
844 Your autobuild on %s failed after %.1f minutes
845 when trying to test %s with the following error:
846
847    %s
848
849 the autobuild has been abandoned. Please fix the error and resubmit.
850
851 A summary of the autobuild process is here:
852
853   %s/autobuild.log
854 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
855
856     if options.restrict_tests:
857         text += """
858 The build was restricted to tests matching %s\n""" % options.restrict_tests
859
860     if failed_task != 'rebase':
861         text += '''
862 You can see logs of the failed task here:
863
864   %s/%s.stdout
865   %s/%s.stderr
866
867 or you can get full logs of all tasks in this job here:
868
869   %s/logs.tar.gz
870
871 The top commit for the tree that was built was:
872
873 %s
874
875 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
876
877     if add_log_tail:
878         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
879         lines = f.readlines()
880         log_tail = "".join(lines[-50:])
881         num_lines = len(lines)
882         if num_lines < 50:
883             # Also include stderr (compile failures) if < 50 lines of stdout
884             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
885             log_tail += "".join(f.readlines()[-(50 - num_lines):])
886
887         text += '''
888 The last 50 lines of log messages:
889
890 %s
891     ''' % log_tail
892         f.close()
893
894     logs = os.path.join(gitroot, 'logs.tar.gz')
895     send_email('autobuild[%s] failure on %s for task %s during %s'
896                % (options.branch, platform.node(), failed_task, failed_stage),
897                text, logs)
898
899
900 def email_success(elapsed_time, log_base=None):
901     '''send an email to options.email about a successful build'''
902     if log_base is None:
903         log_base = gitroot
904     text = '''
905 Dear Developer,
906
907 Your autobuild on %s has succeeded after %.1f minutes.
908
909 ''' % (platform.node(), elapsed_time / 60.)
910
911     if options.restrict_tests:
912         text += """
913 The build was restricted to tests matching %s\n""" % options.restrict_tests
914
915     if options.keeplogs:
916         text += '''
917
918 you can get full logs of all tasks in this job here:
919
920   %s/logs.tar.gz
921
922 ''' % log_base
923
924     text += '''
925 The top commit for the tree that was built was:
926
927 %s
928 ''' % top_commit_msg
929
930     logs = os.path.join(gitroot, 'logs.tar.gz')
931     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
932                text, logs)
933
934
935 (options, args) = parser.parse_args()
936
937 if options.retry:
938     if options.rebase is None:
939         raise Exception('You can only use --retry if you also rebase')
940
941 testbase = "%s/b%u" % (options.testbase, os.getpid())
942 test_master = "%s/master" % testbase
943 test_prefix = "%s/prefix" % testbase
944 test_tmpdir = "%s/tmp" % testbase
945 os.environ['TMPDIR'] = test_tmpdir
946
947 # get the top commit message, for emails
948 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
949 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
950
951 try:
952     os.makedirs(testbase)
953 except Exception as reason:
954     raise Exception("Unable to create %s : %s" % (testbase, reason))
955 cleanup_list.append(testbase)
956
957 if options.daemon:
958     logfile = os.path.join(testbase, "log")
959     do_print("Forking into the background, writing progress to %s" % logfile)
960     daemonize(logfile)
961
962 write_pidfile(gitroot + "/autobuild.pid")
963
964 start_time = time.time()
965
966 while True:
967     try:
968         run_cmd("rm -rf %s" % test_tmpdir, show=True)
969         os.makedirs(test_tmpdir)
970         # The waf uninstall code removes empty directories all the way
971         # up the tree.  Creating a file in test_tmpdir stops it from
972         # being removed.
973         run_cmd("touch %s" % os.path.join(test_tmpdir,
974                                           ".directory-is-not-empty"), show=True)
975         run_cmd("stat %s" % test_tmpdir, show=True)
976         run_cmd("stat %s" % testbase, show=True)
977         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
978     except Exception:
979         cleanup()
980         raise
981
982     try:
983         try:
984             if options.rebase is not None:
985                 rebase_tree(options.rebase, rebase_branch=options.branch)
986         except Exception:
987             cleanup_list.append(gitroot + "/autobuild.pid")
988             cleanup()
989             elapsed_time = time.time() - start_time
990             email_failure(-1, 'rebase', 'rebase', 'rebase',
991                           'rebase on %s failed' % options.branch,
992                           elapsed_time, log_base=options.log_base)
993             sys.exit(1)
994         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
995         if options.tail:
996             blist.start_tail()
997         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
998         if status != 0 or errstr != "retry":
999             break
1000         cleanup()
1001     except Exception:
1002         cleanup()
1003         raise
1004
1005 cleanup_list.append(gitroot + "/autobuild.pid")
1006
1007 do_print(errstr)
1008
1009 blist.kill_kids()
1010 if options.tail:
1011     do_print("waiting for tail to flush")
1012     time.sleep(1)
1013
1014 elapsed_time = time.time() - start_time
1015 if status == 0:
1016     if options.passcmd is not None:
1017         do_print("Running passcmd: %s" % options.passcmd)
1018         run_cmd(options.passcmd, dir=test_master)
1019     if options.pushto is not None:
1020         push_to(options.pushto, push_branch=options.branch)
1021     if options.keeplogs or options.attach_logs:
1022         blist.tarlogs("logs.tar.gz")
1023         do_print("Logs in logs.tar.gz")
1024     if options.always_email:
1025         email_success(elapsed_time, log_base=options.log_base)
1026     blist.remove_logs()
1027     cleanup()
1028     do_print(errstr)
1029     sys.exit(0)
1030
1031 # something failed, gather a tar of the logs
1032 blist.tarlogs("logs.tar.gz")
1033
1034 if options.email is not None:
1035     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1036                   elapsed_time, log_base=options.log_base)
1037 else:
1038     elapsed_minutes = elapsed_time / 60.0
1039     print('''
1040
1041 ####################################################################
1042
1043 AUTOBUILD FAILURE
1044
1045 Your autobuild[%s] on %s failed after %.1f minutes
1046 when trying to test %s with the following error:
1047
1048    %s
1049
1050 the autobuild has been abandoned. Please fix the error and resubmit.
1051
1052 ####################################################################
1053
1054 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1055
1056 cleanup()
1057 do_print(errstr)
1058 do_print("Logs in logs.tar.gz")
1059 sys.exit(status)