selftest: Add basic sanity-check tests for nopython target
[sfrench/samba-autobuild/.git] / script / autobuild.py
1 #!/usr/bin/env python3
2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # released under GNU GPL v3 or later
5
6 from __future__ import print_function
7 from subprocess import call, check_call, Popen, PIPE
8 import os
9 import tarfile
10 import sys
11 import time
12 from optparse import OptionParser
13 import smtplib
14 import email
15 from email.mime.text import MIMEText
16 from email.mime.base import MIMEBase
17 from email.mime.application import MIMEApplication
18 from email.mime.multipart import MIMEMultipart
19 from distutils.sysconfig import get_python_lib
20 import platform
21
22 try:
23     from waflib.Build import CACHE_SUFFIX
24 except ImportError:
25     sys.path.insert(0, "./third_party/waf")
26     from waflib.Build import CACHE_SUFFIX
27
28
29 os.environ["PYTHONUNBUFFERED"] = "1"
30
31 # This speeds up testing remarkably.
32 os.environ['TDB_NO_FSYNC'] = '1'
33
34 cleanup_list = []
35
36 builddirs = {
37     "ctdb": "ctdb",
38     "samba": ".",
39     "samba-nt4": ".",
40     "samba-fileserver": ".",
41     "samba-xc": ".",
42     "samba-o3": ".",
43     "samba-ctdb": ".",
44     "samba-libs": ".",
45     "samba-static": ".",
46     "samba-none-env": ".",
47     "samba-ad-dc": ".",
48     "samba-ad-dc-ntvfs": ".",
49     "samba-ad-dc-2": ".",
50     "samba-ad-dc-backup": ".",
51     "samba-systemkrb5": ".",
52     "samba-nopython": ".",
53     "samba-nopython-py2": ".",
54     "ldb": "lib/ldb",
55     "tdb": "lib/tdb",
56     "talloc": "lib/talloc",
57     "replace": "lib/replace",
58     "tevent": "lib/tevent",
59     "pidl": "pidl"
60 }
61
62 defaulttasks = builddirs.keys()
63
64 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
65     defaulttasks.remove("samba-o3")
66
67 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
68 samba_configure_params = " --picky-developer ${PREFIX} --with-profiling-data"
69
70 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
71 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
72 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
73 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
74 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
75 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
76 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
77
78 tasks = {
79     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
80                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
81                ("make", "make all", "text/plain"),
82                ("install", "make install", "text/plain"),
83                ("test", "make autotest", "text/plain"),
84                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
85                ("clean", "make clean", "text/plain")],
86
87     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
88     "samba": [
89                 ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
90                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
91                 ("make", "make -j", "text/plain"),
92                 ("test", "make test FAIL_IMMEDIATELY=1 "
93                  "TESTS='--exclude-env=none "
94                  "--exclude-env=nt4_dc "
95                  "--exclude-env=nt4_member "
96                  "--exclude-env=ad_dc "
97                  "--exclude-env=ad_dc_ntvfs "
98                  "--exclude-env=ad_dc_no_nss "
99                  "--exclude-env=fl2003dc "
100                  "--exclude-env=fl2008r2dc "
101                  "--exclude-env=ad_member "
102                  "--exclude-env=ad_member_idmap_rid "
103                  "--exclude-env=ad_member_idmap_ad "
104                  "--exclude-env=chgdcpass "
105                  "--exclude-env=vampire_2000_dc "
106                  "--exclude-env=fl2000dc "
107                  "--exclude-env=fileserver "
108                  "--exclude-env=backupfromdc "
109                  "--exclude-env=restoredc "
110                  "--exclude-env=renamedc "
111                  "--exclude-env=offlinebackupdc "
112                  "--exclude-env=labdc "
113                  "'",
114                  "text/plain"),
115                 ("install", "make install", "text/plain"),
116                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
117                 ("clean", "make clean", "text/plain")],
118
119     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
120                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
121                     ("make", "make -j", "text/plain"),
122                     ("test", "make test FAIL_IMMEDIATELY=1 "
123                      "TESTS='--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
124                     ("install", "make install", "text/plain"),
125                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
126                     ("clean", "make clean", "text/plain")],
127
128     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
129                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
130                            ("make", "make -j", "text/plain"),
131                            ("test", "make test FAIL_IMMEDIATELY=1 "
132                             "TESTS='--include-env=fileserver'", "text/plain"),
133                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
134
135     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
136                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
137                       ("make", "make -j", "text/plain"),
138                       ("test", "make test FAIL_IMMEDIATELY=1 "
139                        "TESTS='--include-env=ad_dc "
140                        "--include-env=fl2003dc "
141                        "--include-env=fl2008r2dc "
142                        "--include-env=ad_member "
143                        "--include-env=ad_member_idmap_rid "
144                        "--include-env=ad_member_idmap_ad'", "text/plain"),
145                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
146
147     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
148                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
149                         ("make", "make -j", "text/plain"),
150                         ("test", "make test FAIL_IMMEDIATELY=1 "
151                          "TESTS='--include-env=chgdcpass "
152                          "--include-env=vampire_2000_dc "
153                          "--include-env=fl2000dc "
154                          "--include-env=ad_dc_no_nss "
155                          "'",
156                          "text/plain"),
157                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
158
159     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
160     # This is currently the longest task, so we don't randomly delay it.
161     "samba-ad-dc-ntvfs": [
162                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
163                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
164                       ("make", "make -j", "text/plain"),
165                       ("test", "make test FAIL_IMMEDIATELY=1 "
166                        "TESTS='--include-env=ad_dc_ntvfs'",
167                        "text/plain"),
168                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
169
170     # run the backup/restore testenvs separately as they're fairly standalone
171     # (and CI seems to max out at ~8 different DCs running at once)
172     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
173                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
174                         ("make", "make -j", "text/plain"),
175                         ("test", "make test FAIL_IMMEDIATELY=1 "
176                          "TESTS='--include-env=backupfromdc "
177                          "--include-env=restoredc "
178                          "--include-env=renamedc "
179                          "--include-env=offlinebackupdc "
180                          "--include-env=labdc "
181                          "'",
182                          "text/plain"),
183                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
184
185     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
186                           ("make", "make -j", "text/plain"),
187                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
188
189     # Test cross-compile infrastructure
190     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
191                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
192                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
193                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
194                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
195                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
196                    ("compare-results", "script/compare_cc_results.py "
197                     "./bin/c4che/default{} "
198                     "./bin-xe/c4che/default{} "
199                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
200
201     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
202     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
203                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
204                    ("make", "make -j", "text/plain"),
205                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
206                     "TESTS='--include-env=ad_dc'", "text/plain"),
207                    ("install", "make install", "text/plain"),
208                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
209                    ("clean", "make clean", "text/plain")],
210
211     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
212
213                      # make sure we have tdb around:
214                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
215                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
216                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
217
218
219                      # build samba with cluster support (also building ctdb):
220                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
221                      ("samba-make", "make", "text/plain"),
222                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
223                      ("samba-install", "make install", "text/plain"),
224                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
225
226                      # clean up:
227                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
228                      ("clean", "make clean", "text/plain"),
229                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
230
231     "samba-libs": [
232                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
233                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
234                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
235                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
236
237                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
238                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
239                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
240
241                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
242                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
243                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
244
245                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
246                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
247                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
248
249                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
250                       ("nondevel-make", "make -j", "text/plain"),
251                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
252                       ("nondevel-install", "make install", "text/plain"),
253                       ("nondevel-dist", "make dist", "text/plain"),
254
255                       # retry with all modules shared
256                       ("allshared-distclean", "make distclean", "text/plain"),
257                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
258                       ("allshared-make", "make -j", "text/plain")],
259
260     "samba-none-env": [
261                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
262                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
263                       ("make", "make -j", "text/plain"),
264                       ("test", "make test "
265                        "FAIL_IMMEDIATELY=1 "
266                        "TESTS='--include-env=none'",
267                        "text/plain")],
268
269     "samba-static": [
270                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
271                       # build with all modules static
272                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
273                       ("allstatic-make", "make -j", "text/plain"),
274                       ("allstatic-test", "make test "
275                        "FAIL_IMMEDIATELY=1 "
276                        "TESTS='samba3.smb2.create.*nt4_dc'",
277                        "text/plain"),
278
279                       # retry without any required modules
280                       ("none-distclean", "make distclean", "text/plain"),
281                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
282                       ("none-make", "make -j", "text/plain"),
283
284                       # retry with nonshared smbd and smbtorture
285                       ("nonshared-distclean", "make distclean", "text/plain"),
286                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
287                       ("nonshared-make", "make -j", "text/plain")],
288
289     "samba-systemkrb5": [
290                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
291                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
292                       ("make", "make -j", "text/plain"),
293                       # we currently cannot run a full make test, a limited list of tests could be run
294                       # via "make test TESTS=sometests"
295                       ("test", "make test FAIL_IMMEDIATELY=1 "
296                        "TESTS='--include-env=ktest'", "text/plain"),
297                       ("install", "make install", "text/plain"),
298                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
299                       ("clean", "make clean", "text/plain")
300                       ],
301
302     # Test Samba without python still builds.  When this test fails
303     # due to more use of Python, the expectations is that the newly
304     # failing part of the code should be disabled when
305     # --disable-python is set (rather than major work being done to
306     # support this environment).  The target here is for vendors
307     # shipping a minimal smbd.
308     "samba-nopython": [
309                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
310                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
311                       ("make", "make -j", "text/plain"),
312                       ("install", "make install", "text/plain"),
313                       ("test", "make test-nopython", "text/plain"),
314                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
315                       ("clean", "make clean", "text/plain"),
316
317                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
318                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
319                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
320
321                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
322                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
323                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
324
325                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
326                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
327                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
328
329                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
330                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
331                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
332
333                       # retry against installed library packages
334                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
335                       ("libs-make", "make -j", "text/plain"),
336                       ("libs-install", "make install", "text/plain"),
337                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
338                       ("libs-clean", "make clean", "text/plain")
339                       ],
340
341     # check we can do the same thing using python2
342     "samba-nopython-py2": [
343                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
344                       ("configure", "PYTHON=python2 ./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
345                       ("make", "PYTHON=python2 make -j", "text/plain"),
346                       ("install", "PYTHON=python2 make install", "text/plain"),
347                       ("test", "make test-nopython", "text/plain"),
348                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
349                       ("clean", "PYTHON=python2 make clean", "text/plain"),
350
351                       ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
352                       ("talloc-make", "cd lib/talloc && PYTHON=python2 make", "text/plain"),
353                       ("talloc-install", "cd lib/talloc && PYTHON=python2 make install", "text/plain"),
354
355                       ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
356                       ("tdb-make", "cd lib/tdb && PYTHON=python2 make", "text/plain"),
357                       ("tdb-install", "cd lib/tdb && PYTHON=python2 make install", "text/plain"),
358
359                       ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
360                       ("tevent-make", "cd lib/tevent && PYTHON=python2 make", "text/plain"),
361                       ("tevent-install", "cd lib/tevent && PYTHON=python2 make install", "text/plain"),
362
363                       ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
364                       ("ldb-make", "cd lib/ldb && PYTHON=python2 make", "text/plain"),
365                       ("ldb-install", "cd lib/ldb && PYTHON=python2 make install", "text/plain"),
366
367                       # retry against installed library packages
368                       ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
369                       ("libs-make", "PYTHON=python2 make -j", "text/plain"),
370                       ("libs-install", "PYTHON=python2 make install", "text/plain"),
371                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
372                       ("libs-clean", "PYTHON=python2 make clean", "text/plain")
373                       ],
374
375     "ldb": [
376               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
377               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
378               ("make", "make", "text/plain"),
379               ("install", "make install", "text/plain"),
380               ("test", "make test", "text/plain"),
381               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX}", "text/plain"),
382               ("make-no-lmdb", "make", "text/plain"),
383               ("install-no-lmdb", "make install", "text/plain"),
384               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
385               ("distcheck", "make distcheck", "text/plain"),
386               ("clean", "make clean", "text/plain")],
387
388     "tdb": [
389               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
390               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
391               ("make", "make", "text/plain"),
392               ("install", "make install", "text/plain"),
393               ("test", "make test", "text/plain"),
394               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
395               ("distcheck", "make distcheck", "text/plain"),
396               ("clean", "make clean", "text/plain")],
397
398     "talloc": [
399                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
400                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
401                  ("make", "make", "text/plain"),
402                  ("install", "make install", "text/plain"),
403                  ("test", "make test", "text/plain"),
404                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
405                  ("distcheck", "make distcheck", "text/plain"),
406                  ("clean", "make clean", "text/plain")],
407
408     "replace": [
409                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
410                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
411                   ("make", "make", "text/plain"),
412                   ("install", "make install", "text/plain"),
413                   ("test", "make test", "text/plain"),
414                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
415                   ("distcheck", "make distcheck", "text/plain"),
416                   ("clean", "make clean", "text/plain")],
417
418     "tevent": [
419                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
420                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
421                  ("make", "make", "text/plain"),
422                  ("install", "make install", "text/plain"),
423                  ("test", "make test", "text/plain"),
424                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
425                  ("distcheck", "make distcheck", "text/plain"),
426                  ("clean", "make clean", "text/plain")],
427
428     "pidl": [
429         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
430         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
431         ("touch", "touch *.yp", "text/plain"),
432         ("make", "make", "text/plain"),
433         ("test", "make test", "text/plain"),
434         ("install", "make install", "text/plain"),
435         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
436         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
437         ("clean", "make clean", "text/plain")],
438
439
440     # these are useful for debugging autobuild
441     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
442     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
443
444
445 }
446
447
448 def do_print(msg):
449     print("%s" % msg)
450     sys.stdout.flush()
451     sys.stderr.flush()
452
453
454 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
455     if show is None:
456         show = options.verbose
457     if show:
458         do_print("Running: '%s' in '%s'" % (cmd, dir))
459     if output:
460         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
461     elif checkfail:
462         return check_call(cmd, shell=True, cwd=dir)
463     else:
464         return call(cmd, shell=True, cwd=dir)
465
466
467 class builder(object):
468     '''handle build of one directory'''
469
470     def __init__(self, name, sequence, cp=True):
471         self.name = name
472         if name in builddirs:
473             self.dir = builddirs[name]
474         else:
475             self.dir = "."
476
477         self.tag = self.name.replace('/', '_')
478         self.sequence = sequence
479         self.next = 0
480         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
481         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
482         if options.verbose:
483             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
484             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
485         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
486         self.stdout = open(self.stdout_path, 'w')
487         self.stderr = open(self.stderr_path, 'w')
488         self.stdin  = open("/dev/null", 'r')
489         self.sdir = "%s/%s" % (testbase, self.tag)
490         self.prefix = "%s/%s" % (test_prefix, self.tag)
491         run_cmd("rm -rf %s" % self.sdir)
492         run_cmd("rm -rf %s" % self.prefix)
493         if cp:
494             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
495         else:
496             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
497         self.start_next()
498
499     def start_next(self):
500         if self.next == len(self.sequence):
501             if not options.nocleanup:
502                 run_cmd("rm -rf %s" % self.sdir)
503                 run_cmd("rm -rf %s" % self.prefix)
504             do_print('%s: Completed OK' % self.name)
505             self.done = True
506             return
507         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
508         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
509         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
510         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
511         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
512 #        if self.output_mime_type == "text/x-subunit":
513 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
514         cwd = os.getcwd()
515         os.chdir("%s/%s" % (self.sdir, self.dir))
516         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
517         self.proc = Popen(self.cmd, shell=True,
518                           close_fds=True,
519                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
520         os.chdir(cwd)
521         self.next += 1
522
523
524 class buildlist(object):
525     '''handle build of multiple directories'''
526
527     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
528         global tasks
529         self.tlist = []
530         self.tail_proc = None
531         self.retry = None
532         if tasknames == []:
533             if options.restrict_tests:
534                 tasknames = ["samba-test-only"]
535             else:
536                 tasknames = defaulttasks
537         else:
538             # If we are only running one test,
539             # do not sleep randomly to wait for it to start
540             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
541
542         for n in tasknames:
543             b = builder(n, tasks[n], cp=n is not "pidl")
544             self.tlist.append(b)
545         if options.retry:
546             rebase_remote = "rebaseon"
547             retry_task = [("retry",
548                             '''set -e
549                             git remote add -t %s %s %s
550                             git fetch %s
551                             while :; do
552                               sleep 60
553                               git describe %s/%s > old_remote_branch.desc
554                               git fetch %s
555                               git describe %s/%s > remote_branch.desc
556                               diff old_remote_branch.desc remote_branch.desc
557                             done
558                            ''' % (
559                                 rebase_branch, rebase_remote, rebase_url,
560                                rebase_remote,
561                                rebase_remote, rebase_branch,
562                                rebase_remote,
563                                rebase_remote, rebase_branch
564                             ),
565                             "test/plain")]
566
567             self.retry = builder('retry', retry_task, cp=False)
568             self.need_retry = False
569
570     def kill_kids(self):
571         if self.tail_proc is not None:
572             self.tail_proc.terminate()
573             self.tail_proc.wait()
574             self.tail_proc = None
575         if self.retry is not None:
576             self.retry.proc.terminate()
577             self.retry.proc.wait()
578             self.retry = None
579         for b in self.tlist:
580             if b.proc is not None:
581                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
582                 b.proc.terminate()
583                 b.proc.wait()
584                 b.proc = None
585
586     def wait_one(self):
587         while True:
588             none_running = True
589             for b in self.tlist:
590                 if b.proc is None:
591                     continue
592                 none_running = False
593                 b.status = b.proc.poll()
594                 if b.status is None:
595                     continue
596                 b.proc = None
597                 return b
598             if options.retry:
599                 ret = self.retry.proc.poll()
600                 if ret is not None:
601                     self.need_retry = True
602                     self.retry = None
603                     return None
604             if none_running:
605                 return None
606             time.sleep(0.1)
607
608     def run(self):
609         while True:
610             b = self.wait_one()
611             if options.retry and self.need_retry:
612                 self.kill_kids()
613                 do_print("retry needed")
614                 return (0, None, None, None, "retry")
615             if b is None:
616                 break
617             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
618                 self.kill_kids()
619                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
620             b.start_next()
621         self.kill_kids()
622         return (0, None, None, None, "All OK")
623
624     def write_system_info(self):
625         filename = 'system-info.txt'
626         f = open(filename, 'w')
627         for cmd in ['uname -a',
628                     'lsb_release -a',
629                     'free',
630                     'mount',
631                     'cat /proc/cpuinfo',
632                     'cc --version',
633                     'df -m .',
634                     'df -m %s' % testbase]:
635             out = run_cmd(cmd, output=True, checkfail=False)
636             print('### %s' % cmd, file=f)
637             print(out.decode('utf8', 'backslashreplace'), file=f)
638             print(file=f)
639         f.close()
640         return filename
641
642     def tarlogs(self, fname):
643         tar = tarfile.open(fname, "w:gz")
644         for b in self.tlist:
645             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
646             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
647         if os.path.exists("autobuild.log"):
648             tar.add("autobuild.log")
649         sys_info = self.write_system_info()
650         tar.add(sys_info)
651         tar.close()
652
653     def remove_logs(self):
654         for b in self.tlist:
655             os.unlink(b.stdout_path)
656             os.unlink(b.stderr_path)
657
658     def start_tail(self):
659         cmd = ["tail", "-f"]
660         for b in self.tlist:
661             cmd.append(b.stdout_path)
662             cmd.append(b.stderr_path)
663         self.tail_proc = Popen(cmd, close_fds=True)
664
665
666 def cleanup():
667     if options.nocleanup:
668         return
669     run_cmd("stat %s || true" % test_tmpdir, show=True)
670     run_cmd("stat %s" % testbase, show=True)
671     do_print("Cleaning up %r" % cleanup_list)
672     for d in cleanup_list:
673         run_cmd("rm -rf %s" % d)
674
675
676 def find_git_root():
677     '''get to the top of the git repo'''
678     p = os.getcwd()
679     while p != '/':
680         if os.path.isdir(os.path.join(p, ".git")):
681             return p
682         p = os.path.abspath(os.path.join(p, '..'))
683     return None
684
685
686 def daemonize(logfile):
687     pid = os.fork()
688     if pid == 0:  # Parent
689         os.setsid()
690         pid = os.fork()
691         if pid != 0:  # Actual daemon
692             os._exit(0)
693     else:  # Grandparent
694         os._exit(0)
695
696     import resource      # Resource usage information.
697     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
698     if maxfd == resource.RLIM_INFINITY:
699         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
700     for fd in range(0, maxfd):
701         try:
702             os.close(fd)
703         except OSError:
704             pass
705     os.open(logfile, os.O_RDWR | os.O_CREAT)
706     os.dup2(0, 1)
707     os.dup2(0, 2)
708
709
710 def write_pidfile(fname):
711     '''write a pid file, cleanup on exit'''
712     f = open(fname, mode='w')
713     f.write("%u\n" % os.getpid())
714     f.close()
715
716
717 def rebase_tree(rebase_url, rebase_branch="master"):
718     rebase_remote = "rebaseon"
719     do_print("Rebasing on %s" % rebase_url)
720     run_cmd("git describe HEAD", show=True, dir=test_master)
721     run_cmd("git remote add -t %s %s %s" %
722             (rebase_branch, rebase_remote, rebase_url),
723             show=True, dir=test_master)
724     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
725     if options.fix_whitespace:
726         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
727                 (rebase_remote, rebase_branch),
728                 show=True, dir=test_master)
729     else:
730         run_cmd("git rebase --force-rebase %s/%s" %
731                 (rebase_remote, rebase_branch),
732                 show=True, dir=test_master)
733     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
734                    (rebase_remote, rebase_branch),
735                    dir=test_master, output=True)
736     if diff == '':
737         do_print("No differences between HEAD and %s/%s - exiting" %
738                  (rebase_remote, rebase_branch))
739         sys.exit(0)
740     run_cmd("git describe %s/%s" %
741             (rebase_remote, rebase_branch),
742             show=True, dir=test_master)
743     run_cmd("git describe HEAD", show=True, dir=test_master)
744     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
745             (rebase_remote, rebase_branch),
746             show=True, dir=test_master)
747
748
749 def push_to(push_url, push_branch="master"):
750     push_remote = "pushto"
751     do_print("Pushing to %s" % push_url)
752     if options.mark:
753         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
754         run_cmd("git commit --amend -c HEAD", dir=test_master)
755         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
756         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
757     run_cmd("git remote add -t %s %s %s" %
758             (push_branch, push_remote, push_url),
759             show=True, dir=test_master)
760     run_cmd("git push %s +HEAD:%s" %
761             (push_remote, push_branch),
762             show=True, dir=test_master)
763
764
765 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
766
767 gitroot = find_git_root()
768 if gitroot is None:
769     raise Exception("Failed to find git root")
770
771 parser = OptionParser()
772 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
773 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
774 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
775 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
776                   default=def_testbase)
777 parser.add_option("", "--passcmd", help="command to run on success", default=None)
778 parser.add_option("", "--verbose", help="show all commands as they are run",
779                   default=False, action="store_true")
780 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
781                   default=None, type='str')
782 parser.add_option("", "--pushto", help="push to a git url on success",
783                   default=None, type='str')
784 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
785                   default=False, action="store_true")
786 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
787                   default=False, action="store_true")
788 parser.add_option("", "--retry", help="automatically retry if master changes",
789                   default=False, action="store_true")
790 parser.add_option("", "--email", help="send email to the given address on failure",
791                   type='str', default=None)
792 parser.add_option("", "--email-from", help="send email from the given address",
793                   type='str', default="autobuild@samba.org")
794 parser.add_option("", "--email-server", help="send email via the given server",
795                   type='str', default='localhost')
796 parser.add_option("", "--always-email", help="always send email, even on success",
797                   action="store_true")
798 parser.add_option("", "--daemon", help="daemonize after initial setup",
799                   action="store_true")
800 parser.add_option("", "--branch", help="the branch to work on (default=master)",
801                   default="master", type='str')
802 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
803                   default=gitroot, type='str')
804 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
805                   default=False, action="store_true")
806 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
807                   default='')
808
809
810 def send_email(subject, text, log_tar):
811     if options.email is None:
812         do_print("not sending email because the recipient is not set")
813         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
814                  (subject, text))
815         return
816     outer = MIMEMultipart()
817     outer['Subject'] = subject
818     outer['To'] = options.email
819     outer['From'] = options.email_from
820     outer['Date'] = email.utils.formatdate(localtime=True)
821     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
822     outer.attach(MIMEText(text, 'plain'))
823     if options.attach_logs:
824         fp = open(log_tar, 'rb')
825         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
826         fp.close()
827         # Set the filename parameter
828         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
829         outer.attach(msg)
830     content = outer.as_string()
831     s = smtplib.SMTP(options.email_server)
832     s.sendmail(options.email_from, [options.email], content)
833     s.set_debuglevel(1)
834     s.quit()
835
836
837 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
838                   elapsed_time, log_base=None, add_log_tail=True):
839     '''send an email to options.email about the failure'''
840     elapsed_minutes = elapsed_time / 60.0
841     if log_base is None:
842         log_base = gitroot
843     text = '''
844 Dear Developer,
845
846 Your autobuild on %s failed after %.1f minutes
847 when trying to test %s with the following error:
848
849    %s
850
851 the autobuild has been abandoned. Please fix the error and resubmit.
852
853 A summary of the autobuild process is here:
854
855   %s/autobuild.log
856 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
857
858     if options.restrict_tests:
859         text += """
860 The build was restricted to tests matching %s\n""" % options.restrict_tests
861
862     if failed_task != 'rebase':
863         text += '''
864 You can see logs of the failed task here:
865
866   %s/%s.stdout
867   %s/%s.stderr
868
869 or you can get full logs of all tasks in this job here:
870
871   %s/logs.tar.gz
872
873 The top commit for the tree that was built was:
874
875 %s
876
877 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
878
879     if add_log_tail:
880         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
881         lines = f.readlines()
882         log_tail = "".join(lines[-50:])
883         num_lines = len(lines)
884         if num_lines < 50:
885             # Also include stderr (compile failures) if < 50 lines of stdout
886             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
887             log_tail += "".join(f.readlines()[-(50 - num_lines):])
888
889         text += '''
890 The last 50 lines of log messages:
891
892 %s
893     ''' % log_tail
894         f.close()
895
896     logs = os.path.join(gitroot, 'logs.tar.gz')
897     send_email('autobuild[%s] failure on %s for task %s during %s'
898                % (options.branch, platform.node(), failed_task, failed_stage),
899                text, logs)
900
901
902 def email_success(elapsed_time, log_base=None):
903     '''send an email to options.email about a successful build'''
904     if log_base is None:
905         log_base = gitroot
906     text = '''
907 Dear Developer,
908
909 Your autobuild on %s has succeeded after %.1f minutes.
910
911 ''' % (platform.node(), elapsed_time / 60.)
912
913     if options.restrict_tests:
914         text += """
915 The build was restricted to tests matching %s\n""" % options.restrict_tests
916
917     if options.keeplogs:
918         text += '''
919
920 you can get full logs of all tasks in this job here:
921
922   %s/logs.tar.gz
923
924 ''' % log_base
925
926     text += '''
927 The top commit for the tree that was built was:
928
929 %s
930 ''' % top_commit_msg
931
932     logs = os.path.join(gitroot, 'logs.tar.gz')
933     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
934                text, logs)
935
936
937 (options, args) = parser.parse_args()
938
939 if options.retry:
940     if options.rebase is None:
941         raise Exception('You can only use --retry if you also rebase')
942
943 testbase = "%s/b%u" % (options.testbase, os.getpid())
944 test_master = "%s/master" % testbase
945 test_prefix = "%s/prefix" % testbase
946 test_tmpdir = "%s/tmp" % testbase
947 os.environ['TMPDIR'] = test_tmpdir
948
949 # get the top commit message, for emails
950 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
951 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
952
953 try:
954     os.makedirs(testbase)
955 except Exception as reason:
956     raise Exception("Unable to create %s : %s" % (testbase, reason))
957 cleanup_list.append(testbase)
958
959 if options.daemon:
960     logfile = os.path.join(testbase, "log")
961     do_print("Forking into the background, writing progress to %s" % logfile)
962     daemonize(logfile)
963
964 write_pidfile(gitroot + "/autobuild.pid")
965
966 start_time = time.time()
967
968 while True:
969     try:
970         run_cmd("rm -rf %s" % test_tmpdir, show=True)
971         os.makedirs(test_tmpdir)
972         # The waf uninstall code removes empty directories all the way
973         # up the tree.  Creating a file in test_tmpdir stops it from
974         # being removed.
975         run_cmd("touch %s" % os.path.join(test_tmpdir,
976                                           ".directory-is-not-empty"), show=True)
977         run_cmd("stat %s" % test_tmpdir, show=True)
978         run_cmd("stat %s" % testbase, show=True)
979         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
980     except Exception:
981         cleanup()
982         raise
983
984     try:
985         try:
986             if options.rebase is not None:
987                 rebase_tree(options.rebase, rebase_branch=options.branch)
988         except Exception:
989             cleanup_list.append(gitroot + "/autobuild.pid")
990             cleanup()
991             elapsed_time = time.time() - start_time
992             email_failure(-1, 'rebase', 'rebase', 'rebase',
993                           'rebase on %s failed' % options.branch,
994                           elapsed_time, log_base=options.log_base)
995             sys.exit(1)
996         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
997         if options.tail:
998             blist.start_tail()
999         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1000         if status != 0 or errstr != "retry":
1001             break
1002         cleanup()
1003     except Exception:
1004         cleanup()
1005         raise
1006
1007 cleanup_list.append(gitroot + "/autobuild.pid")
1008
1009 do_print(errstr)
1010
1011 blist.kill_kids()
1012 if options.tail:
1013     do_print("waiting for tail to flush")
1014     time.sleep(1)
1015
1016 elapsed_time = time.time() - start_time
1017 if status == 0:
1018     if options.passcmd is not None:
1019         do_print("Running passcmd: %s" % options.passcmd)
1020         run_cmd(options.passcmd, dir=test_master)
1021     if options.pushto is not None:
1022         push_to(options.pushto, push_branch=options.branch)
1023     if options.keeplogs or options.attach_logs:
1024         blist.tarlogs("logs.tar.gz")
1025         do_print("Logs in logs.tar.gz")
1026     if options.always_email:
1027         email_success(elapsed_time, log_base=options.log_base)
1028     blist.remove_logs()
1029     cleanup()
1030     do_print(errstr)
1031     sys.exit(0)
1032
1033 # something failed, gather a tar of the logs
1034 blist.tarlogs("logs.tar.gz")
1035
1036 if options.email is not None:
1037     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1038                   elapsed_time, log_base=options.log_base)
1039 else:
1040     elapsed_minutes = elapsed_time / 60.0
1041     print('''
1042
1043 ####################################################################
1044
1045 AUTOBUILD FAILURE
1046
1047 Your autobuild[%s] on %s failed after %.1f minutes
1048 when trying to test %s with the following error:
1049
1050    %s
1051
1052 the autobuild has been abandoned. Please fix the error and resubmit.
1053
1054 ####################################################################
1055
1056 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1057
1058 cleanup()
1059 do_print(errstr)
1060 do_print("Logs in logs.tar.gz")
1061 sys.exit(status)