-
Notifications
You must be signed in to change notification settings - Fork 7
/
Copy pathtrun
executable file
·206 lines (178 loc) · 9.36 KB
/
trun
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <[email protected]>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
""" `trun ...` - run `...` while testing pygolang
For example it is not possible to import sanitized libgolang.so if non-sanitized
python was used to start tests - it will fail with
ImportError: /usr/lib/x86_64-linux-gnu/libtsan.so.0: cannot allocate memory in static TLS block
trun cares to run python with LD_PRELOAD set appropriately to /path/to/libtsan.so
"""
# TODO integrate startup code into e.g. `gpython -race` which should arrange
# for _golang.xxx-race.so to be loaded instead of _golang.xxx.so, and also
# arrange for initial $LD_PRELOAD. Always build both _golang.xxx.so and
# _golang.xxx-race.so as part of standard pygolang build.
from __future__ import print_function, absolute_import
import os, os.path, sys, re, subprocess, platform, types
PY3 = (bytes is not str)
if PY3:
from importlib import machinery as imp_machinery
else:
import imp, pkgutil
# env_prepend prepends value to ${name} environment variable.
#
# the value is prepended with " " separator.
# the value is prepended - not appended - so that defaults set by trun can be overridden by user.
def env_prepend(name, value):
_ = os.environ.get(name, "")
if _ != "":
value += " " + _
os.environ[name] = value
# grep1 searches for the first line, that matches pattern, from text.
def grep1(pattern, text): # -> re.Match|None
p = re.compile(pattern)
for l in text.splitlines():
m = p.search(l)
if m is not None:
return m
return None
# ximport_empty_golangmod injects synthetic golang package in order to be able
# to import e.g. golang.pyx.build, or locate golang._golang, without built/working golang.
def ximport_empty_golangmod():
assert 'golang' not in sys.modules
golang = types.ModuleType('golang')
golang.__package__ = 'golang'
golang.__path__ = ['golang']
golang.__file__ = 'golang/__init__.py'
if PY3:
golang.__loader__ = imp_machinery.SourceFileLoader('golang', 'golang/__init__.py')
else:
golang.__loader__ = pkgutil.ImpLoader('golang', None, 'golang/__init__.py',
[None, None, imp.PY_SOURCE])
sys.modules['golang'] = golang
def main():
# install synthetic golang package so that we can import golang.X even if
# golang._golang (which is imported by golang/__init__.py) is not built/functional.
# Use golang.pyx.build._findpkg to locate _golang.so that corresponds to our python.
ximport_empty_golangmod()
from golang.pyx import build
_golang_so = build._findpkg('golang._golang')
# determine if _golang.so is linked to a sanitizer, and if yes, to which
# particular sanitizer DSO. Set LD_PRELOAD appropriately.
libxsan = None
ld_preload = None
if 'linux' in sys.platform:
p = subprocess.Popen(["ldd", _golang_so.path], stdout=subprocess.PIPE)
out, _ = p.communicate()
if PY3:
out = out.decode('utf-8')
_ = grep1(r"lib.san\.so\.. => ([^\s]+)", out)
if _ is not None:
libxsan = _.group(1)
# Some distributions (e.g. Debian testing as of 20200918) build
# libtsan/libasan with --as-needed, which removes libstdc++ from
# linked-in DSOs and lead to runtime assert failure inside
# sanitizer library on e.g. exception throw. Work it around with
# explicitly including libstdc++ into LD_PRELOAD as well.
# https://github.com/google/sanitizers/issues/934
# https://github.com/google/sanitizers/issues/934#issuecomment-649516500
_ = grep1(r"libstdc\+\+\.so\.. => ([^\s]+)", out)
if _ is None:
print("trun %r: cannot detect to which libstdc++ %s is linked" % (sys.argv[1:], _golang_so.path), file=sys.stderr)
sys.exit(2)
libstdcxx = _.group(1)
ld_preload = ("LD_PRELOAD", "%s %s" % (libxsan, libstdcxx))
elif 'darwin' in sys.platform:
# on darwin there is no ready out-of-the box analog of ldd, but
# sanitizer runtimes print instruction what to preload, e.g.
# ==973==ERROR: Interceptors are not working. This may be because ThreadSanitizer is loaded too late (e.g. via dlopen). Please launch the executable with:
# DYLD_INSERT_LIBRARIES=/Library/Developer/CommandLineTools/usr/lib/clang/10.0.1/lib/darwin/libclang_rt.tsan_osx_dynamic.dylib
# "interceptors not installed" && 0./test.sh: line 6: 973 Abort trap: 6 ./trun python -m pytest "$@"
# try to `import golang` to retrieve that.
p = subprocess.Popen(["python", "-c", "import golang"], stderr=subprocess.PIPE)
_, err = p.communicate()
if p.returncode != 0:
if PY3:
err = err.decode('utf-8')
_ = grep1("DYLD_INSERT_LIBRARIES=(.*)$", err)
if _ is not None:
libxsan = _.group(1)
ld_preload = ("DYLD_INSERT_LIBRARIES", libxsan)
else:
print("trun %r: `import golang` failed with unexpected error:" % sys.argv[1:], file=sys.stderr)
print(err, file=sys.stderr)
sys.exit(2)
# ld_preload has e.g. ("LD_PRELOAD", "/usr/lib/x86_64-linux-gnu/libtsan.so.0")
if ld_preload is not None:
#print('env <-', ld_preload)
env_prepend(*ld_preload)
# $LD_PRELOAD setup; ready to exec `...`
# if TSAN/ASAN detects a bug - make it fail loudly on the first bug
env_prepend("TSAN_OPTIONS", "halt_on_error=1")
env_prepend("ASAN_OPTIONS", "halt_on_error=1")
# tweak TSAN/ASAN/LSAN defaults:
# enable TSAN deadlock detector
# (unfortunately it caughts only few _potential_ deadlocks and actually
# gets stuck on any real deadlock)
env_prepend("TSAN_OPTIONS", "detect_deadlocks=1")
env_prepend("TSAN_OPTIONS", "second_deadlock_stack=1")
# tune ASAN to check more aggressively by default
env_prepend("ASAN_OPTIONS", "detect_stack_use_after_return=1")
# enable ASAN/LSAN leak detector.
#
# Do it only on CPython ≥ 3.11 because on py2 and on earlier py3 versions
# there are many many python allocations, whose lifetime coincide with
# python interpreter lifetime, and which are not explicitly freed on python
# shutdown. For py3 they significantly improved this step by step and
# starting from 3.11 it becomes practical to silence some still-leaks with
# suppressions, while for earlier py3 versions and especially for py2 it
# is, unfortunately, not manageable. Do not spend engineering time with
# activating LSAN on PyPy as that is tier 2 platform and bug tail history
# of memory leaks is very long even only on cpython.
if sys.version_info < (3,11):
env_prepend("ASAN_OPTIONS", "detect_leaks=0")
if libxsan is not None:
if 'asan' in libxsan.lower():
print("W: trun %r: asan: leak detection deactivated on %s %s" % (
sys.argv[1:], platform.python_implementation(), platform.python_version()),
file=sys.stderr)
else:
env_prepend("ASAN_OPTIONS", "detect_leaks=1")
env_prepend("LSAN_OPTIONS", "suppressions=%s" % os.path.abspath(os.path.join(
os.path.dirname(__file__), ".lsan-ignore.txt")))
# do not print statistics for suppressed leaks - else it breaks tests that verify program output
env_prepend("LSAN_OPTIONS", "print_suppressions=0")
# enable DWARF-based unwinding.
# else, if python is not compiled with -fno-omit-frame-pointer, it can show
# the whole traceback as e.g. just
# Direct leak of 32 byte(s) in 1 object(s) allocated from:
# #0 0x7f88522f3bd7 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
# #1 0x55f910a3d9a4 in PyThread_allocate_lock Python/thread_pthread.h:385
# and our leak suppressions won't work.
# this is slower compared to default frame-pointer based unwinding, but
# still works reasonably timely when run with just tests.
env_prepend("ASAN_OPTIONS", "fast_unwind_on_malloc=0")
# leak suppression also needs full tracebacks to work correctly, since with
# python there are many levels of call nesting at C level, and to filter-out e.g.
# top-level PyImport_Import we need to go really deep.
env_prepend("ASAN_OPTIONS", "malloc_context_size=255")
# exec `...`
os.execvp(sys.argv[1], sys.argv[1:])
if __name__ == '__main__':
main()