Python code coverage for Lib/_osx_support.py

#	count	content
1	n/a	"""Shared OS X support functions."""
2	n/a
3	n/a	import os
4	n/a	import re
5	n/a	import sys
6	n/a
7	n/a	__all__ = [
8	n/a	'compiler_fixup',
9	n/a	'customize_config_vars',
10	n/a	'customize_compiler',
11	n/a	'get_platform_osx',
12	n/a	]
13	n/a
14	n/a	# configuration variables that may contain universal build flags,
15	n/a	# like "-arch" or "-isdkroot", that may need customization for
16	n/a	# the user environment
17	n/a	_UNIVERSAL_CONFIG_VARS = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS', 'BASECFLAGS',
18	n/a	'BLDSHARED', 'LDSHARED', 'CC', 'CXX',
19	n/a	'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS',
20	n/a	'PY_CORE_CFLAGS')
21	n/a
22	n/a	# configuration variables that may contain compiler calls
23	n/a	_COMPILER_CONFIG_VARS = ('BLDSHARED', 'LDSHARED', 'CC', 'CXX')
24	n/a
25	n/a	# prefix added to original configuration variable names
26	n/a	_INITPRE = '_OSX_SUPPORT_INITIAL_'
27	n/a
28	n/a
29	n/a	def _find_executable(executable, path=None):
30	n/a	"""Tries to find 'executable' in the directories listed in 'path'.
31	n/a
32	n/a	A string listing directories separated by 'os.pathsep'; defaults to
33	n/a	os.environ['PATH']. Returns the complete filename or None if not found.
34	n/a	"""
35	n/a	if path is None:
36	n/a	path = os.environ['PATH']
37	n/a
38	n/a	paths = path.split(os.pathsep)
39	n/a	base, ext = os.path.splitext(executable)
40	n/a
41	n/a	if (sys.platform == 'win32') and (ext != '.exe'):
42	n/a	executable = executable + '.exe'
43	n/a
44	n/a	if not os.path.isfile(executable):
45	n/a	for p in paths:
46	n/a	f = os.path.join(p, executable)
47	n/a	if os.path.isfile(f):
48	n/a	# the file exists, we have a shot at spawn working
49	n/a	return f
50	n/a	return None
51	n/a	else:
52	n/a	return executable
53	n/a
54	n/a
55	n/a	def _read_output(commandstring):
56	n/a	"""Output from successful command execution or None"""
57	n/a	# Similar to os.popen(commandstring, "r").read(),
58	n/a	# but without actually using os.popen because that
59	n/a	# function is not usable during python bootstrap.
60	n/a	# tempfile is also not available then.
61	n/a	import contextlib
62	n/a	try:
63	n/a	import tempfile
64	n/a	fp = tempfile.NamedTemporaryFile()
65	n/a	except ImportError:
66	n/a	fp = open("/tmp/_osx_support.%s"%(
67	n/a	os.getpid(),), "w+b")
68	n/a
69	n/a	with contextlib.closing(fp) as fp:
70	n/a	cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name)
71	n/a	return fp.read().decode('utf-8').strip() if not os.system(cmd) else None
72	n/a
73	n/a
74	n/a	def _find_build_tool(toolname):
75	n/a	"""Find a build tool on current path or using xcrun"""
76	n/a	return (_find_executable(toolname)
77	n/a	or _read_output("/usr/bin/xcrun -find %s" % (toolname,))
78	n/a	or ''
79	n/a	)
80	n/a
81	n/a	_SYSTEM_VERSION = None
82	n/a
83	n/a	def _get_system_version():
84	n/a	"""Return the OS X system version as a string"""
85	n/a	# Reading this plist is a documented way to get the system
86	n/a	# version (see the documentation for the Gestalt Manager)
87	n/a	# We avoid using platform.mac_ver to avoid possible bootstrap issues during
88	n/a	# the build of Python itself (distutils is used to build standard library
89	n/a	# extensions).
90	n/a
91	n/a	global _SYSTEM_VERSION
92	n/a
93	n/a	if _SYSTEM_VERSION is None:
94	n/a	_SYSTEM_VERSION = ''
95	n/a	try:
96	n/a	f = open('/System/Library/CoreServices/SystemVersion.plist')
97	n/a	except OSError:
98	n/a	# We're on a plain darwin box, fall back to the default
99	n/a	# behaviour.
100	n/a	pass
101	n/a	else:
102	n/a	try:
103	n/a	m = re.search(r'<key>ProductUserVisibleVersion</key>\s*'
104	n/a	r'<string>(.*?)</string>', f.read())
105	n/a	finally:
106	n/a	f.close()
107	n/a	if m is not None:
108	n/a	_SYSTEM_VERSION = '.'.join(m.group(1).split('.')[:2])
109	n/a	# else: fall back to the default behaviour
110	n/a
111	n/a	return _SYSTEM_VERSION
112	n/a
113	n/a	def _remove_original_values(_config_vars):
114	n/a	"""Remove original unmodified values for testing"""
115	n/a	# This is needed for higher-level cross-platform tests of get_platform.
116	n/a	for k in list(_config_vars):
117	n/a	if k.startswith(_INITPRE):
118	n/a	del _config_vars[k]
119	n/a
120	n/a	def _save_modified_value(_config_vars, cv, newvalue):
121	n/a	"""Save modified and original unmodified value of configuration var"""
122	n/a
123	n/a	oldvalue = _config_vars.get(cv, '')
124	n/a	if (oldvalue != newvalue) and (_INITPRE + cv not in _config_vars):
125	n/a	_config_vars[_INITPRE + cv] = oldvalue
126	n/a	_config_vars[cv] = newvalue
127	n/a
128	n/a	def _supports_universal_builds():
129	n/a	"""Returns True if universal builds are supported on this system"""
130	n/a	# As an approximation, we assume that if we are running on 10.4 or above,
131	n/a	# then we are running with an Xcode environment that supports universal
132	n/a	# builds, in particular -isysroot and -arch arguments to the compiler. This
133	n/a	# is in support of allowing 10.4 universal builds to run on 10.3.x systems.
134	n/a
135	n/a	osx_version = _get_system_version()
136	n/a	if osx_version:
137	n/a	try:
138	n/a	osx_version = tuple(int(i) for i in osx_version.split('.'))
139	n/a	except ValueError:
140	n/a	osx_version = ''
141	n/a	return bool(osx_version >= (10, 4)) if osx_version else False
142	n/a
143	n/a
144	n/a	def _find_appropriate_compiler(_config_vars):
145	n/a	"""Find appropriate C compiler for extension module builds"""
146	n/a
147	n/a	# Issue #13590:
148	n/a	# The OSX location for the compiler varies between OSX
149	n/a	# (or rather Xcode) releases. With older releases (up-to 10.5)
150	n/a	# the compiler is in /usr/bin, with newer releases the compiler
151	n/a	# can only be found inside Xcode.app if the "Command Line Tools"
152	n/a	# are not installed.
153	n/a	#
154	n/a	# Furthermore, the compiler that can be used varies between
155	n/a	# Xcode releases. Up to Xcode 4 it was possible to use 'gcc-4.2'
156	n/a	# as the compiler, after that 'clang' should be used because
157	n/a	# gcc-4.2 is either not present, or a copy of 'llvm-gcc' that
158	n/a	# miscompiles Python.
159	n/a
160	n/a	# skip checks if the compiler was overridden with a CC env variable
161	n/a	if 'CC' in os.environ:
162	n/a	return _config_vars
163	n/a
164	n/a	# The CC config var might contain additional arguments.
165	n/a	# Ignore them while searching.
166	n/a	cc = oldcc = _config_vars['CC'].split()[0]
167	n/a	if not _find_executable(cc):
168	n/a	# Compiler is not found on the shell search PATH.
169	n/a	# Now search for clang, first on PATH (if the Command LIne
170	n/a	# Tools have been installed in / or if the user has provided
171	n/a	# another location via CC). If not found, try using xcrun
172	n/a	# to find an uninstalled clang (within a selected Xcode).
173	n/a
174	n/a	# NOTE: Cannot use subprocess here because of bootstrap
175	n/a	# issues when building Python itself (and os.popen is
176	n/a	# implemented on top of subprocess and is therefore not
177	n/a	# usable as well)
178	n/a
179	n/a	cc = _find_build_tool('clang')
180	n/a
181	n/a	elif os.path.basename(cc).startswith('gcc'):
182	n/a	# Compiler is GCC, check if it is LLVM-GCC
183	n/a	data = _read_output("'%s' --version"
184	n/a	% (cc.replace("'", "'\"'\"'"),))
185	n/a	if data and 'llvm-gcc' in data:
186	n/a	# Found LLVM-GCC, fall back to clang
187	n/a	cc = _find_build_tool('clang')
188	n/a
189	n/a	if not cc:
190	n/a	raise SystemError(
191	n/a	"Cannot locate working compiler")
192	n/a
193	n/a	if cc != oldcc:
194	n/a	# Found a replacement compiler.
195	n/a	# Modify config vars using new compiler, if not already explicitly
196	n/a	# overridden by an env variable, preserving additional arguments.
197	n/a	for cv in _COMPILER_CONFIG_VARS:
198	n/a	if cv in _config_vars and cv not in os.environ:
199	n/a	cv_split = _config_vars[cv].split()
200	n/a	cv_split[0] = cc if cv != 'CXX' else cc + '++'
201	n/a	_save_modified_value(_config_vars, cv, ' '.join(cv_split))
202	n/a
203	n/a	return _config_vars
204	n/a
205	n/a
206	n/a	def _remove_universal_flags(_config_vars):
207	n/a	"""Remove all universal build arguments from config vars"""
208	n/a
209	n/a	for cv in _UNIVERSAL_CONFIG_VARS:
210	n/a	# Do not alter a config var explicitly overridden by env var
211	n/a	if cv in _config_vars and cv not in os.environ:
212	n/a	flags = _config_vars[cv]
213	n/a	flags = re.sub(r'-arch\s+\w+\s', ' ', flags, re.ASCII)
214	n/a	flags = re.sub('-isysroot [^ \t]*', ' ', flags)
215	n/a	_save_modified_value(_config_vars, cv, flags)
216	n/a
217	n/a	return _config_vars
218	n/a
219	n/a
220	n/a	def _remove_unsupported_archs(_config_vars):
221	n/a	"""Remove any unsupported archs from config vars"""
222	n/a	# Different Xcode releases support different sets for '-arch'
223	n/a	# flags. In particular, Xcode 4.x no longer supports the
224	n/a	# PPC architectures.
225	n/a	#
226	n/a	# This code automatically removes '-arch ppc' and '-arch ppc64'
227	n/a	# when these are not supported. That makes it possible to
228	n/a	# build extensions on OSX 10.7 and later with the prebuilt
229	n/a	# 32-bit installer on the python.org website.
230	n/a
231	n/a	# skip checks if the compiler was overridden with a CC env variable
232	n/a	if 'CC' in os.environ:
233	n/a	return _config_vars
234	n/a
235	n/a	if re.search(r'-arch\s+ppc', _config_vars['CFLAGS']) is not None:
236	n/a	# NOTE: Cannot use subprocess here because of bootstrap
237	n/a	# issues when building Python itself
238	n/a	status = os.system(
239	n/a	"""echo 'int main{};' \| """
240	n/a	"""'%s' -c -arch ppc -x c -o /dev/null /dev/null 2>/dev/null"""
241	n/a	%(_config_vars['CC'].replace("'", "'\"'\"'"),))
242	n/a	if status:
243	n/a	# The compile failed for some reason. Because of differences
244	n/a	# across Xcode and compiler versions, there is no reliable way
245	n/a	# to be sure why it failed. Assume here it was due to lack of
246	n/a	# PPC support and remove the related '-arch' flags from each
247	n/a	# config variables not explicitly overridden by an environment
248	n/a	# variable. If the error was for some other reason, we hope the
249	n/a	# failure will show up again when trying to compile an extension
250	n/a	# module.
251	n/a	for cv in _UNIVERSAL_CONFIG_VARS:
252	n/a	if cv in _config_vars and cv not in os.environ:
253	n/a	flags = _config_vars[cv]
254	n/a	flags = re.sub(r'-arch\s+ppc\w*\s', ' ', flags)
255	n/a	_save_modified_value(_config_vars, cv, flags)
256	n/a
257	n/a	return _config_vars
258	n/a
259	n/a
260	n/a	def _override_all_archs(_config_vars):
261	n/a	"""Allow override of all archs with ARCHFLAGS env var"""
262	n/a	# NOTE: This name was introduced by Apple in OSX 10.5 and
263	n/a	# is used by several scripting languages distributed with
264	n/a	# that OS release.
265	n/a	if 'ARCHFLAGS' in os.environ:
266	n/a	arch = os.environ['ARCHFLAGS']
267	n/a	for cv in _UNIVERSAL_CONFIG_VARS:
268	n/a	if cv in _config_vars and '-arch' in _config_vars[cv]:
269	n/a	flags = _config_vars[cv]
270	n/a	flags = re.sub(r'-arch\s+\w+\s', ' ', flags)
271	n/a	flags = flags + ' ' + arch
272	n/a	_save_modified_value(_config_vars, cv, flags)
273	n/a
274	n/a	return _config_vars
275	n/a
276	n/a
277	n/a	def _check_for_unavailable_sdk(_config_vars):
278	n/a	"""Remove references to any SDKs not available"""
279	n/a	# If we're on OSX 10.5 or later and the user tries to
280	n/a	# compile an extension using an SDK that is not present
281	n/a	# on the current machine it is better to not use an SDK
282	n/a	# than to fail. This is particularly important with
283	n/a	# the standalone Command Line Tools alternative to a
284	n/a	# full-blown Xcode install since the CLT packages do not
285	n/a	# provide SDKs. If the SDK is not present, it is assumed
286	n/a	# that the header files and dev libs have been installed
287	n/a	# to /usr and /System/Library by either a standalone CLT
288	n/a	# package or the CLT component within Xcode.
289	n/a	cflags = _config_vars.get('CFLAGS', '')
290	n/a	m = re.search(r'-isysroot\s+(\S+)', cflags)
291	n/a	if m is not None:
292	n/a	sdk = m.group(1)
293	n/a	if not os.path.exists(sdk):
294	n/a	for cv in _UNIVERSAL_CONFIG_VARS:
295	n/a	# Do not alter a config var explicitly overridden by env var
296	n/a	if cv in _config_vars and cv not in os.environ:
297	n/a	flags = _config_vars[cv]
298	n/a	flags = re.sub(r'-isysroot\s+\S+(?:\s\|$)', ' ', flags)
299	n/a	_save_modified_value(_config_vars, cv, flags)
300	n/a
301	n/a	return _config_vars
302	n/a
303	n/a
304	n/a	def compiler_fixup(compiler_so, cc_args):
305	n/a	"""
306	n/a	This function will strip '-isysroot PATH' and '-arch ARCH' from the
307	n/a	compile flags if the user has specified one them in extra_compile_flags.
308	n/a
309	n/a	This is needed because '-arch ARCH' adds another architecture to the
310	n/a	build, without a way to remove an architecture. Furthermore GCC will
311	n/a	barf if multiple '-isysroot' arguments are present.
312	n/a	"""
313	n/a	stripArch = stripSysroot = False
314	n/a
315	n/a	compiler_so = list(compiler_so)
316	n/a
317	n/a	if not _supports_universal_builds():
318	n/a	# OSX before 10.4.0, these don't support -arch and -isysroot at
319	n/a	# all.
320	n/a	stripArch = stripSysroot = True
321	n/a	else:
322	n/a	stripArch = '-arch' in cc_args
323	n/a	stripSysroot = '-isysroot' in cc_args
324	n/a
325	n/a	if stripArch or 'ARCHFLAGS' in os.environ:
326	n/a	while True:
327	n/a	try:
328	n/a	index = compiler_so.index('-arch')
329	n/a	# Strip this argument and the next one:
330	n/a	del compiler_so[index:index+2]
331	n/a	except ValueError:
332	n/a	break
333	n/a
334	n/a	if 'ARCHFLAGS' in os.environ and not stripArch:
335	n/a	# User specified different -arch flags in the environ,
336	n/a	# see also distutils.sysconfig
337	n/a	compiler_so = compiler_so + os.environ['ARCHFLAGS'].split()
338	n/a
339	n/a	if stripSysroot:
340	n/a	while True:
341	n/a	try:
342	n/a	index = compiler_so.index('-isysroot')
343	n/a	# Strip this argument and the next one:
344	n/a	del compiler_so[index:index+2]
345	n/a	except ValueError:
346	n/a	break
347	n/a
348	n/a	# Check if the SDK that is used during compilation actually exists,
349	n/a	# the universal build requires the usage of a universal SDK and not all
350	n/a	# users have that installed by default.
351	n/a	sysroot = None
352	n/a	if '-isysroot' in cc_args:
353	n/a	idx = cc_args.index('-isysroot')
354	n/a	sysroot = cc_args[idx+1]
355	n/a	elif '-isysroot' in compiler_so:
356	n/a	idx = compiler_so.index('-isysroot')
357	n/a	sysroot = compiler_so[idx+1]
358	n/a
359	n/a	if sysroot and not os.path.isdir(sysroot):
360	n/a	from distutils import log
361	n/a	log.warn("Compiling with an SDK that doesn't seem to exist: %s",
362	n/a	sysroot)
363	n/a	log.warn("Please check your Xcode installation")
364	n/a
365	n/a	return compiler_so
366	n/a
367	n/a
368	n/a	def customize_config_vars(_config_vars):
369	n/a	"""Customize Python build configuration variables.
370	n/a
371	n/a	Called internally from sysconfig with a mutable mapping
372	n/a	containing name/value pairs parsed from the configured
373	n/a	makefile used to build this interpreter. Returns
374	n/a	the mapping updated as needed to reflect the environment
375	n/a	in which the interpreter is running; in the case of
376	n/a	a Python from a binary installer, the installed
377	n/a	environment may be very different from the build
378	n/a	environment, i.e. different OS levels, different
379	n/a	built tools, different available CPU architectures.
380	n/a
381	n/a	This customization is performed whenever
382	n/a	distutils.sysconfig.get_config_vars() is first
383	n/a	called. It may be used in environments where no
384	n/a	compilers are present, i.e. when installing pure
385	n/a	Python dists. Customization of compiler paths
386	n/a	and detection of unavailable archs is deferred
387	n/a	until the first extension module build is
388	n/a	requested (in distutils.sysconfig.customize_compiler).
389	n/a
390	n/a	Currently called from distutils.sysconfig
391	n/a	"""
392	n/a
393	n/a	if not _supports_universal_builds():
394	n/a	# On Mac OS X before 10.4, check if -arch and -isysroot
395	n/a	# are in CFLAGS or LDFLAGS and remove them if they are.
396	n/a	# This is needed when building extensions on a 10.3 system
397	n/a	# using a universal build of python.
398	n/a	_remove_universal_flags(_config_vars)
399	n/a
400	n/a	# Allow user to override all archs with ARCHFLAGS env var
401	n/a	_override_all_archs(_config_vars)
402	n/a
403	n/a	# Remove references to sdks that are not found
404	n/a	_check_for_unavailable_sdk(_config_vars)
405	n/a
406	n/a	return _config_vars
407	n/a
408	n/a
409	n/a	def customize_compiler(_config_vars):
410	n/a	"""Customize compiler path and configuration variables.
411	n/a
412	n/a	This customization is performed when the first
413	n/a	extension module build is requested
414	n/a	in distutils.sysconfig.customize_compiler).
415	n/a	"""
416	n/a
417	n/a	# Find a compiler to use for extension module builds
418	n/a	_find_appropriate_compiler(_config_vars)
419	n/a
420	n/a	# Remove ppc arch flags if not supported here
421	n/a	_remove_unsupported_archs(_config_vars)
422	n/a
423	n/a	# Allow user to override all archs with ARCHFLAGS env var
424	n/a	_override_all_archs(_config_vars)
425	n/a
426	n/a	return _config_vars
427	n/a
428	n/a
429	n/a	def get_platform_osx(_config_vars, osname, release, machine):
430	n/a	"""Filter values for get_platform()"""
431	n/a	# called from get_platform() in sysconfig and distutils.util
432	n/a	#
433	n/a	# For our purposes, we'll assume that the system version from
434	n/a	# distutils' perspective is what MACOSX_DEPLOYMENT_TARGET is set
435	n/a	# to. This makes the compatibility story a bit more sane because the
436	n/a	# machine is going to compile and link as if it were
437	n/a	# MACOSX_DEPLOYMENT_TARGET.
438	n/a
439	n/a	macver = _config_vars.get('MACOSX_DEPLOYMENT_TARGET', '')
440	n/a	macrelease = _get_system_version() or macver
441	n/a	macver = macver or macrelease
442	n/a
443	n/a	if macver:
444	n/a	release = macver
445	n/a	osname = "macosx"
446	n/a
447	n/a	# Use the original CFLAGS value, if available, so that we
448	n/a	# return the same machine type for the platform string.
449	n/a	# Otherwise, distutils may consider this a cross-compiling
450	n/a	# case and disallow installs.
451	n/a	cflags = _config_vars.get(_INITPRE+'CFLAGS',
452	n/a	_config_vars.get('CFLAGS', ''))
453	n/a	if macrelease:
454	n/a	try:
455	n/a	macrelease = tuple(int(i) for i in macrelease.split('.')[0:2])
456	n/a	except ValueError:
457	n/a	macrelease = (10, 0)
458	n/a	else:
459	n/a	# assume no universal support
460	n/a	macrelease = (10, 0)
461	n/a
462	n/a	if (macrelease >= (10, 4)) and '-arch' in cflags.strip():
463	n/a	# The universal build will build fat binaries, but not on
464	n/a	# systems before 10.4
465	n/a
466	n/a	machine = 'fat'
467	n/a
468	n/a	archs = re.findall(r'-arch\s+(\S+)', cflags)
469	n/a	archs = tuple(sorted(set(archs)))
470	n/a
471	n/a	if len(archs) == 1:
472	n/a	machine = archs[0]
473	n/a	elif archs == ('i386', 'ppc'):
474	n/a	machine = 'fat'
475	n/a	elif archs == ('i386', 'x86_64'):
476	n/a	machine = 'intel'
477	n/a	elif archs == ('i386', 'ppc', 'x86_64'):
478	n/a	machine = 'fat3'
479	n/a	elif archs == ('ppc64', 'x86_64'):
480	n/a	machine = 'fat64'
481	n/a	elif archs == ('i386', 'ppc', 'ppc64', 'x86_64'):
482	n/a	machine = 'universal'
483	n/a	else:
484	n/a	raise ValueError(
485	n/a	"Don't know machine value for archs=%r" % (archs,))
486	n/a
487	n/a	elif machine == 'i386':
488	n/a	# On OSX the machine type returned by uname is always the
489	n/a	# 32-bit variant, even if the executable architecture is
490	n/a	# the 64-bit variant
491	n/a	if sys.maxsize >= 2**32:
492	n/a	machine = 'x86_64'
493	n/a
494	n/a	elif machine in ('PowerPC', 'Power_Macintosh'):
495	n/a	# Pick a sane name for the PPC architecture.
496	n/a	# See 'i386' case
497	n/a	if sys.maxsize >= 2**32:
498	n/a	machine = 'ppc64'
499	n/a	else:
500	n/a	machine = 'ppc'
501	n/a
502	n/a	return (osname, release, machine)