551 lines
17 KiB
Python
551 lines
17 KiB
Python
from __future__ import print_function
|
|
|
|
import errno
|
|
import itertools
|
|
import math
|
|
import numbers
|
|
import os
|
|
import platform
|
|
import re
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
|
|
|
|
def is_string(value):
|
|
try:
|
|
# Python 2 and Python 3 are different here.
|
|
return isinstance(value, basestring)
|
|
except NameError:
|
|
return isinstance(value, str)
|
|
|
|
|
|
def pythonize_bool(value):
|
|
if value is None:
|
|
return False
|
|
if type(value) is bool:
|
|
return value
|
|
if isinstance(value, numbers.Number):
|
|
return value != 0
|
|
if is_string(value):
|
|
if value.lower() in ("1", "true", "on", "yes"):
|
|
return True
|
|
if value.lower() in ("", "0", "false", "off", "no"):
|
|
return False
|
|
raise ValueError('"{}" is not a valid boolean'.format(value))
|
|
|
|
|
|
def make_word_regex(word):
|
|
return r"\b" + word + r"\b"
|
|
|
|
|
|
def to_bytes(s):
|
|
"""Return the parameter as type 'bytes', possibly encoding it.
|
|
|
|
In Python2, the 'bytes' type is the same as 'str'. In Python3, they
|
|
are distinct.
|
|
|
|
"""
|
|
if isinstance(s, bytes):
|
|
# In Python2, this branch is taken for both 'str' and 'bytes'.
|
|
# In Python3, this branch is taken only for 'bytes'.
|
|
return s
|
|
# In Python2, 's' is a 'unicode' object.
|
|
# In Python3, 's' is a 'str' object.
|
|
# Encode to UTF-8 to get 'bytes' data.
|
|
return s.encode("utf-8")
|
|
|
|
|
|
def to_string(b):
|
|
"""Return the parameter as type 'str', possibly encoding it.
|
|
|
|
In Python2, the 'str' type is the same as 'bytes'. In Python3, the
|
|
'str' type is (essentially) Python2's 'unicode' type, and 'bytes' is
|
|
distinct.
|
|
|
|
"""
|
|
if isinstance(b, str):
|
|
# In Python2, this branch is taken for types 'str' and 'bytes'.
|
|
# In Python3, this branch is taken only for 'str'.
|
|
return b
|
|
if isinstance(b, bytes):
|
|
# In Python2, this branch is never taken ('bytes' is handled as 'str').
|
|
# In Python3, this is true only for 'bytes'.
|
|
try:
|
|
return b.decode("utf-8")
|
|
except UnicodeDecodeError:
|
|
# If the value is not valid Unicode, return the default
|
|
# repr-line encoding.
|
|
return str(b)
|
|
|
|
# By this point, here's what we *don't* have:
|
|
#
|
|
# - In Python2:
|
|
# - 'str' or 'bytes' (1st branch above)
|
|
# - In Python3:
|
|
# - 'str' (1st branch above)
|
|
# - 'bytes' (2nd branch above)
|
|
#
|
|
# The last type we might expect is the Python2 'unicode' type. There is no
|
|
# 'unicode' type in Python3 (all the Python3 cases were already handled). In
|
|
# order to get a 'str' object, we need to encode the 'unicode' object.
|
|
try:
|
|
return b.encode("utf-8")
|
|
except AttributeError:
|
|
raise TypeError("not sure how to convert %s to %s" % (type(b), str))
|
|
|
|
|
|
def to_unicode(s):
|
|
"""Return the parameter as type which supports unicode, possibly decoding
|
|
it.
|
|
|
|
In Python2, this is the unicode type. In Python3 it's the str type.
|
|
|
|
"""
|
|
if isinstance(s, bytes):
|
|
# In Python2, this branch is taken for both 'str' and 'bytes'.
|
|
# In Python3, this branch is taken only for 'bytes'.
|
|
return s.decode("utf-8")
|
|
return s
|
|
|
|
|
|
def usable_core_count():
|
|
"""Return the number of cores the current process can use, if supported.
|
|
Otherwise, return the total number of cores (like `os.cpu_count()`).
|
|
Default to 1 if undetermined.
|
|
|
|
"""
|
|
try:
|
|
n = len(os.sched_getaffinity(0))
|
|
except AttributeError:
|
|
n = os.cpu_count() or 1
|
|
|
|
# On Windows with more than 60 processes, multiprocessing's call to
|
|
# _winapi.WaitForMultipleObjects() prints an error and lit hangs.
|
|
if platform.system() == "Windows":
|
|
return min(n, 60)
|
|
|
|
return n
|
|
|
|
def abs_path_preserve_drive(path):
|
|
"""Return the absolute path without resolving drive mappings on Windows.
|
|
|
|
"""
|
|
if platform.system() == "Windows":
|
|
# Windows has limitations on path length (MAX_PATH) that
|
|
# can be worked around using substitute drives, which map
|
|
# a drive letter to a longer path on another drive.
|
|
# Since Python 3.8, os.path.realpath resolves sustitute drives,
|
|
# so we should not use it. In Python 3.7, os.path.realpath
|
|
# was implemented as os.path.abspath.
|
|
return os.path.abspath(path)
|
|
else:
|
|
# On UNIX, the current directory always has symbolic links resolved,
|
|
# so any program accepting relative paths cannot preserve symbolic
|
|
# links in paths and we should always use os.path.realpath.
|
|
return os.path.realpath(path)
|
|
|
|
def mkdir(path):
|
|
try:
|
|
if platform.system() == "Windows":
|
|
from ctypes import windll
|
|
from ctypes import GetLastError, WinError
|
|
|
|
path = os.path.abspath(path)
|
|
# Make sure that the path uses backslashes here, in case
|
|
# python would have happened to use forward slashes, as the
|
|
# NT path format only supports backslashes.
|
|
path = path.replace("/", "\\")
|
|
NTPath = to_unicode(r"\\?\%s" % path)
|
|
if not windll.kernel32.CreateDirectoryW(NTPath, None):
|
|
raise WinError(GetLastError())
|
|
else:
|
|
os.mkdir(path)
|
|
except OSError:
|
|
e = sys.exc_info()[1]
|
|
# ignore EEXIST, which may occur during a race condition
|
|
if e.errno != errno.EEXIST:
|
|
raise
|
|
|
|
|
|
def mkdir_p(path):
|
|
"""mkdir_p(path) - Make the "path" directory, if it does not exist; this
|
|
will also make directories for any missing parent directories."""
|
|
if not path or os.path.exists(path):
|
|
return
|
|
|
|
parent = os.path.dirname(path)
|
|
if parent != path:
|
|
mkdir_p(parent)
|
|
|
|
mkdir(path)
|
|
|
|
|
|
def listdir_files(dirname, suffixes=None, exclude_filenames=None):
|
|
"""Yields files in a directory.
|
|
|
|
Filenames that are not excluded by rules below are yielded one at a time, as
|
|
basenames (i.e., without dirname).
|
|
|
|
Files starting with '.' are always skipped.
|
|
|
|
If 'suffixes' is not None, then only filenames ending with one of its
|
|
members will be yielded. These can be extensions, like '.exe', or strings,
|
|
like 'Test'. (It is a lexicographic check; so an empty sequence will yield
|
|
nothing, but a single empty string will yield all filenames.)
|
|
|
|
If 'exclude_filenames' is not None, then none of the file basenames in it
|
|
will be yielded.
|
|
|
|
If specified, the containers for 'suffixes' and 'exclude_filenames' must
|
|
support membership checking for strs.
|
|
|
|
Args:
|
|
dirname: a directory path.
|
|
suffixes: (optional) a sequence of strings (set, list, etc.).
|
|
exclude_filenames: (optional) a sequence of strings.
|
|
|
|
Yields:
|
|
Filenames as returned by os.listdir (generally, str).
|
|
|
|
"""
|
|
if exclude_filenames is None:
|
|
exclude_filenames = set()
|
|
if suffixes is None:
|
|
suffixes = {""}
|
|
for filename in os.listdir(dirname):
|
|
if (
|
|
os.path.isdir(os.path.join(dirname, filename))
|
|
or filename.startswith(".")
|
|
or filename in exclude_filenames
|
|
or not any(filename.endswith(sfx) for sfx in suffixes)
|
|
):
|
|
continue
|
|
yield filename
|
|
|
|
|
|
def which(command, paths=None):
|
|
"""which(command, [paths]) - Look up the given command in the paths string
|
|
(or the PATH environment variable, if unspecified)."""
|
|
|
|
if paths is None:
|
|
paths = os.environ.get("PATH", "")
|
|
|
|
# Check for absolute match first.
|
|
if os.path.isabs(command) and os.path.isfile(command):
|
|
return os.path.normcase(os.path.normpath(command))
|
|
|
|
# Would be nice if Python had a lib function for this.
|
|
if not paths:
|
|
paths = os.defpath
|
|
|
|
# Get suffixes to search.
|
|
# On Cygwin, 'PATHEXT' may exist but it should not be used.
|
|
if os.pathsep == ";":
|
|
pathext = os.environ.get("PATHEXT", "").split(";")
|
|
else:
|
|
pathext = [""]
|
|
|
|
# Search the paths...
|
|
for path in paths.split(os.pathsep):
|
|
for ext in pathext:
|
|
p = os.path.join(path, command + ext)
|
|
if os.path.exists(p) and not os.path.isdir(p):
|
|
return os.path.normcase(os.path.abspath(p))
|
|
|
|
return None
|
|
|
|
|
|
def checkToolsPath(dir, tools):
|
|
for tool in tools:
|
|
if not os.path.exists(os.path.join(dir, tool)):
|
|
return False
|
|
return True
|
|
|
|
|
|
def whichTools(tools, paths):
|
|
for path in paths.split(os.pathsep):
|
|
if checkToolsPath(path, tools):
|
|
return path
|
|
return None
|
|
|
|
|
|
def printHistogram(items, title="Items"):
|
|
items.sort(key=lambda item: item[1])
|
|
|
|
maxValue = max([v for _, v in items])
|
|
|
|
# Select first "nice" bar height that produces more than 10 bars.
|
|
power = int(math.ceil(math.log(maxValue, 10)))
|
|
for inc in itertools.cycle((5, 2, 2.5, 1)):
|
|
barH = inc * 10**power
|
|
N = int(math.ceil(maxValue / barH))
|
|
if N > 10:
|
|
break
|
|
elif inc == 1:
|
|
power -= 1
|
|
|
|
histo = [set() for i in range(N)]
|
|
for name, v in items:
|
|
bin = min(int(N * v / maxValue), N - 1)
|
|
histo[bin].add(name)
|
|
|
|
barW = 40
|
|
hr = "-" * (barW + 34)
|
|
print("Slowest %s:" % title)
|
|
print(hr)
|
|
for name, value in reversed(items[-20:]):
|
|
print("%.2fs: %s" % (value, name))
|
|
print("\n%s Times:" % title)
|
|
print(hr)
|
|
pDigits = int(math.ceil(math.log(maxValue, 10)))
|
|
pfDigits = max(0, 3 - pDigits)
|
|
if pfDigits:
|
|
pDigits += pfDigits + 1
|
|
cDigits = int(math.ceil(math.log(len(items), 10)))
|
|
print(
|
|
"[%s] :: [%s] :: [%s]"
|
|
% (
|
|
"Range".center((pDigits + 1) * 2 + 3),
|
|
"Percentage".center(barW),
|
|
"Count".center(cDigits * 2 + 1),
|
|
)
|
|
)
|
|
print(hr)
|
|
for i, row in reversed(list(enumerate(histo))):
|
|
pct = float(len(row)) / len(items)
|
|
w = int(barW * pct)
|
|
print(
|
|
"[%*.*fs,%*.*fs) :: [%s%s] :: [%*d/%*d]"
|
|
% (
|
|
pDigits,
|
|
pfDigits,
|
|
i * barH,
|
|
pDigits,
|
|
pfDigits,
|
|
(i + 1) * barH,
|
|
"*" * w,
|
|
" " * (barW - w),
|
|
cDigits,
|
|
len(row),
|
|
cDigits,
|
|
len(items),
|
|
)
|
|
)
|
|
print(hr)
|
|
|
|
|
|
class ExecuteCommandTimeoutException(Exception):
|
|
def __init__(self, msg, out, err, exitCode):
|
|
assert isinstance(msg, str)
|
|
assert isinstance(out, str)
|
|
assert isinstance(err, str)
|
|
assert isinstance(exitCode, int)
|
|
self.msg = msg
|
|
self.out = out
|
|
self.err = err
|
|
self.exitCode = exitCode
|
|
|
|
|
|
# Close extra file handles on UNIX (on Windows this cannot be done while
|
|
# also redirecting input).
|
|
kUseCloseFDs = not (platform.system() == "Windows")
|
|
|
|
|
|
def executeCommand(
|
|
command, cwd=None, env=None, input=None, timeout=0, redirect_stderr=False
|
|
):
|
|
"""Execute command ``command`` (list of arguments or string) with.
|
|
|
|
* working directory ``cwd`` (str), use None to use the current
|
|
working directory
|
|
* environment ``env`` (dict), use None for none
|
|
* Input to the command ``input`` (str), use string to pass
|
|
no input.
|
|
* Max execution time ``timeout`` (int) seconds. Use 0 for no timeout.
|
|
* ``redirect_stderr`` (bool), use True if redirect stderr to stdout
|
|
|
|
Returns a tuple (out, err, exitCode) where
|
|
* ``out`` (str) is the standard output of running the command
|
|
* ``err`` (str) is the standard error of running the command
|
|
* ``exitCode`` (int) is the exitCode of running the command
|
|
|
|
If the timeout is hit an ``ExecuteCommandTimeoutException``
|
|
is raised.
|
|
|
|
"""
|
|
if input is not None:
|
|
input = to_bytes(input)
|
|
err_out = subprocess.STDOUT if redirect_stderr else subprocess.PIPE
|
|
p = subprocess.Popen(
|
|
command,
|
|
cwd=cwd,
|
|
stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE,
|
|
stderr=err_out,
|
|
env=env,
|
|
close_fds=kUseCloseFDs,
|
|
)
|
|
timerObject = None
|
|
# FIXME: Because of the way nested function scopes work in Python 2.x we
|
|
# need to use a reference to a mutable object rather than a plain
|
|
# bool. In Python 3 we could use the "nonlocal" keyword but we need
|
|
# to support Python 2 as well.
|
|
hitTimeOut = [False]
|
|
try:
|
|
if timeout > 0:
|
|
|
|
def killProcess():
|
|
# We may be invoking a shell so we need to kill the
|
|
# process and all its children.
|
|
hitTimeOut[0] = True
|
|
killProcessAndChildren(p.pid)
|
|
|
|
timerObject = threading.Timer(timeout, killProcess)
|
|
timerObject.start()
|
|
|
|
out, err = p.communicate(input=input)
|
|
exitCode = p.wait()
|
|
finally:
|
|
if timerObject != None:
|
|
timerObject.cancel()
|
|
|
|
# Ensure the resulting output is always of string type.
|
|
out = to_string(out)
|
|
err = "" if redirect_stderr else to_string(err)
|
|
|
|
if hitTimeOut[0]:
|
|
raise ExecuteCommandTimeoutException(
|
|
msg="Reached timeout of {} seconds".format(timeout),
|
|
out=out,
|
|
err=err,
|
|
exitCode=exitCode,
|
|
)
|
|
|
|
# Detect Ctrl-C in subprocess.
|
|
if exitCode == -signal.SIGINT:
|
|
raise KeyboardInterrupt
|
|
|
|
return out, err, exitCode
|
|
|
|
|
|
def isAIXTriple(target_triple):
|
|
"""Whether the given target triple is for AIX,
|
|
e.g. powerpc64-ibm-aix
|
|
"""
|
|
return "aix" in target_triple
|
|
|
|
|
|
def addAIXVersion(target_triple):
|
|
"""Add the AIX version to the given target triple,
|
|
e.g. powerpc64-ibm-aix7.2.5.6
|
|
"""
|
|
os_cmd = "oslevel -s | awk -F\'-\' \'{printf \"%.1f.%d.%d\", $1/1000, $2, $3}\'"
|
|
os_version = subprocess.run(os_cmd, capture_output=True, shell=True).stdout.decode()
|
|
return re.sub("aix", "aix" + os_version, target_triple)
|
|
|
|
|
|
def isMacOSTriple(target_triple):
|
|
"""Whether the given target triple is for macOS,
|
|
e.g. x86_64-apple-darwin, arm64-apple-macos
|
|
"""
|
|
return "darwin" in target_triple or "macos" in target_triple
|
|
|
|
|
|
def usePlatformSdkOnDarwin(config, lit_config):
|
|
# On Darwin, support relocatable SDKs by providing Clang with a
|
|
# default system root path.
|
|
if isMacOSTriple(config.target_triple):
|
|
try:
|
|
cmd = subprocess.Popen(
|
|
["xcrun", "--show-sdk-path", "--sdk", "macosx"],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
out, err = cmd.communicate()
|
|
out = out.strip()
|
|
res = cmd.wait()
|
|
except OSError:
|
|
res = -1
|
|
if res == 0 and out:
|
|
sdk_path = out.decode()
|
|
lit_config.note("using SDKROOT: %r" % sdk_path)
|
|
config.environment["SDKROOT"] = sdk_path
|
|
|
|
|
|
def findPlatformSdkVersionOnMacOS(config, lit_config):
|
|
if isMacOSTriple(config.target_triple):
|
|
try:
|
|
cmd = subprocess.Popen(
|
|
["xcrun", "--show-sdk-version", "--sdk", "macosx"],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
out, err = cmd.communicate()
|
|
out = out.strip()
|
|
res = cmd.wait()
|
|
except OSError:
|
|
res = -1
|
|
if res == 0 and out:
|
|
return out.decode()
|
|
return None
|
|
|
|
|
|
def killProcessAndChildrenIsSupported():
|
|
"""
|
|
Returns a tuple (<supported> , <error message>)
|
|
where
|
|
`<supported>` is True if `killProcessAndChildren()` is supported on
|
|
the current host, returns False otherwise.
|
|
`<error message>` is an empty string if `<supported>` is True,
|
|
otherwise is contains a string describing why the function is
|
|
not supported.
|
|
"""
|
|
if platform.system() == "AIX":
|
|
return (True, "")
|
|
try:
|
|
import psutil # noqa: F401
|
|
|
|
return (True, "")
|
|
except ImportError:
|
|
return (
|
|
False,
|
|
"Requires the Python psutil module but it could"
|
|
" not be found. Try installing it via pip or via"
|
|
" your operating system's package manager.",
|
|
)
|
|
|
|
|
|
def killProcessAndChildren(pid):
|
|
"""This function kills a process with ``pid`` and all its running children
|
|
(recursively). It is currently implemented using the psutil module on some
|
|
platforms which provides a simple platform neutral implementation.
|
|
|
|
TODO: Reimplement this without using psutil on all platforms so we can
|
|
remove our dependency on it.
|
|
|
|
"""
|
|
if platform.system() == "AIX":
|
|
subprocess.call("kill -kill $(ps -o pid= -L{})".format(pid), shell=True)
|
|
else:
|
|
import psutil
|
|
|
|
try:
|
|
psutilProc = psutil.Process(pid)
|
|
# Handle the different psutil API versions
|
|
try:
|
|
# psutil >= 2.x
|
|
children_iterator = psutilProc.children(recursive=True)
|
|
except AttributeError:
|
|
# psutil 1.x
|
|
children_iterator = psutilProc.get_children(recursive=True)
|
|
for child in children_iterator:
|
|
try:
|
|
child.kill()
|
|
except psutil.NoSuchProcess:
|
|
pass
|
|
psutilProc.kill()
|
|
except psutil.NoSuchProcess:
|
|
pass
|