| from __future__ import print_function |
| |
| import errno |
| import itertools |
| import math |
| import numbers |
| import os |
| import platform |
| import signal |
| import subprocess |
| import sys |
| import threading |
| |
| |
| def norm_path(path): |
| path = os.path.realpath(path) |
| path = os.path.normpath(path) |
| path = os.path.normcase(path) |
| return path |
| |
| |
| def is_string(value): |
| try: |
| # Python 2 and Python 3 are different here. |
| return isinstance(value, basestring) |
| except NameError: |
| return isinstance(value, str) |
| |
| |
| def pythonize_bool(value): |
| if value is None: |
| return False |
| if type(value) is bool: |
| return value |
| if isinstance(value, numbers.Number): |
| return value != 0 |
| if is_string(value): |
| if value.lower() in ('1', 'true', 'on', 'yes'): |
| return True |
| if value.lower() in ('', '0', 'false', 'off', 'no'): |
| return False |
| raise ValueError('"{}" is not a valid boolean'.format(value)) |
| |
| |
| def make_word_regex(word): |
| return r'\b' + word + r'\b' |
| |
| |
| def to_bytes(s): |
| """Return the parameter as type 'bytes', possibly encoding it. |
| |
| In Python2, the 'bytes' type is the same as 'str'. In Python3, they |
| are distinct. |
| |
| """ |
| if isinstance(s, bytes): |
| # In Python2, this branch is taken for both 'str' and 'bytes'. |
| # In Python3, this branch is taken only for 'bytes'. |
| return s |
| # In Python2, 's' is a 'unicode' object. |
| # In Python3, 's' is a 'str' object. |
| # Encode to UTF-8 to get 'bytes' data. |
| return s.encode('utf-8') |
| |
| |
| def to_string(b): |
| """Return the parameter as type 'str', possibly encoding it. |
| |
| In Python2, the 'str' type is the same as 'bytes'. In Python3, the |
| 'str' type is (essentially) Python2's 'unicode' type, and 'bytes' is |
| distinct. |
| |
| """ |
| if isinstance(b, str): |
| # In Python2, this branch is taken for types 'str' and 'bytes'. |
| # In Python3, this branch is taken only for 'str'. |
| return b |
| if isinstance(b, bytes): |
| # In Python2, this branch is never taken ('bytes' is handled as 'str'). |
| # In Python3, this is true only for 'bytes'. |
| try: |
| return b.decode('utf-8') |
| except UnicodeDecodeError: |
| # If the value is not valid Unicode, return the default |
| # repr-line encoding. |
| return str(b) |
| |
| # By this point, here's what we *don't* have: |
| # |
| # - In Python2: |
| # - 'str' or 'bytes' (1st branch above) |
| # - In Python3: |
| # - 'str' (1st branch above) |
| # - 'bytes' (2nd branch above) |
| # |
| # The last type we might expect is the Python2 'unicode' type. There is no |
| # 'unicode' type in Python3 (all the Python3 cases were already handled). In |
| # order to get a 'str' object, we need to encode the 'unicode' object. |
| try: |
| return b.encode('utf-8') |
| except AttributeError: |
| raise TypeError('not sure how to convert %s to %s' % (type(b), str)) |
| |
| |
| def to_unicode(s): |
| """Return the parameter as type which supports unicode, possibly decoding |
| it. |
| |
| In Python2, this is the unicode type. In Python3 it's the str type. |
| |
| """ |
| if isinstance(s, bytes): |
| # In Python2, this branch is taken for both 'str' and 'bytes'. |
| # In Python3, this branch is taken only for 'bytes'. |
| return s.decode('utf-8') |
| return s |
| |
| |
| def detectCPUs(): |
| """Detects the number of CPUs on a system. |
| |
| Cribbed from pp. |
| |
| """ |
| # Linux, Unix and MacOS: |
| if hasattr(os, 'sysconf'): |
| if 'SC_NPROCESSORS_ONLN' in os.sysconf_names: |
| # Linux & Unix: |
| ncpus = os.sysconf('SC_NPROCESSORS_ONLN') |
| if isinstance(ncpus, int) and ncpus > 0: |
| return ncpus |
| else: # OSX: |
| return int(subprocess.check_output(['sysctl', '-n', 'hw.ncpu'], |
| stderr=subprocess.STDOUT)) |
| # Windows: |
| if 'NUMBER_OF_PROCESSORS' in os.environ: |
| ncpus = int(os.environ['NUMBER_OF_PROCESSORS']) |
| if ncpus > 0: |
| # With more than 32 processes, process creation often fails with |
| # "Too many open files". FIXME: Check if there's a better fix. |
| return min(ncpus, 32) |
| return 1 # Default |
| |
| |
| def mkdir(path): |
| try: |
| if platform.system() == 'Windows': |
| from ctypes import windll |
| from ctypes import GetLastError, WinError |
| |
| path = os.path.abspath(path) |
| NTPath = to_unicode(r'\\?\%s' % path) |
| if not windll.kernel32.CreateDirectoryW(NTPath, None): |
| raise WinError(GetLastError()) |
| else: |
| os.mkdir(path) |
| except OSError: |
| e = sys.exc_info()[1] |
| # ignore EEXIST, which may occur during a race condition |
| if e.errno != errno.EEXIST: |
| raise |
| |
| |
| def mkdir_p(path): |
| """mkdir_p(path) - Make the "path" directory, if it does not exist; this |
| will also make directories for any missing parent directories.""" |
| if not path or os.path.exists(path): |
| return |
| |
| parent = os.path.dirname(path) |
| if parent != path: |
| mkdir_p(parent) |
| |
| mkdir(path) |
| |
| |
| def listdir_files(dirname, suffixes=None, exclude_filenames=None): |
| """Yields files in a directory. |
| |
| Filenames that are not excluded by rules below are yielded one at a time, as |
| basenames (i.e., without dirname). |
| |
| Files starting with '.' are always skipped. |
| |
| If 'suffixes' is not None, then only filenames ending with one of its |
| members will be yielded. These can be extensions, like '.exe', or strings, |
| like 'Test'. (It is a lexicographic check; so an empty sequence will yield |
| nothing, but a single empty string will yield all filenames.) |
| |
| If 'exclude_filenames' is not None, then none of the file basenames in it |
| will be yielded. |
| |
| If specified, the containers for 'suffixes' and 'exclude_filenames' must |
| support membership checking for strs. |
| |
| Args: |
| dirname: a directory path. |
| suffixes: (optional) a sequence of strings (set, list, etc.). |
| exclude_filenames: (optional) a sequence of strings. |
| |
| Yields: |
| Filenames as returned by os.listdir (generally, str). |
| |
| """ |
| if exclude_filenames is None: |
| exclude_filenames = set() |
| if suffixes is None: |
| suffixes = {''} |
| for filename in os.listdir(dirname): |
| if (os.path.isdir(os.path.join(dirname, filename)) or |
| filename.startswith('.') or |
| filename in exclude_filenames or |
| not any(filename.endswith(sfx) for sfx in suffixes)): |
| continue |
| yield filename |
| |
| |
| def which(command, paths=None): |
| """which(command, [paths]) - Look up the given command in the paths string |
| (or the PATH environment variable, if unspecified).""" |
| |
| if paths is None: |
| paths = os.environ.get('PATH', '') |
| |
| # Check for absolute match first. |
| if os.path.isabs(command) and os.path.isfile(command): |
| return os.path.normcase(os.path.normpath(command)) |
| |
| # Would be nice if Python had a lib function for this. |
| if not paths: |
| paths = os.defpath |
| |
| # Get suffixes to search. |
| # On Cygwin, 'PATHEXT' may exist but it should not be used. |
| if os.pathsep == ';': |
| pathext = os.environ.get('PATHEXT', '').split(';') |
| else: |
| pathext = [''] |
| |
| # Search the paths... |
| for path in paths.split(os.pathsep): |
| for ext in pathext: |
| p = os.path.join(path, command + ext) |
| if os.path.exists(p) and not os.path.isdir(p): |
| return os.path.normcase(os.path.normpath(p)) |
| |
| return None |
| |
| |
| def checkToolsPath(dir, tools): |
| for tool in tools: |
| if not os.path.exists(os.path.join(dir, tool)): |
| return False |
| return True |
| |
| |
| def whichTools(tools, paths): |
| for path in paths.split(os.pathsep): |
| if checkToolsPath(path, tools): |
| return path |
| return None |
| |
| |
| def printHistogram(items, title='Items'): |
| items.sort(key=lambda item: item[1]) |
| |
| maxValue = max([v for _, v in items]) |
| |
| # Select first "nice" bar height that produces more than 10 bars. |
| power = int(math.ceil(math.log(maxValue, 10))) |
| for inc in itertools.cycle((5, 2, 2.5, 1)): |
| barH = inc * 10**power |
| N = int(math.ceil(maxValue / barH)) |
| if N > 10: |
| break |
| elif inc == 1: |
| power -= 1 |
| |
| histo = [set() for i in range(N)] |
| for name, v in items: |
| bin = min(int(N * v / maxValue), N - 1) |
| histo[bin].add(name) |
| |
| barW = 40 |
| hr = '-' * (barW + 34) |
| print('\nSlowest %s:' % title) |
| print(hr) |
| for name, value in items[-20:]: |
| print('%.2fs: %s' % (value, name)) |
| print('\n%s Times:' % title) |
| print(hr) |
| pDigits = int(math.ceil(math.log(maxValue, 10))) |
| pfDigits = max(0, 3 - pDigits) |
| if pfDigits: |
| pDigits += pfDigits + 1 |
| cDigits = int(math.ceil(math.log(len(items), 10))) |
| print('[%s] :: [%s] :: [%s]' % ('Range'.center((pDigits + 1) * 2 + 3), |
| 'Percentage'.center(barW), |
| 'Count'.center(cDigits * 2 + 1))) |
| print(hr) |
| for i, row in enumerate(histo): |
| pct = float(len(row)) / len(items) |
| w = int(barW * pct) |
| print('[%*.*fs,%*.*fs) :: [%s%s] :: [%*d/%*d]' % ( |
| pDigits, pfDigits, i * barH, pDigits, pfDigits, (i + 1) * barH, |
| '*' * w, ' ' * (barW - w), cDigits, len(row), cDigits, len(items))) |
| |
| |
| class ExecuteCommandTimeoutException(Exception): |
| def __init__(self, msg, out, err, exitCode): |
| assert isinstance(msg, str) |
| assert isinstance(out, str) |
| assert isinstance(err, str) |
| assert isinstance(exitCode, int) |
| self.msg = msg |
| self.out = out |
| self.err = err |
| self.exitCode = exitCode |
| |
| |
| # Close extra file handles on UNIX (on Windows this cannot be done while |
| # also redirecting input). |
| kUseCloseFDs = not (platform.system() == 'Windows') |
| |
| |
| def executeCommand(command, cwd=None, env=None, input=None, timeout=0): |
| """Execute command ``command`` (list of arguments or string) with. |
| |
| * working directory ``cwd`` (str), use None to use the current |
| working directory |
| * environment ``env`` (dict), use None for none |
| * Input to the command ``input`` (str), use string to pass |
| no input. |
| * Max execution time ``timeout`` (int) seconds. Use 0 for no timeout. |
| |
| Returns a tuple (out, err, exitCode) where |
| * ``out`` (str) is the standard output of running the command |
| * ``err`` (str) is the standard error of running the command |
| * ``exitCode`` (int) is the exitCode of running the command |
| |
| If the timeout is hit an ``ExecuteCommandTimeoutException`` |
| is raised. |
| |
| """ |
| if input is not None: |
| input = to_bytes(input) |
| p = subprocess.Popen(command, cwd=cwd, |
| stdin=subprocess.PIPE, |
| stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE, |
| env=env, close_fds=kUseCloseFDs) |
| timerObject = None |
| # FIXME: Because of the way nested function scopes work in Python 2.x we |
| # need to use a reference to a mutable object rather than a plain |
| # bool. In Python 3 we could use the "nonlocal" keyword but we need |
| # to support Python 2 as well. |
| hitTimeOut = [False] |
| try: |
| if timeout > 0: |
| def killProcess(): |
| # We may be invoking a shell so we need to kill the |
| # process and all its children. |
| hitTimeOut[0] = True |
| killProcessAndChildren(p.pid) |
| |
| timerObject = threading.Timer(timeout, killProcess) |
| timerObject.start() |
| |
| out, err = p.communicate(input=input) |
| exitCode = p.wait() |
| finally: |
| if timerObject != None: |
| timerObject.cancel() |
| |
| # Ensure the resulting output is always of string type. |
| out = to_string(out) |
| err = to_string(err) |
| |
| if hitTimeOut[0]: |
| raise ExecuteCommandTimeoutException( |
| msg='Reached timeout of {} seconds'.format(timeout), |
| out=out, |
| err=err, |
| exitCode=exitCode |
| ) |
| |
| # Detect Ctrl-C in subprocess. |
| if exitCode == -signal.SIGINT: |
| raise KeyboardInterrupt |
| |
| return out, err, exitCode |
| |
| |
| def usePlatformSdkOnDarwin(config, lit_config): |
| # On Darwin, support relocatable SDKs by providing Clang with a |
| # default system root path. |
| if 'darwin' in config.target_triple: |
| try: |
| cmd = subprocess.Popen(['xcrun', '--show-sdk-path', '--sdk', 'macosx'], |
| stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| out, err = cmd.communicate() |
| out = out.strip() |
| res = cmd.wait() |
| except OSError: |
| res = -1 |
| if res == 0 and out: |
| sdk_path = out |
| lit_config.note('using SDKROOT: %r' % sdk_path) |
| config.environment['SDKROOT'] = sdk_path |
| |
| |
| def findPlatformSdkVersionOnMacOS(config, lit_config): |
| if 'darwin' in config.target_triple: |
| try: |
| cmd = subprocess.Popen(['xcrun', '--show-sdk-version', '--sdk', 'macosx'], |
| stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| out, err = cmd.communicate() |
| out = out.strip() |
| res = cmd.wait() |
| except OSError: |
| res = -1 |
| if res == 0 and out: |
| return out |
| return None |
| |
| |
| def killProcessAndChildren(pid): |
| """This function kills a process with ``pid`` and all its running children |
| (recursively). It is currently implemented using the psutil module which |
| provides a simple platform neutral implementation. |
| |
| TODO: Reimplement this without using psutil so we can remove |
| our dependency on it. |
| |
| """ |
| import psutil |
| try: |
| psutilProc = psutil.Process(pid) |
| # Handle the different psutil API versions |
| try: |
| # psutil >= 2.x |
| children_iterator = psutilProc.children(recursive=True) |
| except AttributeError: |
| # psutil 1.x |
| children_iterator = psutilProc.get_children(recursive=True) |
| for child in children_iterator: |
| try: |
| child.kill() |
| except psutil.NoSuchProcess: |
| pass |
| psutilProc.kill() |
| except psutil.NoSuchProcess: |
| pass |
| |
| |
| try: |
| import win32api |
| except ImportError: |
| win32api = None |
| |
| def abort_now(): |
| """Abort the current process without doing any exception teardown""" |
| sys.stdout.flush() |
| if win32api: |
| win32api.TerminateProcess(win32api.GetCurrentProcess(), 3) |
| else: |
| os.kill(0, 9) |