libcxx/utils/adb_run.py - llvm-project - Git at Google


            
            #!/usr/bin/env python3

            
            # ===----------------------------------------------------------------------===##

            
            #

            
            # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

            
            # See https://llvm.org/LICENSE.txt for license information.

            
            # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

            
            #

            
            # ===----------------------------------------------------------------------===##


            """adb_run.py is a utility for running a libc++ test program via adb.

            
            """


            import argparse

            
            import hashlib

            
            import os

            
            import re

            
            import shlex

            
            import socket

            
            import subprocess

            
            import sys

            
            from typing import List, Tuple


            # Sync a host file /path/to/dir/file to ${REMOTE_BASE_DIR}/run-${HASH}/dir/file.

            
            REMOTE_BASE_DIR = "/data/local/tmp/adb_run"


            g_job_limit_socket = None

            
            g_verbose = False


            def run_adb_sync_command(command: List[str]) -> None:

            
                """Run an adb command and discard the output, unless the command fails. If

            
                the command fails, dump the output instead, and exit the script with

            
                failure.

            
                """

            
                if g_verbose:

            
                    sys.stderr.write(f"running: {shlex.join(command)}\n")

            
                proc = subprocess.run(command, universal_newlines=True,

            
                                      stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,

            
                                      stderr=subprocess.STDOUT, encoding="utf-8")

            
                if proc.returncode != 0:

            
                    # adb's stdout (e.g. for adb push) should normally be discarded, but

            
                    # on failure, it should be shown. Print it to stderr because it's

            
                    # unrelated to the test program's stdout output. A common error caught

            
                    # here is "No space left on device".

            
                    sys.stderr.write(f"{proc.stdout}\n"

            
                                     f"error: adb command exited with {proc.returncode}: "

            
                                     f"{shlex.join(command)}\n")

            
                    sys.exit(proc.returncode)


            def sync_test_dir(local_dir: str, remote_dir: str) -> None:

            
                """Sync the libc++ test directory on the host to the remote device."""


                # Optimization: The typical libc++ test directory has only a single

            
                # *.tmp.exe file in it. In that case, skip the `mkdir` command, which is

            
                # normally necessary because we don't know if the target directory already

            
                # exists on the device.

            
                local_files = os.listdir(local_dir)

            
                if len(local_files) == 1:

            
                    local_file = os.path.join(local_dir, local_files[0])

            
                    remote_file = os.path.join(remote_dir, local_files[0])

            
                    if not os.path.islink(local_file) and os.path.isfile(local_file):

            
                        run_adb_sync_command(["adb", "push", "--sync", local_file,

            
                                              remote_file])

            
                        return


                assert os.path.basename(local_dir) == os.path.basename(remote_dir)

            
                run_adb_sync_command(["adb", "shell", "mkdir", "-p", remote_dir])

            
                run_adb_sync_command(["adb", "push", "--sync", local_dir,

            
                                      os.path.dirname(remote_dir)])


            def build_env_arg(env_args: List[str], prepend_path_args: List[Tuple[str, str]]) -> str:

            
                components = []

            
                for arg in env_args:

            
                    k, v = arg.split("=", 1)

            
                    components.append(f"export {k}={shlex.quote(v)}; ")

            
                for k, v in prepend_path_args:

            
                    components.append(f"export {k}={shlex.quote(v)}${{{k}:+:${k}}}; ")

            
                return "".join(components)


            def run_command(args: argparse.Namespace) -> int:

            
                local_dir = args.execdir

            
                assert local_dir.startswith("/")

            
                assert not local_dir.endswith("/")


                # Copy each execdir to a subdir of REMOTE_BASE_DIR. Name the directory using

            
                # a hash of local_dir so that concurrent adb_run invocations don't create

            
                # the same intermediate parent directory. At least `adb push` has trouble

            
                # with concurrent mkdir syscalls on common parent directories. (Somehow

            
                # mkdir fails with EAGAIN/EWOULDBLOCK, see internal Google bug,

            
                # b/289311228.)

            
                local_dir_hash = hashlib.sha1(local_dir.encode()).hexdigest()

            
                remote_dir = f"{REMOTE_BASE_DIR}/run-{local_dir_hash}/{os.path.basename(local_dir)}"

            
                sync_test_dir(local_dir, remote_dir)


                adb_shell_command = (

            
                    # Set the environment early so that PATH can be overridden. Overriding

            
                    # PATH is useful for:

            
                    #  - Replacing older shell utilities with toybox (e.g. on old devices).

            
                    #  - Adding a `bash` command that delegates to `sh` (mksh).

            
                    f"{build_env_arg(args.env, args.prepend_path_env)}"


                    # Set a high oom_score_adj so that, if the test program uses too much

            
                    # memory, it is killed before anything else on the device. The default

            
                    # oom_score_adj is -1000, so a test using too much memory typically

            
                    # crashes the device.

            
                    "echo 1000 >/proc/self/oom_score_adj; "


                    # If we're running as root, switch to the shell user. The libc++

            
                    # filesystem tests require running without root permissions. Some x86

            
                    # emulator devices (before Android N) do not have a working `adb unroot`

            
                    # and always run as root. Non-debug builds typically lack `su` and only

            
                    # run as the shell user.

            
                    #

            
                    # Some libc++ tests create temporary files in the working directory,

            
                    # which might be owned by root. Before switching to shell, make the

            
                    # cwd writable (and readable+executable) to every user.

            
                    #

            
                    # N.B.:

            
                    #  - Avoid "id -u" because it wasn't supported until Android M.

            
                    #  - The `env` and `which` commands were also added in Android M.

            
                    #  - Starting in Android M, su from root->shell resets PATH, so we need

            
                    #    to modify it again in the new environment.

            
                    #  - Avoid chmod's "a+rwx" syntax because it's not supported until

            
                    #    Android N.

            
                    #  - Defining this function allows specifying the arguments to the test

            
                    #    program (i.e. "$@") only once.

            
                    "run_without_root() {"

            
                    "  chmod 777 .;"

            
                    "  case \"$(id)\" in"

            
                    "    *\"uid=0(root)\"*)"

            
                    "    if command -v env >/dev/null; then"

            
                    "      su shell \"$(command -v env)\" PATH=\"$PATH\" \"$@\";"

            
                    "    else"

            
                    "      su shell \"$@\";"

            
                    "    fi;;"

            
                    "    *) \"$@\";;"

            
                    "  esac;"

            
                    "}; "

            
                )


                # Older versions of Bionic limit the length of argv[0] to 127 bytes

            
                # (SOINFO_NAME_LEN-1), and the path to libc++ tests tend to exceed this

            
                # limit. Changing the working directory works around this limit. The limit

            
                # is increased to 4095 (PATH_MAX-1) in Android M (API 23).

            
                command_line = [arg.replace(local_dir + "/", "./") for arg in args.command]


                # Prior to the adb feature "shell_v2" (added in Android N), `adb shell`

            
                # always created a pty:

            
                #  - This merged stdout and stderr together.

            
                #  - The pty converts LF to CRLF.

            
                #  - The exit code of the shell command wasn't propagated.

            
                # Work around all three limitations, unless "shell_v2" is present.

            
                proc = subprocess.run(["adb", "features"], check=True,

            
                                      stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,

            
                                      encoding="utf-8")

            
                adb_features = set(proc.stdout.strip().split())

            
                has_shell_v2 = "shell_v2" in adb_features

            
                if has_shell_v2:

            
                    adb_shell_command += (

            
                        f"cd {remote_dir} && run_without_root {shlex.join(command_line)}"

            
                    )

            
                else:

            
                    adb_shell_command += (

            
                        f"{{"

            
                        f"  stdout=$("

            
                        f"    cd {remote_dir} && run_without_root {shlex.join(command_line)};"

            
                        f"    echo -n __libcxx_adb_exit__=$?"

            
                        f"  ); "

            
                        f"}} 2>&1; "

            
                        f"echo -n __libcxx_adb_stdout__\"$stdout\""

            
                    )


                adb_command_line = ["adb", "shell", adb_shell_command]

            
                if g_verbose:

            
                    sys.stderr.write(f"running: {shlex.join(adb_command_line)}\n")


                if has_shell_v2:

            
                    proc = subprocess.run(adb_command_line, shell=False, check=False,

            
                                          encoding="utf-8")

            
                    return proc.returncode

            
                else:

            
                    proc = subprocess.run(adb_command_line, shell=False, check=False,

            
                                          stdout=subprocess.PIPE, stderr=subprocess.STDOUT,

            
                                          encoding="utf-8")

            
                    # The old `adb shell` mode used a pty, which converted LF to CRLF.

            
                    # Convert it back.

            
                    output = proc.stdout.replace("\r\n", "\n")


                    if proc.returncode:

            
                        sys.stderr.write(f"error: adb failed:\n"

            
                                         f"  command: {shlex.join(adb_command_line)}\n"

            
                                         f"  output: {output}\n")

            
                        return proc.returncode


                    match = re.match(r"(.*)__libcxx_adb_stdout__(.*)__libcxx_adb_exit__=(\d+)$",

            
                                 output, re.DOTALL)

            
                    if not match:

            
                        sys.stderr.write(f"error: could not parse adb output:\n"

            
                                         f"  command: {shlex.join(adb_command_line)}\n"

            
                                         f"  output: {output}\n")

            
                        return 1


                    sys.stderr.write(match.group(1))

            
                    sys.stdout.write(match.group(2))

            
                    return int(match.group(3))


            def connect_to_job_limiter_server(sock_addr: str) -> None:

            
                sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)


                try:

            
                    sock.connect(sock_addr)

            
                except (FileNotFoundError, ConnectionRefusedError) as e:

            
                    # Copying-and-pasting an adb_run.py command-line from a lit test failure

            
                    # is likely to fail because the socket no longer exists (or is

            
                    # inactive), so just give a warning.

            
                    sys.stderr.write(f"warning: could not connect to {sock_addr}: {e}\n")

            
                    return


                # The connect call can succeed before the server has called accept, because

            
                # of the listen backlog, so wait for the server to send a byte.

            
                sock.recv(1)


                # Keep the socket open until this process ends, then let the OS close the

            
                # connection automatically.

            
                global g_job_limit_socket

            
                g_job_limit_socket = sock


            def main() -> int:

            
                """Main function (pylint wants this docstring)."""

            
                parser = argparse.ArgumentParser()

            
                parser.add_argument("--execdir", type=str, required=True)

            
                parser.add_argument("--env", type=str, required=False, action="append",

            
                                    default=[], metavar="NAME=VALUE")

            
                parser.add_argument("--prepend-path-env", type=str, nargs=2, required=False,

            
                                    action="append", default=[],

            
                                    metavar=("NAME", "PATH"))

            
                parser.add_argument("--job-limit-socket")

            
                parser.add_argument("--verbose", "-v", default=False, action="store_true")

            
                parser.add_argument("command", nargs=argparse.ONE_OR_MORE)

            
                args = parser.parse_args()


                global g_verbose

            
                g_verbose = args.verbose

            
                if args.job_limit_socket is not None:

            
                    connect_to_job_limiter_server(args.job_limit_socket)

            
                return run_command(args)


            if __name__ == '__main__':

            
                sys.exit(main())
	#!/usr/bin/env python3
	# ===----------------------------------------------------------------------===##
	#
	# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	# See https://llvm.org/LICENSE.txt for license information.
	# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	#
	# ===----------------------------------------------------------------------===##

	"""adb_run.py is a utility for running a libc++ test program via adb.
	"""

	import argparse
	import hashlib
	import os
	import re
	import shlex
	import socket
	import subprocess
	import sys
	from typing import List, Tuple


	# Sync a host file /path/to/dir/file to ${REMOTE_BASE_DIR}/run-${HASH}/dir/file.
	REMOTE_BASE_DIR = "/data/local/tmp/adb_run"

	g_job_limit_socket = None
	g_verbose = False


	def run_adb_sync_command(command: List[str]) -> None:
	"""Run an adb command and discard the output, unless the command fails. If
	the command fails, dump the output instead, and exit the script with
	failure.
	"""
	if g_verbose:
	sys.stderr.write(f"running: {shlex.join(command)}\n")
	proc = subprocess.run(command, universal_newlines=True,
	stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
	stderr=subprocess.STDOUT, encoding="utf-8")
	if proc.returncode != 0:
	# adb's stdout (e.g. for adb push) should normally be discarded, but
	# on failure, it should be shown. Print it to stderr because it's
	# unrelated to the test program's stdout output. A common error caught
	# here is "No space left on device".
	sys.stderr.write(f"{proc.stdout}\n"
	f"error: adb command exited with {proc.returncode}: "
	f"{shlex.join(command)}\n")
	sys.exit(proc.returncode)


	def sync_test_dir(local_dir: str, remote_dir: str) -> None:
	"""Sync the libc++ test directory on the host to the remote device."""

	# Optimization: The typical libc++ test directory has only a single
	# *.tmp.exe file in it. In that case, skip the `mkdir` command, which is
	# normally necessary because we don't know if the target directory already
	# exists on the device.
	local_files = os.listdir(local_dir)
	if len(local_files) == 1:
	local_file = os.path.join(local_dir, local_files[0])
	remote_file = os.path.join(remote_dir, local_files[0])
	if not os.path.islink(local_file) and os.path.isfile(local_file):
	run_adb_sync_command(["adb", "push", "--sync", local_file,
	remote_file])
	return

	assert os.path.basename(local_dir) == os.path.basename(remote_dir)
	run_adb_sync_command(["adb", "shell", "mkdir", "-p", remote_dir])
	run_adb_sync_command(["adb", "push", "--sync", local_dir,
	os.path.dirname(remote_dir)])


	def build_env_arg(env_args: List[str], prepend_path_args: List[Tuple[str, str]]) -> str:
	components = []
	for arg in env_args:
	k, v = arg.split("=", 1)
	components.append(f"export {k}={shlex.quote(v)}; ")
	for k, v in prepend_path_args:
	components.append(f"export {k}={shlex.quote(v)}${{{k}:+:${k}}}; ")
	return "".join(components)


	def run_command(args: argparse.Namespace) -> int:
	local_dir = args.execdir
	assert local_dir.startswith("/")
	assert not local_dir.endswith("/")

	# Copy each execdir to a subdir of REMOTE_BASE_DIR. Name the directory using
	# a hash of local_dir so that concurrent adb_run invocations don't create
	# the same intermediate parent directory. At least `adb push` has trouble
	# with concurrent mkdir syscalls on common parent directories. (Somehow
	# mkdir fails with EAGAIN/EWOULDBLOCK, see internal Google bug,
	# b/289311228.)
	local_dir_hash = hashlib.sha1(local_dir.encode()).hexdigest()
	remote_dir = f"{REMOTE_BASE_DIR}/run-{local_dir_hash}/{os.path.basename(local_dir)}"
	sync_test_dir(local_dir, remote_dir)

	adb_shell_command = (
	# Set the environment early so that PATH can be overridden. Overriding
	# PATH is useful for:
	# - Replacing older shell utilities with toybox (e.g. on old devices).
	# - Adding a `bash` command that delegates to `sh` (mksh).
	f"{build_env_arg(args.env, args.prepend_path_env)}"

	# Set a high oom_score_adj so that, if the test program uses too much
	# memory, it is killed before anything else on the device. The default
	# oom_score_adj is -1000, so a test using too much memory typically
	# crashes the device.
	"echo 1000 >/proc/self/oom_score_adj; "

	# If we're running as root, switch to the shell user. The libc++
	# filesystem tests require running without root permissions. Some x86
	# emulator devices (before Android N) do not have a working `adb unroot`
	# and always run as root. Non-debug builds typically lack `su` and only
	# run as the shell user.
	#
	# Some libc++ tests create temporary files in the working directory,
	# which might be owned by root. Before switching to shell, make the
	# cwd writable (and readable+executable) to every user.
	#
	# N.B.:
	# - Avoid "id -u" because it wasn't supported until Android M.
	# - The `env` and `which` commands were also added in Android M.
	# - Starting in Android M, su from root->shell resets PATH, so we need
	# to modify it again in the new environment.
	# - Avoid chmod's "a+rwx" syntax because it's not supported until
	# Android N.
	# - Defining this function allows specifying the arguments to the test
	# program (i.e. "$@") only once.
	"run_without_root() {"
	" chmod 777 .;"
	" case \"$(id)\" in"
	" \"uid=0(root)\")"
	" if command -v env >/dev/null; then"
	" su shell \"$(command -v env)\" PATH=\"$PATH\" \"$@\";"
	" else"
	" su shell \"$@\";"
	" fi;;"
	" *) \"$@\";;"
	" esac;"
	"}; "
	)

	# Older versions of Bionic limit the length of argv[0] to 127 bytes
	# (SOINFO_NAME_LEN-1), and the path to libc++ tests tend to exceed this
	# limit. Changing the working directory works around this limit. The limit
	# is increased to 4095 (PATH_MAX-1) in Android M (API 23).
	command_line = [arg.replace(local_dir + "/", "./") for arg in args.command]

	# Prior to the adb feature "shell_v2" (added in Android N), `adb shell`
	# always created a pty:
	# - This merged stdout and stderr together.
	# - The pty converts LF to CRLF.
	# - The exit code of the shell command wasn't propagated.
	# Work around all three limitations, unless "shell_v2" is present.
	proc = subprocess.run(["adb", "features"], check=True,
	stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
	encoding="utf-8")
	adb_features = set(proc.stdout.strip().split())
	has_shell_v2 = "shell_v2" in adb_features
	if has_shell_v2:
	adb_shell_command += (
	f"cd {remote_dir} && run_without_root {shlex.join(command_line)}"
	)
	else:
	adb_shell_command += (
	f"{{"
	f" stdout=$("
	f" cd {remote_dir} && run_without_root {shlex.join(command_line)};"
	f" echo -n __libcxx_adb_exit__=$?"
	f" ); "
	f"}} 2>&1; "
	f"echo -n __libcxx_adb_stdout__\"$stdout\""
	)

	adb_command_line = ["adb", "shell", adb_shell_command]
	if g_verbose:
	sys.stderr.write(f"running: {shlex.join(adb_command_line)}\n")

	if has_shell_v2:
	proc = subprocess.run(adb_command_line, shell=False, check=False,
	encoding="utf-8")
	return proc.returncode
	else:
	proc = subprocess.run(adb_command_line, shell=False, check=False,
	stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
	encoding="utf-8")
	# The old `adb shell` mode used a pty, which converted LF to CRLF.
	# Convert it back.
	output = proc.stdout.replace("\r\n", "\n")

	if proc.returncode:
	sys.stderr.write(f"error: adb failed:\n"
	f" command: {shlex.join(adb_command_line)}\n"
	f" output: {output}\n")
	return proc.returncode

	match = re.match(r"(.)__libcxx_adb_stdout__(.)__libcxx_adb_exit__=(\d+)$",
	output, re.DOTALL)
	if not match:
	sys.stderr.write(f"error: could not parse adb output:\n"
	f" command: {shlex.join(adb_command_line)}\n"
	f" output: {output}\n")
	return 1

	sys.stderr.write(match.group(1))
	sys.stdout.write(match.group(2))
	return int(match.group(3))


	def connect_to_job_limiter_server(sock_addr: str) -> None:
	sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)

	try:
	sock.connect(sock_addr)
	except (FileNotFoundError, ConnectionRefusedError) as e:
	# Copying-and-pasting an adb_run.py command-line from a lit test failure
	# is likely to fail because the socket no longer exists (or is
	# inactive), so just give a warning.
	sys.stderr.write(f"warning: could not connect to {sock_addr}: {e}\n")
	return

	# The connect call can succeed before the server has called accept, because
	# of the listen backlog, so wait for the server to send a byte.
	sock.recv(1)

	# Keep the socket open until this process ends, then let the OS close the
	# connection automatically.
	global g_job_limit_socket
	g_job_limit_socket = sock


	def main() -> int:
	"""Main function (pylint wants this docstring)."""
	parser = argparse.ArgumentParser()
	parser.add_argument("--execdir", type=str, required=True)
	parser.add_argument("--env", type=str, required=False, action="append",
	default=[], metavar="NAME=VALUE")
	parser.add_argument("--prepend-path-env", type=str, nargs=2, required=False,
	action="append", default=[],
	metavar=("NAME", "PATH"))
	parser.add_argument("--job-limit-socket")
	parser.add_argument("--verbose", "-v", default=False, action="store_true")
	parser.add_argument("command", nargs=argparse.ONE_OR_MORE)
	args = parser.parse_args()

	global g_verbose
	g_verbose = args.verbose
	if args.job_limit_socket is not None:
	connect_to_job_limiter_server(args.job_limit_socket)
	return run_command(args)


	if __name__ == '__main__':
	sys.exit(main())