[lit] Move increase_process_limit to ParallelRun
Increasing the process limit only makes sense when we use multiple
processes.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@375474 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/utils/lit/lit/main.py b/utils/lit/lit/main.py
index 249dfbc..e50c706 100755
--- a/utils/lit/lit/main.py
+++ b/utils/lit/lit/main.py
@@ -182,32 +182,7 @@
fname = test.getFilePath()
os.utime(fname, None)
-def increase_process_limit(litConfig, opts):
- # Because some tests use threads internally, and at least on Linux each
- # of these threads counts toward the current process limit, try to
- # raise the (soft) process limit so that tests don't fail due to
- # resource exhaustion.
- try:
- cpus = lit.util.detectCPUs()
- desired_limit = opts.numWorkers * cpus * 2 # the 2 is a safety factor
-
- # Import the resource module here inside this try block because it
- # will likely fail on Windows.
- import resource
-
- max_procs_soft, max_procs_hard = resource.getrlimit(resource.RLIMIT_NPROC)
- desired_limit = min(desired_limit, max_procs_hard)
-
- if max_procs_soft < desired_limit:
- resource.setrlimit(resource.RLIMIT_NPROC, (desired_limit, max_procs_hard))
- litConfig.note('raised the process limit from %d to %d' % \
- (max_procs_soft, desired_limit))
- except:
- pass
-
def run_tests(tests, litConfig, opts, numTotalTests):
- increase_process_limit(litConfig, opts)
-
display = lit.display.create_display(opts, len(tests), numTotalTests,
opts.numWorkers)
def progress_callback(test):
diff --git a/utils/lit/lit/run.py b/utils/lit/lit/run.py
index d39b4b1..88ce445 100644
--- a/utils/lit/lit/run.py
+++ b/utils/lit/lit/run.py
@@ -109,6 +109,8 @@
multiprocessing.BoundedSemaphore(v) for k, v in
self.lit_config.parallelism_groups.items()}
+ self._increase_process_limit()
+
# Start a process pool. Copy over the data shared between all test runs.
# FIXME: Find a way to capture the worker process stderr. If the user
# interrupts the workers before we make it into our task callback, they
@@ -144,3 +146,25 @@
if self.hit_max_failures:
pool.terminate()
break
+
+ # Some tests use threads internally, and at least on Linux each of these
+ # threads counts toward the current process limit. Try to raise the (soft)
+ # process limit so that tests don't fail due to resource exhaustion.
+ def _increase_process_limit(self):
+ ncpus = lit.util.detectCPUs()
+ desired_limit = self.workers * ncpus * 2 # the 2 is a safety factor
+
+ # Importing the resource module will likely fail on Windows.
+ try:
+ import resource
+ NPROC = resource.RLIMIT_NPROC
+
+ soft_limit, hard_limit = resource.getrlimit(NPROC)
+ desired_limit = min(desired_limit, hard_limit)
+
+ if soft_limit < desired_limit:
+ resource.setrlimit(NPROC, (desired_limit, hard_limit))
+ self.lit_config.note('Raised process limit from %d to %d' % \
+ (soft_limit, desired_limit))
+ except Exception as ex:
+ self.lit_config.warning('Failed to raise process limit: %s' % ex)
