[libFuzzer] Guard symbolization with try-lock.

Summary:
When out-of-memory or timeout occurs, threads can be stopped during
symbolization, thereby causing a deadlock when the OOM/TO handlers
attempt symbolization.  We avoid this deadlock by skipping symbolization
if another thread is symbolizing.

Reviewers: kcc

Reviewed By: kcc

Subscribers: llvm-commits

Differential Revision: https://reviews.llvm.org/D46605

llvm-svn: 331825
GitOrigin-RevId: 14cf71a3a505eea44bcea188df510107d9655205
diff --git a/FuzzerLoop.cpp b/FuzzerLoop.cpp
index 6cc220d..1915563 100644
--- a/FuzzerLoop.cpp
+++ b/FuzzerLoop.cpp
@@ -105,7 +105,7 @@
       return;
     Printf("MALLOC[%zd] %p %zd\n", N, ptr, size);
     if (TraceLevel >= 2 && EF)
-      EF->__sanitizer_print_stack_trace();
+      PrintStackTrace();
   }
 }
 
@@ -118,7 +118,7 @@
       return;
     Printf("FREE[%zd]   %p\n", N, ptr);
     if (TraceLevel >= 2 && EF)
-      EF->__sanitizer_print_stack_trace();
+      PrintStackTrace();
   }
 }
 
@@ -129,8 +129,7 @@
   Printf("==%d== ERROR: libFuzzer: out-of-memory (malloc(%zd))\n", GetPid(),
          Size);
   Printf("   To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
-  if (EF->__sanitizer_print_stack_trace)
-    EF->__sanitizer_print_stack_trace();
+  PrintStackTrace();
   DumpCurrentUnit("oom-");
   Printf("SUMMARY: libFuzzer: out-of-memory\n");
   PrintFinalStats();
@@ -231,8 +230,7 @@
   if (EF->__sanitizer_acquire_crash_state)
     EF->__sanitizer_acquire_crash_state();
   Printf("==%lu== ERROR: libFuzzer: deadly signal\n", GetPid());
-  if (EF->__sanitizer_print_stack_trace)
-    EF->__sanitizer_print_stack_trace();
+  PrintStackTrace();
   Printf("NOTE: libFuzzer has rudimentary signal handlers.\n"
          "      Combine libFuzzer with AddressSanitizer or similar for better "
          "crash reports.\n");
@@ -249,8 +247,7 @@
       !EF->__sanitizer_acquire_crash_state())
     return;
   Printf("==%lu== ERROR: libFuzzer: fuzz target exited\n", GetPid());
-  if (EF->__sanitizer_print_stack_trace)
-    EF->__sanitizer_print_stack_trace();
+  PrintStackTrace();
   Printf("SUMMARY: libFuzzer: fuzz target exited\n");
   DumpCurrentUnit("crash-");
   PrintFinalStats();
@@ -296,8 +293,7 @@
     DumpCurrentUnit("timeout-");
     Printf("==%lu== ERROR: libFuzzer: timeout after %d seconds\n", GetPid(),
            Seconds);
-    if (EF->__sanitizer_print_stack_trace)
-      EF->__sanitizer_print_stack_trace();
+    PrintStackTrace();
     Printf("SUMMARY: libFuzzer: timeout\n");
     PrintFinalStats();
     _Exit(Options.TimeoutExitCode); // Stop right now.
@@ -312,8 +308,7 @@
       "==%lu== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %zdMb)\n",
       GetPid(), GetPeakRSSMb(), Options.RssLimitMb);
   Printf("   To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
-  if (EF->__sanitizer_print_memory_profile)
-    EF->__sanitizer_print_memory_profile(95, 8);
+  PrintMemoryProfile();
   DumpCurrentUnit("oom-");
   Printf("SUMMARY: libFuzzer: out-of-memory\n");
   PrintFinalStats();
diff --git a/FuzzerUtil.cpp b/FuzzerUtil.cpp
index 96b37d3..6286f9a 100644
--- a/FuzzerUtil.cpp
+++ b/FuzzerUtil.cpp
@@ -16,6 +16,7 @@
 #include <chrono>
 #include <cstring>
 #include <errno.h>
+#include <mutex>
 #include <signal.h>
 #include <sstream>
 #include <stdio.h>
@@ -179,8 +180,12 @@
   return Res;
 }
 
+static std::mutex SymbolizeMutex;
+
 std::string DescribePC(const char *SymbolizedFMT, uintptr_t PC) {
-  if (!EF->__sanitizer_symbolize_pc) return "<can not symbolize>";
+  std::unique_lock<std::mutex> l(SymbolizeMutex, std::try_to_lock);
+  if (!EF->__sanitizer_symbolize_pc || !l.owns_lock())
+    return "<can not symbolize>";
   char PcDescr[1024] = {};
   EF->__sanitizer_symbolize_pc(reinterpret_cast<void*>(PC),
                                SymbolizedFMT, PcDescr, sizeof(PcDescr));
@@ -195,6 +200,18 @@
     Printf(FallbackFMT, PC);
 }
 
+void PrintStackTrace() {
+  std::unique_lock<std::mutex> l(SymbolizeMutex, std::try_to_lock);
+  if (EF->__sanitizer_print_stack_trace && l.owns_lock())
+    EF->__sanitizer_print_stack_trace();
+}
+
+void PrintMemoryProfile() {
+  std::unique_lock<std::mutex> l(SymbolizeMutex, std::try_to_lock);
+  if (EF->__sanitizer_print_memory_profile && l.owns_lock())
+    EF->__sanitizer_print_memory_profile(95, 8);
+}
+
 unsigned NumberOfCpuCores() {
   unsigned N = std::thread::hardware_concurrency();
   if (!N) {
diff --git a/FuzzerUtil.h b/FuzzerUtil.h
index f2ed028..8c5c57c 100644
--- a/FuzzerUtil.h
+++ b/FuzzerUtil.h
@@ -40,6 +40,10 @@
 
 std::string DescribePC(const char *SymbolizedFMT, uintptr_t PC);
 
+void PrintStackTrace();
+
+void PrintMemoryProfile();
+
 unsigned NumberOfCpuCores();
 
 // Platform specific functions.