[libFuzzer] Handle unstable edges by disregarding unstable edges

Summary:
Added a new mode within flag -handle_unstable for new unstable handling algorithm that does the following:
    When an edge is shown as unstable, copy to UnstableCounters the value 0.
    During ApplyUnstableCounters we copy back the value 0 to ModuleInline8bitCounters if the edge was unstable.

This way we would be ignoring completely features that were collected through non-determinism.
Unstable hits would be counted as if it never hit.

Reviewers: metzman, Dor1s, kcc, morehouse

Reviewed By: metzman, morehouse

Subscribers: delcypher, llvm-commits, #sanitizers

Differential Revision: https://reviews.llvm.org/D49684

llvm-svn: 337853
GitOrigin-RevId: 8db0befc6d34c6db33ce8210faccdb5da6c2f670
diff --git a/FuzzerDriver.cpp b/FuzzerDriver.cpp
index 4f40d6a..783474a 100644
--- a/FuzzerDriver.cpp
+++ b/FuzzerDriver.cpp
@@ -619,7 +619,8 @@
   Options.PrintCorpusStats = Flags.print_corpus_stats;
   Options.PrintCoverage = Flags.print_coverage;
   Options.PrintUnstableStats = Flags.print_unstable_stats;
-  if (Flags.handle_unstable)
+  if (Flags.handle_unstable == TracePC::MinUnstable ||
+      Flags.handle_unstable == TracePC::ZeroUnstable)
     Options.HandleUnstable = Flags.handle_unstable;
   Options.DumpCoverage = Flags.dump_coverage;
   if (Flags.exit_on_src_pos)
diff --git a/FuzzerFlags.def b/FuzzerFlags.def
index aa3a987..ba04bc2 100644
--- a/FuzzerFlags.def
+++ b/FuzzerFlags.def
@@ -114,7 +114,9 @@
                    " Executes every input 3 times in total if a unique feature"
                    " is found during the first execution."
                    " If 1, we only use the minimum hit count from the 3 runs"
-                   " to determine whether an input is interesting.")
+                   " to determine whether an input is interesting."
+                   " If 2, we disregard edges that are found unstable for"
+                   " feature collection.")
 FUZZER_FLAG_INT(print_unstable_stats, 0, "Experimental."
 				  " If 1, print unstable statistics at exit.")
 FUZZER_FLAG_INT(handle_segv, 1, "If 1, try to intercept SIGSEGV.")
diff --git a/FuzzerLoop.cpp b/FuzzerLoop.cpp
index e63ee73..4bc8836 100644
--- a/FuzzerLoop.cpp
+++ b/FuzzerLoop.cpp
@@ -472,7 +472,8 @@
   TPC.UpdateUnstableCounters(Options.HandleUnstable);
 
   // Move minimum hit counts back to ModuleInline8bitCounters
-  if (Options.HandleUnstable)
+  if (Options.HandleUnstable == TracePC::MinUnstable ||
+      Options.HandleUnstable == TracePC::ZeroUnstable)
     TPC.ApplyUnstableCounters();
 }
 
diff --git a/FuzzerTracePC.cpp b/FuzzerTracePC.cpp
index 859ff63..29ffc8e 100644
--- a/FuzzerTracePC.cpp
+++ b/FuzzerTracePC.cpp
@@ -83,11 +83,14 @@
 // and records differences as unstable edges.
 void TracePC::UpdateUnstableCounters(int UnstableMode) {
   IterateInline8bitCounters([&](int i, int j, int UnstableIdx) {
-    if (ModuleCounters[i].Start[j] != UnstableCounters[UnstableIdx].Counter)
+    if (ModuleCounters[i].Start[j] != UnstableCounters[UnstableIdx].Counter) {
       UnstableCounters[UnstableIdx].IsUnstable = true;
-    if (UnstableMode &&
-        ModuleCounters[i].Start[j] < UnstableCounters[UnstableIdx].Counter)
-      UnstableCounters[UnstableIdx].Counter = ModuleCounters[i].Start[j];
+      if (UnstableMode == ZeroUnstable)
+        UnstableCounters[UnstableIdx].Counter = 0;
+      else if (UnstableMode == MinUnstable)
+        UnstableCounters[UnstableIdx].Counter = std::min(
+            ModuleCounters[i].Start[j], UnstableCounters[UnstableIdx].Counter);
+    }
   });
 }
 
diff --git a/FuzzerTracePC.h b/FuzzerTracePC.h
index a051568..097ba69 100644
--- a/FuzzerTracePC.h
+++ b/FuzzerTracePC.h
@@ -74,6 +74,11 @@
   // How many bits of PC are used from __sanitizer_cov_trace_pc.
   static const size_t kTracePcBits = 18;
 
+  enum HandleUnstableOptions {
+    MinUnstable = 1,
+    ZeroUnstable = 2,
+  };
+
   void HandleInit(uint32_t *Start, uint32_t *Stop);
   void HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop);
   void HandlePCsInit(const uintptr_t *Start, const uintptr_t *Stop);