[libFuzzer] Mutation tracking and logging implemented.
Summary:
Code now exists to track number of mutations that are used in fuzzing in total
and ones that produce new coverage. The stats are currently being dumped to the
command line.
Patch by Kodé Williams (@kodewilliams).
Reviewers: metzman, Dor1s, morehouse, kcc
Reviewed By: Dor1s, morehouse, kcc
Subscribers: delcypher, kubamracek, kcc, morehouse, llvm-commits, #sanitizers, mgorny
Differential Revision: https://reviews.llvm.org/D48054
llvm-svn: 337324
GitOrigin-RevId: 061b4af998fc4a211566505555403806c373886a
diff --git a/FuzzerDriver.cpp b/FuzzerDriver.cpp
index 9375925..eb849fc 100644
--- a/FuzzerDriver.cpp
+++ b/FuzzerDriver.cpp
@@ -615,6 +615,7 @@
Options.PrintNewCovPcs = Flags.print_pcs;
Options.PrintNewCovFuncs = Flags.print_funcs;
Options.PrintFinalStats = Flags.print_final_stats;
+ Options.PrintMutationStats = Flags.print_mutation_stats;
Options.PrintCorpusStats = Flags.print_corpus_stats;
Options.PrintCoverage = Flags.print_coverage;
Options.PrintUnstableStats = Flags.print_unstable_stats;
diff --git a/FuzzerFlags.def b/FuzzerFlags.def
index e50b82a..dc92758 100644
--- a/FuzzerFlags.def
+++ b/FuzzerFlags.def
@@ -155,3 +155,4 @@
FUZZER_FLAG_INT(analyze_dict, 0, "Experimental")
FUZZER_DEPRECATED_FLAG(use_clang_coverage)
FUZZER_FLAG_STRING(data_flow_trace, "Experimental: use the data flow trace")
+FUZZER_FLAG_INT(print_mutation_stats, 0, "Experimental")
diff --git a/FuzzerLoop.cpp b/FuzzerLoop.cpp
index a2d53ee..1ba0765 100644
--- a/FuzzerLoop.cpp
+++ b/FuzzerLoop.cpp
@@ -360,6 +360,7 @@
TPC.DumpCoverage();
if (Options.PrintCorpusStats)
Corpus.PrintStats();
+ if (Options.PrintMutationStats) MD.PrintMutationStats();
if (!Options.PrintFinalStats)
return;
size_t ExecPerSec = execPerSec();
diff --git a/FuzzerMutate.cpp b/FuzzerMutate.cpp
index 6f6ce07..e89e1a4 100644
--- a/FuzzerMutate.cpp
+++ b/FuzzerMutate.cpp
@@ -30,34 +30,36 @@
DefaultMutators.insert(
DefaultMutators.begin(),
{
- {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes"},
- {&MutationDispatcher::Mutate_InsertByte, "InsertByte"},
+ {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes", 0, 0},
+ {&MutationDispatcher::Mutate_InsertByte, "InsertByte", 0, 0},
{&MutationDispatcher::Mutate_InsertRepeatedBytes,
- "InsertRepeatedBytes"},
- {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte"},
- {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit"},
- {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes"},
- {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt"},
- {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt"},
- {&MutationDispatcher::Mutate_CopyPart, "CopyPart"},
- {&MutationDispatcher::Mutate_CrossOver, "CrossOver"},
+ "InsertRepeatedBytes", 0, 0},
+ {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte", 0, 0},
+ {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit", 0, 0},
+ {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes", 0, 0},
+ {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt", 0,
+ 0},
+ {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt", 0,
+ 0},
+ {&MutationDispatcher::Mutate_CopyPart, "CopyPart", 0, 0},
+ {&MutationDispatcher::Mutate_CrossOver, "CrossOver", 0, 0},
{&MutationDispatcher::Mutate_AddWordFromManualDictionary,
- "ManualDict"},
+ "ManualDict", 0, 0},
{&MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary,
- "PersAutoDict"},
+ "PersAutoDict", 0, 0},
});
if(Options.UseCmp)
DefaultMutators.push_back(
- {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP"});
+ {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP", 0, 0});
if (EF->LLVMFuzzerCustomMutator)
- Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom"});
+ Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom", 0, 0});
else
Mutators = DefaultMutators;
if (EF->LLVMFuzzerCustomCrossOver)
Mutators.push_back(
- {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver"});
+ {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver", 0, 0});
}
static char RandCh(Random &Rand) {
@@ -464,6 +466,7 @@
if (!PersistentAutoDictionary.ContainsWord(DE->GetW()))
PersistentAutoDictionary.push_back({DE->GetW(), 1});
}
+ RecordUsefulMutations();
}
void MutationDispatcher::PrintRecommendedDictionary() {
@@ -484,8 +487,7 @@
void MutationDispatcher::PrintMutationSequence() {
Printf("MS: %zd ", CurrentMutatorSequence.size());
- for (auto M : CurrentMutatorSequence)
- Printf("%s-", M.Name);
+ for (auto M : CurrentMutatorSequence) Printf("%s-", M->Name);
if (!CurrentDictionaryEntrySequence.empty()) {
Printf(" DE: ");
for (auto DE : CurrentDictionaryEntrySequence) {
@@ -513,12 +515,13 @@
// in which case they will return 0.
// Try several times before returning un-mutated data.
for (int Iter = 0; Iter < 100; Iter++) {
- auto M = Mutators[Rand(Mutators.size())];
- size_t NewSize = (this->*(M.Fn))(Data, Size, MaxSize);
+ auto M = &Mutators[Rand(Mutators.size())];
+ size_t NewSize = (this->*(M->Fn))(Data, Size, MaxSize);
if (NewSize && NewSize <= MaxSize) {
if (Options.OnlyASCII)
ToASCII(Data, NewSize);
CurrentMutatorSequence.push_back(M);
+ M->TotalCount++;
return NewSize;
}
}
@@ -531,4 +534,21 @@
{W, std::numeric_limits<size_t>::max()});
}
+void MutationDispatcher::RecordUsefulMutations() {
+ for (auto M : CurrentMutatorSequence) M->UsefulCount++;
+}
+
+void MutationDispatcher::PrintMutationStats() {
+ Printf("\nstat::mutation_usefulness: ");
+ for (size_t i = 0; i < Mutators.size(); i++) {
+ double UsefulPercentage =
+ Mutators[i].TotalCount
+ ? (100.0 * Mutators[i].UsefulCount) / Mutators[i].TotalCount
+ : 0;
+ Printf("%.3f", UsefulPercentage);
+ if (i < Mutators.size() - 1) Printf(",");
+ }
+ Printf("\n");
+}
+
} // namespace fuzzer
diff --git a/FuzzerMutate.h b/FuzzerMutate.h
index 996d756..d8c1ddc 100644
--- a/FuzzerMutate.h
+++ b/FuzzerMutate.h
@@ -86,11 +86,16 @@
Random &GetRand() { return Rand; }
-private:
+ void PrintMutationStats();
+ void RecordUsefulMutations();
+
+ private:
struct Mutator {
size_t (MutationDispatcher::*Fn)(uint8_t *Data, size_t Size, size_t Max);
const char *Name;
+ uint64_t UsefulCount;
+ uint64_t TotalCount;
};
size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size,
@@ -128,8 +133,8 @@
// entries that led to successful discoveries in the past mutations.
Dictionary PersistentAutoDictionary;
- Vector<Mutator> CurrentMutatorSequence;
Vector<DictionaryEntry *> CurrentDictionaryEntrySequence;
+ Vector<Mutator *> CurrentMutatorSequence;
static const size_t kCmpDictionaryEntriesDequeSize = 16;
DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize];
diff --git a/FuzzerOptions.h b/FuzzerOptions.h
index e32b7d5..daa9104 100644
--- a/FuzzerOptions.h
+++ b/FuzzerOptions.h
@@ -52,6 +52,7 @@
bool PrintNewCovPcs = false;
int PrintNewCovFuncs = 0;
bool PrintFinalStats = false;
+ bool PrintMutationStats = false;
bool PrintCorpusStats = false;
bool PrintCoverage = false;
bool PrintUnstableStats = false;
