[PATCH] D100161: Redistribute energy for Corpus

[taotao gu via Phabricator via cfe-commits]

gtt1995 created this revision.
gtt1995 added reviewers: 01alchemist, 0b01.
gtt1995 requested review of this revision.
Herald added projects: clang, Sanitizers.
Herald added subscribers: Sanitizers, cfe-commits.

Divide the corpus into n parts according to size. Each job executes each corpus 
in turn, Job one executes the corpus with the smallest size, Job two executes 
the relatively larger corpus,...Job N executes the seed of the largest corpus, 
in turn,. i.e. each job choose some seeds from corpus 1, corpus2 2,..., corpus 
N, corpus1,corpus2...corpus N .....
that is, allocate more energy to the small seeds, trigger the common path in 
advance, and prefer to keep the small seeds.
In my experiment , It is found that the bugs rate is greatly accelerated, the 
cov is greatly increased (equal to the effect of entropic improvement), and the 
size of the newly generated interesting seeds is very small.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D100161

Files:
  clang/tools/clang-format/clang-format.py
  compiler-rt/lib/fuzzer/FuzzerDriver.cpp
  compiler-rt/lib/fuzzer/FuzzerFlags.def
  compiler-rt/lib/fuzzer/FuzzerFork.cpp
  compiler-rt/lib/fuzzer/FuzzerFork.h


Index: compiler-rt/lib/fuzzer/FuzzerFork.h
===================================================================
--- compiler-rt/lib/fuzzer/FuzzerFork.h
+++ compiler-rt/lib/fuzzer/FuzzerFork.h
@@ -18,7 +18,7 @@
 namespace fuzzer {
 void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
                   const Vector<std::string> &Args,
-                  const Vector<std::string> &CorpusDirs, int NumJobs);
+                  const Vector<std::string> &CorpusDirs, int NumJobs, int NumCorpuses);
 } // namespace fuzzer
 
 #endif // LLVM_FUZZER_FORK_H
Index: compiler-rt/lib/fuzzer/FuzzerFork.cpp
===================================================================
--- compiler-rt/lib/fuzzer/FuzzerFork.cpp
+++ compiler-rt/lib/fuzzer/FuzzerFork.cpp
@@ -114,7 +114,7 @@
         .count();
   }
 
-  FuzzJob *CreateNewJob(size_t JobId) {
+  FuzzJob *CreateNewJob(size_t JobId, int NumCorpuses) {
     Command Cmd(Args);
     Cmd.removeFlag("fork");
     Cmd.removeFlag("runs");
@@ -135,11 +135,23 @@
     std::string Seeds;
     if (size_t CorpusSubsetSize =
             std::min(Files.size(), (size_t)sqrt(Files.size() + 2))) {
+      size_t AverageSize = Files.size()/NumCorpuses +1;
       auto Time1 = std::chrono::system_clock::now();
+      size_t StartIndex = ((JobId-1)%NumCorpuses) *  AverageSize;
+      printf("\n Job %d Choose Corpus  %d ",JobId,(JobId)%NumCorpuses);
       for (size_t i = 0; i < CorpusSubsetSize; i++) {
-        auto &SF = Files[Rand->SkewTowardsLast(Files.size())];
-        Seeds += (Seeds.empty() ? "" : ",") + SF;
-        CollectDFT(SF);
+        size_t j = Rand->SkewTowardsLast(AverageSize);
+        size_t m = j + StartIndex;
+        if (m < Files.size()) {
+                auto &SF = Files[m];
+                Seeds += (Seeds.empty() ? "" : ",") + SF;
+                CollectDFT(SF);
+        }
+        else  {
+                auto &SF = Files[Rand->SkewTowardsLast(Files.size())];
+                Seeds += (Seeds.empty() ? "" : ",") + SF;
+                CollectDFT(SF);
+        }
       }
       auto Time2 = std::chrono::system_clock::now();
       auto DftTimeInSeconds = duration_cast<seconds>(Time2 - Time1).count();
@@ -284,7 +296,7 @@
 // This is just a skeleton of an experimental -fork=1 feature.
 void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
                   const Vector<std::string> &Args,
-                  const Vector<std::string> &CorpusDirs, int NumJobs) {
+                  const Vector<std::string> &CorpusDirs, int NumJobs, int NumCorpuses) {
   Printf("INFO: -fork=%d: fuzzing in separate process(s)\n", NumJobs);
 
   GlobalEnv Env;
@@ -341,8 +353,9 @@
   Vector<std::thread> Threads;
   for (int t = 0; t < NumJobs; t++) {
     Threads.push_back(std::thread(WorkerThread, &FuzzQ, &MergeQ));
-    FuzzQ.Push(Env.CreateNewJob(JobId++));
+    FuzzQ.Push(Env.CreateNewJob(JobId++, NumCorpuses));
   }
+  //printf("\n 创建%d个jobs\n",NumJobs);
 
   while (true) {
     std::unique_ptr<FuzzJob> Job(MergeQ.Pop());
@@ -399,7 +412,7 @@
       break;
     }
 
-    FuzzQ.Push(Env.CreateNewJob(JobId++));
+    FuzzQ.Push(Env.CreateNewJob(JobId++, NumCorpuses));
   }
 
   for (auto &T : Threads)
Index: compiler-rt/lib/fuzzer/FuzzerFlags.def
===================================================================
--- compiler-rt/lib/fuzzer/FuzzerFlags.def
+++ compiler-rt/lib/fuzzer/FuzzerFlags.def
@@ -56,6 +56,7 @@
 FUZZER_FLAG_INT(max_total_time, 0, "If positive, indicates the maximal total "
                                    "time in seconds to run the fuzzer.")
 FUZZER_FLAG_INT(help, 0, "Print help.")
+FUZZER_FLAG_INT(NumCorpuses, 1, "Divide the corpus into N parts according to size.")
 FUZZER_FLAG_INT(fork, 0, "Experimental mode where fuzzing happens "
                 "in a subprocess")
 FUZZER_FLAG_INT(ignore_timeouts, 1, "Ignore timeouts in fork mode")
Index: compiler-rt/lib/fuzzer/FuzzerDriver.cpp
===================================================================
--- compiler-rt/lib/fuzzer/FuzzerDriver.cpp
+++ compiler-rt/lib/fuzzer/FuzzerDriver.cpp
@@ -867,7 +867,7 @@
   }
 
   if (Flags.fork)
-    FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork);
+    FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork, Flags.NumCorpuses);
 
   if (Flags.merge)
     Merge(F, Options, Args, *Inputs, Flags.merge_control_file);

_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits