blob: 52a233f05ed4c3f28f964ee8456425e17b0aeb6f [file] [log] [blame]
kcc2e6ca5c2019-02-12 22:48:55 +00001//===- FuzzerFork.cpp - run fuzzing in separate subprocesses --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
kcc64bcb922019-02-13 04:04:45 +00008// Spawn and orchestrate separate fuzzing processes.
kcc2e6ca5c2019-02-12 22:48:55 +00009//===----------------------------------------------------------------------===//
10
11#include "FuzzerCommand.h"
12#include "FuzzerFork.h"
13#include "FuzzerIO.h"
14#include "FuzzerMerge.h"
15#include "FuzzerSHA1.h"
kcc9c0ed932019-02-15 01:22:00 +000016#include "FuzzerTracePC.h"
kcc2e6ca5c2019-02-12 22:48:55 +000017#include "FuzzerUtil.h"
18
kccb1fa9e02019-02-14 01:11:29 +000019#include <atomic>
kcc55e54ed2019-02-15 21:51:15 +000020#include <chrono>
kccdd391142019-02-14 21:09:32 +000021#include <fstream>
kccedefdf32019-02-16 00:14:16 +000022#include <memory>
kcc6526f1d2019-02-14 00:25:43 +000023#include <mutex>
kcc6526f1d2019-02-14 00:25:43 +000024#include <queue>
kccdd391142019-02-14 21:09:32 +000025#include <sstream>
26#include <thread>
kcc6526f1d2019-02-14 00:25:43 +000027
kcc2e6ca5c2019-02-12 22:48:55 +000028namespace fuzzer {
29
kccdd391142019-02-14 21:09:32 +000030struct Stats {
31 size_t number_of_executed_units = 0;
32 size_t peak_rss_mb = 0;
33 size_t average_exec_per_sec = 0;
34};
35
36static Stats ParseFinalStatsFromLog(const std::string &LogPath) {
37 std::ifstream In(LogPath);
38 std::string Line;
39 Stats Res;
40 struct {
41 const char *Name;
42 size_t *Var;
43 } NameVarPairs[] = {
44 {"stat::number_of_executed_units:", &Res.number_of_executed_units},
45 {"stat::peak_rss_mb:", &Res.peak_rss_mb},
46 {"stat::average_exec_per_sec:", &Res.average_exec_per_sec},
47 {nullptr, nullptr},
48 };
49 while (std::getline(In, Line, '\n')) {
50 if (Line.find("stat::") != 0) continue;
51 std::istringstream ISS(Line);
52 std::string Name;
53 size_t Val;
54 ISS >> Name >> Val;
55 for (size_t i = 0; NameVarPairs[i].Name; i++)
56 if (Name == NameVarPairs[i].Name)
57 *NameVarPairs[i].Var = Val;
58 }
59 return Res;
60}
61
kcc64bcb922019-02-13 04:04:45 +000062struct FuzzJob {
63 // Inputs.
64 Command Cmd;
kcc64bcb922019-02-13 04:04:45 +000065 std::string CorpusDir;
66 std::string LogPath;
67 std::string CFPath;
kcc64bcb922019-02-13 04:04:45 +000068
69 // Fuzzing Outputs.
70 int ExitCode;
kccedefdf32019-02-16 00:14:16 +000071
72 ~FuzzJob() {
73 RemoveFile(CFPath);
74 RemoveFile(LogPath);
75 RmDirRecursive(CorpusDir);
76 }
kcc64bcb922019-02-13 04:04:45 +000077};
78
79struct GlobalEnv {
kcc6526f1d2019-02-14 00:25:43 +000080 Vector<std::string> Args;
81 Vector<std::string> CorpusDirs;
kcc64bcb922019-02-13 04:04:45 +000082 std::string MainCorpusDir;
kcc6526f1d2019-02-14 00:25:43 +000083 std::string TempDir;
kcc98a86242019-02-15 00:08:16 +000084 Set<uint32_t> Features, Cov;
kcc64bcb922019-02-13 04:04:45 +000085 Vector<std::string> Files;
kcc6526f1d2019-02-14 00:25:43 +000086 Random *Rand;
kcc55e54ed2019-02-15 21:51:15 +000087 std::chrono::system_clock::time_point ProcessStartTime;
kcc6526f1d2019-02-14 00:25:43 +000088 int Verbosity = 0;
89
kcc55e54ed2019-02-15 21:51:15 +000090 size_t NumTimeouts = 0;
91 size_t NumOOMs = 0;
92 size_t NumCrashes = 0;
93
94
kccdd391142019-02-14 21:09:32 +000095 size_t NumRuns = 0;
96
kcc55e54ed2019-02-15 21:51:15 +000097 size_t secondsSinceProcessStartUp() const {
98 return std::chrono::duration_cast<std::chrono::seconds>(
99 std::chrono::system_clock::now() - ProcessStartTime)
100 .count();
101 }
102
kcc6526f1d2019-02-14 00:25:43 +0000103 FuzzJob *CreateNewJob(size_t JobId) {
104 Command Cmd(Args);
105 Cmd.removeFlag("fork");
106 for (auto &C : CorpusDirs) // Remove all corpora from the args.
107 Cmd.removeArgument(C);
108 Cmd.addFlag("reload", "0"); // working in an isolated dir, no reload.
kccdd391142019-02-14 21:09:32 +0000109 Cmd.addFlag("print_final_stats", "1");
kcc9c0ed932019-02-15 01:22:00 +0000110 Cmd.addFlag("print_funcs", "0"); // no need to spend time symbolizing.
kcc6526f1d2019-02-14 00:25:43 +0000111 Cmd.addFlag("max_total_time", std::to_string(std::min((size_t)300, JobId)));
112
113 auto Job = new FuzzJob;
114 std::string Seeds;
kcc9c0ed932019-02-15 01:22:00 +0000115 if (size_t CorpusSubsetSize =
116 std::min(Files.size(), (size_t)sqrt(Files.size() + 2)))
kcc6526f1d2019-02-14 00:25:43 +0000117 for (size_t i = 0; i < CorpusSubsetSize; i++)
118 Seeds += (Seeds.empty() ? "" : ",") +
119 Files[Rand->SkewTowardsLast(Files.size())];
120 if (!Seeds.empty())
121 Cmd.addFlag("seed_inputs", Seeds);
122 Job->LogPath = DirPlusFile(TempDir, std::to_string(JobId) + ".log");
123 Job->CorpusDir = DirPlusFile(TempDir, "C" + std::to_string(JobId));
124 Job->CFPath = DirPlusFile(TempDir, std::to_string(JobId) + ".merge");
125
126
127 Cmd.addArgument(Job->CorpusDir);
128 RmDirRecursive(Job->CorpusDir);
129 MkDir(Job->CorpusDir);
130
131 Cmd.setOutputFile(Job->LogPath);
132 Cmd.combineOutAndErr();
133
134 Job->Cmd = Cmd;
135
136 if (Verbosity >= 2)
137 Printf("Job %zd/%p Created: %s\n", JobId, Job,
138 Job->Cmd.toString().c_str());
139 // Start from very short runs and gradually increase them.
140 return Job;
141 }
142
143 void RunOneMergeJob(FuzzJob *Job) {
144 Vector<SizedFile> TempFiles;
145 GetSizedFilesFromDir(Job->CorpusDir, &TempFiles);
146
147 Vector<std::string> FilesToAdd;
kcc98a86242019-02-15 00:08:16 +0000148 Set<uint32_t> NewFeatures, NewCov;
kcc6526f1d2019-02-14 00:25:43 +0000149 CrashResistantMerge(Args, {}, TempFiles, &FilesToAdd, Features,
kcc98a86242019-02-15 00:08:16 +0000150 &NewFeatures, Cov, &NewCov, Job->CFPath, false);
kcc6526f1d2019-02-14 00:25:43 +0000151 for (auto &Path : FilesToAdd) {
152 auto U = FileToVector(Path);
153 auto NewPath = DirPlusFile(MainCorpusDir, Hash(U));
154 WriteToFile(U, NewPath);
155 Files.push_back(NewPath);
156 }
kcc6526f1d2019-02-14 00:25:43 +0000157 Features.insert(NewFeatures.begin(), NewFeatures.end());
kcc98a86242019-02-15 00:08:16 +0000158 Cov.insert(NewCov.begin(), NewCov.end());
kcc9c0ed932019-02-15 01:22:00 +0000159 for (auto Idx : NewCov)
160 if (auto *TE = TPC.PCTableEntryByIdx(Idx))
161 if (TPC.PcIsFuncEntry(TE))
162 PrintPC(" NEW_FUNC: %p %F %L\n", "",
163 TPC.GetNextInstructionPc(TE->PC));
164
kccdd391142019-02-14 21:09:32 +0000165 auto Stats = ParseFinalStatsFromLog(Job->LogPath);
166 NumRuns += Stats.number_of_executed_units;
kcc55e54ed2019-02-15 21:51:15 +0000167 if (!FilesToAdd.empty() || Job->ExitCode != 0)
168 Printf("#%zd: cov: %zd ft: %zd corp: %zd exec/s %zd "
169 "oom/timeout/crash: %zd/%zd/%zd time: %zds\n", NumRuns,
kcc98a86242019-02-15 00:08:16 +0000170 Cov.size(), Features.size(), Files.size(),
kcc55e54ed2019-02-15 21:51:15 +0000171 Stats.average_exec_per_sec,
172 NumOOMs, NumTimeouts, NumCrashes, secondsSinceProcessStartUp());
kcc6526f1d2019-02-14 00:25:43 +0000173 }
kcc64bcb922019-02-13 04:04:45 +0000174};
175
kcc6526f1d2019-02-14 00:25:43 +0000176struct JobQueue {
177 std::queue<FuzzJob *> Qu;
178 std::mutex Mu;
kcc64bcb922019-02-13 04:04:45 +0000179
kcc6526f1d2019-02-14 00:25:43 +0000180 void Push(FuzzJob *Job) {
181 std::lock_guard<std::mutex> Lock(Mu);
182 Qu.push(Job);
kcc64bcb922019-02-13 04:04:45 +0000183 }
kcc6526f1d2019-02-14 00:25:43 +0000184 FuzzJob *Pop() {
185 std::lock_guard<std::mutex> Lock(Mu);
186 if (Qu.empty()) return nullptr;
187 auto Job = Qu.front();
188 Qu.pop();
189 return Job;
190 }
191};
192
193void WorkerThread(std::atomic<bool> *Stop, JobQueue *FuzzQ, JobQueue *MergeQ) {
kccb1fa9e02019-02-14 01:11:29 +0000194 while (!Stop->load()) {
kcc6526f1d2019-02-14 00:25:43 +0000195 auto Job = FuzzQ->Pop();
196 // Printf("WorkerThread: job %p\n", Job);
197 if (!Job) {
198 SleepSeconds(1);
199 continue;
200 }
201 Job->ExitCode = ExecuteCommand(Job->Cmd);
202 MergeQ->Push(Job);
203 }
kcc64bcb922019-02-13 04:04:45 +0000204}
205
kcc2e6ca5c2019-02-12 22:48:55 +0000206// This is just a skeleton of an experimental -fork=1 feature.
207void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
208 const Vector<std::string> &Args,
kcc6526f1d2019-02-14 00:25:43 +0000209 const Vector<std::string> &CorpusDirs, int NumJobs) {
kcc55e54ed2019-02-15 21:51:15 +0000210 Printf("INFO: -fork=%d: fuzzing in separate process(s)\n", NumJobs);
kcc2e6ca5c2019-02-12 22:48:55 +0000211
kcc64bcb922019-02-13 04:04:45 +0000212 GlobalEnv Env;
kcc6526f1d2019-02-14 00:25:43 +0000213 Env.Args = Args;
214 Env.CorpusDirs = CorpusDirs;
215 Env.Rand = &Rand;
216 Env.Verbosity = Options.Verbosity;
kcc55e54ed2019-02-15 21:51:15 +0000217 Env.ProcessStartTime = std::chrono::system_clock::now();
kcc64bcb922019-02-13 04:04:45 +0000218
kcc2e6ca5c2019-02-12 22:48:55 +0000219 Vector<SizedFile> SeedFiles;
220 for (auto &Dir : CorpusDirs)
221 GetSizedFilesFromDir(Dir, &SeedFiles);
222 std::sort(SeedFiles.begin(), SeedFiles.end());
kcc6526f1d2019-02-14 00:25:43 +0000223 Env.TempDir = TempPath(".dir");
224 RmDirRecursive(Env.TempDir); // in case there is a leftover from old runs.
225 MkDir(Env.TempDir);
kcc2e6ca5c2019-02-12 22:48:55 +0000226
kcc2e6ca5c2019-02-12 22:48:55 +0000227
kcc64bcb922019-02-13 04:04:45 +0000228 if (CorpusDirs.empty())
kcc6526f1d2019-02-14 00:25:43 +0000229 MkDir(Env.MainCorpusDir = DirPlusFile(Env.TempDir, "C"));
kcc64bcb922019-02-13 04:04:45 +0000230 else
231 Env.MainCorpusDir = CorpusDirs[0];
232
kcc6526f1d2019-02-14 00:25:43 +0000233 auto CFPath = DirPlusFile(Env.TempDir, "merge.txt");
234 CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, {}, &Env.Features,
kcc98a86242019-02-15 00:08:16 +0000235 {}, &Env.Cov,
kcc64bcb922019-02-13 04:04:45 +0000236 CFPath, false);
237 RemoveFile(CFPath);
kcc55e54ed2019-02-15 21:51:15 +0000238 Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs,
239 Env.Files.size(), Env.TempDir.c_str());
kcc64bcb922019-02-13 04:04:45 +0000240
kcc2e6ca5c2019-02-12 22:48:55 +0000241 int ExitCode = 0;
kcc64bcb922019-02-13 04:04:45 +0000242
kcc6526f1d2019-02-14 00:25:43 +0000243 JobQueue FuzzQ, MergeQ;
244 std::atomic<bool> Stop(false);
kcc64bcb922019-02-13 04:04:45 +0000245
kcc6526f1d2019-02-14 00:25:43 +0000246 size_t JobId = 1;
247 Vector<std::thread> Threads;
248 for (int t = 0; t < NumJobs; t++) {
249 Threads.push_back(std::thread(WorkerThread, &Stop, &FuzzQ, &MergeQ));
250 FuzzQ.Push(Env.CreateNewJob(JobId++));
kcc2e6ca5c2019-02-12 22:48:55 +0000251 }
252
kcc55e54ed2019-02-15 21:51:15 +0000253 while (true) {
kccedefdf32019-02-16 00:14:16 +0000254 std::unique_ptr<FuzzJob> Job(MergeQ.Pop());
kcc6526f1d2019-02-14 00:25:43 +0000255 if (!Job) {
kcc55e54ed2019-02-15 21:51:15 +0000256 if (Stop)
257 break;
kcc6526f1d2019-02-14 00:25:43 +0000258 SleepSeconds(1);
259 continue;
260 }
261 ExitCode = Job->ExitCode;
kccedefdf32019-02-16 00:14:16 +0000262 if (ExitCode == Options.InterruptExitCode) {
263 Printf("==%lu== libFuzzer: a child was interrupted; exiting\n", GetPid());
264 Stop = true;
265 break;
266 }
267
268 Env.RunOneMergeJob(Job.get());
kcc6526f1d2019-02-14 00:25:43 +0000269
270 // Continue if our crash is one of the ignorred ones.
271 if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode)
kcc55e54ed2019-02-15 21:51:15 +0000272 Env.NumTimeouts++;
kcc6526f1d2019-02-14 00:25:43 +0000273 else if (Options.IgnoreOOMs && ExitCode == Options.OOMExitCode)
kcc55e54ed2019-02-15 21:51:15 +0000274 Env.NumOOMs++;
kcc6526f1d2019-02-14 00:25:43 +0000275 else if (ExitCode != 0) {
kcc55e54ed2019-02-15 21:51:15 +0000276 Env.NumCrashes++;
277 if (Options.IgnoreCrashes) {
278 std::ifstream In(Job->LogPath);
279 std::string Line;
280 while (std::getline(In, Line, '\n'))
281 if (Line.find("ERROR:") != Line.npos)
282 Printf("%s\n", Line.c_str());
283 } else {
284 // And exit if we don't ignore this crash.
285 Printf("INFO: log from the inner process:\n%s",
286 FileToString(Job->LogPath).c_str());
287 Stop = true;
288 }
kcc6526f1d2019-02-14 00:25:43 +0000289 }
kcc55e54ed2019-02-15 21:51:15 +0000290
291 // Stop if we are over the time budget.
292 // This is not precise, since other threads are still running
293 // and we will wait while joining them.
294 // We also don't stop instantly: other jobs need to finish.
295 if (Options.MaxTotalTimeSec > 0 && !Stop &&
296 Env.secondsSinceProcessStartUp() >= (size_t)Options.MaxTotalTimeSec) {
297 Printf("INFO: fuzzed for %zd seconds, wrapping up soon\n",
298 Env.secondsSinceProcessStartUp());
299 Stop = true;
300 }
301
302 if (!Stop)
303 FuzzQ.Push(Env.CreateNewJob(JobId++));
kcc6526f1d2019-02-14 00:25:43 +0000304 }
305 Stop = true;
306
307 for (auto &T : Threads)
308 T.join();
309
metzmane847d8a2019-02-27 19:27:16 +0000310 // The workers have terminated. Don't try to remove the directory before they
311 // terminate to avoid a race condition preventing cleanup on Windows.
312 RmDirRecursive(Env.TempDir);
313
kcc2e6ca5c2019-02-12 22:48:55 +0000314 // Use the exit code from the last child process.
kcc55e54ed2019-02-15 21:51:15 +0000315 Printf("INFO: exiting: %d time: %zds\n", ExitCode,
316 Env.secondsSinceProcessStartUp());
kcc2e6ca5c2019-02-12 22:48:55 +0000317 exit(ExitCode);
318}
319
320} // namespace fuzzer