kcc | 2e6ca5c | 2019-02-12 22:48:55 +0000 | [diff] [blame] | 1 | //===- FuzzerFork.cpp - run fuzzing in separate subprocesses --------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
kcc | 64bcb92 | 2019-02-13 04:04:45 +0000 | [diff] [blame] | 8 | // Spawn and orchestrate separate fuzzing processes. |
kcc | 2e6ca5c | 2019-02-12 22:48:55 +0000 | [diff] [blame] | 9 | //===----------------------------------------------------------------------===// |
| 10 | |
| 11 | #include "FuzzerCommand.h" |
| 12 | #include "FuzzerFork.h" |
| 13 | #include "FuzzerIO.h" |
| 14 | #include "FuzzerMerge.h" |
| 15 | #include "FuzzerSHA1.h" |
| 16 | #include "FuzzerUtil.h" |
| 17 | |
kcc | 6526f1d | 2019-02-14 00:25:43 +0000 | [diff] [blame^] | 18 | #include <mutex> |
| 19 | #include <thread> |
| 20 | #include <queue> |
| 21 | |
kcc | 2e6ca5c | 2019-02-12 22:48:55 +0000 | [diff] [blame] | 22 | namespace fuzzer { |
| 23 | |
kcc | 64bcb92 | 2019-02-13 04:04:45 +0000 | [diff] [blame] | 24 | struct FuzzJob { |
| 25 | // Inputs. |
| 26 | Command Cmd; |
kcc | 64bcb92 | 2019-02-13 04:04:45 +0000 | [diff] [blame] | 27 | std::string CorpusDir; |
| 28 | std::string LogPath; |
| 29 | std::string CFPath; |
kcc | 64bcb92 | 2019-02-13 04:04:45 +0000 | [diff] [blame] | 30 | |
| 31 | // Fuzzing Outputs. |
| 32 | int ExitCode; |
| 33 | }; |
| 34 | |
| 35 | struct GlobalEnv { |
kcc | 6526f1d | 2019-02-14 00:25:43 +0000 | [diff] [blame^] | 36 | Vector<std::string> Args; |
| 37 | Vector<std::string> CorpusDirs; |
kcc | 64bcb92 | 2019-02-13 04:04:45 +0000 | [diff] [blame] | 38 | std::string MainCorpusDir; |
kcc | 6526f1d | 2019-02-14 00:25:43 +0000 | [diff] [blame^] | 39 | std::string TempDir; |
kcc | 64bcb92 | 2019-02-13 04:04:45 +0000 | [diff] [blame] | 40 | Set<uint32_t> Features; |
| 41 | Vector<std::string> Files; |
kcc | 6526f1d | 2019-02-14 00:25:43 +0000 | [diff] [blame^] | 42 | Random *Rand; |
| 43 | int Verbosity = 0; |
| 44 | |
| 45 | FuzzJob *CreateNewJob(size_t JobId) { |
| 46 | Command Cmd(Args); |
| 47 | Cmd.removeFlag("fork"); |
| 48 | for (auto &C : CorpusDirs) // Remove all corpora from the args. |
| 49 | Cmd.removeArgument(C); |
| 50 | Cmd.addFlag("reload", "0"); // working in an isolated dir, no reload. |
| 51 | Cmd.addFlag("max_total_time", std::to_string(std::min((size_t)300, JobId))); |
| 52 | |
| 53 | auto Job = new FuzzJob; |
| 54 | std::string Seeds; |
| 55 | if (size_t CorpusSubsetSize = std::min(Files.size(), (size_t)100)) |
| 56 | for (size_t i = 0; i < CorpusSubsetSize; i++) |
| 57 | Seeds += (Seeds.empty() ? "" : ",") + |
| 58 | Files[Rand->SkewTowardsLast(Files.size())]; |
| 59 | if (!Seeds.empty()) |
| 60 | Cmd.addFlag("seed_inputs", Seeds); |
| 61 | Job->LogPath = DirPlusFile(TempDir, std::to_string(JobId) + ".log"); |
| 62 | Job->CorpusDir = DirPlusFile(TempDir, "C" + std::to_string(JobId)); |
| 63 | Job->CFPath = DirPlusFile(TempDir, std::to_string(JobId) + ".merge"); |
| 64 | |
| 65 | |
| 66 | Cmd.addArgument(Job->CorpusDir); |
| 67 | RmDirRecursive(Job->CorpusDir); |
| 68 | MkDir(Job->CorpusDir); |
| 69 | |
| 70 | Cmd.setOutputFile(Job->LogPath); |
| 71 | Cmd.combineOutAndErr(); |
| 72 | |
| 73 | Job->Cmd = Cmd; |
| 74 | |
| 75 | if (Verbosity >= 2) |
| 76 | Printf("Job %zd/%p Created: %s\n", JobId, Job, |
| 77 | Job->Cmd.toString().c_str()); |
| 78 | // Start from very short runs and gradually increase them. |
| 79 | return Job; |
| 80 | } |
| 81 | |
| 82 | void RunOneMergeJob(FuzzJob *Job) { |
| 83 | Vector<SizedFile> TempFiles; |
| 84 | GetSizedFilesFromDir(Job->CorpusDir, &TempFiles); |
| 85 | |
| 86 | Vector<std::string> FilesToAdd; |
| 87 | Set<uint32_t> NewFeatures; |
| 88 | CrashResistantMerge(Args, {}, TempFiles, &FilesToAdd, Features, |
| 89 | &NewFeatures, Job->CFPath, false); |
| 90 | RemoveFile(Job->CFPath); |
| 91 | for (auto &Path : FilesToAdd) { |
| 92 | auto U = FileToVector(Path); |
| 93 | auto NewPath = DirPlusFile(MainCorpusDir, Hash(U)); |
| 94 | WriteToFile(U, NewPath); |
| 95 | Files.push_back(NewPath); |
| 96 | } |
| 97 | Printf("Removing %s\n", Job->CorpusDir.c_str()); |
| 98 | RmDirRecursive(Job->CorpusDir); |
| 99 | Features.insert(NewFeatures.begin(), NewFeatures.end()); |
| 100 | Printf("INFO: temp_files: %zd files_added: %zd newft: %zd ft: %zd\n", |
| 101 | TempFiles.size(), FilesToAdd.size(), NewFeatures.size(), |
| 102 | Features.size()); |
| 103 | } |
kcc | 64bcb92 | 2019-02-13 04:04:45 +0000 | [diff] [blame] | 104 | }; |
| 105 | |
kcc | 6526f1d | 2019-02-14 00:25:43 +0000 | [diff] [blame^] | 106 | struct JobQueue { |
| 107 | std::queue<FuzzJob *> Qu; |
| 108 | std::mutex Mu; |
kcc | 64bcb92 | 2019-02-13 04:04:45 +0000 | [diff] [blame] | 109 | |
kcc | 6526f1d | 2019-02-14 00:25:43 +0000 | [diff] [blame^] | 110 | void Push(FuzzJob *Job) { |
| 111 | std::lock_guard<std::mutex> Lock(Mu); |
| 112 | Qu.push(Job); |
kcc | 64bcb92 | 2019-02-13 04:04:45 +0000 | [diff] [blame] | 113 | } |
kcc | 6526f1d | 2019-02-14 00:25:43 +0000 | [diff] [blame^] | 114 | FuzzJob *Pop() { |
| 115 | std::lock_guard<std::mutex> Lock(Mu); |
| 116 | if (Qu.empty()) return nullptr; |
| 117 | auto Job = Qu.front(); |
| 118 | Qu.pop(); |
| 119 | return Job; |
| 120 | } |
| 121 | }; |
| 122 | |
| 123 | void WorkerThread(std::atomic<bool> *Stop, JobQueue *FuzzQ, JobQueue *MergeQ) { |
| 124 | while (!*Stop) { |
| 125 | auto Job = FuzzQ->Pop(); |
| 126 | // Printf("WorkerThread: job %p\n", Job); |
| 127 | if (!Job) { |
| 128 | SleepSeconds(1); |
| 129 | continue; |
| 130 | } |
| 131 | Job->ExitCode = ExecuteCommand(Job->Cmd); |
| 132 | MergeQ->Push(Job); |
| 133 | } |
kcc | 64bcb92 | 2019-02-13 04:04:45 +0000 | [diff] [blame] | 134 | } |
| 135 | |
kcc | 2e6ca5c | 2019-02-12 22:48:55 +0000 | [diff] [blame] | 136 | // This is just a skeleton of an experimental -fork=1 feature. |
| 137 | void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, |
| 138 | const Vector<std::string> &Args, |
kcc | 6526f1d | 2019-02-14 00:25:43 +0000 | [diff] [blame^] | 139 | const Vector<std::string> &CorpusDirs, int NumJobs) { |
| 140 | Printf("INFO: -fork=%d: doing fuzzing in a separate process in order to " |
| 141 | "be more resistant to crashes, timeouts, and OOMs\n", NumJobs); |
kcc | 2e6ca5c | 2019-02-12 22:48:55 +0000 | [diff] [blame] | 142 | |
kcc | 64bcb92 | 2019-02-13 04:04:45 +0000 | [diff] [blame] | 143 | GlobalEnv Env; |
kcc | 6526f1d | 2019-02-14 00:25:43 +0000 | [diff] [blame^] | 144 | Env.Args = Args; |
| 145 | Env.CorpusDirs = CorpusDirs; |
| 146 | Env.Rand = &Rand; |
| 147 | Env.Verbosity = Options.Verbosity; |
kcc | 64bcb92 | 2019-02-13 04:04:45 +0000 | [diff] [blame] | 148 | |
kcc | 2e6ca5c | 2019-02-12 22:48:55 +0000 | [diff] [blame] | 149 | Vector<SizedFile> SeedFiles; |
| 150 | for (auto &Dir : CorpusDirs) |
| 151 | GetSizedFilesFromDir(Dir, &SeedFiles); |
| 152 | std::sort(SeedFiles.begin(), SeedFiles.end()); |
kcc | 6526f1d | 2019-02-14 00:25:43 +0000 | [diff] [blame^] | 153 | Env.TempDir = TempPath(".dir"); |
| 154 | RmDirRecursive(Env.TempDir); // in case there is a leftover from old runs. |
| 155 | MkDir(Env.TempDir); |
kcc | 2e6ca5c | 2019-02-12 22:48:55 +0000 | [diff] [blame] | 156 | |
kcc | 2e6ca5c | 2019-02-12 22:48:55 +0000 | [diff] [blame] | 157 | |
kcc | 64bcb92 | 2019-02-13 04:04:45 +0000 | [diff] [blame] | 158 | if (CorpusDirs.empty()) |
kcc | 6526f1d | 2019-02-14 00:25:43 +0000 | [diff] [blame^] | 159 | MkDir(Env.MainCorpusDir = DirPlusFile(Env.TempDir, "C")); |
kcc | 64bcb92 | 2019-02-13 04:04:45 +0000 | [diff] [blame] | 160 | else |
| 161 | Env.MainCorpusDir = CorpusDirs[0]; |
| 162 | |
kcc | 6526f1d | 2019-02-14 00:25:43 +0000 | [diff] [blame^] | 163 | auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); |
| 164 | CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, {}, &Env.Features, |
kcc | 64bcb92 | 2019-02-13 04:04:45 +0000 | [diff] [blame] | 165 | CFPath, false); |
| 166 | RemoveFile(CFPath); |
kcc | 6526f1d | 2019-02-14 00:25:43 +0000 | [diff] [blame^] | 167 | Printf("INFO: -fork=%d: %zd seeds, starting to fuzz; scratch: %s\n", |
| 168 | NumJobs, Env.Files.size(), Env.TempDir.c_str()); |
kcc | 64bcb92 | 2019-02-13 04:04:45 +0000 | [diff] [blame] | 169 | |
kcc | 2e6ca5c | 2019-02-12 22:48:55 +0000 | [diff] [blame] | 170 | int ExitCode = 0; |
kcc | 64bcb92 | 2019-02-13 04:04:45 +0000 | [diff] [blame] | 171 | |
kcc | 6526f1d | 2019-02-14 00:25:43 +0000 | [diff] [blame^] | 172 | JobQueue FuzzQ, MergeQ; |
| 173 | std::atomic<bool> Stop(false); |
kcc | 64bcb92 | 2019-02-13 04:04:45 +0000 | [diff] [blame] | 174 | |
kcc | 6526f1d | 2019-02-14 00:25:43 +0000 | [diff] [blame^] | 175 | size_t JobId = 1; |
| 176 | Vector<std::thread> Threads; |
| 177 | for (int t = 0; t < NumJobs; t++) { |
| 178 | Threads.push_back(std::thread(WorkerThread, &Stop, &FuzzQ, &MergeQ)); |
| 179 | FuzzQ.Push(Env.CreateNewJob(JobId++)); |
kcc | 2e6ca5c | 2019-02-12 22:48:55 +0000 | [diff] [blame] | 180 | } |
| 181 | |
kcc | 6526f1d | 2019-02-14 00:25:43 +0000 | [diff] [blame^] | 182 | while (!Stop) { |
| 183 | auto Job = MergeQ.Pop(); |
| 184 | if (!Job) { |
| 185 | SleepSeconds(1); |
| 186 | continue; |
| 187 | } |
| 188 | ExitCode = Job->ExitCode; |
| 189 | if (ExitCode != Options.InterruptExitCode) |
| 190 | Env.RunOneMergeJob(Job); |
| 191 | |
| 192 | // Continue if our crash is one of the ignorred ones. |
| 193 | if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode) |
| 194 | ; |
| 195 | else if (Options.IgnoreOOMs && ExitCode == Options.OOMExitCode) |
| 196 | ; |
| 197 | else if (ExitCode == Options.InterruptExitCode) |
| 198 | Stop = true; |
| 199 | else if (ExitCode != 0) { |
| 200 | // And exit if we don't ignore this crash. |
| 201 | Printf("INFO: log from the inner process:\n%s", |
| 202 | FileToString(Job->LogPath).c_str()); |
| 203 | Stop = true; |
| 204 | } |
| 205 | RemoveFile(Job->LogPath); |
| 206 | delete Job; |
| 207 | FuzzQ.Push(Env.CreateNewJob(JobId++)); |
| 208 | } |
| 209 | Stop = true; |
| 210 | |
| 211 | for (auto &T : Threads) |
| 212 | T.join(); |
| 213 | |
| 214 | RmDirRecursive(Env.TempDir); |
kcc | 2e6ca5c | 2019-02-12 22:48:55 +0000 | [diff] [blame] | 215 | |
| 216 | // Use the exit code from the last child process. |
| 217 | Printf("Fork: exiting: %d\n", ExitCode); |
| 218 | exit(ExitCode); |
| 219 | } |
| 220 | |
| 221 | } // namespace fuzzer |
| 222 | |