FuzzerDriver.cpp 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939
  1. //===- FuzzerDriver.cpp - FuzzerDriver function and flags -----------------===//
  2. //
  3. // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
  4. // See https://llvm.org/LICENSE.txt for license information.
  5. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  6. //
  7. //===----------------------------------------------------------------------===//
  8. // FuzzerDriver and flag parsing.
  9. //===----------------------------------------------------------------------===//
  10. #include "FuzzerCommand.h"
  11. #include "FuzzerCorpus.h"
  12. #include "FuzzerFork.h"
  13. #include "FuzzerIO.h"
  14. #include "FuzzerInterface.h"
  15. #include "FuzzerInternal.h"
  16. #include "FuzzerMerge.h"
  17. #include "FuzzerMutate.h"
  18. #include "FuzzerPlatform.h"
  19. #include "FuzzerRandom.h"
  20. #include "FuzzerTracePC.h"
  21. #include <algorithm>
  22. #include <atomic>
  23. #include <chrono>
  24. #include <cstdlib>
  25. #include <cstring>
  26. #include <mutex>
  27. #include <string>
  28. #include <thread>
  29. #include <fstream>
  30. // This function should be present in the libFuzzer so that the client
  31. // binary can test for its existence.
  32. #if LIBFUZZER_MSVC
  33. extern "C" void __libfuzzer_is_present() {}
  34. #if defined(_M_IX86) || defined(__i386__)
  35. #pragma comment(linker, "/include:___libfuzzer_is_present")
  36. #else
  37. #pragma comment(linker, "/include:__libfuzzer_is_present")
  38. #endif
  39. #else
  40. extern "C" __attribute__((used)) void __libfuzzer_is_present() {}
  41. #endif // LIBFUZZER_MSVC
  42. namespace fuzzer {
  43. // Program arguments.
  44. struct FlagDescription {
  45. const char *Name;
  46. const char *Description;
  47. int Default;
  48. int *IntFlag;
  49. const char **StrFlag;
  50. unsigned int *UIntFlag;
  51. };
  52. struct {
  53. #define FUZZER_DEPRECATED_FLAG(Name)
  54. #define FUZZER_FLAG_INT(Name, Default, Description) int Name;
  55. #define FUZZER_FLAG_UNSIGNED(Name, Default, Description) unsigned int Name;
  56. #define FUZZER_FLAG_STRING(Name, Description) const char *Name;
  57. #include "FuzzerFlags.def"
  58. #undef FUZZER_DEPRECATED_FLAG
  59. #undef FUZZER_FLAG_INT
  60. #undef FUZZER_FLAG_UNSIGNED
  61. #undef FUZZER_FLAG_STRING
  62. } Flags;
  63. static const FlagDescription FlagDescriptions [] {
  64. #define FUZZER_DEPRECATED_FLAG(Name) \
  65. {#Name, "Deprecated; don't use", 0, nullptr, nullptr, nullptr},
  66. #define FUZZER_FLAG_INT(Name, Default, Description) \
  67. {#Name, Description, Default, &Flags.Name, nullptr, nullptr},
  68. #define FUZZER_FLAG_UNSIGNED(Name, Default, Description) \
  69. {#Name, Description, static_cast<int>(Default), \
  70. nullptr, nullptr, &Flags.Name},
  71. #define FUZZER_FLAG_STRING(Name, Description) \
  72. {#Name, Description, 0, nullptr, &Flags.Name, nullptr},
  73. #include "FuzzerFlags.def"
  74. #undef FUZZER_DEPRECATED_FLAG
  75. #undef FUZZER_FLAG_INT
  76. #undef FUZZER_FLAG_UNSIGNED
  77. #undef FUZZER_FLAG_STRING
  78. };
  79. static const size_t kNumFlags =
  80. sizeof(FlagDescriptions) / sizeof(FlagDescriptions[0]);
  81. static std::vector<std::string> *Inputs;
  82. static std::string *ProgName;
  83. static void PrintHelp() {
  84. Printf("Usage:\n");
  85. auto Prog = ProgName->c_str();
  86. Printf("\nTo run fuzzing pass 0 or more directories.\n");
  87. Printf("%s [-flag1=val1 [-flag2=val2 ...] ] [dir1 [dir2 ...] ]\n", Prog);
  88. Printf("\nTo run individual tests without fuzzing pass 1 or more files:\n");
  89. Printf("%s [-flag1=val1 [-flag2=val2 ...] ] file1 [file2 ...]\n", Prog);
  90. Printf("\nFlags: (strictly in form -flag=value)\n");
  91. size_t MaxFlagLen = 0;
  92. for (size_t F = 0; F < kNumFlags; F++)
  93. MaxFlagLen = std::max(strlen(FlagDescriptions[F].Name), MaxFlagLen);
  94. for (size_t F = 0; F < kNumFlags; F++) {
  95. const auto &D = FlagDescriptions[F];
  96. if (strstr(D.Description, "internal flag") == D.Description) continue;
  97. Printf(" %s", D.Name);
  98. for (size_t i = 0, n = MaxFlagLen - strlen(D.Name); i < n; i++)
  99. Printf(" ");
  100. Printf("\t");
  101. Printf("%d\t%s\n", D.Default, D.Description);
  102. }
  103. Printf("\nFlags starting with '--' will be ignored and "
  104. "will be passed verbatim to subprocesses.\n");
  105. }
  106. static const char *FlagValue(const char *Param, const char *Name) {
  107. size_t Len = strlen(Name);
  108. if (Param[0] == '-' && strstr(Param + 1, Name) == Param + 1 &&
  109. Param[Len + 1] == '=')
  110. return &Param[Len + 2];
  111. return nullptr;
  112. }
  113. // Avoid calling stol as it triggers a bug in clang/glibc build.
  114. static long MyStol(const char *Str) {
  115. long Res = 0;
  116. long Sign = 1;
  117. if (*Str == '-') {
  118. Str++;
  119. Sign = -1;
  120. }
  121. for (size_t i = 0; Str[i]; i++) {
  122. char Ch = Str[i];
  123. if (Ch < '0' || Ch > '9')
  124. return Res;
  125. Res = Res * 10 + (Ch - '0');
  126. }
  127. return Res * Sign;
  128. }
  129. static bool ParseOneFlag(const char *Param) {
  130. if (Param[0] != '-') return false;
  131. if (Param[1] == '-') {
  132. static bool PrintedWarning = false;
  133. if (!PrintedWarning) {
  134. PrintedWarning = true;
  135. Printf("INFO: libFuzzer ignores flags that start with '--'\n");
  136. }
  137. for (size_t F = 0; F < kNumFlags; F++)
  138. if (FlagValue(Param + 1, FlagDescriptions[F].Name))
  139. Printf("WARNING: did you mean '%s' (single dash)?\n", Param + 1);
  140. return true;
  141. }
  142. for (size_t F = 0; F < kNumFlags; F++) {
  143. const char *Name = FlagDescriptions[F].Name;
  144. const char *Str = FlagValue(Param, Name);
  145. if (Str) {
  146. if (FlagDescriptions[F].IntFlag) {
  147. auto Val = MyStol(Str);
  148. *FlagDescriptions[F].IntFlag = static_cast<int>(Val);
  149. if (Flags.verbosity >= 2)
  150. Printf("Flag: %s %d\n", Name, Val);
  151. return true;
  152. } else if (FlagDescriptions[F].UIntFlag) {
  153. auto Val = std::stoul(Str);
  154. *FlagDescriptions[F].UIntFlag = static_cast<unsigned int>(Val);
  155. if (Flags.verbosity >= 2)
  156. Printf("Flag: %s %u\n", Name, Val);
  157. return true;
  158. } else if (FlagDescriptions[F].StrFlag) {
  159. *FlagDescriptions[F].StrFlag = Str;
  160. if (Flags.verbosity >= 2)
  161. Printf("Flag: %s %s\n", Name, Str);
  162. return true;
  163. } else { // Deprecated flag.
  164. Printf("Flag: %s: deprecated, don't use\n", Name);
  165. return true;
  166. }
  167. }
  168. }
  169. Printf("\n\nWARNING: unrecognized flag '%s'; "
  170. "use -help=1 to list all flags\n\n", Param);
  171. return true;
  172. }
  173. // We don't use any library to minimize dependencies.
  174. static void ParseFlags(const std::vector<std::string> &Args,
  175. const ExternalFunctions *EF) {
  176. for (size_t F = 0; F < kNumFlags; F++) {
  177. if (FlagDescriptions[F].IntFlag)
  178. *FlagDescriptions[F].IntFlag = FlagDescriptions[F].Default;
  179. if (FlagDescriptions[F].UIntFlag)
  180. *FlagDescriptions[F].UIntFlag =
  181. static_cast<unsigned int>(FlagDescriptions[F].Default);
  182. if (FlagDescriptions[F].StrFlag)
  183. *FlagDescriptions[F].StrFlag = nullptr;
  184. }
  185. // Disable len_control by default, if LLVMFuzzerCustomMutator is used.
  186. if (EF->LLVMFuzzerCustomMutator) {
  187. Flags.len_control = 0;
  188. Printf("INFO: found LLVMFuzzerCustomMutator (%p). "
  189. "Disabling -len_control by default.\n", EF->LLVMFuzzerCustomMutator);
  190. }
  191. Inputs = new std::vector<std::string>;
  192. for (size_t A = 1; A < Args.size(); A++) {
  193. if (ParseOneFlag(Args[A].c_str())) {
  194. if (Flags.ignore_remaining_args)
  195. break;
  196. continue;
  197. }
  198. Inputs->push_back(Args[A]);
  199. }
  200. }
  201. static std::mutex Mu;
  202. static void PulseThread() {
  203. while (true) {
  204. SleepSeconds(600);
  205. std::lock_guard<std::mutex> Lock(Mu);
  206. Printf("pulse...\n");
  207. }
  208. }
  209. static void WorkerThread(const Command &BaseCmd, std::atomic<unsigned> *Counter,
  210. unsigned NumJobs, std::atomic<bool> *HasErrors) {
  211. while (true) {
  212. unsigned C = (*Counter)++;
  213. if (C >= NumJobs) break;
  214. std::string Log = "fuzz-" + std::to_string(C) + ".log";
  215. Command Cmd(BaseCmd);
  216. Cmd.setOutputFile(Log);
  217. Cmd.combineOutAndErr();
  218. if (Flags.verbosity) {
  219. std::string CommandLine = Cmd.toString();
  220. Printf("%s\n", CommandLine.c_str());
  221. }
  222. int ExitCode = ExecuteCommand(Cmd);
  223. if (ExitCode != 0)
  224. *HasErrors = true;
  225. std::lock_guard<std::mutex> Lock(Mu);
  226. Printf("================== Job %u exited with exit code %d ============\n",
  227. C, ExitCode);
  228. fuzzer::CopyFileToErr(Log);
  229. }
  230. }
  231. static void ValidateDirectoryExists(const std::string &Path,
  232. bool CreateDirectory) {
  233. if (Path.empty()) {
  234. Printf("ERROR: Provided directory path is an empty string\n");
  235. exit(1);
  236. }
  237. if (IsDirectory(Path))
  238. return;
  239. if (CreateDirectory) {
  240. if (!MkDirRecursive(Path)) {
  241. Printf("ERROR: Failed to create directory \"%s\"\n", Path.c_str());
  242. exit(1);
  243. }
  244. return;
  245. }
  246. Printf("ERROR: The required directory \"%s\" does not exist\n", Path.c_str());
  247. exit(1);
  248. }
  249. std::string CloneArgsWithoutX(const std::vector<std::string> &Args,
  250. const char *X1, const char *X2) {
  251. std::string Cmd;
  252. for (auto &S : Args) {
  253. if (FlagValue(S.c_str(), X1) || FlagValue(S.c_str(), X2))
  254. continue;
  255. Cmd += S + " ";
  256. }
  257. return Cmd;
  258. }
  259. static int RunInMultipleProcesses(const std::vector<std::string> &Args,
  260. unsigned NumWorkers, unsigned NumJobs) {
  261. std::atomic<unsigned> Counter(0);
  262. std::atomic<bool> HasErrors(false);
  263. Command Cmd(Args);
  264. Cmd.removeFlag("jobs");
  265. Cmd.removeFlag("workers");
  266. std::vector<std::thread> V;
  267. std::thread Pulse(PulseThread);
  268. Pulse.detach();
  269. for (unsigned i = 0; i < NumWorkers; i++)
  270. V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs,
  271. &HasErrors));
  272. for (auto &T : V)
  273. T.join();
  274. return HasErrors ? 1 : 0;
  275. }
  276. static void RssThread(Fuzzer *F, size_t RssLimitMb) {
  277. while (true) {
  278. SleepSeconds(1);
  279. size_t Peak = GetPeakRSSMb();
  280. if (Peak > RssLimitMb)
  281. F->RssLimitCallback();
  282. }
  283. }
  284. static void StartRssThread(Fuzzer *F, size_t RssLimitMb) {
  285. if (!RssLimitMb)
  286. return;
  287. std::thread T(RssThread, F, RssLimitMb);
  288. T.detach();
  289. }
  290. int RunOneTest(Fuzzer *F, const char *InputFilePath, size_t MaxLen) {
  291. Unit U = FileToVector(InputFilePath);
  292. if (MaxLen && MaxLen < U.size())
  293. U.resize(MaxLen);
  294. F->ExecuteCallback(U.data(), U.size());
  295. if (Flags.print_full_coverage) {
  296. // Leak detection is not needed when collecting full coverage data.
  297. F->TPCUpdateObservedPCs();
  298. } else {
  299. F->TryDetectingAMemoryLeak(U.data(), U.size(), true);
  300. }
  301. return 0;
  302. }
  303. static bool AllInputsAreFiles() {
  304. if (Inputs->empty()) return false;
  305. for (auto &Path : *Inputs)
  306. if (!IsFile(Path))
  307. return false;
  308. return true;
  309. }
  310. static std::string GetDedupTokenFromCmdOutput(const std::string &S) {
  311. auto Beg = S.find("DEDUP_TOKEN:");
  312. if (Beg == std::string::npos)
  313. return "";
  314. auto End = S.find('\n', Beg);
  315. if (End == std::string::npos)
  316. return "";
  317. return S.substr(Beg, End - Beg);
  318. }
  319. int CleanseCrashInput(const std::vector<std::string> &Args,
  320. const FuzzingOptions &Options) {
  321. if (Inputs->size() != 1 || !Flags.exact_artifact_path) {
  322. Printf("ERROR: -cleanse_crash should be given one input file and"
  323. " -exact_artifact_path\n");
  324. exit(1);
  325. }
  326. std::string InputFilePath = Inputs->at(0);
  327. std::string OutputFilePath = Flags.exact_artifact_path;
  328. Command Cmd(Args);
  329. Cmd.removeFlag("cleanse_crash");
  330. assert(Cmd.hasArgument(InputFilePath));
  331. Cmd.removeArgument(InputFilePath);
  332. auto TmpFilePath = TempPath("CleanseCrashInput", ".repro");
  333. Cmd.addArgument(TmpFilePath);
  334. Cmd.setOutputFile(getDevNull());
  335. Cmd.combineOutAndErr();
  336. std::string CurrentFilePath = InputFilePath;
  337. auto U = FileToVector(CurrentFilePath);
  338. size_t Size = U.size();
  339. const std::vector<uint8_t> ReplacementBytes = {' ', 0xff};
  340. for (int NumAttempts = 0; NumAttempts < 5; NumAttempts++) {
  341. bool Changed = false;
  342. for (size_t Idx = 0; Idx < Size; Idx++) {
  343. Printf("CLEANSE[%d]: Trying to replace byte %zd of %zd\n", NumAttempts,
  344. Idx, Size);
  345. uint8_t OriginalByte = U[Idx];
  346. if (ReplacementBytes.end() != std::find(ReplacementBytes.begin(),
  347. ReplacementBytes.end(),
  348. OriginalByte))
  349. continue;
  350. for (auto NewByte : ReplacementBytes) {
  351. U[Idx] = NewByte;
  352. WriteToFile(U, TmpFilePath);
  353. auto ExitCode = ExecuteCommand(Cmd);
  354. RemoveFile(TmpFilePath);
  355. if (!ExitCode) {
  356. U[Idx] = OriginalByte;
  357. } else {
  358. Changed = true;
  359. Printf("CLEANSE: Replaced byte %zd with 0x%x\n", Idx, NewByte);
  360. WriteToFile(U, OutputFilePath);
  361. break;
  362. }
  363. }
  364. }
  365. if (!Changed) break;
  366. }
  367. return 0;
  368. }
  369. int MinimizeCrashInput(const std::vector<std::string> &Args,
  370. const FuzzingOptions &Options) {
  371. if (Inputs->size() != 1) {
  372. Printf("ERROR: -minimize_crash should be given one input file\n");
  373. exit(1);
  374. }
  375. std::string InputFilePath = Inputs->at(0);
  376. Command BaseCmd(Args);
  377. BaseCmd.removeFlag("minimize_crash");
  378. BaseCmd.removeFlag("exact_artifact_path");
  379. assert(BaseCmd.hasArgument(InputFilePath));
  380. BaseCmd.removeArgument(InputFilePath);
  381. if (Flags.runs <= 0 && Flags.max_total_time == 0) {
  382. Printf("INFO: you need to specify -runs=N or "
  383. "-max_total_time=N with -minimize_crash=1\n"
  384. "INFO: defaulting to -max_total_time=600\n");
  385. BaseCmd.addFlag("max_total_time", "600");
  386. }
  387. BaseCmd.combineOutAndErr();
  388. std::string CurrentFilePath = InputFilePath;
  389. while (true) {
  390. Unit U = FileToVector(CurrentFilePath);
  391. Printf("CRASH_MIN: minimizing crash input: '%s' (%zd bytes)\n",
  392. CurrentFilePath.c_str(), U.size());
  393. Command Cmd(BaseCmd);
  394. Cmd.addArgument(CurrentFilePath);
  395. Printf("CRASH_MIN: executing: %s\n", Cmd.toString().c_str());
  396. std::string CmdOutput;
  397. bool Success = ExecuteCommand(Cmd, &CmdOutput);
  398. if (Success) {
  399. Printf("ERROR: the input %s did not crash\n", CurrentFilePath.c_str());
  400. exit(1);
  401. }
  402. Printf("CRASH_MIN: '%s' (%zd bytes) caused a crash. Will try to minimize "
  403. "it further\n",
  404. CurrentFilePath.c_str(), U.size());
  405. auto DedupToken1 = GetDedupTokenFromCmdOutput(CmdOutput);
  406. if (!DedupToken1.empty())
  407. Printf("CRASH_MIN: DedupToken1: %s\n", DedupToken1.c_str());
  408. std::string ArtifactPath =
  409. Flags.exact_artifact_path
  410. ? Flags.exact_artifact_path
  411. : Options.ArtifactPrefix + "minimized-from-" + Hash(U);
  412. Cmd.addFlag("minimize_crash_internal_step", "1");
  413. Cmd.addFlag("exact_artifact_path", ArtifactPath);
  414. Printf("CRASH_MIN: executing: %s\n", Cmd.toString().c_str());
  415. CmdOutput.clear();
  416. Success = ExecuteCommand(Cmd, &CmdOutput);
  417. Printf("%s", CmdOutput.c_str());
  418. if (Success) {
  419. if (Flags.exact_artifact_path) {
  420. CurrentFilePath = Flags.exact_artifact_path;
  421. WriteToFile(U, CurrentFilePath);
  422. }
  423. Printf("CRASH_MIN: failed to minimize beyond %s (%d bytes), exiting\n",
  424. CurrentFilePath.c_str(), U.size());
  425. break;
  426. }
  427. auto DedupToken2 = GetDedupTokenFromCmdOutput(CmdOutput);
  428. if (!DedupToken2.empty())
  429. Printf("CRASH_MIN: DedupToken2: %s\n", DedupToken2.c_str());
  430. if (DedupToken1 != DedupToken2) {
  431. if (Flags.exact_artifact_path) {
  432. CurrentFilePath = Flags.exact_artifact_path;
  433. WriteToFile(U, CurrentFilePath);
  434. }
  435. Printf("CRASH_MIN: mismatch in dedup tokens"
  436. " (looks like a different bug). Won't minimize further\n");
  437. break;
  438. }
  439. CurrentFilePath = ArtifactPath;
  440. Printf("*********************************\n");
  441. }
  442. return 0;
  443. }
  444. int MinimizeCrashInputInternalStep(Fuzzer *F, InputCorpus *Corpus) {
  445. assert(Inputs->size() == 1);
  446. std::string InputFilePath = Inputs->at(0);
  447. Unit U = FileToVector(InputFilePath);
  448. Printf("INFO: Starting MinimizeCrashInputInternalStep: %zd\n", U.size());
  449. if (U.size() < 2) {
  450. Printf("INFO: The input is small enough, exiting\n");
  451. exit(0);
  452. }
  453. F->SetMaxInputLen(U.size());
  454. F->SetMaxMutationLen(U.size() - 1);
  455. F->MinimizeCrashLoop(U);
  456. Printf("INFO: Done MinimizeCrashInputInternalStep, no crashes found\n");
  457. exit(0);
  458. return 0;
  459. }
  460. void Merge(Fuzzer *F, FuzzingOptions &Options,
  461. const std::vector<std::string> &Args,
  462. const std::vector<std::string> &Corpora, const char *CFPathOrNull) {
  463. if (Corpora.size() < 2) {
  464. Printf("INFO: Merge requires two or more corpus dirs\n");
  465. exit(0);
  466. }
  467. std::vector<SizedFile> OldCorpus, NewCorpus;
  468. GetSizedFilesFromDir(Corpora[0], &OldCorpus);
  469. for (size_t i = 1; i < Corpora.size(); i++)
  470. GetSizedFilesFromDir(Corpora[i], &NewCorpus);
  471. std::sort(OldCorpus.begin(), OldCorpus.end());
  472. std::sort(NewCorpus.begin(), NewCorpus.end());
  473. std::string CFPath = CFPathOrNull ? CFPathOrNull : TempPath("Merge", ".txt");
  474. std::vector<std::string> NewFiles;
  475. std::set<uint32_t> NewFeatures, NewCov;
  476. CrashResistantMerge(Args, OldCorpus, NewCorpus, &NewFiles, {}, &NewFeatures,
  477. {}, &NewCov, CFPath, true, Flags.set_cover_merge);
  478. for (auto &Path : NewFiles)
  479. F->WriteToOutputCorpus(FileToVector(Path, Options.MaxLen));
  480. // We are done, delete the control file if it was a temporary one.
  481. if (!Flags.merge_control_file)
  482. RemoveFile(CFPath);
  483. exit(0);
  484. }
  485. int AnalyzeDictionary(Fuzzer *F, const std::vector<Unit> &Dict,
  486. UnitVector &Corpus) {
  487. Printf("Started dictionary minimization (up to %d tests)\n",
  488. Dict.size() * Corpus.size() * 2);
  489. // Scores and usage count for each dictionary unit.
  490. std::vector<int> Scores(Dict.size());
  491. std::vector<int> Usages(Dict.size());
  492. std::vector<size_t> InitialFeatures;
  493. std::vector<size_t> ModifiedFeatures;
  494. for (auto &C : Corpus) {
  495. // Get coverage for the testcase without modifications.
  496. F->ExecuteCallback(C.data(), C.size());
  497. InitialFeatures.clear();
  498. TPC.CollectFeatures([&](size_t Feature) {
  499. InitialFeatures.push_back(Feature);
  500. });
  501. for (size_t i = 0; i < Dict.size(); ++i) {
  502. std::vector<uint8_t> Data = C;
  503. auto StartPos = std::search(Data.begin(), Data.end(),
  504. Dict[i].begin(), Dict[i].end());
  505. // Skip dictionary unit, if the testcase does not contain it.
  506. if (StartPos == Data.end())
  507. continue;
  508. ++Usages[i];
  509. while (StartPos != Data.end()) {
  510. // Replace all occurrences of dictionary unit in the testcase.
  511. auto EndPos = StartPos + Dict[i].size();
  512. for (auto It = StartPos; It != EndPos; ++It)
  513. *It ^= 0xFF;
  514. StartPos = std::search(EndPos, Data.end(),
  515. Dict[i].begin(), Dict[i].end());
  516. }
  517. // Get coverage for testcase with masked occurrences of dictionary unit.
  518. F->ExecuteCallback(Data.data(), Data.size());
  519. ModifiedFeatures.clear();
  520. TPC.CollectFeatures([&](size_t Feature) {
  521. ModifiedFeatures.push_back(Feature);
  522. });
  523. if (InitialFeatures == ModifiedFeatures)
  524. --Scores[i];
  525. else
  526. Scores[i] += 2;
  527. }
  528. }
  529. Printf("###### Useless dictionary elements. ######\n");
  530. for (size_t i = 0; i < Dict.size(); ++i) {
  531. // Dictionary units with positive score are treated as useful ones.
  532. if (Scores[i] > 0)
  533. continue;
  534. Printf("\"");
  535. PrintASCII(Dict[i].data(), Dict[i].size(), "\"");
  536. Printf(" # Score: %d, Used: %d\n", Scores[i], Usages[i]);
  537. }
  538. Printf("###### End of useless dictionary elements. ######\n");
  539. return 0;
  540. }
  541. std::vector<std::string> ParseSeedInuts(const char *seed_inputs) {
  542. // Parse -seed_inputs=file1,file2,... or -seed_inputs=@seed_inputs_file
  543. std::vector<std::string> Files;
  544. if (!seed_inputs) return Files;
  545. std::string SeedInputs;
  546. if (Flags.seed_inputs[0] == '@')
  547. SeedInputs = FileToString(Flags.seed_inputs + 1); // File contains list.
  548. else
  549. SeedInputs = Flags.seed_inputs; // seed_inputs contains the list.
  550. if (SeedInputs.empty()) {
  551. Printf("seed_inputs is empty or @file does not exist.\n");
  552. exit(1);
  553. }
  554. // Parse SeedInputs.
  555. size_t comma_pos = 0;
  556. while ((comma_pos = SeedInputs.find_last_of(',')) != std::string::npos) {
  557. Files.push_back(SeedInputs.substr(comma_pos + 1));
  558. SeedInputs = SeedInputs.substr(0, comma_pos);
  559. }
  560. Files.push_back(SeedInputs);
  561. return Files;
  562. }
  563. static std::vector<SizedFile>
  564. ReadCorpora(const std::vector<std::string> &CorpusDirs,
  565. const std::vector<std::string> &ExtraSeedFiles) {
  566. std::vector<SizedFile> SizedFiles;
  567. size_t LastNumFiles = 0;
  568. for (auto &Dir : CorpusDirs) {
  569. GetSizedFilesFromDir(Dir, &SizedFiles);
  570. Printf("INFO: % 8zd files found in %s\n", SizedFiles.size() - LastNumFiles,
  571. Dir.c_str());
  572. LastNumFiles = SizedFiles.size();
  573. }
  574. for (auto &File : ExtraSeedFiles)
  575. if (auto Size = FileSize(File))
  576. SizedFiles.push_back({File, Size});
  577. return SizedFiles;
  578. }
  579. void FuzzerExit(int status) {
  580. if (EF->LLVMFuzzerCleanup)
  581. EF->LLVMFuzzerCleanup();
  582. (exit)(status);
  583. }
  584. int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
  585. using namespace fuzzer;
  586. assert(argc && argv && "Argument pointers cannot be nullptr");
  587. std::string Argv0((*argv)[0]);
  588. EF = new ExternalFunctions();
  589. if (EF->LLVMFuzzerInitialize)
  590. EF->LLVMFuzzerInitialize(argc, argv);
  591. if (EF->__msan_scoped_disable_interceptor_checks)
  592. EF->__msan_scoped_disable_interceptor_checks();
  593. const std::vector<std::string> Args(*argv, *argv + *argc);
  594. assert(!Args.empty());
  595. ProgName = new std::string(Args[0]);
  596. if (Argv0 != *ProgName) {
  597. Printf("ERROR: argv[0] has been modified in LLVMFuzzerInitialize\n");
  598. exit(1);
  599. }
  600. ParseFlags(Args, EF);
  601. if (Flags.help) {
  602. PrintHelp();
  603. return 0;
  604. }
  605. if (Flags.close_fd_mask & 2)
  606. DupAndCloseStderr();
  607. if (Flags.close_fd_mask & 1)
  608. CloseStdout();
  609. if (Flags.jobs > 0 && Flags.workers == 0) {
  610. Flags.workers = std::min(NumberOfCpuCores() / 2, Flags.jobs);
  611. if (Flags.workers > 1)
  612. Printf("Running %u workers\n", Flags.workers);
  613. }
  614. if (Flags.workers > 0 && Flags.jobs > 0)
  615. return RunInMultipleProcesses(Args, Flags.workers, Flags.jobs);
  616. FuzzingOptions Options;
  617. Options.Verbosity = Flags.verbosity;
  618. Options.MaxLen = Flags.max_len;
  619. Options.LenControl = Flags.len_control;
  620. Options.KeepSeed = Flags.keep_seed;
  621. Options.UnitTimeoutSec = Flags.timeout;
  622. Options.ErrorExitCode = Flags.error_exitcode;
  623. Options.TimeoutExitCode = Flags.timeout_exitcode;
  624. Options.InterruptExitCode = Flags.interrupted_exitcode;
  625. Options.DumpInterrupted = Flags.dump_interrupted;
  626. Options.IgnoreTimeouts = Flags.ignore_timeouts;
  627. Options.IgnoreOOMs = Flags.ignore_ooms;
  628. Options.IgnoreCrashes = Flags.ignore_crashes;
  629. Options.MaxTotalTimeSec = Flags.max_total_time;
  630. Options.DoCrossOver = Flags.cross_over;
  631. Options.CrossOverUniformDist = Flags.cross_over_uniform_dist;
  632. Options.MutateDepth = Flags.mutate_depth;
  633. Options.ReduceDepth = Flags.reduce_depth;
  634. Options.UseCounters = Flags.use_counters;
  635. Options.UseMemmem = Flags.use_memmem;
  636. Options.UseCmp = Flags.use_cmp;
  637. Options.UseValueProfile = Flags.use_value_profile;
  638. Options.Shrink = Flags.shrink;
  639. Options.ReduceInputs = Flags.reduce_inputs;
  640. Options.ShuffleAtStartUp = Flags.shuffle;
  641. Options.PreferSmall = Flags.prefer_small;
  642. Options.ReloadIntervalSec = Flags.reload;
  643. Options.OnlyASCII = Flags.only_ascii;
  644. Options.DetectLeaks = Flags.detect_leaks;
  645. Options.PurgeAllocatorIntervalSec = Flags.purge_allocator_interval;
  646. Options.TraceMalloc = Flags.trace_malloc;
  647. Options.RssLimitMb = Flags.rss_limit_mb;
  648. Options.MallocLimitMb = Flags.malloc_limit_mb;
  649. if (!Options.MallocLimitMb)
  650. Options.MallocLimitMb = Options.RssLimitMb;
  651. if (Flags.runs >= 0)
  652. Options.MaxNumberOfRuns = Flags.runs;
  653. if (!Inputs->empty() && !Flags.minimize_crash_internal_step) {
  654. // Ensure output corpus assumed to be the first arbitrary argument input
  655. // is not a path to an existing file.
  656. std::string OutputCorpusDir = (*Inputs)[0];
  657. if (!IsFile(OutputCorpusDir)) {
  658. Options.OutputCorpus = OutputCorpusDir;
  659. ValidateDirectoryExists(Options.OutputCorpus, Flags.create_missing_dirs);
  660. }
  661. }
  662. Options.ReportSlowUnits = Flags.report_slow_units;
  663. if (Flags.artifact_prefix) {
  664. Options.ArtifactPrefix = Flags.artifact_prefix;
  665. // Since the prefix could be a full path to a file name prefix, assume
  666. // that if the path ends with the platform's separator that a directory
  667. // is desired
  668. std::string ArtifactPathDir = Options.ArtifactPrefix;
  669. if (!IsSeparator(ArtifactPathDir[ArtifactPathDir.length() - 1])) {
  670. ArtifactPathDir = DirName(ArtifactPathDir);
  671. }
  672. ValidateDirectoryExists(ArtifactPathDir, Flags.create_missing_dirs);
  673. }
  674. if (Flags.exact_artifact_path) {
  675. Options.ExactArtifactPath = Flags.exact_artifact_path;
  676. ValidateDirectoryExists(DirName(Options.ExactArtifactPath),
  677. Flags.create_missing_dirs);
  678. }
  679. std::vector<Unit> Dictionary;
  680. if (Flags.dict)
  681. if (!ParseDictionaryFile(FileToString(Flags.dict), &Dictionary))
  682. return 1;
  683. if (Flags.verbosity > 0 && !Dictionary.empty())
  684. Printf("Dictionary: %zd entries\n", Dictionary.size());
  685. bool RunIndividualFiles = AllInputsAreFiles();
  686. Options.SaveArtifacts =
  687. !RunIndividualFiles || Flags.minimize_crash_internal_step;
  688. Options.PrintNewCovPcs = Flags.print_pcs;
  689. Options.PrintNewCovFuncs = Flags.print_funcs;
  690. Options.PrintFinalStats = Flags.print_final_stats;
  691. Options.PrintCorpusStats = Flags.print_corpus_stats;
  692. Options.PrintCoverage = Flags.print_coverage;
  693. Options.PrintFullCoverage = Flags.print_full_coverage;
  694. if (Flags.exit_on_src_pos)
  695. Options.ExitOnSrcPos = Flags.exit_on_src_pos;
  696. if (Flags.exit_on_item)
  697. Options.ExitOnItem = Flags.exit_on_item;
  698. if (Flags.focus_function)
  699. Options.FocusFunction = Flags.focus_function;
  700. if (Flags.data_flow_trace)
  701. Options.DataFlowTrace = Flags.data_flow_trace;
  702. if (Flags.features_dir) {
  703. Options.FeaturesDir = Flags.features_dir;
  704. ValidateDirectoryExists(Options.FeaturesDir, Flags.create_missing_dirs);
  705. }
  706. if (Flags.mutation_graph_file)
  707. Options.MutationGraphFile = Flags.mutation_graph_file;
  708. if (Flags.collect_data_flow)
  709. Options.CollectDataFlow = Flags.collect_data_flow;
  710. if (Flags.stop_file)
  711. Options.StopFile = Flags.stop_file;
  712. Options.Entropic = Flags.entropic;
  713. Options.EntropicFeatureFrequencyThreshold =
  714. (size_t)Flags.entropic_feature_frequency_threshold;
  715. Options.EntropicNumberOfRarestFeatures =
  716. (size_t)Flags.entropic_number_of_rarest_features;
  717. Options.EntropicScalePerExecTime = Flags.entropic_scale_per_exec_time;
  718. if (!Options.FocusFunction.empty())
  719. Options.Entropic = false; // FocusFunction overrides entropic scheduling.
  720. if (Options.Entropic)
  721. Printf("INFO: Running with entropic power schedule (0x%X, %d).\n",
  722. Options.EntropicFeatureFrequencyThreshold,
  723. Options.EntropicNumberOfRarestFeatures);
  724. struct EntropicOptions Entropic;
  725. Entropic.Enabled = Options.Entropic;
  726. Entropic.FeatureFrequencyThreshold =
  727. Options.EntropicFeatureFrequencyThreshold;
  728. Entropic.NumberOfRarestFeatures = Options.EntropicNumberOfRarestFeatures;
  729. Entropic.ScalePerExecTime = Options.EntropicScalePerExecTime;
  730. unsigned Seed = Flags.seed;
  731. // Initialize Seed.
  732. if (Seed == 0)
  733. Seed = static_cast<unsigned>(
  734. std::chrono::system_clock::now().time_since_epoch().count() + GetPid());
  735. if (Flags.verbosity)
  736. Printf("INFO: Seed: %u\n", Seed);
  737. if (Flags.collect_data_flow && !Flags.fork &&
  738. !(Flags.merge || Flags.set_cover_merge)) {
  739. if (RunIndividualFiles)
  740. return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace,
  741. ReadCorpora({}, *Inputs));
  742. else
  743. return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace,
  744. ReadCorpora(*Inputs, {}));
  745. }
  746. Random Rand(Seed);
  747. auto *MD = new MutationDispatcher(Rand, Options);
  748. auto *Corpus = new InputCorpus(Options.OutputCorpus, Entropic);
  749. auto *F = new Fuzzer(Callback, *Corpus, *MD, Options);
  750. for (auto &U: Dictionary)
  751. if (U.size() <= Word::GetMaxSize())
  752. MD->AddWordToManualDictionary(Word(U.data(), U.size()));
  753. // Threads are only supported by Chrome. Don't use them with emscripten
  754. // for now.
  755. #if !LIBFUZZER_EMSCRIPTEN
  756. StartRssThread(F, Flags.rss_limit_mb);
  757. #endif // LIBFUZZER_EMSCRIPTEN
  758. Options.HandleAbrt = Flags.handle_abrt;
  759. Options.HandleAlrm = !Flags.minimize_crash;
  760. Options.HandleBus = Flags.handle_bus;
  761. Options.HandleFpe = Flags.handle_fpe;
  762. Options.HandleIll = Flags.handle_ill;
  763. Options.HandleInt = Flags.handle_int;
  764. Options.HandleSegv = Flags.handle_segv;
  765. Options.HandleTerm = Flags.handle_term;
  766. Options.HandleXfsz = Flags.handle_xfsz;
  767. Options.HandleUsr1 = Flags.handle_usr1;
  768. Options.HandleUsr2 = Flags.handle_usr2;
  769. Options.HandleWinExcept = Flags.handle_winexcept;
  770. SetSignalHandler(Options);
  771. std::atexit(Fuzzer::StaticExitCallback);
  772. if (Flags.minimize_crash)
  773. return MinimizeCrashInput(Args, Options);
  774. if (Flags.minimize_crash_internal_step)
  775. return MinimizeCrashInputInternalStep(F, Corpus);
  776. if (Flags.cleanse_crash)
  777. return CleanseCrashInput(Args, Options);
  778. if (RunIndividualFiles) {
  779. Options.SaveArtifacts = false;
  780. int Runs = std::max(1, Flags.runs);
  781. Printf("%s: Running %zd inputs %d time(s) each.\n", ProgName->c_str(),
  782. Inputs->size(), Runs);
  783. for (auto &Path : *Inputs) {
  784. auto StartTime = system_clock::now();
  785. Printf("Running: %s\n", Path.c_str());
  786. for (int Iter = 0; Iter < Runs; Iter++)
  787. RunOneTest(F, Path.c_str(), Options.MaxLen);
  788. auto StopTime = system_clock::now();
  789. auto MS = duration_cast<milliseconds>(StopTime - StartTime).count();
  790. Printf("Executed %s in %zd ms\n", Path.c_str(), (long)MS);
  791. }
  792. Printf("***\n"
  793. "*** NOTE: fuzzing was not performed, you have only\n"
  794. "*** executed the target code on a fixed set of inputs.\n"
  795. "***\n");
  796. F->PrintFinalStats();
  797. exit(0);
  798. }
  799. Options.ForkCorpusGroups = Flags.fork_corpus_groups;
  800. if (Flags.fork)
  801. FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork);
  802. if (Flags.merge || Flags.set_cover_merge)
  803. Merge(F, Options, Args, *Inputs, Flags.merge_control_file);
  804. if (Flags.merge_inner) {
  805. const size_t kDefaultMaxMergeLen = 1 << 20;
  806. if (Options.MaxLen == 0)
  807. F->SetMaxInputLen(kDefaultMaxMergeLen);
  808. assert(Flags.merge_control_file);
  809. F->CrashResistantMergeInternalStep(Flags.merge_control_file,
  810. !strncmp(Flags.merge_inner, "2", 1));
  811. exit(0);
  812. }
  813. if (Flags.analyze_dict) {
  814. size_t MaxLen = INT_MAX; // Large max length.
  815. UnitVector InitialCorpus;
  816. for (auto &Inp : *Inputs) {
  817. Printf("Loading corpus dir: %s\n", Inp.c_str());
  818. ReadDirToVectorOfUnits(Inp.c_str(), &InitialCorpus, nullptr,
  819. MaxLen, /*ExitOnError=*/false);
  820. }
  821. if (Dictionary.empty() || Inputs->empty()) {
  822. Printf("ERROR: can't analyze dict without dict and corpus provided\n");
  823. return 1;
  824. }
  825. if (AnalyzeDictionary(F, Dictionary, InitialCorpus)) {
  826. Printf("Dictionary analysis failed\n");
  827. exit(1);
  828. }
  829. Printf("Dictionary analysis succeeded\n");
  830. exit(0);
  831. }
  832. auto CorporaFiles = ReadCorpora(*Inputs, ParseSeedInuts(Flags.seed_inputs));
  833. F->Loop(CorporaFiles);
  834. if (Flags.verbosity)
  835. Printf("Done %zd runs in %zd second(s)\n", F->getTotalNumberOfRuns(),
  836. F->secondsSinceProcessStartUp());
  837. F->PrintFinalStats();
  838. exit(0); // Don't let F destroy itself.
  839. }
  840. extern "C" ATTRIBUTE_INTERFACE int
  841. LLVMFuzzerRunDriver(int *argc, char ***argv,
  842. int (*UserCb)(const uint8_t *Data, size_t Size)) {
  843. return FuzzerDriver(argc, argv, UserCb);
  844. }
  845. // Storage for global ExternalFunctions object.
  846. ExternalFunctions *EF = nullptr;
  847. } // namespace fuzzer