add -F option to sync to foreign fuzzer queues
MODIFY
GNUmakefile
GNUmakefile
@@ -455,10 +455,10 @@ code-format:
./.custom-format.py -i llvm_mode/*.h
./.custom-format.py -i llvm_mode/*.cc
./.custom-format.py -i gcc_plugin/*.c
- #./.custom-format.py -i gcc_plugin/*.h
+ @#./.custom-format.py -i gcc_plugin/*.h
./.custom-format.py -i gcc_plugin/*.cc
./.custom-format.py -i custom_mutators/*/*.c
- ./.custom-format.py -i custom_mutators/*/*.h
+ @#./.custom-format.py -i custom_mutators/*/*.h # destroys input.h :-(
./.custom-format.py -i examples/*/*.c
./.custom-format.py -i examples/*/*.h
./.custom-format.py -i test/*.c

MODIFY
TODO.md
TODO.md
## Roadmap 2.67+
- -i - + foreign fuzzer sync support: scandir with time sort
- - pre_save custom module example to save away test cases
- expand on AFL_LLVM_INSTRUMENT_FILE to also support sancov allowlist format
- - allow to sync against honggfuzz and libfuzzer
- AFL_MAP_SIZE for qemu_mode and unicorn_mode
- namespace for targets? e.g. network
- learn from honggfuzz (mutations, maybe ptrace?)

MODIFY
docs/Changelog.md
docs/Changelog.md
@@ -11,6 +11,8 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
### Version ++2.66d (devel)
- afl-fuzz:
+ - added -F option to allow -M main fuzzers to sync to foreign fuzzers,
+ e.g. honggfuzz or libfuzzer
- eliminated CPU affinity race condition for -S/-M runs
- llvm_mode:
- fixes for laf-intel float splitting (thanks to mark-griffin for

MODIFY
docs/parallel_fuzzing.md
docs/parallel_fuzzing.md
@@ -99,7 +99,15 @@ example may be:
This is not a concern if you use @@ without -f and let afl-fuzz come up with the
file name.
-## 3) Multi-system parallelization
+## 3) Syncing with non-afl fuzzers or independant instances
+
+A -M main node can be told with the `-F other_fuzzer_queue_directory` option
+to sync results from other fuzzers, e.g. libfuzzer or honggfuzz.
+
+Only the specified directory will by synced into afl, not subdirectories.
+The specified directories do not need to exist yet at the start of afl.
+
+## 4) Multi-system parallelization
The basic operating principle for multi-system parallelization is similar to
the mechanism explained in section 2. The key difference is that you need to
@@ -176,7 +184,7 @@ It is *not* advisable to skip the synchronization script and run the fuzzers
directly on a network filesystem; unexpected latency and unkillable processes
in I/O wait state can mess things up.
-## 4) Remote monitoring and data collection
+## 5) Remote monitoring and data collection
You can use screen, nohup, tmux, or something equivalent to run remote
instances of afl-fuzz. If you redirect the program's output to a file, it will
@@ -200,7 +208,7 @@ Keep in mind that crashing inputs are *not* automatically propagated to the
main instance, so you may still want to monitor for crashes fleet-wide
from within your synchronization or health checking scripts (see afl-whatsup).
-## 5) Asymmetric setups
+## 6) Asymmetric setups
It is perhaps worth noting that all of the following is permitted:

MODIFY
include/afl-fuzz.h
include/afl-fuzz.h
@@ -347,6 +347,13 @@ struct afl_pass_stat {
};
+struct foreign_sync {
+
+ u8 * dir;
+ time_t ctime;
+
+};
+
typedef struct afl_state {
/* Position of this state in the global states list */
@@ -574,6 +581,11 @@ typedef struct afl_state {
u8 describe_op_buf_256[256]; /* describe_op will use this to return a string
up to 256 */
+/* foreign sync */
+#define FOREIGN_SYNCS_MAX 32
+ u8 foreign_sync_cnt;
+ struct foreign_sync foreign_syncs[FOREIGN_SYNCS_MAX];
+
#ifdef _AFL_DOCUMENT_MUTATIONS
u8 do_document;
u32 document_counter;
@@ -937,6 +949,7 @@ void fix_up_banner(afl_state_t *, u8 *);
void check_if_tty(afl_state_t *);
void setup_signal_handlers(void);
void save_cmdline(afl_state_t *, u32, char **);
+void read_foreign_testcases(afl_state_t *, int);
/* CmpLog */

MODIFY
src/afl-fuzz-init.c
src/afl-fuzz-init.c
@@ -438,6 +438,159 @@ static void shuffle_ptrs(afl_state_t *afl, void **ptrs, u32 cnt) {
}
+/* Read all testcases from foreign input directories, then queue them for
+ testing. Called at startup and at sync intervals.
+ Does not descend into subdirectories! */
+
+void read_foreign_testcases(afl_state_t *afl, int first) {
+
+ if (!afl->foreign_sync_cnt) return;
+
+ struct dirent **nl;
+ s32 nl_cnt;
+ u32 i, iter;
+
+ u8 val_buf[2][STRINGIFY_VAL_SIZE_MAX];
+
+ for (iter = 0; iter < afl->foreign_sync_cnt; iter++) {
+
+ if (afl->foreign_syncs[iter].dir != NULL &&
+ afl->foreign_syncs[iter].dir[0] != 0) {
+
+ if (first) ACTF("Scanning '%s'...", afl->foreign_syncs[iter].dir);
+ time_t ctime_max = 0;
+
+ /* We use scandir() + alphasort() rather than readdir() because otherwise,
+ the ordering of test cases would vary somewhat randomly and would be
+ difficult to control. */
+
+ nl_cnt = scandir(afl->foreign_syncs[iter].dir, &nl, NULL, NULL);
+
+ if (nl_cnt < 0) {
+
+ if (first) {
+
+ WARNF("Unable to open directory '%s'", afl->foreign_syncs[iter].dir);
+ sleep(1);
+
+ }
+
+ continue;
+
+ }
+
+ if (nl_cnt == 0) {
+
+ if (first)
+ WARNF("directory %s is currently empty",
+ afl->foreign_syncs[iter].dir);
+ continue;
+
+ }
+
+ /* Show stats */
+
+ snprintf(afl->stage_name_buf, STAGE_BUF_SIZE, "foreign sync %u", iter);
+
+ afl->stage_name = afl->stage_name_buf;
+ afl->stage_cur = 0;
+ afl->stage_max = 0;
+
+ for (i = 0; i < nl_cnt; ++i) {
+
+ struct stat st;
+
+ u8 *fn2 =
+ alloc_printf("%s/%s", afl->foreign_syncs[iter].dir, nl[i]->d_name);
+
+ free(nl[i]); /* not tracked */
+
+ if (unlikely(lstat(fn2, &st) || access(fn2, R_OK))) {
+
+ if (first) PFATAL("Unable to access '%s'", fn2);
+ continue;
+
+ }
+
+ /* we detect new files by their ctime */
+ if (likely(st.st_ctime <= afl->foreign_syncs[iter].ctime)) {
+
+ ck_free(fn2);
+ continue;
+
+ }
+
+ /* This also takes care of . and .. */
+
+ if (!S_ISREG(st.st_mode) || !st.st_size || strstr(fn2, "/README.txt")) {
+
+ ck_free(fn2);
+ continue;
+
+ }
+
+ if (st.st_size > MAX_FILE) {
+
+ if (first)
+ WARNF(
+ "Test case '%s' is too big (%s, limit is %s), skipping", fn2,
+ stringify_mem_size(val_buf[0], sizeof(val_buf[0]), st.st_size),
+ stringify_mem_size(val_buf[1], sizeof(val_buf[1]), MAX_FILE));
+ ck_free(fn2);
+ continue;
+
+ }
+
+ // lets do not use add_to_queue(afl, fn2, st.st_size, 0);
+ // as this could add duplicates of the startup input corpus
+
+ int fd = open(fn2, O_RDONLY);
+ if (fd < 0) {
+
+ ck_free(fn2);
+ continue;
+
+ }
+
+ u8 fault;
+ u8 *mem = mmap(0, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
+
+ if (mem == MAP_FAILED) {
+
+ ck_free(fn2);
+ continue;
+
+ }
+
+ write_to_testcase(afl, mem, st.st_size);
+ fault = fuzz_run_target(afl, &afl->fsrv, afl->fsrv.exec_tmout);
+ afl->syncing_party = "foreign";
+ afl->queued_imported +=
+ save_if_interesting(afl, mem, st.st_size, fault);
+ afl->syncing_party = 0;
+ munmap(mem, st.st_size);
+ close(fd);
+
+ if (st.st_ctime > ctime_max) ctime_max = st.st_ctime;
+
+ }
+
+ afl->foreign_syncs[iter].ctime = ctime_max;
+ free(nl); /* not tracked */
+
+ }
+
+ }
+
+ if (first) {
+
+ afl->last_path_time = 0;
+ afl->queued_at_start = afl->queued_paths;
+
+ }
+
+}
+
/* Read all testcases from the input directory, then queue them for testing.
Called at startup. */
@@ -530,6 +683,7 @@ void read_testcases(afl_state_t *afl) {
WARNF("Test case '%s' is too big (%s, limit is %s), skipping", fn2,
stringify_mem_size(val_buf[0], sizeof(val_buf[0]), st.st_size),
stringify_mem_size(val_buf[1], sizeof(val_buf[1]), MAX_FILE));
+ ck_free(fn2);
continue;
}

MODIFY
src/afl-fuzz-run.c
src/afl-fuzz-run.c
@@ -612,6 +612,8 @@ void sync_fuzzers(afl_state_t *afl) {
}
+ if (afl->foreign_sync_cnt) read_foreign_testcases(afl, 0);
+
}
/* Trim all new test cases to save cycles when doing deterministic checks. The

MODIFY
src/afl-fuzz.c
src/afl-fuzz.c
@@ -131,10 +131,13 @@ static void usage(afl_state_t *afl, u8 *argv0, int more_help) {
"executions.\n\n"
"Other stuff:\n"
- " -T text - text banner to show on the screen\n"
" -M/-S id - distributed mode (see docs/parallel_fuzzing.md)\n"
" use -D to force -S secondary to perform deterministic "
"fuzzing\n"
+ " -F path - sync to a foreign fuzzer queue directory (requires "
+ "-M, can\n"
+ " be specified up to %u times)\n"
+ " -T text - text banner to show on the screen\n"
" -I command - execute this command/script when a new crash is "
"found\n"
//" -B bitmap.txt - mutate a specific test case, use the out/fuzz_bitmap
@@ -142,7 +145,7 @@ static void usage(afl_state_t *afl, u8 *argv0, int more_help) {
" -C - crash exploration mode (the peruvian rabbit thing)\n"
" -e ext - file extension for the fuzz test input file (if "
"needed)\n\n",
- argv0, EXEC_TIMEOUT, MEM_LIMIT);
+ argv0, EXEC_TIMEOUT, MEM_LIMIT, FOREIGN_SYNCS_MAX);
if (more_help > 1) {
@@ -403,6 +406,19 @@ int main(int argc, char **argv_orig, char **envp) {
afl->use_splicing = 1;
break;
+ case 'F': /* foreign sync dir */
+
+ if (!afl->is_main_node)
+ FATAL(
+ "Option -F can only be specified after the -M option for the "
+ "main fuzzer of a fuzzing campaign");
+ if (afl->foreign_sync_cnt >= FOREIGN_SYNCS_MAX)
+ FATAL("Maximum %u entried of -F option can be specified",
+ FOREIGN_SYNCS_MAX);
+ afl->foreign_syncs[afl->foreign_sync_cnt].dir = optarg;
+ afl->foreign_sync_cnt++;
+ break;
+
case 'f': /* target file */
if (afl->fsrv.out_file) { FATAL("Multiple -f options not supported"); }
@@ -1059,6 +1075,8 @@ int main(int argc, char **argv_orig, char **envp) {
setup_cmdline_file(afl, argv + optind);
read_testcases(afl);
+ // read_foreign_testcases(afl, 1); for the moment dont do this
+
load_auto(afl);
pivot_inputs(afl);

add -F option to sync to foreign fuzzer queues