Finish refactoring APIs for the custom mutator and Python module
Remove AFL_PYTHON_ONLY (env) and python_only (variable)
Unify fuzz API of the custom mutator and Python module
Merge the custom mutator into the old python_stage, which is now renamed to custom_mutator_stage
MODIFY
.gitignore
.gitignore
@@ -18,9 +18,11 @@ afl-qemu-trace
afl-showmap
afl-tmin
afl-analyze.8
+afl-as.8
afl-clang-fast++.8
afl-clang-fast.8
afl-cmin.8
+afl-cmin.bash.8
afl-fuzz.8
afl-gcc.8
afl-gcc-fast.8

MODIFY
docs/custom_mutator.md
docs/custom_mutator.md
@@ -13,7 +13,7 @@ a given grammar.
The custom mutator library is passed to afl-fuzz via the
AFL_CUSTOM_MUTATOR_LIBRARY environment variable. The library must export
-the afl_custom_mutator() function and must be compiled as a shared object.
+the afl_custom_fuzz() function and must be compiled as a shared object.
For example:
```
$CC -shared -Wall -O3 <lib-name>.c -o <lib-name>.so

MODIFY
docs/env_variables.md
docs/env_variables.md
@@ -223,15 +223,15 @@ checks or alter some of the more exotic semantics of the tool:
for more.
- Setting AFL_CUSTOM_MUTATOR_LIBRARY to a shared library with
- afl_custom_mutator() creates additional mutations through this library.
+ afl_custom_fuzz() creates additional mutations through this library.
+ If afl-fuzz is compiled with Python (which is autodetected during builing
+ afl-fuzz), setting AFL_PYTHON_MODULE to a Python module can also provide
+ additional mutations.
If AFL_CUSTOM_MUTATOR_ONLY is also set, all mutations will solely be
- performed with/from the library. See [custom_mutator.md](custom_mutator.md)
-
- - For AFL_PYTHON_MODULE and AFL_PYTHON_ONLY - they require afl-fuzz to
- be compiled with Python (which is autodetected during builing afl-fuzz).
- Please see [python_mutators.md](python_mutators.md).
+ performed with/from the library/Python module.
This feature allows to configure custom mutators which can be very helpful
in e.g. fuzzing XML or other highly flexible structured input.
+ Please see [custom_mutator.md](custom_mutator.md) or [python_mutators.md](python_mutators.md).
- AFL_FAST_CAL keeps the calibration stage about 2.5x faster (albeit less
precise), which can help when starting a session against a slow target.

MODIFY
include/afl-fuzz.h
include/afl-fuzz.h
@@ -276,8 +276,7 @@ extern u8 cal_cycles, /* Calibration cycles defaults */
no_unlink, /* do not unlink cur_input */
use_stdin, /* use stdin for sending data */
debug, /* Debug mode */
- custom_only, /* Custom mutator only mode */
- python_only; /* Python-only mode */
+ custom_only; /* Custom mutator only mode */
extern u32 stats_update_freq; /* Stats update frequency (execs) */
@@ -471,6 +470,8 @@ struct custom_mutator {
* Initialize the custom mutator.
*
* (Optional)
+ *
+ * @param seed Seed used for the mutation.
*/
void (*afl_custom_init)(unsigned int seed);
@@ -479,17 +480,18 @@ struct custom_mutator {
*
* (Optional for now. Required in the future)
*
- * @param[in] data Input data to be mutated
- * @param[in] size Size of input data
+ * @param[in] buf Input data to be mutated
+ * @param[in] buf_size Size of input data
+ * @param[in] add_buf Buffer containing the additional test case
+ * @param[in] add_buf_size Size of the additional test case
* @param[out] mutated_out Buffer to store the mutated input
* @param[in] max_size Maximum size of the mutated output. The mutation must not
* produce data larger than max_size.
- * @param[in] seed Seed used for the mutation. The mutation should produce the
- * same output given the same seed.
* @return Size of the mutated output.
*/
- size_t (*afl_custom_fuzz)(u8* data, size_t size, u8* mutated_out,
- size_t max_size, unsigned int seed);
+ size_t (*afl_custom_fuzz)(u8* buf, size_t buf_size,
+ u8* add_buf, size_t add_buf_size,
+ u8* mutated_out, size_t max_size);
/**
* A post-processing function to use right before AFL writes the test case to
@@ -498,12 +500,14 @@ struct custom_mutator {
* (Optional) If this functionality is not needed, simply don't define this
* function.
*
- * @param[in] data Buffer containing the test case to be executed
- * @param[in] size Size of the test case
- * @param[out] new_data Buffer to store the test case after processing
- * @return Size of data after processing
+ * @param[in] buf Buffer containing the test case to be executed
+ * @param[in] buf_size Size of the test case
+ * @param[out] out_buf Pointer to the buffer of storing the test case after
+ * processing. External library should allocate memory for out_buf. AFL++
+ * will release the memory after saving the test case.
+ * @return Size of the output buffer after processing
*/
- size_t (*afl_custom_pre_save)(u8* data, size_t size, u8** new_data);
+ size_t (*afl_custom_pre_save)(u8* buf, size_t buf_size, u8** out_buf);
/**
* This method is called at the start of each trimming operation and receives
@@ -521,11 +525,11 @@ struct custom_mutator {
*
* (Optional)
*
- * @param data Buffer containing the test case
- * @param size Size of the test case
+ * @param buf Buffer containing the test case
+ * @param buf_size Size of the test case
* @return The amount of possible iteration steps to trim the input
*/
- u32 (*afl_custom_init_trim)(u8* data, size_t size);
+ u32 (*afl_custom_init_trim)(u8* buf, size_t buf_size);
/**
* This method is called for each trimming operation. It doesn't have any
@@ -538,10 +542,12 @@ struct custom_mutator {
*
* (Optional)
*
- * @param[out] ret Buffer containing the trimmed test case
- * @param[out] ret_len Size of the trimmed test case
+ * @param[out] out_buf Pointer to the buffer containing the trimmed test case.
+ * External library should allocate memory for out_buf. AFL++ will release
+ * the memory after saving the test case.
+ * @param[out] out_buf_size Pointer to the size of the trimmed test case
*/
- void (*afl_custom_trim)(u8** ret, size_t* ret_len);
+ void (*afl_custom_trim)(u8** out_buf, size_t* out_buf_size);
/**
* This method is called after each trim operation to inform you if your
@@ -627,9 +633,9 @@ int init_py_module(u8*);
void finalize_py_module();
void init_py(unsigned int seed);
-/* TODO: unify fuzz interface for custom mutator and Python mutator */
-size_t fuzz_py(u8*, size_t, u8*, size_t, unsigned int);
-void fuzz_py_original(char*, size_t, char*, size_t, char**, size_t*);
+size_t fuzz_py(u8* buf, size_t buf_size,
+ u8* add_buf, size_t add_buf_size,
+ u8* mutated_out, size_t max_size);
size_t pre_save_py(u8* data, size_t size, u8** new_data);
u32 init_trim_py(u8*, size_t);
u32 post_trim_py(u8);

MODIFY
include/envs.h
include/envs.h
@@ -24,7 +24,7 @@ const char *afl_environment_variables[] = {
"AFL_NO_X86", // not really an env but we dont want to warn on it
"AFL_PATH", "AFL_PERFORMANCE_FILE",
//"AFL_PERSISTENT", // not implemented anymore, so warn additionally
- "AFL_POST_LIBRARY", "AFL_PRELOAD", "AFL_PYTHON_MODULE", "AFL_PYTHON_ONLY",
+ "AFL_POST_LIBRARY", "AFL_PRELOAD", "AFL_PYTHON_MODULE",
"AFL_QEMU_COMPCOV", "AFL_QEMU_COMPCOV_DEBUG", "AFL_QEMU_DEBUG_MAPS",
"AFL_QEMU_DISABLE_CACHE", "AFL_QEMU_PERSISTENT_ADDR",
"AFL_QEMU_PERSISTENT_CNT", "AFL_QEMU_PERSISTENT_GPR",

MODIFY
src/afl-fuzz-globals.c
src/afl-fuzz-globals.c
@@ -88,8 +88,7 @@ u8 cal_cycles = CAL_CYCLES, /* Calibration cycles defaults */
no_unlink, /* do not unlink cur_input */
use_stdin = 1, /* use stdin for sending data */
be_quiet, /* is AFL_QUIET set? */
- custom_only, /* Custom mutator only mode */
- python_only; /* Python-only mode */
+ custom_only; /* Custom mutator only mode */
u32 stats_update_freq = 1; /* Stats update frequency (execs) */

MODIFY
src/afl-fuzz-mutators.c
src/afl-fuzz-mutators.c
void setup_custom_mutator(void) {
+ /* Try mutator library first */
u8* fn = getenv("AFL_CUSTOM_MUTATOR_LIBRARY");
if (fn) {
@@ -41,6 +42,7 @@ void setup_custom_mutator(void) {
return;
}
+ /* Try Python module */
#ifdef USE_PYTHON
u8* module_name = getenv("AFL_PYTHON_MODULE");
@@ -286,7 +288,7 @@ void load_custom_mutator_py(const char* module_name) {
/* "afl_custom_fuzz" should not be NULL, but the interface of Python mutator
is quite different from the custom mutator. */
- mutator->afl_custom_fuzz = NULL;
+ mutator->afl_custom_fuzz = fuzz_py;
if (py_functions[PY_FUNC_PRE_SAVE])
mutator->afl_custom_pre_save = pre_save_py;

MODIFY
src/afl-fuzz-one.c
src/afl-fuzz-one.c
@@ -482,56 +482,6 @@ u8 fuzz_one_original(char** argv) {
if (use_radamsa > 1) goto radamsa_stage;
- // custom_stage: // not used - yet
-
- if (mutator->afl_custom_fuzz) {
-
- stage_short = "custom";
- stage_name = "custom mutator";
- stage_max = len << 3;
- stage_val_type = STAGE_VAL_NONE;
-
- const u32 max_seed_size = 4096 * 4096;
- u8* mutated_buf = ck_alloc(max_seed_size);
-
- orig_hit_cnt = queued_paths + unique_crashes;
-
- for (stage_cur = 0; stage_cur < stage_max; ++stage_cur) {
-
- size_t orig_size = (size_t)len;
- size_t mutated_size = mutator->afl_custom_fuzz(in_buf, orig_size,
- mutated_buf, max_seed_size,
- UR(UINT32_MAX));
- if (mutated_size > 0) {
-
- out_buf = ck_realloc(out_buf, mutated_size);
- memcpy(out_buf, mutated_buf, mutated_size);
- if (common_fuzz_stuff(argv, out_buf, (u32)mutated_size)) {
-
- goto abandon_entry;
-
- }
-
- }
-
- }
-
- ck_free(mutated_buf);
- new_hit_cnt = queued_paths + unique_crashes;
-
- stage_finds[STAGE_CUSTOM_MUTATOR] += new_hit_cnt - orig_hit_cnt;
- stage_cycles[STAGE_CUSTOM_MUTATOR] += stage_max;
-
- if (custom_only) {
-
- /* Skip other stages */
- ret_val = 0;
- goto abandon_entry;
-
- }
-
- }
-
if (cmplog_mode) {
if (input_to_state_stage(argv, in_buf, out_buf, len, queue_cur->exec_cksum))
@@ -551,11 +501,7 @@ u8 fuzz_one_original(char** argv) {
: havoc_max_mult * 100)) ||
queue_cur->passed_det) {
-#ifdef USE_PYTHON
- goto python_stage;
-#else
- goto havoc_stage;
-#endif
+ goto custom_mutator_stage;
}
@@ -564,11 +510,7 @@ u8 fuzz_one_original(char** argv) {
if (master_max && (queue_cur->exec_cksum % master_max) != master_id - 1) {
-#ifdef USE_PYTHON
- goto python_stage;
-#else
- goto havoc_stage;
-#endif
+ goto custom_mutator_stage;
}
@@ -1583,24 +1525,25 @@ skip_extras:
if (!queue_cur->passed_det) mark_as_det_done(queue_cur);
-#ifdef USE_PYTHON
-python_stage:
- /**********************************
- * EXTERNAL MUTATORS (Python API) *
- **********************************/
+custom_mutator_stage:
+ /*******************
+ * CUSTOM MUTATORS *
+ *******************/
- if (!py_module) goto havoc_stage;
+ if (!mutator) goto havoc_stage;
+ if (!mutator->afl_custom_fuzz) goto havoc_stage;
- stage_name = "python";
- stage_short = "python";
+ stage_name = "custom mutator";
+ stage_short = "custom";
stage_max = HAVOC_CYCLES * perf_score / havoc_div / 100;
+ stage_val_type = STAGE_VAL_NONE;
if (stage_max < HAVOC_MIN) stage_max = HAVOC_MIN;
- orig_hit_cnt = queued_paths + unique_crashes;
+ const u32 max_seed_size = 4096 * 4096;
+ u8* mutated_buf = ck_alloc(max_seed_size);
- char* retbuf = NULL;
- size_t retlen = 0;
+ orig_hit_cnt = queued_paths + unique_crashes;
for (stage_cur = 0; stage_cur < stage_max; ++stage_cur) {
@@ -1647,26 +1590,24 @@ python_stage:
ck_read(fd, new_buf, target->len, target->fname);
close(fd);
- fuzz_py_original(out_buf, len, new_buf, target->len, &retbuf, &retlen);
+ size_t mutated_size = mutator->afl_custom_fuzz(out_buf, len,
+ new_buf, target->len,
+ mutated_buf, max_seed_size);
ck_free(new_buf);
- if (retbuf) {
+ if (mutated_size > 0) {
- if (!retlen) goto abandon_entry;
+ out_buf = ck_realloc(out_buf, mutated_size);
+ memcpy(out_buf, mutated_buf, mutated_size);
- if (common_fuzz_stuff(argv, retbuf, retlen)) {
+ if (common_fuzz_stuff(argv, out_buf, (u32)mutated_size)) {
- free(retbuf);
+ ck_free(mutated_buf);
goto abandon_entry;
}
- /* Reset retbuf/retlen */
- free(retbuf);
- retbuf = NULL;
- retlen = 0;
-
/* If we're finding new stuff, let's run for a bit longer, limits
permitting. */
@@ -1687,12 +1628,13 @@ python_stage:
}
+ ck_free(mutated_buf);
new_hit_cnt = queued_paths + unique_crashes;
- stage_finds[STAGE_PYTHON] += new_hit_cnt - orig_hit_cnt;
- stage_cycles[STAGE_PYTHON] += stage_max;
+ stage_finds[STAGE_CUSTOM_MUTATOR] += new_hit_cnt - orig_hit_cnt;
+ stage_cycles[STAGE_CUSTOM_MUTATOR] += stage_max;
- if (python_only) {
+ if (custom_only) {
/* Skip other stages */
ret_val = 0;
@@ -1700,8 +1642,6 @@ python_stage:
}
-#endif
-
/****************
* RANDOM HAVOC *
****************/
@@ -2270,11 +2210,10 @@ retry_splicing:
out_buf = ck_alloc_nozero(len);
memcpy(out_buf, in_buf, len);
-#ifdef USE_PYTHON
- goto python_stage;
-#else
- goto havoc_stage;
-#endif
+ goto custom_mutator_stage;
+ /* ???: While integrating Python module, the author decided to jump to
+ python stage, but the reason behind this is not clear.*/
+ // goto havoc_stage;
}

MODIFY
src/afl-fuzz-python.c
src/afl-fuzz-python.c
@@ -159,67 +159,16 @@ void init_py(unsigned int seed) {
}
}
-void fuzz_py_original(char* buf, size_t buflen,
- char* add_buf, size_t add_buflen,
- char** ret, size_t* retlen) {
-
- if (py_module != NULL) {
+size_t fuzz_py(u8* buf, size_t buf_size,
+ u8* add_buf, size_t add_buf_size,
+ u8* mutated_out, size_t max_size) {
- PyObject *py_args, *py_value;
- py_args = PyTuple_New(2);
- py_value = PyByteArray_FromStringAndSize(buf, buflen);
- if (!py_value) {
-
- Py_DECREF(py_args);
- fprintf(stderr, "Cannot convert argument\n");
- return;
-
- }
-
- PyTuple_SetItem(py_args, 0, py_value);
-
- py_value = PyByteArray_FromStringAndSize(add_buf, add_buflen);
- if (!py_value) {
-
- Py_DECREF(py_args);
- fprintf(stderr, "Cannot convert argument\n");
- return;
-
- }
-
- PyTuple_SetItem(py_args, 1, py_value);
-
- py_value = PyObject_CallObject(py_functions[PY_FUNC_FUZZ], py_args);
-
- Py_DECREF(py_args);
-
- if (py_value != NULL) {
-
- *retlen = PyByteArray_Size(py_value);
- *ret = malloc(*retlen);
- memcpy(*ret, PyByteArray_AsString(py_value), *retlen);
- Py_DECREF(py_value);
-
- } else {
-
- PyErr_Print();
- fprintf(stderr, "Call failed\n");
- return;
-
- }
-
- }
-
-}
-
-size_t fuzz_py(u8* data, size_t size, u8* mutated_out, size_t max_size,
- unsigned int seed) {
-
- size_t out_size;
+ size_t mutated_size;
PyObject *py_args, *py_value;
py_args = PyTuple_New(3);
- py_value = PyByteArray_FromStringAndSize(data, size);
+ /* buf */
+ py_value = PyByteArray_FromStringAndSize(buf, buf_size);
if (!py_value) {
Py_DECREF(py_args);
@@ -229,11 +178,8 @@ size_t fuzz_py(u8* data, size_t size, u8* mutated_out, size_t max_size,
PyTuple_SetItem(py_args, 0, py_value);
-#if PY_MAJOR_VERSION >= 3
- py_value = PyLong_FromLong(max_size);
-#else
- py_value = PyInt_FromLong(max_size);
-#endif
+ /* add_buf */
+ py_value = PyByteArray_FromStringAndSize(add_buf, add_buf_size);
if (!py_value) {
Py_DECREF(py_args);
@@ -243,10 +189,11 @@ size_t fuzz_py(u8* data, size_t size, u8* mutated_out, size_t max_size,
PyTuple_SetItem(py_args, 1, py_value);
+ /* max_size */
#if PY_MAJOR_VERSION >= 3
- py_value = PyLong_FromLong(seed);
+ py_value = PyLong_FromLong(max_size);
#else
- py_value = PyInt_FromLong(seed);
+ py_value = PyInt_FromLong(max_size);
#endif
if (!py_value) {
@@ -263,11 +210,10 @@ size_t fuzz_py(u8* data, size_t size, u8* mutated_out, size_t max_size,
if (py_value != NULL) {
- out_size = PyByteArray_Size(py_value);
- memcpy(mutated_out, PyByteArray_AsString(py_value), out_size);
+ mutated_size = PyByteArray_Size(py_value);
+ memcpy(mutated_out, PyByteArray_AsString(py_value), mutated_size);
Py_DECREF(py_value);
-
- return out_size;
+ return mutated_size;
} else {
@@ -278,12 +224,12 @@ size_t fuzz_py(u8* data, size_t size, u8* mutated_out, size_t max_size,
}
-size_t pre_save_py(u8* data, size_t size, u8** new_data) {
+size_t pre_save_py(u8* buf, size_t buf_size, u8** out_buf) {
- size_t new_size;
+ size_t out_buf_size;
PyObject *py_args, *py_value;
py_args = PyTuple_New(2);
- py_value = PyByteArray_FromStringAndSize(data, size);
+ py_value = PyByteArray_FromStringAndSize(buf, buf_size);
if (!py_value) {
Py_DECREF(py_args);
@@ -299,11 +245,11 @@ size_t pre_save_py(u8* data, size_t size, u8** new_data) {
if (py_value != NULL) {
- new_size = PyByteArray_Size(py_value);
- *new_data = malloc(new_size);
- memcpy(*new_data, PyByteArray_AsString(py_value), new_size);
+ out_buf_size = PyByteArray_Size(py_value);
+ *out_buf = malloc(out_buf_size);
+ memcpy(*out_buf, PyByteArray_AsString(py_value), out_buf_size);
Py_DECREF(py_value);
- return new_size;
+ return out_buf_size;
} else {
@@ -314,12 +260,12 @@ size_t pre_save_py(u8* data, size_t size, u8** new_data) {
}
-u32 init_trim_py(u8* buf, size_t buflen) {
+u32 init_trim_py(u8* buf, size_t buf_size) {
PyObject *py_args, *py_value;
py_args = PyTuple_New(1);
- py_value = PyByteArray_FromStringAndSize(buf, buflen);
+ py_value = PyByteArray_FromStringAndSize(buf, buf_size);
if (!py_value) {
Py_DECREF(py_args);
@@ -389,7 +335,7 @@ u32 post_trim_py(u8 success) {
}
-void trim_py(u8** ret, size_t* retlen) {
+void trim_py(u8** out_buf, size_t* out_buf_size) {
PyObject *py_args, *py_value;
@@ -399,9 +345,9 @@ void trim_py(u8** ret, size_t* retlen) {
if (py_value != NULL) {
- *retlen = PyByteArray_Size(py_value);
- *ret = malloc(*retlen);
- memcpy(*ret, PyByteArray_AsString(py_value), *retlen);
+ *out_buf_size = PyByteArray_Size(py_value);
+ *out_buf = malloc(*out_buf_size);
+ memcpy(*out_buf, PyByteArray_AsString(py_value), *out_buf_size);
Py_DECREF(py_value);
} else {

MODIFY
src/afl-fuzz-run.c
src/afl-fuzz-run.c
@@ -309,11 +309,12 @@ void write_to_testcase(void* mem, u32 len) {
lseek(fd, 0, SEEK_SET);
- if (mutator->afl_custom_pre_save) {
+ if (mutator && mutator->afl_custom_pre_save) {
u8* new_data;
size_t new_size = mutator->afl_custom_pre_save(mem, len, &new_data);
ck_write(fd, new_data, new_size, out_file);
+ ck_free(new_data);
} else {

MODIFY
src/afl-fuzz.c
src/afl-fuzz.c
@@ -158,7 +158,6 @@ static void usage(u8* argv0, int more_help) {
"AFL_CUSTOM_MUTATOR_LIBRARY: lib with afl_custom_fuzz() to mutate inputs\n"
"AFL_CUSTOM_MUTATOR_ONLY: avoid AFL++'s internal mutators\n"
"AFL_PYTHON_MODULE: mutate and trim inputs with the specified Python module\n"
- "AFL_PYTHON_ONLY: skip AFL++'s own mutators\n"
"AFL_DEBUG: extra debugging output for Python mode trimming\n"
"AFL_DISABLE_TRIM: disable the trimming of test cases\n"
"AFL_NO_UI: switch status screen off\n"
@@ -658,11 +657,10 @@ int main(int argc, char** argv, char** envp) {
OKF("afl-tmin fork server patch from github.com/nccgroup/TriforceAFL");
OKF("MOpt Mutator from github.com/puppet-meteor/MOpt-AFL");
- if (sync_id && force_deterministic &&
- (getenv("AFL_CUSTOM_MUTATOR_ONLY") || getenv("AFL_PYTHON_ONLY")))
+ if (sync_id && force_deterministic && getenv("AFL_CUSTOM_MUTATOR_ONLY"))
WARNF(
- "Using -M master with the AFL_..._ONLY mutator options will result in "
- "no deterministic mutations being done!");
+ "Using -M master with the AFL_CUSTOM_MUTATOR_ONLY mutator options will "
+ "result in no deterministic mutations being done!");
check_environment_vars(envp);
@@ -832,16 +830,6 @@ int main(int argc, char** argv, char** envp) {
if (get_afl_env("AFL_DEBUG")) debug = 1;
- if (get_afl_env("AFL_PYTHON_ONLY")) {
-
- /* This ensures we don't proceed to havoc/splice */
- python_only = 1;
-
- /* Ensure we also skip all deterministic steps */
- skip_deterministic = 1;
-
- }
-
if (get_afl_env("AFL_CUSTOM_MUTATOR_ONLY")) {
/* This ensures we don't proceed to havoc/splice */

Finish refactoring APIs for the custom mutator and Python module

- Remove AFL_PYTHON_ONLY (env) and python_only (variable)
- Unify fuzz API of the custom mutator and Python module
- Merge the custom mutator into the old python_stage, which is now renamed to custom_mutator_stage