add afl-ld-lto for LTO
MODIFY
docs/Changelog.md
docs/Changelog.md
@@ -22,6 +22,8 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
better coverage. The original afl instrumentation can be set via
AFL_LLVM_INSTRUMENT=AFL. This is automatically done when the WHITELIST
feature is used.
+ - some targets want a ld variant for LD that is not gcc/clang but ld, added
+ afl-ld-lto to solve this
- lowered minimum required llvm version to 3.4 (except LLVMInsTrim,
which needs 3.8.0)
- WHITELIST feature now supports wildcards (thanks to sirmc)

MODIFY
llvm_mode/GNUmakefile
llvm_mode/GNUmakefile
@@ -246,7 +246,7 @@ ifeq "$(TEST_MMAP)" "1"
LDFLAGS += -Wno-deprecated-declarations
endif
- PROGS = ../afl-clang-fast ../afl-llvm-pass.so ../afl-llvm-lto-whitelist.so ../afl-llvm-lto-instrumentation.so ../afl-llvm-lto-instrim.so ../libLLVMInsTrim.so ../afl-llvm-rt.o ../afl-llvm-rt-32.o ../afl-llvm-rt-64.o ../compare-transform-pass.so ../split-compares-pass.so ../split-switches-pass.so ../cmplog-routines-pass.so ../cmplog-instructions-pass.so
+ PROGS = ../afl-clang-fast ../afl-llvm-pass.so ../afl-ld-lto ../afl-llvm-lto-whitelist.so ../afl-llvm-lto-instrumentation.so ../afl-llvm-lto-instrim.so ../libLLVMInsTrim.so ../afl-llvm-rt.o ../afl-llvm-rt-32.o ../afl-llvm-rt-64.o ../compare-transform-pass.so ../split-compares-pass.so ../split-switches-pass.so ../cmplog-routines-pass.so ../cmplog-instructions-pass.so
# If prerequisites are not given, warn, do not build anything, and exit with code 0
ifeq "$(LLVMVER)" ""
@@ -304,7 +304,7 @@ afl-common.o: ../src/afl-common.c
$(CC) $(CFLAGS) -c $< -o $@ $(LDFLAGS)
../afl-clang-fast: afl-clang-fast.c afl-common.o | test_deps
- $(CC) $(CFLAGS) $< afl-common.o -o $@ $(LDFLAGS) -DCFLAGS_OPT=\"$(CFLAGS_OPT)\" -Dxxx
+ $(CC) $(CFLAGS) $< afl-common.o -o $@ $(LDFLAGS) -DCFLAGS_OPT=\"$(CFLAGS_OPT)\"
ln -sf afl-clang-fast ../afl-clang-fast++
ifneq "$(AFL_CLANG_FLTO)" ""
ifeq "$(LLVM_LTO)" "1"
@@ -330,6 +330,11 @@ ifeq "$(LLVM_LTO)" "1"
$(CXX) $(CLANG_CFL) -fno-rtti -fPIC -std=$(LLVM_STDCXX) -shared $< -o $@ $(CLANG_LFL) afl-llvm-common.o
endif
+../afl-ld-lto: afl-ld-lto.c
+ifeq "$(LLVM_LTO)" "1"
+ $(CC) $(CFLAGS) $< -o $@
+endif
+
../afl-llvm-lto-instrumentation.so: afl-llvm-lto-instrumentation.so.cc afl-llvm-common.o
ifeq "$(LLVM_LTO)" "1"
$(CXX) $(CLANG_CFL) -Wno-writable-strings -fno-rtti -fPIC -std=$(LLVM_STDCXX) -shared $< -o $@ $(CLANG_LFL) afl-llvm-common.o
@@ -392,6 +397,7 @@ install: all
install -d -m 755 $${DESTDIR}$(BIN_PATH) $${DESTDIR}$(HELPER_PATH) $${DESTDIR}$(DOC_PATH) $${DESTDIR}$(MISC_PATH)
if [ -f ../afl-clang-fast -a -f ../libLLVMInsTrim.so -a -f ../afl-llvm-rt.o ]; then set -e; install -m 755 ../afl-clang-fast $${DESTDIR}$(BIN_PATH); ln -sf afl-clang-fast $${DESTDIR}$(BIN_PATH)/afl-clang-fast++; install -m 755 ../libLLVMInsTrim.so ../afl-llvm-pass.so ../afl-llvm-rt.o $${DESTDIR}$(HELPER_PATH); fi
if [ -f ../afl-clang-lto ]; then set -e; ln -sf afl-clang-fast $${DESTDIR}$(BIN_PATH)/afl-clang-lto; ln -sf afl-clang-fast $${DESTDIR}$(BIN_PATH)/afl-clang-lto++; install -m 755 ../afl-llvm-lto-instrumentation.so ../afl-llvm-lto-instrim.so ../afl-llvm-rt-lto*.o ../afl-llvm-lto-whitelist.so $${DESTDIR}$(HELPER_PATH); fi
+ if [ -f ../afl-ld-lto ]; then set -e; install -m 755 ../afl-ld-lto $${DESTDIR}$(BIN_PATH); fi
if [ -f ../afl-llvm-rt-32.o ]; then set -e; install -m 755 ../afl-llvm-rt-32.o $${DESTDIR}$(HELPER_PATH); fi
if [ -f ../afl-llvm-rt-64.o ]; then set -e; install -m 755 ../afl-llvm-rt-64.o $${DESTDIR}$(HELPER_PATH); fi
if [ -f ../compare-transform-pass.so ]; then set -e; install -m 755 ../compare-transform-pass.so $${DESTDIR}$(HELPER_PATH); fi

MODIFY
llvm_mode/README.lto.md
llvm_mode/README.lto.md
@@ -14,9 +14,11 @@ This version requires a current llvm 11 compiled from the github master.
4. AUTODICTIONARY feature! see below
-5. If any problems arise be sure to set `AR=llvm-ar RANLIB=llvm-ranlib` also
- note that if that target uses _init functions or early constructors then
- also set `AFL_LLVM_MAP_DYNAMIC=1` as your target will crash otherwise
+5. If any problems arise be sure to set `AR=llvm-ar RANLIB=llvm-ranlib`.
+ Some targets might need `LD=afl-clang-lto` and others `LD=afl-ld-lto`.
+
+6. If a target uses _init functions or early constructors then additionally
+ set `AFL_LLVM_MAP_DYNAMIC=1` as your target will crash otherwise!
## Introduction and problem description
@@ -61,7 +63,8 @@ AUTODICTIONARY: 11 strings found
## Getting llvm 11
-### Installing llvm 11
+### Installing llvm 11 from the llvm repository
+
Installing the llvm snapshot builds is easy and mostly painless:
In the follow line change `NAME` for your Debian or Ubuntu release name
@@ -80,7 +83,7 @@ apt-get install -y clang-11 clang-tools-11 libc++1-11 libc++-11-dev \
libomp5-11 lld-11 lldb-11 llvm-11 llvm-11-dev llvm-11-runtime llvm-11-tools
```
-### Building llvm 11
+### Building llvm 11 yourself
Building llvm from github takes quite some long time and is not painless:
```
@@ -117,6 +120,9 @@ export AFL_LLVM_INSTRUMENT=CFG
make
```
+NOTE: some targets also need to set the linker, try both `afl-clang-lto` and
+`afl-ld-lto` for this for `LD=` for `configure`.
+
## AUTODICTIONARY feature
Setting `AFL_LLVM_LTO_AUTODICTIONARY` will generate a dictionary in the
@@ -135,6 +141,51 @@ to be dynamic - the original afl way, which is slower).
AFL_LLVM_MAP_DYNAMIC can be set so the shared memory address is dynamic (which
is safer but also slower).
+## Solving difficult targets
+
+Some targets are difficult because the configure script does unusual stuff that
+is unexpected for afl. See the next chapter `Potential issues` how to solve
+these.
+
+An example of a hard to solve target is ffmpeg. Here is how to successfully
+instrument it:
+
+1. Get and extract the current ffmpeg and change to it's directory
+
+2. Running configure with --cc=clang fails and various other items will fail
+ when compiling, so we have to trick configure:
+
+```
+./configure --enable-lto --disable-shared
+```
+
+3. Now the configuration is done - and we edit the settings in `./ffbuild/config.mak`
+ (-: the original line, +: what to change it into):
+```
+-CC=gcc
++CC=afl-clang-lto
+-CXX=g++
++CXX=afl-clang-lto++
+-AS=gcc
++AS=llvm-as
+-LD=gcc
++LD=afl-clang-lto++
+-DEPCC=gcc
++DEPCC=afl-clang-lto
+-DEPAS=gcc
++DEPAS=afl-clang-lto++
+-AR=ar
++AR=llvm-ar
+-AR_CMD=ar
++AR_CMD=llvm-ar
+-NM_CMD=nm -g
++NM_CMD=llvm-nm -g
+-RANLIB=ranlib -D
++RANLIB=llvm-ranlib -D
+```
+
+4. Then type make, wait for a long time and you are done :)
+
## Potential issues
### compiling libraries fails
@@ -154,6 +205,16 @@ and on some target you have to to AR=/RANLIB= even for make as the configure scr
Other targets ignore environment variables and need the parameters set via
`./configure --cc=... --cxx= --ranlib= ...` etc. (I am looking at you ffmpeg!).
+
+If you see this message
+```
+assembler command failed ...
+```
+then try setting `llvm-as` for configure:
+```
+AS=llvm-as ...
+```
+
### compiling programs still fail
afl-clang-lto is still work in progress.
@@ -166,11 +227,12 @@ Hence if building a target with afl-clang-lto fails try to build it with llvm11
and LTO enabled (`CC=clang-11` `CXX=clang++-11` `CFLAGS=-flto=full` and
`CXXFLAGS=-flto=full`).
-An example that does not build with llvm 11 and LTO is ffmpeg.
-
If this succeeeds then there is an issue with afl-clang-lto. Please report at
[https://github.com/AFLplusplus/AFLplusplus/issues/226](https://github.com/AFLplusplus/AFLplusplus/issues/226)
+Even some targets where clang-11 fails can be build if the fail is just in
+`./configure`, see `Solving difficult targets` above.
+
### Target crashes immediately
If the target is using early constructors (priority values smaller than 6)

MODIFY
src/afl-common.c
src/afl-common.c
@@ -61,8 +61,9 @@ char *afl_environment_variables[] = {
"AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES", "AFL_IMPORT_FIRST",
"AFL_INST_LIBS", "AFL_INST_RATIO", "AFL_KEEP_TRACES", "AFL_KEEP_ASSEMBLY",
"AFL_LD_HARD_FAIL", "AFL_LD_LIMIT_MB", "AFL_LD_NO_CALLOC_OVER",
- "AFL_LD_PRELOAD", "AFL_LD_VERBOSE", "AFL_LLVM_CMPLOG", "AFL_LLVM_INSTRIM",
- "AFL_LLVM_CTX", "AFL_LLVM_INSTRUMENT", "AFL_LLVM_INSTRIM_LOOPHEAD",
+ "AFL_LD_PASSTHROUGH", "AFL_REAL_LD", "AFL_LD_PRELOAD", "AFL_LD_VERBOSE",
+ "AFL_LLVM_CMPLOG", "AFL_LLVM_INSTRIM", "AFL_LLVM_CTX",
+ "AFL_LLVM_INSTRUMENT", "AFL_LLVM_INSTRIM_LOOPHEAD",
"AFL_LLVM_LTO_AUTODICTIONARY", "AFL_LLVM_AUTODICTIONARY",
"AFL_LLVM_SKIPSINGLEBLOCK", "AFL_LLVM_INSTRIM_SKIPSINGLEBLOCK",
"AFL_LLVM_LAF_SPLIT_COMPARES", "AFL_LLVM_LAF_SPLIT_COMPARES_BITW",

add afl-ld-lto for LTO