title: Avro Schema Evolution: Can't add or remove fields
body: <p>I'm currently trying to evolve my avro schema which should be no big deal according to the docs. However, when adding or removing a field Avro can't deserialize the bytes.</p>
  
  <p>I'm using the following schema:</p>
  
  <p><strong>AvroSchemas.avsc:</strong></p>
  
  <pre><code>[
    {
      "namespace": "stackoverflow.example.avro",
      "type": "record",
      "name": "Record_1_1",
      "fields": [
        {"name": "value0", "type": "string"}
      ]
    },
    {
      "namespace": "stackoverflow.example.avro",
      "type": "record",
      "name": "Record_1_2",
      "fields": [
        {"name": "value0", "type": "string"},
        {"name": "value1", "type": "string", "default": "Hello World"}
      ]
    },
    {
      "namespace": "stackoverflow.example.avro",
      "type": "record",
      "name": "Record_2_1",
      "fields": [
        {"name": "someList", "type": {"type": "array", "items": "int"}}
      ]
    },
    {
      "namespace": "stackoverflow.example.avro",
      "type": "record",
      "name": "Record_2_2",
      "fields": [
        {"name": "someBool", "type": "boolean", "default": "false"},
        {"name": "someList", "type": {"type": "array", "items": "int"}}
      ]
    }
  ]
  </code></pre>
  
  <p>The classes are generated using the following Maven Build-Plugin:</p>
  
  <pre><code>  &lt;plugin&gt;
      &lt;groupId&gt;org.apache.avro&lt;/groupId&gt;
      &lt;artifactId&gt;avro-maven-plugin&lt;/artifactId&gt;
      &lt;version&gt;1.8.2&lt;/version&gt;
      &lt;executions&gt;
        &lt;execution&gt;
          &lt;phase&gt;generate-sources&lt;/phase&gt;
          &lt;goals&gt;
            &lt;goal&gt;schema&lt;/goal&gt;
          &lt;/goals&gt;
          &lt;configuration&gt;
            &lt;sourceDirectory&gt;${project.basedir}/src/main/avro/&lt;/sourceDirectory&gt;
            &lt;outputDirectory&gt;${project.basedir}/src/main/java/&lt;/outputDirectory&gt;
            &lt;stringType&gt;String&lt;/stringType&gt;
          &lt;/configuration&gt;
        &lt;/execution&gt;
      &lt;/executions&gt;
    &lt;/plugin&gt;
  </code></pre>
  
  <p>This is the code I use to test my evolution:</p>
  
  <p><strong>AvroTest.java:</strong></p>
  
  <pre><code>package stackoverflow.example;
  
  import java.io.ByteArrayOutputStream;
  import java.util.ArrayList;
  import java.util.Objects;
  import org.apache.avro.io.DatumReader;
  import org.apache.avro.io.DatumWriter;
  import org.apache.avro.io.Decoder;
  import org.apache.avro.io.DecoderFactory;
  import org.apache.avro.io.Encoder;
  import org.apache.avro.io.EncoderFactory;
  import org.apache.avro.specific.SpecificDatumReader;
  import org.apache.avro.specific.SpecificDatumWriter;
  import stackoverflow.example.avro.Record_1_1;
  import stackoverflow.example.avro.Record_1_2;
  import stackoverflow.example.avro.Record_2_1;
  import stackoverflow.example.avro.Record_2_2;
  
  public class AvroTest {
  
      public static void main(String[] args) throws Exception {
          executeTest0();
          executeTest1();
          executeTest2();
      }
  
      /**
       * Test if read and write methods work
       */
      private static void executeTest0() {
          Record_1_1 source1 = new Record_1_1("A");
          Record_1_1 dest1 = trySerializeDeserialize(source1, Record_1_1.class, Record_1_1.class);
          if (dest1 == null || !Objects.equals(source1.getValue0(), dest1.getValue0())) {
              throw new RuntimeException("Record_1_1 Test 0 failed");
          }
  
          Record_1_2 source2 = new Record_1_2("A", "B");
          Record_1_2 dest2 = trySerializeDeserialize(source2, Record_1_2.class, Record_1_2.class);
          if (dest2 == null || !Objects.equals(source2.getValue0(), dest2.getValue0()) || !Objects.equals(source2.getValue1(), dest2.getValue1())) {
              throw new RuntimeException("Record_1_2 Test 0 failed");
          }
  
          Record_2_1 source3 = new Record_2_1(new ArrayList&lt;&gt;());
          Record_2_1 dest3 = trySerializeDeserialize(source3, Record_2_1.class, Record_2_1.class);
          if (dest3 == null || !Objects.equals(source3.getSomeList(), dest3.getSomeList())) {
              throw new RuntimeException("Record_2_1 Test 0 failed");
          }
  
          Record_2_2 source4 = new Record_2_2(true, new ArrayList&lt;&gt;());
          Record_2_2 dest4 = trySerializeDeserialize(source4, Record_2_2.class, Record_2_2.class);
          if (dest4 == null || !Objects.equals(source4.getSomeBool(), dest4.getSomeBool()) || !Objects.equals(source4.getSomeList(), dest4.getSomeList())) {
              throw new RuntimeException("Record_2_2 Test 0 failed");
          }
      }
  
      private static void executeTest1() {
          Record_1_1 source1 = new Record_1_1("Test");
          Record_1_2 dest1 = trySerializeDeserialize(source1, Record_1_1.class, Record_1_2.class);
          if (dest1 == null || !Objects.equals(dest1.getValue1(), "Hello World")) {
              System.out.println("adding field with default value failed: " + dest1);
          }
  
          Record_1_2 source2 = new Record_1_2("Test0", "Test1");
          Record_1_1 dest2 = trySerializeDeserialize(source2, Record_1_2.class, Record_1_1.class);
          if (dest2 == null || !Objects.equals(source2.getValue0(), dest2.getValue0())) {
              System.out.println("removing field failed: " + dest2);
          }
      }
  
      private static void executeTest2() {
          Record_2_1 source1 = new Record_2_1(new ArrayList&lt;&gt;());
          Record_2_2 dest1 = trySerializeDeserialize(source1, Record_2_1.class, Record_2_2.class);
          if (dest1 == null || !Objects.equals(source1.getSomeList(), dest1.getSomeList())) {
              System.out.println("adding boolean field with default value failed: " + dest1);
          }
  
          Record_2_2 source2 = new Record_2_2(true, new ArrayList&lt;&gt;());
          Record_2_1 dest2 = trySerializeDeserialize(source2, Record_2_2.class, Record_2_1.class);
          if (dest2 == null || !Objects.equals(source2.getSomeList(), dest2.getSomeList())) {
              System.out.println("removing boolean field failed: " + dest2);
          }
      }
  
      private static &lt;T, E&gt; E trySerializeDeserialize(T source, Class&lt;T&gt; sourceClass, Class&lt;E&gt; destClass) {
          E result;
  
          try {
              byte[] bytes = write(source, sourceClass);
              result = read(bytes, destClass);
          } catch (Exception e) {
              result = null;
          }
  
          return result;
      }
  
      private static &lt;T&gt; byte[] write(T value, Class&lt;T&gt; clazz) throws Exception {
          byte[] bytes;
  
          try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
              Encoder encoder = EncoderFactory.get().binaryEncoder(bos, null);
              DatumWriter&lt;T&gt; writer = new SpecificDatumWriter&lt;&gt;(clazz);
              writer.write(value, encoder);
  
              encoder.flush();
              bytes = bos.toByteArray();
          }
  
          return bytes;
      }
  
      private static &lt;T&gt; T read(byte[] bytes, Class&lt;T&gt; clazz) throws Exception {
          Decoder decoder = DecoderFactory.get().binaryDecoder(bytes, null);
          DatumReader&lt;T&gt; reader = new SpecificDatumReader&lt;&gt;(clazz);
  
          return reader.read(null, decoder);
      }
  }
  </code></pre>
  
  <p><strong>Output:</strong></p>
  
  <pre><code>adding field with default value failed: null
  adding boolean field with default value failed: null
  removing boolean field failed: null
  </code></pre>
  
  <p>According to the docs all of my test should work (adding a field with a default-value or removing a field on receiver-side). But I don't think that the docs were just written for fun so is there maybe some setting that I'm missing?</p>