Annotate a class's important string value via an attribute
It's considered bad form for a property getter to throw an exception. I would do so in a protected constructor of your FieldBase class. At that point, you can cache the retrieved value in a class member and not have to do any reflection in the property getter at all.

I'm also a fan of immutable semantics where available and appropriate and have modified the attribute itself to be such:

[FieldTag("Base")]
public abstract class FieldBase {
    protected FieldBase() {
        FieldTagAttribute fieldTagAttribute = (FieldTagAttribute) (GetType().GetCustomAttribute(typeof(FieldTagAttribute), false));

        if (fieldTagAttribute == null) {
            throw new InvalidOperationException(String.Format("Class {0} has no attribute [FieldTag]. Please add the attribute to this class.", GetType()));
        }

        s_FieldTag = fieldTagAttribute.FieldTag;
    }

    private readonly string s_FieldTag;

    public string FieldTag { get { return s_FieldTag; } }
}

[System.AttributeUsageAttribute(System.AttributeTargets.Class, Inherited = false)]
public sealed class FieldTagAttribute : Attribute {
   public FieldTagAttribute(string fieldTag) {
       s_FieldTag = fieldTag;
   }

   private readonly string s_FieldTag;

   public string FieldTag { get { return s_FieldTag; } }
}

[FieldTag("32A")]
public class Field32A : FieldBase {
    public string Value { get; set; }
    public override string ToString() {
        return ":" + FieldTag + ":" + Value;
    }
}

public class Program {
    public static void Main() {
        Field32A field32A = new Field32A { Value = "12345" };
        field32A.ToString().Dump();
    }
}