/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.gora.mongodb.store;

import static org.apache.gora.mongodb.store.MongoMapping.DocumentFieldType;

import java.io.IOException;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;

import javax.xml.bind.DatatypeConverter;

import org.apache.avro.Schema;
import org.apache.avro.Schema.Field;
import org.apache.avro.Schema.Type;
import org.apache.avro.generic.GenericArray;
import org.apache.avro.util.Utf8;
import org.apache.gora.mongodb.filters.MongoFilterUtil;
import org.apache.gora.mongodb.query.MongoDBQuery;
import org.apache.gora.mongodb.query.MongoDBResult;
import org.apache.gora.mongodb.utils.BSONDecorator;
import org.apache.gora.mongodb.utils.GoraDBEncoder;
import org.apache.gora.persistency.Persistent;
import org.apache.gora.persistency.impl.BeanFactoryImpl;
import org.apache.gora.persistency.impl.DirtyListWrapper;
import org.apache.gora.persistency.impl.DirtyMapWrapper;
import org.apache.gora.persistency.impl.PersistentBase;
import org.apache.gora.query.PartitionQuery;
import org.apache.gora.query.Query;
import org.apache.gora.query.Result;
import org.apache.gora.query.impl.PartitionQueryImpl;
import org.apache.gora.store.impl.DataStoreBase;
import org.apache.gora.util.AvroUtils;
import org.apache.gora.util.ClassLoadingUtils;
import org.bson.types.ObjectId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Splitter;
import com.mongodb.*;

/**
 * Implementation of a MongoDB data store to be used by gora.
 * 
 * @param <K>
 *          class to be used for the key
 * @param <T>
 *          class to be persisted within the store
 * @author Fabien Poulard <fpoulard@dictanova.com>
 * @author Damien Raude-Morvan <draudemorvan@dictanova.com>
 */
public class MongoStore<K, T extends PersistentBase> extends
    DataStoreBase<K, T> {

  public static final Logger LOG = LoggerFactory.getLogger(MongoStore.class);

  // Configuration properties

  /**
   * Property indicating if the hadoop configuration has priority or not
   */
  public static final String PROP_OVERRIDING = "gora.mongodb.override_hadoop_configuration";

  /**
   * Property pointing to the file for the mapping
   */
  public static final String PROP_MAPPING_FILE = "gora.mongodb.mapping.file";

  /**
   * Property pointing to the host where the server is running
   */
  public static final String PROP_MONGO_SERVERS = "gora.mongodb.servers";

  /**
   * Property pointing to the username to connect to the server
   */
  public static final String PROP_MONGO_LOGIN = "gora.mongodb.login";

  /**
   * Property pointing to the secret to connect to the server
   */
  public static final String PROP_MONGO_SECRET = "gora.mongodb.secret";

  /**
   * Default value for mapping file
   */
  public static final String DEFAULT_MAPPING_FILE = "/gora-mongodb-mapping.xml";

  /**
   * Property to select the database
   */
  public static String PROP_MONGO_DB = "gora.mongodb.db";

  /**
   * MongoDB client
   */
  private static ConcurrentHashMap<String, MongoClient> mapsOfClients = new ConcurrentHashMap<String, MongoClient>();

  private DB mongoClientDB;

  private DBCollection mongoClientColl;

  /**
   * Mapping definition for MongoDB
   */
  private MongoMapping mapping;

  private MongoFilterUtil<K, T> filterUtil;

  public MongoStore() {
    // Create a default mapping that will be overriden in initialize method
    this.mapping = new MongoMapping();
  }

  /**
   * Initialize the data store by reading the credentials, setting the client's
   * properties up and reading the mapping file.
   */
  public void initialize(final Class<K> keyClass,
      final Class<T> pPersistentClass, final Properties properties) {
    try {
      LOG.debug("Initializing MongoDB store");

      // Prepare the configuration
      String vPropMappingFile = properties.getProperty(PROP_MAPPING_FILE,
          DEFAULT_MAPPING_FILE);
      String vPropMongoServers = properties.getProperty(PROP_MONGO_SERVERS);
      String vPropMongoLogin = properties.getProperty(PROP_MONGO_LOGIN);
      String vPropMongoSecret = properties.getProperty(PROP_MONGO_SECRET);
      String vPropMongoDb = properties.getProperty(PROP_MONGO_DB);
      String overrideHadoop = properties.getProperty(PROP_OVERRIDING);
      if (!Boolean.parseBoolean(overrideHadoop)) {
        LOG.debug("Hadoop configuration has priority.");
        vPropMappingFile = getConf().get(PROP_MAPPING_FILE, vPropMappingFile);
        vPropMongoServers = getConf()
            .get(PROP_MONGO_SERVERS, vPropMongoServers);
        vPropMongoLogin = getConf().get(PROP_MONGO_LOGIN, vPropMongoLogin);
        vPropMongoSecret = getConf().get(PROP_MONGO_SECRET, vPropMongoSecret);
        vPropMongoDb = getConf().get(PROP_MONGO_DB, vPropMongoDb);
      }
      super.initialize(keyClass, pPersistentClass, properties);

      filterUtil = new MongoFilterUtil<K, T>(getConf());

      // Load the mapping
      MongoMappingBuilder<K, T> builder = new MongoMappingBuilder<K, T>(this);
      LOG.debug("Initializing Mongo store with mapping {}.",
          new Object[] { vPropMappingFile });
      builder.fromFile(vPropMappingFile);
      mapping = builder.build();

      // Prepare MongoDB connection
      mongoClientDB = getDB(vPropMongoServers, vPropMongoDb, vPropMongoLogin,
          vPropMongoSecret);
      mongoClientColl = mongoClientDB
          .getCollection(mapping.getCollectionName());

      LOG.info("Initialized Mongo store for database {} of {}.", new Object[] {
          vPropMongoDb, vPropMongoServers });
    } catch (IOException e) {
      LOG.error("Error while initializing MongoDB store: {}",
          new Object[] { e.getMessage() });
      throw new RuntimeException(e);
    }
  }

  /**
   * Retrieve a client connected to the MongoDB server to be used.
   * 
   * @param servers
   *          This value should specify the host:port (at least one) for
   *          connecting to remote MongoDB. Multiple values must be separated by
   *          coma.
   * @return a {@link Mongo} instance connected to the server
   * @throws UnknownHostException
   */
  private MongoClient getClient(final String servers)
      throws UnknownHostException {
    // Configure options
    MongoClientOptions opts = new MongoClientOptions.Builder()
        .dbEncoderFactory(GoraDBEncoder.FACTORY) // Utf8 serialization!
        .build();
    // Build server address
    List<ServerAddress> addrs = new ArrayList<ServerAddress>();
    Iterable<String> serversArray = Splitter.on(",").split(servers);
    if (serversArray != null) {
      for (String server : serversArray) {
        Iterable<String> params = Splitter.on(":").trimResults().split(server);
        Iterator<String> paramsIterator = params.iterator();
        if (!paramsIterator.hasNext()) {
          // No server, use default
          addrs.add(new ServerAddress());
        } else {
          String host = paramsIterator.next();
          if (paramsIterator.hasNext()) {
            String port = paramsIterator.next();
            addrs.add(new ServerAddress(host, Integer.parseInt(port)));
          } else {
            addrs.add(new ServerAddress(host));
          }
        }
      }
    }
    // Connect to the Mongo server
    return new MongoClient(addrs, opts);
  }

  /**
   * Get reference to Mongo DB, using credentials if not null.
   * 
   * @param servers
   * @param dbname
   *          Name of database to connect to.
   * @param login
   *          Optionnal login for remote database.
   * @param secret
   *          Optional secret for remote database.
   * @return a {@link DB} instance from <tt>mongoClient</tt> or null if
   *         authentication request failed.
   */
  private DB getDB(final String servers, final String dbname,
      final String login, final String secret) throws UnknownHostException {

    // Get reference to Mongo DB
    if (!mapsOfClients.containsKey(servers))
      mapsOfClients.put(servers, getClient(servers));
    DB db = mapsOfClients.get(servers).getDB(dbname);
    // By default, we are authenticated
    boolean auth = true;
    // If configuration contains a login + secret, try to authenticated with DB
    if (login != null && secret != null) {
      auth = db.authenticate(login, secret.toCharArray());
    }

    if (auth) {
      return db;
    } else {
      return null;
    }
  }

  public MongoMapping getMapping() {
    return mapping;
  }

  /**
   * Accessor to the name of the collection used.
   */
  @Override
  public String getSchemaName() {
    return mapping.getCollectionName();
  }

  @Override
  public String getSchemaName(final String mappingSchemaName,
      final Class<?> persistentClass) {
    return super.getSchemaName(mappingSchemaName, persistentClass);
  }

  /**
   * Create a new collection in MongoDB if necessary.
   */
  @Override
  public void createSchema() {
    if (mongoClientDB == null)
      throw new IllegalStateException(
          "Impossible to create the schema as no database has been selected.");
    if (schemaExists()) {
      return;
    }

    // If initialized create the collection
    mongoClientColl = mongoClientDB.createCollection(
        mapping.getCollectionName(), new BasicDBObject()); // send a DBObject to
    // force creation
    // otherwise creation is deferred
    mongoClientColl.setDBEncoderFactory(GoraDBEncoder.FACTORY);

    LOG.info("Collection {} has been created for Mongo instance {}.",
        new Object[] { mapping.getCollectionName(), mongoClientDB.getMongo() });
  }

  /**
   * Drop the collection.
   */
  @Override
  public void deleteSchema() {
    if (mongoClientColl == null)
      throw new IllegalStateException(
          "Impossible to delete the schema as no schema is selected.");
    // If initialized, simply drop the collection
    mongoClientColl.drop();

    LOG.info(
        "Collection {} has been dropped for Mongo instance {}.",
        new Object[] { mongoClientColl.getFullName(), mongoClientDB.getMongo() });
  }

  /**
   * Check if the collection already exists or should be created.
   */
  @Override
  public boolean schemaExists() {
    return mongoClientDB.collectionExists(mapping.getCollectionName());
  }

  /**
   * Ensure the data is synced to disk.
   */
  @Override
  public void flush() {
    for (MongoClient client : mapsOfClients.values()) {
      client.fsync(false);
      LOG.info("Forced synced of database for Mongo instance {}.",
          new Object[] { client });
    }
  }

  /**
   * Release the resources linked to this collection
   */
  @Override
  public void close() {
    mongoClientDB.cleanCursors(true);
    // mongoClient.close();
    // LOG.info("Closed database and connection for Mongo instance {}.",
    // new Object[]{mongoClient});
    LOG.debug("Not closed!!!");
  }

  /**
   * Retrieve an entry from the store with only selected fields.
   * 
   * @param key
   *          identifier of the document in the database
   * @param fields
   *          list of fields to be loaded from the database
   * @throws IOException
   */
  @Override
  public T get(final K key, final String[] fields) {
    String[] dbFields = getFieldsToQuery(fields);
    // Prepare the MongoDB query
    BasicDBObject q = new BasicDBObject("_id", key);
    BasicDBObject proj = new BasicDBObject();
    for (String field : dbFields) {
      String docf = mapping.getDocumentField(field);
      if (docf != null) {
        proj.put(docf, true);
      }
    }
    // Execute the query
    DBObject res = mongoClientColl.findOne(q, proj);
    // Build the corresponding persistent and clears its states
    T persistent = newInstance(res, dbFields);
    if (persistent != null) {
      persistent.clearDirty();
    }
    // Done
    return persistent;
  }

  /**
   * Persist an object into the store.
   * 
   * @param key
   *          identifier of the object in the store
   * @param obj
   *          the object to be inserted
   */
  @Override
  public void put(final K key, final T obj) {
    // Save the object in the database
    if (obj.isDirty()) {
      performPut(key, obj);
    } else {
      LOG.info("Ignored putting object {} in the store as it is neither "
          + "new, neither dirty.", new Object[] { obj });
    }
  }

  /**
   * Update a object that already exists in the store. The object must exist
   * already or the update may fail.
   * 
   * @param key
   *          identifier of the object in the store
   * @param obj
   *          the object to be inserted
   */
  private void performPut(final K key, final T obj) {
    // Build the query to select the object to be updated
    DBObject qSel = new BasicDBObject("_id", key);

    // Build the update query
    BasicDBObject qUpdate = new BasicDBObject();

    BasicDBObject qUpdateSet = newUpdateSetInstance(obj);
    if (qUpdateSet.size() > 0) {
      qUpdate.put("$set", qUpdateSet);
    }

    BasicDBObject qUpdateUnset = newUpdateUnsetInstance(obj);
    if (qUpdateUnset.size() > 0) {
      qUpdate.put("$unset", qUpdateUnset);
    }

    // Execute the update (if there is at least one $set ot $unset
    if (!qUpdate.isEmpty()) {
      mongoClientColl.update(qSel, qUpdate, true, false);
      obj.clearDirty();
    } else {
      LOG.debug("No update to perform, skip {}", key);
    }
  }

  @Override
  public boolean delete(final K key) {
    DBObject removeKey = new BasicDBObject("_id", key);
    WriteResult writeResult = mongoClientColl.remove(removeKey);
    return writeResult != null && writeResult.getN() > 0;
  }

  @Override
  public long deleteByQuery(final Query<K, T> query) {
    // Build the actual MongoDB query
    DBObject q = MongoDBQuery.toDBQuery(query);
    WriteResult writeResult = mongoClientColl.remove(q);
    if (writeResult != null) {
      return writeResult.getN();
    }
    return 0;
  }

  /**
   * Execute the query and return the result.
   */
  @Override
  public Result<K, T> execute(final Query<K, T> query) {

    String[] fields = getFieldsToQuery(query.getFields());
    // Build the actual MongoDB query
    DBObject q = MongoDBQuery.toDBQuery(query);
    DBObject p = MongoDBQuery.toProjection(fields, mapping);

    if (query.getFilter() != null) {
      boolean succeeded = filterUtil.setFilter(q, query.getFilter(), this);
      if (succeeded) {
        // don't need local filter
        query.setLocalFilterEnabled(false);
      }
    }

    // Execute the query on the collection
    DBCursor cursor = mongoClientColl.find(q, p);
    if (query.getLimit() > 0)
      cursor = cursor.limit((int) query.getLimit());
    cursor.batchSize(100);

    // Build the result
    MongoDBResult<K, T> mongoResult = new MongoDBResult<K, T>(this, query);
    mongoResult.setCursor(cursor);

    return mongoResult;
  }

  /**
   * Create a new {@link Query} to query the datastore.
   */
  @Override
  public Query<K, T> newQuery() {
    MongoDBQuery<K, T> query = new MongoDBQuery<K, T>(this);
    query.setFields(getFieldsToQuery(null));
    return query;
  }

  /**
   * Partitions the given query and returns a list of PartitionQuerys, which
   * will execute on local data.
   */
  @Override
  public List<PartitionQuery<K, T>> getPartitions(final Query<K, T> query)
      throws IOException {
    // FIXME: for now, there is only one partition as we do not handle
    // MongoDB sharding configuration
    List<PartitionQuery<K, T>> partitions = new ArrayList<PartitionQuery<K, T>>();
    PartitionQueryImpl<K, T> partitionQuery = new PartitionQueryImpl<K, T>(
        query);
    partitionQuery.setConf(getConf());
    partitions.add(partitionQuery);
    return partitions;
  }

  // //////////////////////////////////////////////////////// DESERIALIZATION

  /**
   * Build a new instance of the persisted class from the {@link DBObject}
   * retrieved from the database.
   * 
   * @param obj
   *          the {@link DBObject} that results from the query to the database
   * @param fields
   *          the list of fields to be mapped to the persistence class instance
   * @return a persistence class instance which content was deserialized from
   *         the {@link DBObject}
   * @throws IOException
   */
  public T newInstance(final DBObject obj, final String[] fields) {
    if (obj == null)
      return null;
    BSONDecorator easybson = new BSONDecorator(obj);
    // Create new empty persistent bean instance
    T persistent = newPersistent();
    String[] dbFields = getFieldsToQuery(fields);

    // Populate each field
    for (String f : dbFields) {
      // Check the field exists in the mapping and in the db
      String docf = mapping.getDocumentField(f);
      if (docf == null || !easybson.containsField(docf))
        continue;

      DocumentFieldType storeType = mapping.getDocumentFieldType(docf);
      Field field = fieldMap.get(f);
      Schema fieldSchema = field.schema();

      LOG.debug(
          "Load from DBObject (MAIN), field:{}, schemaType:{}, docField:{}, storeType:{}",
          new Object[] { field.name(), fieldSchema.getType(), docf, storeType });
      Object result = fromDBObject(fieldSchema, storeType, field, docf,
          easybson);
      persistent.put(field.pos(), result);
    }
    persistent.clearDirty();
    return persistent;
  }

  private Object fromDBObject(final Schema fieldSchema,
      final DocumentFieldType storeType, final Field field, final String docf,
      final BSONDecorator easybson) {
    Object result = null;
    switch (fieldSchema.getType()) {
    case MAP:
      result = fromMongoMap(docf, fieldSchema, easybson, field);
      break;
    case ARRAY:
      result = fromMongoList(docf, fieldSchema, easybson, field);
      break;
    case RECORD:
      DBObject rec = easybson.getDBObject(docf);
      if (rec == null) {
        result = null;
        break;
      }
      result = fromMongoRecord(fieldSchema, docf, rec);
      break;
    case BOOLEAN:
      result = easybson.getBoolean(docf);
      break;
    case DOUBLE:
      result = easybson.getDouble(docf);
      break;
    case FLOAT:
      result = easybson.getDouble(docf).floatValue();
      break;
    case INT:
      result = easybson.getInt(docf);
      break;
    case LONG:
      result = easybson.getLong(docf);
      break;
    case STRING:
      result = fromMongoString(storeType, docf, easybson);
      break;
    case ENUM:
      result = AvroUtils.getEnumValue(fieldSchema, easybson.getUtf8String(docf)
          .toString());
      break;
    case BYTES:
    case FIXED:
      result = easybson.getBytes(docf);
      break;
    case NULL:
      result = null;
      break;
    case UNION:
      result = fromMongoUnion(fieldSchema, storeType, field, docf, easybson);
      break;
    default:
      LOG.warn("Unable to read {}", docf);
      break;
    }
    return result;
  }

  private Object fromMongoUnion(final Schema fieldSchema,
      final DocumentFieldType storeType, final Field field, final String docf,
      final BSONDecorator easybson) {
    Object result;// schema [type0, type1]
    Type type0 = fieldSchema.getTypes().get(0).getType();
    Type type1 = fieldSchema.getTypes().get(1).getType();

    // Check if types are different and there's a "null", like ["null","type"]
    // or ["type","null"]
    if (!type0.equals(type1)
        && (type0.equals(Type.NULL) || type1.equals(Type.NULL))) {
      Schema innerSchema = fieldSchema.getTypes().get(1);
      LOG.debug(
          "Load from DBObject (UNION), schemaType:{}, docField:{}, storeType:{}",
          new Object[] { innerSchema.getType(), docf, storeType });
      // Deserialize as if schema was ["type"]
      result = fromDBObject(innerSchema, storeType, field, docf, easybson);
    } else {
      throw new IllegalStateException(
          "MongoStore doesn't support 3 types union field yet. Please update your mapping");
    }
    return result;
  }

  @SuppressWarnings({ "unchecked", "rawtypes" })
  private Object fromMongoRecord(final Schema fieldSchema, final String docf,
      final DBObject rec) {
    Object result;
    BSONDecorator innerBson = new BSONDecorator(rec);
    Class<?> clazz = null;
    try {
      clazz = ClassLoadingUtils.loadClass(fieldSchema.getFullName());
    } catch (ClassNotFoundException e) {
    }
    Persistent record = new BeanFactoryImpl(keyClass, clazz).newPersistent();
    for (Field recField : fieldSchema.getFields()) {
      Schema innerSchema = recField.schema();
      DocumentFieldType innerStoreType = mapping
          .getDocumentFieldType(innerSchema.getName());
      String innerDocField = mapping.getDocumentField(recField.name()) != null ? mapping
          .getDocumentField(recField.name()) : recField.name();
      String fieldPath = docf + "." + innerDocField;
      LOG.debug(
          "Load from DBObject (RECORD), field:{}, schemaType:{}, docField:{}, storeType:{}",
          new Object[] { recField.name(), innerSchema.getType(), fieldPath,
              innerStoreType });
      record.put(
          recField.pos(),
          fromDBObject(innerSchema, innerStoreType, recField, innerDocField,
              innerBson));
    }
    result = record;
    return result;
  }

  private Object fromMongoList(final String docf, final Schema fieldSchema,
      final BSONDecorator easybson, final Field f) {
    List<Object> list = easybson.getDBList(docf);
    if (list == null) {
      return null;
    }

    List<Object> rlist = new ArrayList<Object>();

    for (Object item : list) {
      DocumentFieldType storeType = mapping.getDocumentFieldType(docf);
      Object o = fromDBObject(fieldSchema.getElementType(), storeType, f,
          "item", new BSONDecorator(new BasicDBObject("item", item)));
      rlist.add(o);
    }
    return new DirtyListWrapper<Object>(rlist);
  }

  private Object fromMongoMap(final String docf, final Schema fieldSchema,
      final BSONDecorator easybson, final Field f) {
    BasicDBObject map = easybson.getDBObject(docf);
    if (map == null) {
      return null;
    }
    Map<Utf8, Object> rmap = new HashMap<Utf8, Object>();
    for (Entry<String, Object> e : map.entrySet()) {
      String mapKey = e.getKey();
      String decodedMapKey = decodeFieldKey(mapKey);

      DocumentFieldType storeType = mapping.getDocumentFieldType(docf);
      Object o = fromDBObject(fieldSchema.getValueType(), storeType, f, mapKey,
          new BSONDecorator(map));
      rmap.put(new Utf8(decodedMapKey), o);
    }
    return new DirtyMapWrapper<Utf8, Object>(rmap);
  }

  private Object fromMongoString(final DocumentFieldType storeType,
      final String docf, final BSONDecorator easybson) {
    Object result;
    if (storeType == DocumentFieldType.OBJECTID) {
      // Try auto-conversion of BSON data to ObjectId
      // It will work if data is stored as String or as ObjectId
      Object bin = easybson.get(docf);
      ObjectId id = ObjectId.massageToObjectId(bin);
      result = new Utf8(id.toString());
    } else if (storeType == DocumentFieldType.DATE) {
      Object bin = easybson.get(docf);
      if (bin instanceof Date) {
        Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
        calendar.setTime((Date) bin);
        result = new Utf8(DatatypeConverter.printDateTime(calendar));
      } else {
        result = new Utf8(bin.toString());
      }
    } else {
      result = easybson.getUtf8String(docf);
    }
    return result;
  }

  // ////////////////////////////////////////////////////////// SERIALIZATION

  /**
   * Build a new instance of {@link DBObject} from the persistence class
   * instance in parameter. Limit the {@link DBObject} to the fields that are
   * dirty and not null, that is the fields that will need to be updated in the
   * store.
   * 
   * @param persistent
   *          a persistence class instance which content is to be serialized as
   *          a {@link DBObject} for use as parameter of a $set operator
   * @return a {@link DBObject} which content corresponds to the fields that
   *         have to be updated... and formatted to be passed in parameter of a
   *         $set operator
   */
  private BasicDBObject newUpdateSetInstance(final T persistent) {
    BasicDBObject result = new BasicDBObject();
    for (Field f : persistent.getSchema().getFields()) {
      if (persistent.isDirty(f.pos()) && (persistent.get(f.pos()) != null)) {
        String docf = mapping.getDocumentField(f.name());
        Object value = persistent.get(f.pos());
        DocumentFieldType storeType = mapping.getDocumentFieldType(docf);
        LOG.debug(
            "Transform value to DBObject (MAIN), docField:{}, schemaType:{}, storeType:{}",
            new Object[] { docf, f.schema().getType(), storeType });
        Object o = toDBObject(docf, f.schema(), f.schema().getType(),
            storeType, value);
        result.put(docf, o);
      }
    }
    return result;
  }

  /**
   * Build a new instance of {@link DBObject} from the persistence class
   * instance in parameter. Limit the {@link DBObject} to the fields that are
   * dirty and null, that is the fields that will need to be updated in the
   * store by being removed.
   * 
   * @param persistent
   *          a persistence class instance which content is to be serialized as
   *          a {@link DBObject} for use as parameter of a $set operator
   * @return a {@link DBObject} which content corresponds to the fields that
   *         have to be updated... and formated to be passed in parameter of a
   *         $unset operator
   */
  private BasicDBObject newUpdateUnsetInstance(final T persistent) {
    BasicDBObject result = new BasicDBObject();
    for (Field f : persistent.getSchema().getFields()) {
      if (persistent.isDirty(f.pos()) && (persistent.get(f.pos()) == null)) {
        String docf = mapping.getDocumentField(f.name());
        Object value = persistent.get(f.pos());
        DocumentFieldType storeType = mapping.getDocumentFieldType(docf);
        LOG.debug(
            "Transform value to DBObject (MAIN), docField:{}, schemaType:{}, storeType:{}",
            new Object[] { docf, f.schema().getType(), storeType });
        Object o = toDBObject(docf, f.schema(), f.schema().getType(),
            storeType, value);
        result.put(docf, o);
      }
    }
    return result;
  }

  @SuppressWarnings("unchecked")
  private Object toDBObject(final String docf, final Schema fieldSchema,
      final Type fieldType, final DocumentFieldType storeType,
      final Object value) {
    Object result = null;
    switch (fieldType) {
    case MAP:
      if (storeType != null && storeType != DocumentFieldType.DOCUMENT) {
        throw new IllegalStateException(
            "Field "
                + fieldSchema.getType()
                + ": to store a Gora 'map', target Mongo mapping have to be of 'document' type");
      }
      Schema valueSchema = fieldSchema.getValueType();
      result = mapToMongo(docf, (Map<CharSequence, ?>) value, valueSchema,
          valueSchema.getType());
      break;
    case ARRAY:
      if (storeType != null && storeType != DocumentFieldType.LIST) {
        throw new IllegalStateException(
            "Field "
                + fieldSchema.getType()
                + ": To store a Gora 'array', target Mongo mapping have to be of 'list' type");
      }
      Schema elementSchema = fieldSchema.getElementType();
      result = listToMongo(docf, (List<?>) value, elementSchema,
          elementSchema.getType());
      break;
    case BYTES:
      if (value != null) {
        result = ((ByteBuffer) value).array();
      }
      break;
    case INT:
    case LONG:
    case FLOAT:
    case DOUBLE:
    case BOOLEAN:
      result = value;
      break;
    case STRING:
      result = stringToMongo(fieldSchema, storeType, value);
      break;
    case ENUM:
      if (value != null)
        result = value.toString();
      break;
    case RECORD:
      if (value == null)
        break;
      result = recordToMongo(docf, fieldSchema, value);
      break;
    case UNION:
      result = unionToMongo(docf, fieldSchema, storeType, value);
      break;
    case FIXED:
      result = value;
      break;

    default:
      LOG.error("Unknown field type: {}", fieldSchema.getType());
      break;
    }

    return result;
  }

  private Object unionToMongo(final String docf, final Schema fieldSchema,
      final DocumentFieldType storeType, final Object value) {
    Object result;// schema [type0, type1]
    Type type0 = fieldSchema.getTypes().get(0).getType();
    Type type1 = fieldSchema.getTypes().get(1).getType();

    // Check if types are different and there's a "null", like ["null","type"]
    // or ["type","null"]
    if (!type0.equals(type1)
        && (type0.equals(Type.NULL) || type1.equals(Type.NULL))) {
      Schema innerSchema = fieldSchema.getTypes().get(1);
      LOG.debug(
          "Transform value to DBObject (UNION), schemaType:{}, type1:{}, storeType:{}",
          new Object[] { innerSchema.getType(), type1, storeType });
      // Deserialize as if schema was ["type"]
      result = toDBObject(docf, innerSchema, type1, storeType, value);
    } else {
      throw new IllegalStateException(
          "MongoStore doesn't support 3 types union field yet. Please update your mapping");
    }
    return result;
  }

  private BasicDBObject recordToMongo(final String docf,
      final Schema fieldSchema, final Object value) {
    BasicDBObject record = new BasicDBObject();
    for (Field member : fieldSchema.getFields()) {
      Object innerValue = ((PersistentBase) value).get(member.pos());
      String innerDoc = mapping.getDocumentField(member.name());
      Type innerType = member.schema().getType();
      DocumentFieldType innerStoreType = mapping.getDocumentFieldType(innerDoc);
      LOG.debug(
          "Transform value to DBObject (RECORD), docField:{}, schemaType:{}, storeType:{}",
          new Object[] { member.name(), member.schema().getType(),
              innerStoreType });
      record.put(
          member.name(),
          toDBObject(docf, member.schema(), innerType, innerStoreType,
              innerValue));
    }
    return record;
  }

  private Object stringToMongo(final Schema fieldSchema,
      final DocumentFieldType storeType, final Object value) {
    Object result = null;
    if (storeType == DocumentFieldType.OBJECTID) {
      if (value != null) {
        ObjectId id;
        try {
          id = new ObjectId(value.toString());
        } catch (IllegalArgumentException e1) {
          // Unable to parse anything from Utf8 value, throw error
          throw new IllegalStateException("Field " + fieldSchema.getType()
              + ": Invalid string: unable to convert to ObjectId");
        }
        result = id;
      }
    } else if (storeType == DocumentFieldType.DATE) {
      if (value != null) {
        // Try to parse date from Utf8 value
        Calendar calendar = null;
        try {
          // Parse as date + time
          calendar = DatatypeConverter.parseDateTime(value.toString());
        } catch (IllegalArgumentException e1) {
          try {
            // Parse as date only
            calendar = DatatypeConverter.parseDate(value.toString());
          } catch (IllegalArgumentException e2) {
            // No-op
          }
        }
        if (calendar == null) {
          // Unable to parse anything from Utf8 value, throw error
          throw new IllegalStateException("Field " + fieldSchema.getType()
              + ": Invalid date format '" + value + "'");
        }
        result = calendar.getTime();
      }
    } else {
      if (value != null) {
        result = value.toString();
      }
    }
    return result;
  }

  /**
   * Convert a Java Map as used in Gora generated classes to a Map that can
   * safely be serialized into MongoDB.
   * 
   * @param value
   *          the Java Map that must be serialized into a MongoDB object
   * @param fieldType
   *          type of the values within the map
   * @return a {@link BasicDBObject} version of the {@link Map} that can be
   *         safely serialized into MongoDB.
   */
  private BasicDBObject mapToMongo(final String docf,
      final Map<CharSequence, ?> value, final Schema fieldSchema,
      final Type fieldType) {
    // Handle null case
    if (value == null)
      return null;

    // Handle regular cases
    BasicDBObject map = new BasicDBObject();
    for (Entry<CharSequence, ?> e : value.entrySet()) {
      String mapKey = e.getKey().toString();
      String encodedMapKey = encodeFieldKey(mapKey);
      Object mapValue = e.getValue();

      DocumentFieldType storeType = mapping.getDocumentFieldType(docf);
      Object result = toDBObject(docf, fieldSchema, fieldType, storeType,
          mapValue);
      map.put(encodedMapKey, result);
    }

    return map;
  }

  /**
   * Convert a Java {@link GenericArray} as used in Gora generated classes to a
   * List that can safely be serialized into MongoDB.
   * 
   * @param array
   *          the {@link GenericArray} to be serialized
   * @param fieldType
   *          type of the elements within the array
   * @return a {@link BasicDBList} version of the {@link GenericArray} that can
   *         be safely serialized into MongoDB.
   */
  private BasicDBList listToMongo(final String docf, final Collection<?> array,
      final Schema fieldSchema, final Type fieldType) {
    // Handle null case
    if (array == null)
      return null;

    // Handle regular cases
    BasicDBList list = new BasicDBList();
    for (Object item : array) {
      DocumentFieldType storeType = mapping.getDocumentFieldType(docf);
      Object result = toDBObject(docf, fieldSchema, fieldType, storeType, item);
      list.add(result);
    }

    return list;
  }

  // //////////////////////////////////////////////////////// CLEANUP

  /**
   * Ensure Key encoding -> dots replaced with middle dots
   * 
   * @param key
   *          char with only dots.
   * @return encoded string with "\u00B7" chars..
   */
  public String encodeFieldKey(final String key) {
    if (key == null) {
      return null;
    }
    return key.replace(".", "\u00B7");
  }

  /**
   * Ensure Key decoding -> middle dots replaced with dots
   * 
   * @param key
   *          encoded string with "\u00B7" chars.
   * @return Cleanup up char with only dots.
   */
  public String decodeFieldKey(final String key) {
    if (key == null) {
      return null;
    }
    return key.replace("\u00B7", ".");
  }
}
