text: /*=============================================================================
  # Filename: Database.h
  # Author: Bookug Lobert
  # Mail: 1181955272@qq.com
  # Last Modified: 2015-10-23 14:20
  # Description: originally written by liyouhuan, modified by zengli and chenjiaqi
  =============================================================================*/
  
  #ifndef _DATABASE_DATABASE_H
  #define _DATABASE_DATABASE_H
  
  #include "../Util/Util.h"
  #include "../Util/Triple.h"
  #include "Join.h"
  #include "../Query/IDList.h"
  #include "../Query/ResultSet.h"
  #include "../Query/SPARQLquery.h"
  #include "../Query/BasicQuery.h"
  #include "../Signature/SigEntry.h"
  #include "../VSTree/VSTree.h"
  #include "../KVstore/KVstore.h"
  #include "../StringIndex/StringIndex.h"
  #include "../Parser/DBparser.h"
  #include "../Parser/RDFParser.h"
  #include "../Parser/SparqlParser.h"
  #include "../Query/QueryCache.h"
  #include "../Query/GeneralEvaluation.h"
  
  class Database {
    public:
    //static const bool only_sub2idpre2id = true;
    //static const int internal = 100 * 1000;
  
    //void test();
    //void test_build_sig();
    //void test_join();
    //void printIDlist(int _i, int* _list, int _len, std::string _log);
    //void printPairList(int _i, int* _list, int _len, std::string _log);
  
    //when encode EntitySig, one way uses STRING-hash, the other one uses ID-hash
    //depending on this->encode_mode
    static const int STRING_MODE = 1;
    static const int ID_MODE = 2;
    Database();
    Database(std::string _name);
    void release(FILE* fp0);
    ~Database();
  
    bool save();
    bool load();
    bool unload();
    void clear();
    int query(const string _query, ResultSet& _result_set, FILE* _fp = stdout, bool update_flag = true, bool export_flag = false);
  
    //1. if subject of _triple doesn't exist,
    //then assign a new subid, and insert a new SigEntry
    //2. assign new tuple_id to tuple, if predicate or object doesn't exist before too;
    //3. if subject exist, update SigEntry, and update spo, ops... etc. if needed
  
    bool build(const string& _rdf_file);
    //interfaces to insert/delete from given rdf file
    bool insert(std::string _rdf_file, bool _is_restore = false);
    bool remove(std::string _rdf_file, bool _is_restore = false);
  
    bool backup();
    bool restore();
  
    //name of this DB
    string getName();
    //get infos
    TYPE_TRIPLE_NUM getTripleNum();
    TYPE_ENTITY_LITERAL_ID getEntityNum();
    TYPE_ENTITY_LITERAL_ID getLiteralNum();
    TYPE_ENTITY_LITERAL_ID getSubNum();
    TYPE_PREDICATE_ID getPreNum();
  
    //root Path of this DB + sixTuplesFile
    string getSixTuplesFile();
  
    //root Path of this DB + signatureBFile
    string getSignatureBFile();
  
    //root Path of this DB + DBInfoFile
    string getDBInfoFile();
  
    //id tuples file
    string getIDTuplesFile();
  
    VSTree* getVSTree();
    KVstore* getKVstore();
    StringIndex* getStringIndex();
    QueryCache* getQueryCache();
    TYPE_TRIPLE_NUM* getpre2num();
    TYPE_TRIPLE_NUM* getpre2sub();
    TYPE_TRIPLE_NUM* getpre2obj();
    TYPE_ENTITY_LITERAL_ID& getlimitID_literal();
    TYPE_ENTITY_LITERAL_ID& getlimitID_entity();
    TYPE_PREDICATE_ID& getlimitID_predicate();
    mutex& get_query_parse_lock();
  
    private:
    string name;
    string store_path;
    bool is_active;
    TYPE_TRIPLE_NUM triples_num;
    TYPE_ENTITY_LITERAL_ID entity_num;
    TYPE_ENTITY_LITERAL_ID sub_num;
    //BETTER: add object num
    TYPE_PREDICATE_ID pre_num;
    TYPE_ENTITY_LITERAL_ID literal_num;
  
    int encode_mode;
    bool if_loaded;
  
    //locks
    mutex query_parse_lock;
    //for read/write, we should use rwlock to improve parallism
    pthread_rwlock_t update_lock;
    //just for debug a block of code
    mutex debug_lock;
    // for getFinalResult
    mutex getFinalResult_lock;
  
    VSTree* vstree;
    KVstore* kvstore;
    StringIndex* stringindex;
    Join* join;
  
    //metadata of this database: sub_num, pre_num, obj_num, literal_num, etc.
    string db_info_file;
  
    //six tuples: <sub pre obj sid pid oid>
    string six_tuples_file;
  
    //B means binary
    string signature_binary_file;
  
    //id tuples file
    string id_tuples_file;
    string update_log;
    string update_log_since_backup;
  
    //pre2num mapping
    TYPE_TRIPLE_NUM* pre2num;
    //pre2subnum mapping
    TYPE_TRIPLE_NUM* pre2sub;
    //pre2objnum mapping
    TYPE_TRIPLE_NUM* pre2obj;
    //valid: check from minNumPID to maxNumPID
    TYPE_PREDICATE_ID maxNumPID, minNumPID;
    void setPreMap();
  
    //TODO: set the buffer capacity as dynamic according to the current memory usage
    //string buffer
    Buffer* entity_buffer;
    //unsigned offset; //maybe let id start from an offset
    unsigned entity_buffer_size;
    Buffer* literal_buffer;
    unsigned literal_buffer_size;
  
    QueryCache* query_cache;
  
    //Trie *trie;
  
    void setStringBuffer();
    void warmUp();
    //BETTER:add a predicate buffer for ?p query
    //However, I think this is not necessary because ?p is rare and the p2xx tree is small enough
  
    void query_stringIndex(int id);
    void check();
    //used for multiple threads
    void load_vstree(unsigned _vstree_size);
    void load_entity2id(int _mode);
    void load_id2entity(int _mode);
    void load_literal2id(int _mode);
    void load_id2literal(int _mode);
    void load_predicate2id(int _mode);
    void load_id2predicate(int _mode);
    void load_sub2values(int _mode);
    void load_obj2values(int _mode);
    void load_pre2values(int _mode);
  
    // functions used to build cache
    void load_cache();
    void get_important_preID();
    std::vector<TYPE_PREDICATE_ID> important_preID;
    void load_important_sub2values();
    void load_important_obj2values();
    void load_candidate_pre2values();
    void build_CacheOfPre2values();
    void build_CacheOfSub2values();
    void build_CacheOfObj2values();
    void get_important_subID();
    void get_important_objID();
    void get_candidate_preID();
    std::priority_queue<KEY_SIZE_VALUE> candidate_preID;
    std::priority_queue<KEY_SIZE_VALUE> important_subID;
    std::priority_queue<KEY_SIZE_VALUE> important_objID;
  
    //triple num per group for insert/delete
    //can not be too high, otherwise the heap will over
    static const int GROUP_SIZE = 1000;
    //manage the ID allocate and garbage
    static const TYPE_ENTITY_LITERAL_ID START_ID_NUM = 0;
    //static const int START_ID_NUM = 1000;
    /////////////////////////////////////////////////////////////////////////////////
    //NOTICE:error if >= LITERAL_FIRST_ID
    string free_id_file_entity;            //the first is limitID, then free id list
    TYPE_ENTITY_LITERAL_ID limitID_entity; //the current maxium ID num(maybe not used so much)
    BlockInfo* freelist_entity;            //free id list, reuse BlockInfo for Storage class
    TYPE_ENTITY_LITERAL_ID allocEntityID();
    void freeEntityID(TYPE_ENTITY_LITERAL_ID _id);
    /////////////////////////////////////////////////////////////////////////////////
    //NOTICE:error if >= 2*LITERAL_FIRST_ID
    string free_id_file_literal;
    TYPE_ENTITY_LITERAL_ID limitID_literal;
    BlockInfo* freelist_literal;
    TYPE_ENTITY_LITERAL_ID allocLiteralID();
    void freeLiteralID(TYPE_ENTITY_LITERAL_ID _id);
    /////////////////////////////////////////////////////////////////////////////////
    //NOTICE:error if >= 2*LITERAL_FIRST_ID
    string free_id_file_predicate;
    TYPE_PREDICATE_ID limitID_predicate;
    BlockInfo* freelist_predicate;
    TYPE_PREDICATE_ID allocPredicateID();
    void freePredicateID(TYPE_PREDICATE_ID _id);
    /////////////////////////////////////////////////////////////////////////////////
    void initIDinfo();  //initialize the members
    void resetIDinfo(); //reset the id info for build
    void readIDinfo();  //read and build the free list
    void writeIDinfo(); //write and empty the free list
    void saveIDinfo();  //write and empty the free list
  
    bool saveDBInfoFile();
    bool loadDBInfoFile();
  
    string getStorePath();
  
    //encode relative signature data of all Basic Graph Query, who union together into SPARQLquery
    void buildSparqlSignature(SPARQLquery& _sparql_q);
  
    //encode Triple into Subject EntityBitSet
    bool encodeTriple2SubEntityBitSet(EntityBitSet& _bitset, const Triple* _p_triple);
    //NOTICE: the encodeTriple with Triple* is invalid now(not enocde the linkage of neighbor-predicate)
    bool encodeTriple2SubEntityBitSet(EntityBitSet& _bitset, TYPE_PREDICATE_ID _pre_id, TYPE_ENTITY_LITERAL_ID _obj_id);
    //encode Triple into Object EntityBitSet
    bool encodeTriple2ObjEntityBitSet(EntityBitSet& _bitset, const Triple* _p_triple);
    bool encodeTriple2ObjEntityBitSet(EntityBitSet& _bitset, TYPE_PREDICATE_ID _pre_id, TYPE_ENTITY_LITERAL_ID _sub_id);
  
    bool calculateEntityBitSet(TYPE_ENTITY_LITERAL_ID _entity_id, EntityBitSet& _bitset);
  
    //check whether the relative 3-tuples exist
    //usually, through sp2olist
    bool exist_triple(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID _pre_id, TYPE_ENTITY_LITERAL_ID _obj_id);
    bool exist_triple(const TripleWithObjType& _triple);
  
    //* _rdf_file denotes the path of the RDF file, where stores the rdf data
    //* there are many step in this function, each one responds to an sub-function
    //* 1. map sub2id and pre2id
    //* 2. map literal2id and encode RDF data into signature,
    //*    storing in binary file: this->getSignatureBFile(), the order responds to subID
    //*    also, store six_tuples in file: this->getSixTuplesFile()
    //* 3. build: subID2objIDlist, <subIDpreID>2objIDlist subID2<preIDobjID>list
    //* 4. build: objID2subIDlist, <objIDpreID>2subIDlist objID2<preIDsubID>list
    //encodeRDF_new invoke new rdfParser to solve task 1 & 2 in one time scan.
    bool encodeRDF_new(const string _rdf_file);
    void readIDTuples(ID_TUPLE*& _p_id_tuples);
    void build_s2xx(ID_TUPLE*);
    void build_o2xx(ID_TUPLE*);
    void build_p2xx(ID_TUPLE*);
  
    //insert and delete, notice that modify is not needed here
    //we can read from file or use sparql syntax
    bool insertTriple(const TripleWithObjType& _triple, vector<unsigned>* _vertices = NULL, vector<unsigned>* _predicates = NULL);
    bool removeTriple(const TripleWithObjType& _triple, vector<unsigned>* _vertices = NULL, vector<unsigned>* _predicates = NULL);
    //NOTICE:one by one is too costly, sort and insert/delete at a time will be better
    unsigned insert(const TripleWithObjType* _triples, TYPE_TRIPLE_NUM _triple_num, bool _is_restore = false);
    //bool insert(const vector<TripleWithObjType>& _triples, vector<int>& _vertices, vector<int>& _predicates);
    unsigned remove(const TripleWithObjType* _triples, TYPE_TRIPLE_NUM _triple_num, bool _is_restore = false);
  
    bool sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file);
    //bool literal2id_RDFintoSignature(const string _rdf_file, int** _p_id_tuples, TYPE_TRIPLE_NUM _id_tuples_max);
  
    bool objIDIsEntityID(TYPE_ENTITY_LITERAL_ID _id);
  
    //* join on the vector of CandidateList, available after retrieve from the VSTREE
    //* and store the resut in _result_set
  
    //bool join(vector<int*>& _result_list, int _var_id, int _pre_id, int _var_id2, const char _edge_type, int _var_num, bool shouldAddLiteral, IDList& _can_list);
  
    //bool select(vector<int*>& _result_list, int _var_id, int _pre_id, int _var_id2, const char _edge_type, int _var_num);
  
    //get the final string result_set from SPARQLquery
    bool getFinalResult(SPARQLquery& _sparql_q, ResultSet& _result_set);
  
    static int read_update_log(const string _path, multiset<string>& _i, multiset<string>& _r);
    bool restore_update(multiset<string>& _i, multiset<string>& _r);
    void clear_update_log();
  };
  
  #endif //_DATABASE_DATABASE_H
  