anchor.diff:    (!foreign 'sense 'sense-id 'id))
   
   (defclass conjugation ()
  -  ((seq :reader seq :col-type integer :initarg :seq)
  +  ((id :reader id :col-type serial)
  +   (seq :reader seq :col-type integer :initarg :seq)
      (from :reader seq-from :col-type integer :initarg :from)
  -   (pos :reader pos :col-type string :initarg :pos)
      )
     (:documentation "conjugation link")
     (:metaclass dao-class)
  -  (:keys seq))
  +  (:keys id))
   
   (deftable conjugation
     (!dao-def)
  +  (!index 'seq)
     (!index 'from)
  -  (!index 'pos)
     (!foreign 'entry 'seq)
     (!foreign 'entry 'from 'seq))
   
  +(defmethod print-object ((obj conjugation) stream)
  +  (print-unreadable-object (obj stream :type t :identity nil)
  +    (format stream "~a -> ~a" (seq-from obj) (seq obj))))
  +
   (defclass conj-prop ()
     ((id :reader id :col-type serial)
  -   (seq :reader seq :col-type integer :initarg :seq)
  -   (neg :reader conj-neg :col-type boolean :initarg :neg)
  -   (fml :reader conj-fml :col-type boolean :initarg :fml))
  +   (conj-id :reader conj-id :col-type integer :initarg :conj-id)
  +   (conj-type :reader conj-type :col-type integer :initarg :conj-type)
  +   (pos :reader pos :col-type string :initarg :pos)
  +   (neg :reader conj-neg :col-type (or db-null boolean) :initarg :neg)
  +   (fml :reader conj-fml :col-type (or db-null boolean) :initarg :fml))
     (:metaclass dao-class)
     (:keys id))
   
   (deftable conj-prop
     (!dao-def)
  -  (!index 'seq)
  -  (!foreign 'entry 'seq))
  -
  +  (!index 'conj-id)
  +  (!foreign 'conjugation 'conj-id 'id))
  +
  +(defmethod print-object ((obj conj-prop) stream)
  +  (print-unreadable-object (obj stream :type t :identity nil)
  +    (format stream "[~a] ~a~[ Affirmative~; Negative~]~[ Plain~; Formal~]"
  +            (pos obj) 
  +            (get-conj-description (conj-type obj))
  +            (case (conj-neg obj) ((nil) 0) ((t) 1))
  +            (case (conj-fml obj) ((nil) 0) ((t) 1))
  +            )))
  +
  +(defun init-tables ()
  +  (with-connection *connection*
  +    (let ((tables '(entry kanji-text kana-text sense gloss sense-prop conjugation conj-prop)))
  +      (loop for table in (reverse tables)
  +         do (query (:drop-table :if-exists table)))
  +      (loop for table in tables
  +         do (create-table table)))))
  +
  +;; Taken from webgunk so that ichiran doesn't depend on it
  +;; strip-whitespace option is removed as it doesn't look necessary in this case
  +(defun node-text (node &rest args &key test)
  +  (let (values result)
  +    (when (or (not test) (funcall test node))
  +      (dom:do-node-list (node (dom:child-nodes node))
  +        (let ((val (case (dom:node-type node)
  +                     (:element (apply #'node-text node args))
  +                     (:text (dom:node-value node)))))
  +          (push val values))))
  +    (setf result (apply #'concatenate 'string (nreverse values)))))
  +
  +(defmacro do-node-list-ord ((ord-var node-var node-list) &body body)
  +  `(let ((,ord-var 0))
  +     (dom:do-node-list (,node-var ,node-list)
  +       ,@body
  +       (incf ,ord-var))))
  +
  +(defun insert-readings (node-list tag table seq pri)
  +  (do-node-list-ord (ord node node-list)
  +    (let* ((reading-node (dom:item (dom:get-elements-by-tag-name node tag) 0))
  +           (reading-text (node-text reading-node))
  +           (common :null))
  +      (dom:do-node-list (node (dom:get-elements-by-tag-name node pri))
  +        (let ((pri-tag (node-text node)))
  +          (if (eql common :null) (setf common 0))
  +          (when (alexandria:starts-with-subseq "nf" pri-tag)
  +            (setf common (parse-integer pri-tag :start 2)))))
  +      (make-dao table :seq seq :text reading-text :ord ord :common common))))
  +
  +(defun insert-sense-traits (sense-node tag sense-id seq)
  +  (do-node-list-ord (ord node (dom:get-elements-by-tag-name sense-node tag))
  +    (make-dao 'sense-prop :sense-id sense-id :tag tag :text (node-text node) :ord ord :seq seq)))
  +
  +(defun insert-senses (node-list seq)
  +  (do-node-list-ord (ord node node-list)
  +    (let ((sense-id (id (make-dao 'sense :seq seq :ord ord))))
  +      (do-node-list-ord (ord node (dom:get-elements-by-tag-name node "gloss"))
  +        (make-dao 'gloss :sense-id sense-id :text (node-text node) :ord ord))
  +      (insert-sense-traits node "pos" sense-id seq)
  +      (insert-sense-traits node "misc" sense-id seq)
  +      (insert-sense-traits node "dial" sense-id seq))))
  +
  +(defun load-entry (content)
  +  (let* ((parsed (cxml:parse content (cxml-dom:make-dom-builder)))
  +         (entseq-node (dom:item (dom:get-elements-by-tag-name parsed "ent_seq") 0))
  +         (seq (parse-integer (node-text entseq-node)))
  +         (entry-obj (make-dao 'entry :seq seq :content content)))
  +    (let* ((kanji-nodes (dom:get-elements-by-tag-name parsed "k_ele"))
  +           (kana-nodes (dom:get-elements-by-tag-name parsed "r_ele"))
  +           (n-kanji (dom:length kanji-nodes))
  +           (n-kana (dom:length kana-nodes))
  +           (sense-nodes (dom:get-elements-by-tag-name parsed "sense")))
  +      (insert-readings kanji-nodes "keb" 'kanji-text seq "ke_pri")
  +      (insert-readings kana-nodes "reb" 'kana-text seq "re_pri")
  +      (insert-senses sense-nodes seq)
  +      (setf (n-kanji entry-obj) n-kanji
  +            (n-kana entry-obj) n-kana)
  +      (update-dao entry-obj))))
  +
  +(defun fix-entities (source)
  +  "replaces entity definitions with abbreviations"
  +  (let ((entity-hash (cxml::dtd-gentities (cxml::dtd (slot-value source 'cxml::context)))))
  +    (maphash 
  +     (lambda (name entdef)
  +       (unless (member name '("lt" "gt" "amp" "apos" "quot") :test #'equal)
  +         (setf (cxml::entdef-value (cdr entdef)) name)))
  +     entity-hash)))
  +    
  +(defun load-jmdict (&key (path *jmdict-path*) (load-conjugations t))
  +  (init-tables)
  +  (with-connection *connection*
  +    (klacks:with-open-source (source (cxml:make-source path))
  +      (klacks:find-element source "JMdict")
  +      (fix-entities source)
  +      (loop for cnt from 1 ;; to 1000
  +         while (klacks:find-element source "entry")
  +         do
  +           (let ((content (klacks:serialize-element source (cxml:make-string-sink))))
  +             (load-entry content))
  +         if (zerop (mod cnt 1000)) do (format t "~a entries loaded~%" cnt)
  +         finally (query "ANALYZE") (format t "~a entries total~%" cnt)))
  +    (when load-conjugations
  +      (format t "Loading conjugations...~%")
  +      (load-conjugations)
  +      (query "ANALYZE"))
  +    ))
  +
  +;;; conjugations generator (warning: terrible code ahead)
   
   (defmacro csv-hash (hash-name (filename &key skip-first) loader-opts &rest accessor-opts-list)
     (let ((base-name (string-trim "*" hash-name))
  
positive.code:     (loop for rule in rules
           collect (cons rule (construct-conjugation word rule)))))
  
  (defun init-tables ()
    (with-connection *connection*
      (let ((tables '(entry kanji-text kana-text sense gloss sense-prop conjugation conj-prop)))
        (loop for table in (reverse tables)
           do (query (:drop-table :if-exists table)))
        (loop for table in tables
           do (create-table table)))))
  
  ;; Taken from webgunk so that ichiran doesn't depend on it
  ;; strip-whitespace option is removed as it doesn't look necessary in this case
  (defun node-text (node &rest args &key test)
    (let (values result)
      (when (or (not test) (funcall test node))
        (dom:do-node-list (node (dom:child-nodes node))
          (let ((val (case (dom:node-type node)
                       (:element (apply #'node-text node args))
                       (:text (dom:node-value node)))))
            (push val values))))
      (setf result (apply #'concatenate 'string (nreverse values)))))
  
  (defmacro do-node-list-ord ((ord-var node-var node-list) &body body)
    `(let ((,ord-var 0))
       (dom:do-node-list (,node-var ,node-list)
         ,@body
         (incf ,ord-var))))
  
  (defun insert-readings (node-list tag table seq pri)
    (do-node-list-ord (ord node node-list)
      (let* ((reading-node (dom:item (dom:get-elements-by-tag-name node tag) 0))
             (reading-text (node-text reading-node))
             (common :null))
        (dom:do-node-list (node (dom:get-elements-by-tag-name node pri))
          (let ((pri-tag (node-text node)))
            (if (eql common :null) (setf common 0))
            (when (alexandria:starts-with-subseq "nf" pri-tag)
              (setf common (parse-integer pri-tag :start 2)))))
        (make-dao table :seq seq :text reading-text :ord ord :common common))
      (incf ord)))
  
  (defun insert-sense-traits (sense-node tag sense-id seq)
    (do-node-list-ord (ord node (dom:get-elements-by-tag-name sense-node tag))
      (make-dao 'sense-prop :sense-id sense-id :tag tag :text (node-text node) :ord ord :seq seq)))
  
  (defun insert-senses (node-list seq)
    (do-node-list-ord (ord node node-list)
      (let ((sense-id (id (make-dao 'sense :seq seq :ord ord))))
        (do-node-list-ord (ord node (dom:get-elements-by-tag-name node "gloss"))
          (make-dao 'gloss :sense-id sense-id :text (node-text node) :ord ord))
        (insert-sense-traits node "pos" sense-id seq)
        (insert-sense-traits node "misc" sense-id seq)
        (insert-sense-traits node "dial" sense-id seq))))
  
  (defun load-entry (content)
    (let* ((parsed (cxml:parse content (cxml-dom:make-dom-builder)))
           (entseq-node (dom:item (dom:get-elements-by-tag-name parsed "ent_seq") 0))
           (seq (parse-integer (node-text entseq-node)))
           (entry-obj (make-dao 'entry :seq seq :content content)))
      (let* ((kanji-nodes (dom:get-elements-by-tag-name parsed "k_ele"))
             (kana-nodes (dom:get-elements-by-tag-name parsed "r_ele"))
             (n-kanji (dom:length kanji-nodes))
             (n-kana (dom:length kana-nodes))
             (sense-nodes (dom:get-elements-by-tag-name parsed "sense")))
        (insert-readings kanji-nodes "keb" 'kanji-text seq "ke_pri")
        (insert-readings kana-nodes "reb" 'kana-text seq "re_pri")
        (insert-senses sense-nodes seq)
        (setf (n-kanji entry-obj) n-kanji
              (n-kana entry-obj) n-kana)
        (update-dao entry-obj))))
  
  (defun fix-entities (source)
    "replaces entity definitions with abbreviations"
    (let ((entity-hash (cxml::dtd-gentities (cxml::dtd (slot-value source 'cxml::context)))))
      (maphash 
       (lambda (name entdef)
         (unless (member name '("lt" "gt" "amp" "apos" "quot") :test #'equal)
           (setf (cxml::entdef-value (cdr entdef)) name)))
       entity-hash)))
      
  (defun load-jmdict (&optional (path *jmdict-path*))
    (init-tables)
    (with-connection *connection*
      (klacks:with-open-source (source (cxml:make-source path))
        (klacks:find-element source "JMdict")
        (fix-entities source)
        (loop for cnt from 1 ;; to 1000
           while (klacks:find-element source "entry")
           do
             (let ((content (klacks:serialize-element source (cxml:make-string-sink))))
               (load-entry content))
           if (zerop (mod cnt 1000)) do (format t "~a entries loaded~%" cnt)
           finally (query "ANALYZE") (return cnt)))))
  
  (defun find-word (word)
    (select-dao 
  
negative.code:           ((pos-id pos description) pos (cons (parse-integer pos-id) description))
            (val (car val)))
  
  (csv-hash *pos-by-index* ("kwpos.csv")
            ((pos-id pos description) (parse-integer pos-id) pos)
            (get-pos val val))
  
  (csv-hash *conj-description* ("conj.csv" :skip-first t)
            ((conj-id description) (parse-integer conj-id) description)
            (val val))
  