# Script which calls different scripts for differing feature extraction.
# See the top-level README.md to see where the raw data should be placed.
#
# Required (language model features derivable):
#     Transcript raw data : disfluency detection corpus (already generated by
#     deep_disfluency.corpus.DisfluencyCorpusCreator.py on swda data
# Optional (if using word timing):
#     Transcript raw data: word timings from Mississippi Swbd files (download)
# Optional (if using audio features and/or ASR):
#     Audio raw data : .sph (or .wav) files (purchase from LDC)
#     OpenSmile (software) + .conf file
# Optional (if using ASR results):
#     IBM Watson ASR account
import argparse
import sys
import subprocess
import os


def extract_features(args):
    corpusName = args.divisionFile[args.divisionFile.rfind("/") + 1:].\
        replace("_ranges.text", "")
    if args.partial_words:
        corpusName += "_partial"

    corpus_filename = corpusName + "_data"
    if args.wordAlignmentFolder:
        corpus_filename += "_timings"
    corpus_filename += ".csv"
    do = False
    if args.wordAlignmentFolder and do:
        # link the word alignments to the disfluency detection corpora
        # also adds laughter
        c = [
            sys.executable,
            os.path.dirname(os.path.realpath(__file__)) +
            '/swbd_map_word_alignments_to_SWDA_words.py',
            '-i', args.corpusLocation + "/" + corpus_filename.
                                                    replace("_timings", ""),
            '-d', args.divisionFile,
            '-a', args.wordAlignmentFolder
            ]
        if args.laughter:
            c.append('-l')
        subprocess.call(c)

    if args.newTags:
        # create the tag representations (normally from the training data
        # not allowed to look into unseen tags in the test/dev set
        c = [
            sys.executable,
            os.path.dirname(os.path.realpath(__file__)) +
            '/create_tag_files.py',
            '-i', args.corpusLocation + "/" + corpus_filename,
            '-tag', args.tagFolder,
            ]
        if args.laughter:
            c.append('-l')
        if args.uttSeg:
            c.append('-u')
        if args.dialogueActs:
            c.append('-d')
        if args.joint:
            c.append('-joint')
        subprocess.call(c)

    if args.ASR:
        raise NotImplementedError("ASR results getting script TODO.")

    if args.posTagger:
        if args.train_pos:
            raise NotImplementedError("POS tag training script TODO")
        raise NotImplementedError("POS tagger extraction needs doing")

    if args.languageModelFolder:
        c = [
            sys.executable,
            os.path.dirname(os.path.realpath(__file__)) +
            '/add_language_model_features.py',
            '-i', args.corpusLocation + "/" + corpus_filename,
            '-lm', args.languageModelFolder,
            '-f', args.matrixFolder + "/lm_matrices",
            '-order', str(3),
            '-xlm',
            # '-tag', args.tagFolder,
            ]
        if args.partial_words:
            c.append("-p")
        c.append('-e')
        if args.uttSeg:
            c.append('-u')
        print c
        subprocess.call(c)

    if args.audioFolder:
        raise NotImplementedError("ASR scripts TODO.")

    # with the extracted separate raw and
    # (optionally) derived (LM, audio) features
    # add these together into single vectors for DNN training format
    # TODO for now this should only run for training and heldout data.
    # test features change based on ASR results.
    c = [
        sys.executable,
        os.path.dirname(os.path.realpath(__file__)) +
        '/save_feature_matrices.py',
        '-i', args.corpusLocation + "/" + corpus_filename,
        '-m', args.matrixFolder,
        '-w', args.tagFolder + "/swbd_word_rep.csv",
        '-p', args.tagFolder + "/swbd_pos_rep.csv",
        '-tag', args.tagFolder + "/swbd_disf1_tags.csv"
        ]
    if args.languageModelFolder:
        c.append('-lm')
        c.append(args.matrixFolder + "/lm_matrices")
    if args.audioFolder:
        c.append('-a')
        c.append(args.matrixFolder + + "/audio_matrices")
    subprocess.call(c)

    extraction_results = {"POS_accuracy": None, "asr_WER": None}
    return extraction_results


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Feature extraction for\
    disfluency and other tagging tasks from disfluency detection corpora and\
    raw data.')
    parser.add_argument('-i', action='store', dest='corpusLocation',
                        default='../data/disfluency_detection',
                        help='location of the disfluency\
                        detection corpus folder')
    parser.add_argument(
        '-m', action='store',
        dest='matrixFolder',
        default='../data/disfluency_detection/feature_matrices',
        help='location of the disfluency annotation csv files'
                        )
    parser.add_argument('-f', action='store', dest='divisionFile',
                        default='../data/disfluency_detection/\
                        swda_divisions_disfluency_detection/\
                        SWDisfTrain_ranges.text',
                        help='location of the file listing the \
                        files used in the corpus')
    parser.add_argument('-p', action='store_true', dest='partial_words',
                        default=False,
                        help='Whether to use partial words or not.')
    parser.add_argument('-a', action='store', dest='wordAlignmentFolder',
                        default=None,
                        help='location of the word alignment files')
    parser.add_argument('-tag', action='store', dest='tagFolder',
                        default=None,
                        help='location of the folder with the tag to\
                        tag index mapping')
    parser.add_argument('-new_tag', action='store_true', dest='newTags',
                        default=False,
                        help='Whether to save a new tag set generated from\
                        the data set to the tag folder.')
    parser.add_argument('-pos', action='store', dest='posTagger',
                        default=None, help='A POSTagger to tag the data.\
                        If None, Gold POS tags assumed.')
    parser.add_argument('-train_pos', action='store_true', dest='trainPOS',
                        default=False,
                        help='Whether to train POS a POS tagger on the data\
                        and save it at pos.')
    parser.add_argument('-u', action='store_true', dest='uttSeg',
                        default=False,
                        help='Whether to annotate with utterance segmentation\
                        tags.')
    parser.add_argument('-d', action='store_true', dest='dialogueActs',
                        default=False,
                        help='Whether to annotate with dialogue acts.')
    parser.add_argument('-l', action='store_true', dest='laughter',
                        default=False,
                        help='Whether to annotate with laughter.')
    parser.add_argument('-joint', action='store_true', dest='joint',
                        default=False,
                        help='Whether to create a joint tag set with the \
                        cross product of the tags (which appear in the data.')
    parser.add_argument('-lm', action='store', dest='languageModelFolder',
                        default=None,
                        help='Location of where to write a clean language\
                        model files out of this corpus.')
    parser.add_argument('-xlm', action='store_true',
                        dest='crossValLanguageModelTraining',
                        default=False,
                        help='Whether to use a cross language model\
                        training to be used for getting lm features on\
                        the same data.')
    parser.add_argument('-asr', action='store_true', dest='ASR',
                        default=False,
                        help='Whether to use IBM ASR to create ASR results.')
    parser.add_argument('-credentials', action='store', dest='credentials',
                        default="1841487c-30f4-4450-90bd-38d1271df295:\
                        EcqA8yIP7HBZ",
                        help="IBM Watson credentials of format username:pword")
    parser.add_argument('-audio', action='store', dest='audioFolder',
                        default=None,
                        help='location of the audio data \
                         files with .sph or wav files.')
    parser.add_argument('-opensmile', action='store', dest='openSmileConfig',
                        default=None,
                        help='location of the OpenSmile config file.')
    args = parser.parse_args()
    r = extract_features(args)
    print r
