Add maximum-sensitivity variation

Add maximum-sensitivity variation

#!/usr/bin/env python3
from collections import OrderedDict, defaultdict
import os
import json
import yaml
import re
from pathlib import Path
import redo
redo.ifchange('comap.py'); import comap

project = redo.base
project_path = Path('projects') / project

with redo.ifchange(evaluation_config=project_path / 'evaluation-config.yaml') \
        as files:
    evaluation_config = yaml.load(files['evaluation_config'])

databases = OrderedDict(evaluation_config['databases'])
outcome_ids = evaluation_config['outcome_ids']
concept_filenames = {
    outcome_id: '{}.{}.concepts.json'.format(project, outcome_id)
    for outcome_id in outcome_ids
}
child_concept_filenames = {
    outcome_id: '{}.{}.child-concepts.json'.format(project, outcome_id)
    for outcome_id in outcome_ids
}
casedef_paths = {
    outcome_id: project_path / 'case-definitions' / (outcome_id + '.yaml')
    for outcome_id in outcome_ids
}

with redo.ifchange(coding_systems='config/coding_systems.yaml',
                   references=(project_path / 'reference.yaml').as_posix(),
                   concepts=concept_filenames,
                   child_concepts=child_concept_filenames,
                   casedefs=casedef_paths) as files:
    coding_systems = yaml.load(files['coding_systems'])
    references = yaml.load(files['references'])
    concepts_by_outcome = {}
    for outcome_id, f in files['concepts'].items():
        concepts_by_outcome[outcome_id] = json.load(f)
    child_concepts_by_outcome = {}
    for outcome_id, f in files['child_concepts'].items():
        child_concepts_by_outcome[outcome_id] = json.load(f).items()
    outcomes = {
        outcome_id: yaml.load(f)
        for outcome_id, f in files['casedefs'].items()
    }

class Variation(object):

    """Identity"""

    def __init__(self):
        self.outcome_id = None
        self.coding_system = None

    def description(self):
        return self.__doc__

    def vary_code(self, code, coding_system):
        """Called on each code of generated concepts and the reference before
        before vary_codes and vary_concepts_and_mapping"""
        return code

    def vary_concepts_and_mapping(self, concepts, mapping, outcome_id):
        """Called for each outcome_id and database before vary_codes. For
        concept-level variations."""
        return concepts, mapping

    def vary_codes(self, generated, reference, coding_systems):
        """Code-level variations."""
        return generated, reference

class MaximumSensitivityVariation(Variation):

    """Maximium sensitivity"""

    def __init__(self):
        self.client = comap.ComapClient()

    def vary_concepts_and_mapping(self, concepts, mapping, outcome_id):
        cuis = set()
        for database_id in mapping.keys():
            coding_system = databases[database_id]
            codes = mapping[database_id]
            if coding_system == 'RCD2':
                coding_system = 'RCD'
                read_translation = comap.translation_read2_to_read3(codes)
                codes = set(code for codes in read_translation.values() for code in codes)
            if not codes:
                continue
            # ... WHERE cui IN (SELECT ... is too slow!
            # query = """
            #     SELECT DISTINCT cui, code FROM MRCONSO
            #     WHERE cui IN (SELECT DISTINCT cui FROM MRCONSO
            #                   WHERE sab = %s AND code IN ({placeholders}))
            #     AND sab = %s
            # """.format(placeholders=', '.join(['%s'] * len(codes)))
            # args = [coding_system] + list(codes) + [coding_system]
            cuis_query = """
                SELECT DISTINCT cui FROM MRCONSO
                WHERE sab = %s AND code IN ({placeholders})
            """.format(placeholders=', '.join(['%s'] * len(codes)))
            with comap.umls_db.cursor() as cursor:
                cursor.execute(cuis_query, tuple([coding_system] + list(codes)))
                cuis_for_codes = []
                for row in cursor.fetchall():
                    cuis_for_codes.append(row[0])
            query = """
                SELECT DISTINCT cui, code FROM MRCONSO
                WHERE cui IN ({placeholders})
            """.format(placeholders=', '.join(['%s'] * len(cuis_for_codes)))
            with comap.umls_db.cursor() as cursor:
                cursor.execute(query, tuple(cuis_for_codes))
                cui_codes = defaultdict(set)
                for row in cursor.fetchall():
                    cui, code = row
                    cui_codes[cui].add(code)
            for code in codes:
                # Find the `cui` that implicates `code` with the least number of irrelevant codes
                cuis_with_count = \
                    [(cui, len(set(codes) - set(codes1)))
                     for cui, codes1 in cui_codes.items()
                     if code in codes1]
                if cuis_with_count:
                    key = lambda x: x[1]
                    cui, _ = sorted(cuis_with_count, key=key)[0]
                    cuis.add(cui)
        return self.client.umls_concepts(cuis, coding_systems), mapping

class VariationChain(Variation):

    """Chain of variations that are applied to the input left-to-right."""

    def __init__(self, *variations):
        self.variations = variations

    def set_context(self, *args, **kwargs):
        for variation in self.variations:
            variation.set_context(*args, **kwargs)

    def description(self):
        res = '; '.join(v.description() for v in self.variations)
        if res:
            return res[0].upper() + res[1:]
        else:
            return ""

    def vary_code(self, code, coding_system):
        for variation in self.variations:
            code = variation.vary_code(code, coding_system)
        return code

    def vary_codes(self, generated, reference, coding_system):
        for variation in self.variations:
            generated, reference = variation.vary_codes(generated, reference, coding_system)
        return generated, reference

    def vary_concepts_and_mapping(self, concepts, mapping, outcome_id):
        for variation in self.variations:
            concepts, mapping = variation.vary_concepts_and_mapping(concepts, mapping, outcome_id)
        return concepts, mapping

class CodeNormalization(Variation):

    def __init__(self, *coding_system_prefixes):
        """`coding_systems` is None or a list of prefixes and this
        normalization is applied only if the list contains a prefix of the
        current coding systems."""
        self.coding_system_prefixes = coding_system_prefixes \
            or [''] # A prefix of any coding system

    def vary_code(self, code, coding_system):
        if any(coding_system.startswith(prefix)
               for prefix in self.coding_system_prefixes):
            return self.normalize(code, coding_system)
        else:
            return code

class NormalizeCodeCase(CodeNormalization):

    """Normalize code case in ICD, ICPC"""

    def __init__(self):
        super().__init__('ICD', 'ICPC')

    def normalize(self, code, coding_system):
        return code.upper()

class NormalizeCodeSuffixDot(CodeNormalization):

    """Drop suffix dots in codes in ICPC"""

    def __init__(self):
        super().__init__('ICPC')

    def normalize(self, code, coding_system):
        if code[-1] == '.':
            return code[:-1]
        else:
            return code

class NormalizeCodeXDD(CodeNormalization):

    """Normalize codes ICD and ICPC to 3 letters"""

    def __init__(self):
        super().__init__('ICD', 'ICPC')

    def normalize(self, code, coding_system):
        code = code.upper()
        m1 = re.match(comap.RegexHierarchy.TYPES['LDD'], code)
        try:
            return m1.group('parent')
        except:
            m2 = re.match(comap.RegexHierarchy.TYPES['DDD'], code)
            try:
                return m2.group('parent')
            except:
                return code

class NormalizeRead2Codes(CodeNormalization):

    """Normalize READ2 codes to 5-byte"""

    READ_RE = re.compile(r'(?P<code>[A-Z\d][A-Za-z0-9.]{4}).*')

    def __init__(self):
        super().__init__('RCD2')

    def normalize(self, code, coding_system):
        m = self.READ_RE.match(code)
        assert m, "Not a READ2 code: {}".format(code)
        return m.group('code')

class IncludeRelatedCodes(Variation):

    """Include related codes"""

    hierarchies = {
        'ICD9CM': comap.RegexHierarchy('DDD'),
        'ICD10CM': comap.RegexHierarchy('LDD'),
        'ICPC': comap.RegexHierarchy('LDD'),
        'RCD2': comap.Read2Hierarchy(),
    }

    def __init__(self, direction):
        """ direction : 'FN-FP' | 'FN' """
        self.direction = direction

    def description(self):
        return super().description() + ' ({})'.format(self.direction)

    def vary_codes(self, generated, reference, coding_system):
        hierarchy = self.hierarchies[coding_system]
        codes = comap.confusion_matrix(generated, reference)
        related_codes, derivated_codes = {
            'FN': comap.include_related,
            'FN-FP': comap.include_related_bidirect,
        }[self.direction](hierarchy, codes=codes)
        return comap.from_confusion_matrix(related_codes)

class IncludeChildConcepts(Variation):

    """Include child concepts"""

    def vary_concepts_and_mapping(self, concepts, mapping, outcome_id):
        cuis = [c['cui'] for c in concepts]
        child_concepts_by_parent = child_concepts_by_outcome[outcome_id]
        child_concepts = [
            child_concept
            for parent_id, concepts_for_parent in child_concepts_by_parent
            for child_concept in concepts_for_parent
            if child_concept not in cuis
        ]
        return concepts + child_concepts, mapping

class IncludeRelatedConcepts(Variation):

    """Include related concepts"""

    def __init__(self, *relations):
        self.relations = list(relations)
        self.client = comap.ComapClient()

    def description(self):
        return super().description() + ' ({})'.format(', '.join(self.relations))

    def vary_concepts_and_mapping(self, concepts, mapping, outcome_id):
        cuis = [c['cui'] for c in concepts]
        coding_systems = list(databases.values())
        related = self.client.related(cuis, self.relations, coding_systems)
        concepts_and_related = {c['cui']: c for c in concepts}
        for cui in related.keys():
            for rel in related[cui].keys():
                for new_concept in related[cui][rel]:
                    new_cui = new_concept['cui']
                    codes = set((sourceConcept['id'], sourceConcept['codingSystem'])
                                for sourceConcept in new_concept['sourceConcepts'])
                    if new_cui not in concepts_and_related:
                        concepts_and_related[new_cui] = new_concept
        return list(concepts_and_related.values()), mapping

def create_results(references, concepts_by_outcome, variations):

    # mapping ::= { database_id: { code } | None }

    # { outcome_id: mapping }
    mappings = {}
    for outcome_id in outcome_ids:
        mappings[outcome_id] = {}
        for database in databases.keys():
            mapping = references[database]['mappings'][outcome_id]
            if 'inclusion' in mapping:
                codes = set(mapping['inclusion'] + mapping.get('exclusion', []))
            else:
                codes = None
            mappings[outcome_id][database] = codes

    # { outcome_id: { variation_id: { "concepts": { concept }, "mapping": mapping } } }
    varied_by_outcome = {}
    for outcome_id in outcome_ids:
        varied_by_outcome[outcome_id] = {}
        concepts = concepts_by_outcome[outcome_id]
        reference_codes = OrderedDict([
            (database_id, mappings[outcome_id][database_id])
            for database_id in databases.keys()
        ])
        for variation_id, variation in variations.items():
            # Map codes in generated concepts with variation.vary_code
            concepts1 = []
            for concept in concepts_by_outcome[outcome_id]:
                concept1 = concept.copy()
                concept1['sourceConcepts'] = []
                for source_concept in concept['sourceConcepts']:
                    source_concept1 = source_concept.copy()
                    id, coding_system = source_concept['id'], source_concept['codingSystem']
                    source_concept1['id'] = variation.vary_code(id, coding_system)
                    concept1['sourceConcepts'].append(source_concept1)
                concepts1.append(concept1)
            # Map reference codes with variation.vary_code
            reference_codes1 = OrderedDict()
            for database_id in databases.keys():
                if reference_codes[database_id] is None:
                    reference_codes1[database_id] = None
                else:
                    reference_codes1[database_id] = [
                        variation.vary_code(code, databases[database_id])
                        for code in mappings[outcome_id][database_id]
                    ]
            concepts2, reference_codes2 = \
                variation.vary_concepts_and_mapping(concepts1, reference_codes1, outcome_id)
            assert concepts2 is not None, (outcome_id, variation_id)
            varied_by_outcome[outcome_id][variation_id] = {
                'concepts': concepts2,
                'reference-codes': reference_codes2,
            }

    res = OrderedDict()
    for outcome_id in outcome_ids:
        res[outcome_id] = OrderedDict()
        for database_id, coding_system in databases.items():
            res[outcome_id][database_id] = OrderedDict([])
            mapping = mappings[outcome_id][database_id]
            if mapping is not None:
                res[outcome_id][database_id]['variations'] = OrderedDict()
                for variation_id, variation in variations.items():
                    varied = varied_by_outcome[outcome_id][variation_id]
                    generated_codes = [
                        variation.vary_code(code['id'], coding_system)
                        for concept in varied['concepts']
                        for code in concept['sourceConcepts']
                        if code['codingSystem'] == coding_system
                    ]
                    reference_codes = [
                        variation.vary_code(code, coding_system)
                        for code in varied['reference-codes'][database_id]
                    ]
                    varied_generated_codes, varied_reference_codes = \
                        variation.vary_codes(generated_codes, reference_codes, coding_system)
                    varied_codes = comap.confusion_matrix(generated=varied_generated_codes,
                                                          reference=varied_reference_codes)
                    measures = comap.measures(codes=varied_codes)

                    res[outcome_id][database_id]['variations'][variation_id] = \
                        OrderedDict([
                            ('name', variation.description()),
                            ('comparison', OrderedDict([
                                ('codes', varied_codes),
                                ('measures', measures),
                            ])),
                    ])
            comment = references[database_id]['mappings'][outcome_id].get('comment')
            if comment is not None:
                res[outcome_id][database_id]['comment'] = comment
    return res

default_variations = VariationChain(NormalizeCodeCase(), NormalizeCodeSuffixDot(), NormalizeRead2Codes())
variations = OrderedDict([
    ('baseline', default_variations),
    ('maximum-sensitivity', VariationChain(default_variations, MaximumSensitivityVariation())),
    ('3-letter-codes', VariationChain(default_variations, NormalizeCodeXDD())),
    ('related-FN', VariationChain(default_variations, IncludeRelatedCodes('FN'))),
    ('related-FN-FP', VariationChain(default_variations, IncludeRelatedCodes('FN-FP'))),
    ('related-concepts-RN', VariationChain(default_variations, IncludeRelatedConcepts('RN'))),
    ('related-concepts-RN-RB', VariationChain(default_variations, IncludeRelatedConcepts('RN', 'RB'))),
    ('related-concepts-CHD', VariationChain(default_variations, IncludeRelatedConcepts('CHD'))),
    ('related-concepts-CHD-PAR', VariationChain(default_variations, IncludeRelatedConcepts('CHD', 'PAR'))),
    ('hyponyms', VariationChain(default_variations, IncludeChildConcepts())),
    ('child-concepts-related-FN-FP', VariationChain(default_variations, IncludeChildConcepts(), IncludeRelatedCodes('FN-FP'))),
])

evaluation = create_results(references, concepts_by_outcome, variations)

# Output results as YAML

with redo.output() as f:
    yaml.dump(evaluation, f)

if os.getenv('DOCTEST'):
    import doctest
    doctest.testmod()
    doctest.testmod(comap)

#!/usr/bin/env python3
from collections import OrderedDict
import os
import json
import yaml
import re
from pathlib import Path
import redo
redo.ifchange('comap.py'); import comap

project = redo.base
project_path = Path('projects') / project

with redo.ifchange(evaluation_config=project_path / 'evaluation-config.yaml') \
        as files:
    evaluation_config = yaml.load(files['evaluation_config'])

databases = OrderedDict(evaluation_config['databases'])
outcome_ids = evaluation_config['outcome_ids']
concept_filenames = {
    outcome_id: '{}.{}.concepts.json'.format(project, outcome_id)
    for outcome_id in outcome_ids
}
child_concept_filenames = {
    outcome_id: '{}.{}.child-concepts.json'.format(project, outcome_id)
    for outcome_id in outcome_ids
}
casedef_paths = {
    outcome_id: project_path / 'case-definitions' / (outcome_id + '.yaml')
    for outcome_id in outcome_ids
}

with redo.ifchange(coding_systems='config/coding_systems.yaml',
                   references=(project_path / 'reference.yaml').as_posix(),
                   concepts=concept_filenames,
                   child_concepts=child_concept_filenames,
                   casedefs=casedef_paths) as files:
    coding_systems = yaml.load(files['coding_systems'])
    references = yaml.load(files['references'])
    concepts_by_outcome = {}
    for outcome_id, f in files['concepts'].items():
        concepts_by_outcome[outcome_id] = json.load(f)
    child_concepts_by_outcome = {}
    for outcome_id, f in files['child_concepts'].items():
        child_concepts_by_outcome[outcome_id] = json.load(f).items()
    outcomes = {
        outcome_id: yaml.load(f)
        for outcome_id, f in files['casedefs'].items()
    }

class Variation(object):

    """Identity"""

    def __init__(self):
        self.outcome_id = None
        self.coding_system = None

    def description(self):
        return self.__doc__

    def vary_code(self, code, coding_system):
        """Called on each code of generated concepts and the reference before
        before vary_codes and vary_concepts_and_mapping"""
        return code

    def vary_concepts_and_mapping(self, concepts, mapping, outcome_id):
        """Called for each outcome_id and database before vary_codes. For
        concept-level variations."""
        return concepts, mapping

    def vary_codes(self, generated, reference, coding_systems):
        """Code-level variations."""
        return generated, reference

class VariationChain(Variation):

    """Chain of variations that are applied to the input left-to-right."""

    def __init__(self, *variations):
        self.variations = variations

    def set_context(self, *args, **kwargs):
        for variation in self.variations:
            variation.set_context(*args, **kwargs)

    def description(self):
        res = '; '.join(v.description() for v in self.variations)
        if res:
            return res[0].upper() + res[1:]
        else:
            return ""

    def vary_code(self, code, coding_system):
        for variation in self.variations:
            code = variation.vary_code(code, coding_system)
        return code

    def vary_codes(self, generated, reference, coding_system):
        for variation in self.variations:
            generated, reference = variation.vary_codes(generated, reference, coding_system)
        return generated, reference

    def vary_concepts_and_mapping(self, concepts, mapping, outcome_id):
        for variation in self.variations:
            concepts, mapping = variation.vary_concepts_and_mapping(concepts, mapping, outcome_id)
        return concepts, mapping

class CodeNormalization(Variation):

    def __init__(self, *coding_system_prefixes):
        """`coding_systems` is None or a list of prefixes and this
        normalization is applied only if the list contains a prefix of the
        current coding systems."""
        self.coding_system_prefixes = coding_system_prefixes \
            or [''] # A prefix of any coding system

    def vary_code(self, code, coding_system):
        if any(coding_system.startswith(prefix)
               for prefix in self.coding_system_prefixes):
            return self.normalize(code, coding_system)
        else:
            return code

class NormalizeCodeCase(CodeNormalization):

    """Normalize code case in ICD, ICPC"""

    def __init__(self):
        super().__init__('ICD', 'ICPC')

    def normalize(self, code, coding_system):
        return code.upper()

class NormalizeCodeSuffixDot(CodeNormalization):

    """Drop suffix dots in codes in ICPC"""

    def __init__(self):
        super().__init__('ICPC')

    def normalize(self, code, coding_system):
        if code[-1] == '.':
            return code[:-1]
        else:
            return code

class NormalizeCodeXDD(CodeNormalization):

    """Normalize codes ICD and ICPC to 3 letters"""

    def __init__(self):
        super().__init__('ICD', 'ICPC')

    def normalize(self, code, coding_system):
        code = code.upper()
        m1 = re.match(comap.RegexHierarchy.TYPES['LDD'], code)
        try:
            return m1.group('parent')
        except:
            m2 = re.match(comap.RegexHierarchy.TYPES['DDD'], code)
            try:
                return m2.group('parent')
            except:
                return code

class NormalizeRead2Codes(CodeNormalization):

    """Normalize READ2 codes to 5-byte"""

    READ_RE = re.compile(r'(?P<code>[A-Z\d][A-Za-z0-9.]{4}).*')

    def __init__(self):
        super().__init__('RCD2')

    def normalize(self, code, coding_system):
        m = self.READ_RE.match(code)
        assert m, "Not a READ2 code: {}".format(code)
        return m.group('code')

class IncludeRelatedCodes(Variation):

    """Include related codes"""

    hierarchies = {
        'ICD9CM': comap.RegexHierarchy('DDD'),
        'ICD10CM': comap.RegexHierarchy('LDD'),
        'ICPC': comap.RegexHierarchy('LDD'),
        'RCD2': comap.Read2Hierarchy(),
    }

    def __init__(self, direction):
        """ direction : 'FN-FP' | 'FN' """
        self.direction = direction

    def description(self):
        return super().description() + ' ({})'.format(self.direction)

    def vary_codes(self, generated, reference, coding_system):
        hierarchy = self.hierarchies[coding_system]
        codes = comap.confusion_matrix(generated, reference)
        related_codes, derivated_codes = {
            'FN': comap.include_related,
            'FN-FP': comap.include_related_bidirect,
        }[self.direction](hierarchy, codes=codes)
        return comap.from_confusion_matrix(related_codes)

class IncludeChildConcepts(Variation):

    """Include child concepts"""

    def vary_concepts_and_mapping(self, concepts, mapping, outcome_id):
        cuis = [c['cui'] for c in concepts]
        child_concepts_by_parent = child_concepts_by_outcome[outcome_id]
        child_concepts = [
            child_concept
            for parent_id, concepts_for_parent in child_concepts_by_parent
            for child_concept in concepts_for_parent
            if child_concept not in cuis
        ]
        return concepts + child_concepts, mapping

class IncludeRelatedConcepts(Variation):

    """Include related concepts"""

    def __init__(self, *relations):
        self.relations = list(relations)
        self.client = comap.ComapClient()

    def description(self):
        return super().description() + ' ({})'.format(', '.join(self.relations))

    def vary_concepts_and_mapping(self, concepts, mapping, outcome_id):
        cuis = [c['cui'] for c in concepts]
        coding_systems = list(databases.values())
        related = self.client.related(cuis, self.relations, coding_systems)
        concepts_and_related = {c['cui']: c for c in concepts}
        for cui in related.keys():
            for rel in related[cui].keys():
                for new_concept in related[cui][rel]:
                    new_cui = new_concept['cui']
                    codes = set((sourceConcept['id'], sourceConcept['codingSystem'])
                                for sourceConcept in new_concept['sourceConcepts'])
                    if new_cui not in concepts_and_related:
                        concepts_and_related[new_cui] = new_concept
        return list(concepts_and_related.values()), mapping

def create_results(references0, concepts_by_outcome, variations):

    # mapping ::= { database_id: { "codes": { code } | None, "comment": str | None } }

    # { outcome_id: mapping }
    mappings = {}
    for outcome_id in outcome_ids:
        mappings[outcome_id] = {}
        for database in databases.keys():
            mapping = references0[database]['mappings'][outcome_id]
            if 'inclusion' in mapping:
                codes = set(mapping['inclusion'] + mapping.get('exclusion', []))
            else:
                codes = None
            mappings[outcome_id][database] = OrderedDict([
                ('codes', codes),
                ('comment', mapping.get('comment')),
            ])

    # { outcome_id: { variation_id: { "concepts": { concept }, "mapping": mapping } } }
    varied_by_outcome = {}
    for outcome_id in outcome_ids:
        varied_by_outcome[outcome_id] = {}
        concepts = concepts_by_outcome[outcome_id]
        for variation_id, variation in variations.items():
            varied_generated, varied_mapping = \
                variation.vary_concepts_and_mapping(concepts, mappings[outcome_id], outcome_id)
            varied_by_outcome[outcome_id][variation_id] = {
                'concepts': varied_generated,
                'mapping': varied_mapping,
            }

    res = OrderedDict()
    for outcome_id in outcome_ids:
        res[outcome_id] = OrderedDict()
        for database, coding_system in databases.items():
            res[outcome_id][database] = OrderedDict([])
            mapping = mappings[outcome_id][database]
            if mapping['codes'] is not None:
                res[outcome_id][database]['variations'] = OrderedDict()
                for variation_id, variation in variations.items():
                    varied = varied_by_outcome[outcome_id][variation_id]
                    generated_codes = [
                        variation.vary_code(code['id'], coding_system)
                        for concept in varied['concepts']
                        for code in concept['sourceConcepts']
                        if code['codingSystem'] == coding_system
                    ]
                    reference_codes = [
                        variation.vary_code(code, coding_system)
                        for code in varied['mapping'][database]['codes']
                    ]
                    varied_generated_codes, varied_reference_codes = \
                        variation.vary_codes(generated_codes, reference_codes, coding_system)
                    varied_codes = comap.confusion_matrix(generated=varied_generated_codes,
                                                          reference=varied_reference_codes)
                    measures = comap.measures(codes=varied_codes)

                    res[outcome_id][database]['variations'][variation_id] = \
                        OrderedDict([
                            ('name', variation.description()),
                            ('comparison', OrderedDict([
                                ('codes', varied_codes),
                                ('measures', measures),
                            ])),
                    ])
            if mapping['comment'] is not None:
                res[outcome_id][database]['comment'] = mapping['comment']
    return res

default_variations = VariationChain(NormalizeCodeCase(), NormalizeCodeSuffixDot(), NormalizeRead2Codes())
variations = OrderedDict([
    ('baseline', default_variations),
    ('3-letter-codes', VariationChain(default_variations, NormalizeCodeXDD())),
    ('related-FN', VariationChain(default_variations, IncludeRelatedCodes('FN'))),
    ('related-FN-FP', VariationChain(default_variations, IncludeRelatedCodes('FN-FP'))),
    ('related-concepts-RN', VariationChain(default_variations, IncludeRelatedConcepts('RN'))),
    ('related-concepts-RN-RB', VariationChain(default_variations, IncludeRelatedConcepts('RN', 'RB'))),
    ('related-concepts-CHD', VariationChain(default_variations, IncludeRelatedConcepts('CHD'))),
    ('related-concepts-CHD-PAR', VariationChain(default_variations, IncludeRelatedConcepts('CHD', 'PAR'))),
    ('hyponyms', VariationChain(default_variations, IncludeChildConcepts())),
    ('child-concepts-related-FN-FP', VariationChain(default_variations, IncludeChildConcepts(), IncludeRelatedCodes('FN-FP'))),
])

evaluation = create_results(references, concepts_by_outcome, variations)

# Output results as YAML

with redo.output() as f:
    yaml.dump(evaluation, f)

if os.getenv('DOCTEST'):
    import doctest
    doctest.testmod()
    doctest.testmod(comap)


redo: add avrdude verify target

redo: add avrdude verify target

#!/bin/bash
#
# automaton
# (c) 2013 Justin Gottula
# The source code of this project is distributed under the terms of the
# simplified BSD license. See the LICENSE file for details.
#
# main redo script

exec >&2

# redo appears to be broken
set -e

TARGET=$1
TARGET_BASE=$2
OUTPUT=$3

PROJ_NAME="automaton"

# configuration files for microcontroller and programmer
CFG_MCU="atmega324a"
CFG_PROG="dragon_jtag"

CFG_FILES=("cfg/mcu/$CFG_MCU" "cfg/prog/$CFG_PROG")
redo-ifchange "${CFG_FILES[@]}"
for CFG_FILE in "${CFG_FILES[@]}"; do
	source "$CFG_FILE"
done

CC="avr-gcc"
AS="avr-as"
LD="avr-ld"

NM="avr-nm"
OBJCOPY="avr-objcopy"
OBJDUMP="avr-objdump"

AVRDUDE="avrdude"

SOURCES=$(find src -type f -iname "*.c")
OBJECTS=${SOURCES//.c/.o}

OUT_ELF="out/$PROJ_NAME.elf"
OUT_BIN="out/$PROJ_NAME.bin"
OUT_MAP="out/$PROJ_NAME.map"
OUT_DUMP="out/$PROJ_NAME.dump"
OUT_LST="out/$PROJ_NAME.lst"

DEFINES=("F_CPU=${CPU_KHZ}000UL")
INC_DIRS=(src)
LIB_DIRS=()
LIBS=()

CFLAGS_STD=(-std=gnu11)
CFLAGS_CPU=("-mmcu=$MCU" -fpack-struct -funsigned-bitfields)
CFLAGS_DEBUG=(-ggdb)
CFLAGS_OPT=(-Os -flto -fuse-linker-plugin -fwhole-program -mcall-prologues \
	-mrelax -mstrict-X -fmerge-all-constants -ffast-math -fno-jump-tables \
	-fshort-enums -Wl,--gc-sections)
CFLAGS_WARN=(-Wall -Wextra -Wno-unused-function -fno-diagnostics-show-caret)
CFLAGS_ETC=(-pipe)

CFLAGS=("${CFLAGS_STD[@]}" "${CFLAGS_CPU[@]}" "${CFLAGS_DEBUG[@]}" \
	"${CFLAGS_OPT[@]}" "${CFLAGS_WARN[@]}" "${CFLAGS_ETC[@]}")
CFLAGS_COMPILE=("${CFLAGS[@]}" "${DEFINES[@]/#/-D}" "${INC_DIRS[@]/#/-I}")
CFLAGS_LINK=("${CFLAGS[@]}" "${LIB_DIRS[@]/#/-L}" "${LIBS[@]/#/-l}")

AVRDUDE_FLAGS=("${AVRDUDE_PART[@]/#/-p}" "${AVRDUDE_PROG[@]/#/-c}" \
	"${AVRDUDE_PORT[@]/#/-P}" "${AVRDUDE_BITCLOCK[@]/#/-B}" \
	"${AVRDUDE_BAUDRATE[@]/#/-b}" "${AVRDUDE_BANGDELAY[@]/#/-i}")

AVRDUDE_FLAGS_LOAD=("${AVRDUDE_FLAGS[@]}" -u \
	-Uflash:w:"$OUT_BIN":r)
AVRDUDE_FLAGS_VERIFY=("${AVRDUDE_FLAGS[@]}" -u \
	-Uflash:v:"$OUT_BIN":r)
AVRDUDE_FLAGS_FUSE=("${AVRDUDE_FLAGS[@]}" -q \
	-Ulfuse:w:"$AVRDUDE_LFUSE":m \
	-Uhfuse:w:"$AVRDUDE_HFUSE":m \
	-Uefuse:w:"$AVRDUDE_EFUSE":m)
AVRDUDE_FLAGS_TERM=("${AVRDUDE_FLAGS[@]}" -t)

case "$TARGET" in
all)
	redo-ifchange $OUT_ELF $OUT_BIN $OUT_MAP $OUT_DUMP $OUT_LST
	;;
$OUT_ELF)
	redo-ifchange $OBJECTS
	$CC "${CFLAGS_LINK[@]}" -o$OUTPUT $OBJECTS
	;;
$OUT_BIN)
	redo-ifchange $OUT_ELF
	$OBJCOPY -Obinary -j.text -j.data $OUT_ELF $OUTPUT

	stat --printf="%n: %s bytes\n" $OUTPUT
	md5sum $OUTPUT
	;;
$OUT_MAP)
	redo-ifchange $OUT_ELF
	$NM -Cn $OUT_ELF >$OUTPUT
	;;
$OUT_DUMP)
	redo-ifchange $OUT_ELF
	$OBJDUMP -aCdfhp $OUT_ELF >$OUTPUT
	;;
$OUT_LST)
	redo-ifchange $OUT_ELF
	$OBJDUMP -CdS -j.text $OUT_ELF >$OUTPUT
	;;
*.o)
	$CC "${CFLAGS_COMPILE[@]}" -o${TARGET//.o/.dep} -MM -MG ${TARGET//.o/.c}

	read DEPS <${TARGET//.o/.dep}
	redo-ifchange ${DEPS#*:}

	$CC "${CFLAGS_COMPILE[@]}" -o$OUTPUT -c ${TARGET//.o/.c}
	;;
clean)
	rm -rf $(find out -type f)
	rm -rf $(find src -type f -iname "*.o")
	rm -rf $(find src -type f -iname "*.dep")
	;;
load)
	redo all
	echo "sudo $AVRDUDE ${AVRDUDE_FLAGS_LOAD[@]}"
	sudo $AVRDUDE "${AVRDUDE_FLAGS_LOAD[@]}"
	;;
verify)
	echo "sudo $AVRDUDE ${AVRDUDE_FLAGS_VERIFY[@]}"
	sudo $AVRDUDE "${AVRDUDE_FLAGS_VERIFY[@]}"
	;;
fuse)
	echo "sudo $AVRDUDE ${AVRDUDE_FLAGS_FUSE[@]}"
	sudo $AVRDUDE "${AVRDUDE_FLAGS_FUSE[@]}"
	;;
term)
	echo "sudo $AVRDUDE ${AVRDUDE_FLAGS_TERM[@]}"
	sudo $AVRDUDE "${AVRDUDE_FLAGS_TERM[@]}"
	;;
*)
	echo "unknown target '$TARGET'"
	exit 1
	;;
esac

#!/bin/bash
#
# automaton
# (c) 2013 Justin Gottula
# The source code of this project is distributed under the terms of the
# simplified BSD license. See the LICENSE file for details.
#
# main redo script

exec >&2

# redo appears to be broken
set -e

TARGET=$1
TARGET_BASE=$2
OUTPUT=$3

PROJ_NAME="automaton"

# configuration files for microcontroller and programmer
CFG_MCU="atmega324a"
CFG_PROG="dragon_jtag"

CFG_FILES=("cfg/mcu/$CFG_MCU" "cfg/prog/$CFG_PROG")
redo-ifchange "${CFG_FILES[@]}"
for CFG_FILE in "${CFG_FILES[@]}"; do
	source "$CFG_FILE"
done

CC="avr-gcc"
AS="avr-as"
LD="avr-ld"

NM="avr-nm"
OBJCOPY="avr-objcopy"
OBJDUMP="avr-objdump"

AVRDUDE="avrdude"

SOURCES=$(find src -type f -iname "*.c")
OBJECTS=${SOURCES//.c/.o}

OUT_ELF="out/$PROJ_NAME.elf"
OUT_BIN="out/$PROJ_NAME.bin"
OUT_MAP="out/$PROJ_NAME.map"
OUT_DUMP="out/$PROJ_NAME.dump"
OUT_LST="out/$PROJ_NAME.lst"

DEFINES=("F_CPU=${CPU_KHZ}000UL")
INC_DIRS=(src)
LIB_DIRS=()
LIBS=()

CFLAGS_STD=(-std=gnu11)
CFLAGS_CPU=("-mmcu=$MCU" -fpack-struct -funsigned-bitfields)
CFLAGS_DEBUG=(-ggdb)
CFLAGS_OPT=(-Os -flto -fuse-linker-plugin -fwhole-program -mcall-prologues \
	-mrelax -mstrict-X -fmerge-all-constants -ffast-math -fno-jump-tables \
	-fshort-enums -Wl,--gc-sections)
CFLAGS_WARN=(-Wall -Wextra -Wno-unused-function -fno-diagnostics-show-caret)
CFLAGS_ETC=(-pipe)

CFLAGS=("${CFLAGS_STD[@]}" "${CFLAGS_CPU[@]}" "${CFLAGS_DEBUG[@]}" \
	"${CFLAGS_OPT[@]}" "${CFLAGS_WARN[@]}" "${CFLAGS_ETC[@]}")
CFLAGS_COMPILE=("${CFLAGS[@]}" "${DEFINES[@]/#/-D}" "${INC_DIRS[@]/#/-I}")
CFLAGS_LINK=("${CFLAGS[@]}" "${LIB_DIRS[@]/#/-L}" "${LIBS[@]/#/-l}")

AVRDUDE_FLAGS=("${AVRDUDE_PART[@]/#/-p}" "${AVRDUDE_PROG[@]/#/-c}" \
	"${AVRDUDE_PORT[@]/#/-P}" "${AVRDUDE_BITCLOCK[@]/#/-B}" \
	"${AVRDUDE_BAUDRATE[@]/#/-b}" "${AVRDUDE_BANGDELAY[@]/#/-i}")

AVRDUDE_FLAGS_LOAD=("${AVRDUDE_FLAGS[@]}" -u \
	-Uflash:w:"$OUT_BIN":r)
AVRDUDE_FLAGS_FUSE=("${AVRDUDE_FLAGS[@]}" -q \
	-Ulfuse:w:"$AVRDUDE_LFUSE":m \
	-Uhfuse:w:"$AVRDUDE_HFUSE":m \
	-Uefuse:w:"$AVRDUDE_EFUSE":m)
AVRDUDE_FLAGS_TERM=("${AVRDUDE_FLAGS[@]}" -t)

case "$TARGET" in
all)
	redo-ifchange $OUT_ELF $OUT_BIN $OUT_MAP $OUT_DUMP $OUT_LST
	;;
$OUT_ELF)
	redo-ifchange $OBJECTS
	$CC "${CFLAGS_LINK[@]}" -o$OUTPUT $OBJECTS
	;;
$OUT_BIN)
	redo-ifchange $OUT_ELF
	$OBJCOPY -Obinary -j.text -j.data $OUT_ELF $OUTPUT

	stat --printf="%n: %s bytes\n" $OUTPUT
	md5sum $OUTPUT
	;;
$OUT_MAP)
	redo-ifchange $OUT_ELF
	$NM -Cn $OUT_ELF >$OUTPUT
	;;
$OUT_DUMP)
	redo-ifchange $OUT_ELF
	$OBJDUMP -aCdfhp $OUT_ELF >$OUTPUT
	;;
$OUT_LST)
	redo-ifchange $OUT_ELF
	$OBJDUMP -CdS -j.text $OUT_ELF >$OUTPUT
	;;
*.o)
	$CC "${CFLAGS_COMPILE[@]}" -o${TARGET//.o/.dep} -MM -MG ${TARGET//.o/.c}

	read DEPS <${TARGET//.o/.dep}
	redo-ifchange ${DEPS#*:}

	$CC "${CFLAGS_COMPILE[@]}" -o$OUTPUT -c ${TARGET//.o/.c}
	;;
clean)
	rm -rf $(find out -type f)
	rm -rf $(find src -type f -iname "*.o")
	rm -rf $(find src -type f -iname "*.dep")
	;;
load)
	redo all
	echo "sudo $AVRDUDE ${AVRDUDE_FLAGS_LOAD[@]}"
	sudo $AVRDUDE "${AVRDUDE_FLAGS_LOAD[@]}"
	;;
fuse)
	echo "sudo $AVRDUDE ${AVRDUDE_FLAGS_FUSE[@]}"
	sudo $AVRDUDE "${AVRDUDE_FLAGS_FUSE[@]}"
	;;
term)
	echo "sudo $AVRDUDE ${AVRDUDE_FLAGS_TERM[@]}"
	sudo $AVRDUDE "${AVRDUDE_FLAGS_TERM[@]}"
	;;
*)
	echo "unknown target '$TARGET'"
	exit 1
	;;
esac


iematch : implemented maxdiff

iematch : implemented maxdiff

implement option maxdiff. If a observation does not have an target
observation within this value then it is not matched.

	clear all

	*command starts here

	cap program drop 	iematch
	    program define	iematch

		syntax  [if] [in]  , ///
			GRPdummy(varname) 		///
			MATCHcont(varname) 		///
			[						///
			idvar(varname)			///
			matchidname(string)		///
			matchdiffname(string)	///
			matchnoname(string)		///
			maxdiff(numlist max = 1 min = 1) ///
			]

		*****
		*1 format errors as smcl
		*2 make match only on obs with value in group dummy
		*3 m:1 match
		*4 option to disable countdown

		noi di "Syntax OK!"

		pause on

		qui {

		********************************
		*
		*	Gen temp sort and merge var
		*
		********************************

		tempvar originalSort

		gen `originalSort' = _n

		preserve

			if "`if'`in'" != "" keep `if'`in'

			********************************
			*
			*	Checking group dummy
			*
			********************************

			*Test that the group dummy is 1, 0 or missing
			cap assert inlist(`grpdummy', 1, 0, .) | `grpdummy' > .
			if _rc != 0 {

				di as error "The variable in grpdummy(`grpdummy') is not a dummy variable. The variable is only allowed to have the values 1, 0 or missing. Observations with missing varaibles in the grpdummy are ignored by this command."
				error _rc
			}

			**Exclude obs with missing value in groupdummy

			*Count number of obs to be dropped and and output that number if more than zero
			count if missing(`grpdummy')
			if `r(N)' > 0 {
				*This is not an error just outputting the number
				noi di "`r(N)' observation(s) were excluded due to missing value in group dummy"
			}
			*Drop obs with missing
			drop if missing(`grpdummy')

			********************************
			*
			*	Checking match continous var
			*
			********************************

			*Test that the continous is numeric
			cap confirm numeric variable `matchcont'
			if _rc != 0 {

				di as error "The variable to match on specified in matchcont(`matchcont') is not a numeric variable."
				error 109

			}

			**Exclude obs with missing value in groupdummy

			*Count number of obs to be dropped and and output that number if more than zero
			count if missing(`matchcont')
			if `r(N)' > 0 {
				*This is not an error just outputting the number
				noi di "`r(N)' observation(s) were excluded due to missing value in match varaible"
			}
			*Drop obs with missing
			drop if missing(`matchcont')

			********************************
			*
			*	Checking duplicates in varaible names in options
			*
			********************************

			*Create a local of all varnames used in any option
			local allnewvars `idvar' `matchidname' `matchdiffname' `matchnoname'

			*Check if there are any duplicates in the local of all varnames in options
			local dupnewvars : list dups allnewvars

			*Throw error if there were any duplicates
			if "`dupnewvars'" != "" {

				di as error "The same variable name was used twice or more in the options. Go back and check syntax"
				error 198

			}

			********************************
			*
			*	Checking ID var
			*
			********************************

			*Checking set-up for the ID var used for _matchID
			if "`idvar'" == "" {

				*Copy this name
				local idvar _ID

				*Make sure that the default name _ID is not already used
				cap confirm variable `idvar'
				if _rc == 0 {

					di as error "A variable with name `idvar' is already defined. Either drop this variable or specify a variable using idvar() that fully and uniquely identifies the data set."
					error 110

				}

				*Generate the ID var from row number
				gen `idvar' = _n

			}
			else {

				*Make sure that user specified ID is uniquely and fully identifying the observations
				cap isid `idvar'
				if _rc != 0 {

					di as error "Variable `idvar' specified in idvar() does not fully and uniquely identify the observations in the data set. Meaning that `idvar' is not allowed to have any duplicates or missing values. Make `idvar' uniquly and fully idnetifying or exclude the varaibleas using if or in."
					error 450

				}
			}

			********************************
			*
			*	Checking and generating the MatchID and the MatchDiff var
			*
			********************************

			*Test input from user or test if deafault value is valiable
			iematchMatchVarCheck _matchID `matchidname'
			local matchIDname "`r(validVarName)'"
			gen  `matchIDname' = .

			iematchMatchVarCheck _matchDiff `matchdiffname'
			local matchDiffName "`r(validVarName)'"
			gen `matchDiffName' = .

			iematchMatchVarCheck _matchOrReason `matchnoname'
			local noMatchReasonName "`r(validVarName)'"
			gen `noMatchReasonName' = .

			*Create a label for missing values that explains no match
			label define matchLabel 0 "Not Matched" 1 "Matched" .n "Obs not included in match" .m "No match within maxdiff"

			*Apply the label
			label value `noMatchReasonName' matchLabel

			********************************
			*
			*	Creating tempvar used
			*
			********************************

			*Initiate the temporary variables used by this command
			tempvar hi_diff lo_diff pref match newMatch rand maxDiffObs maxDiffNow

			gen `hi_diff' = .
			gen `lo_diff' = .

			gen `pref' = .
			gen byte `match' = 0
			gen byte `newMatch' = 0

			** Generate random variable that seperate two obs
			*  with the same value in the continuos variable
			gen `rand' = uniform()

			if "`maxdiff'" != ""  gen byte `maxDiffObs' = 0
			if "`maxdiff'" != ""  gen `maxDiffNow' = 0

			********************************
			*
			*	Start matching
			*
			********************************
			noi di ""
			noi di "{pstd}Observations left to match: {p_end}"
			count if `grpdummy' == 1 & `match' == 0
			local left2Match = `r(N)'
			noi di "{pstd}`left2Match' " _c

			while (`left2Match' > 0) {

				sort `match' `matchcont' `rand'

				replace `lo_diff' 	= .
				replace `hi_diff' 	= .
				replace `pref'		= . if `match' == 0
				replace `newMatch' 	= 0

				replace `lo_diff' = abs(`matchcont' - `matchcont'[_n-1]) if `match' == 0 & `grpdummy' != `grpdummy'[_n-1]
				replace `hi_diff' = abs(`matchcont' - `matchcont'[_n+1]) if `match' == 0 & `grpdummy' != `grpdummy'[_n+1]

				replace `pref' = `idvar'[_n-1] if `match' == 0 & `lo_diff' <  `hi_diff'
				replace `pref' = `idvar'[_n+1] if `match' == 0 & `lo_diff' >= `hi_diff'

				if "`maxdiff'" != "" {

					replace `maxDiffObs' 	= 0
					replace `maxDiffNow'	= 0
					replace `maxDiffNow'	= min(`lo_diff', `hi_diff') if min(`lo_diff', `hi_diff') < .

					replace `maxDiffObs' 	= 1 if `maxDiffNow' > `maxdiff' & `match' == 0 & `grpdummy' == 1

					replace `match' = 1					if `maxDiffObs'  == 1
					replace `pref'  = .					if `maxDiffObs'  == 1
					replace `noMatchReasonName' = .m 	if `maxDiffObs'  == 1

				}

				replace `newMatch' = 1 if `match' == 0 & `pref' == `idvar'[_n-1] & `pref'[_n-1] == `idvar'
				replace `newMatch' = 1 if `match' == 0 & `pref' == `idvar'[_n+1] & `pref'[_n+1] == `idvar'

				replace `match' = 1 								if `newMatch' == 1
				replace `matchDiffName' = min(`lo_diff', `hi_diff') if `newMatch' == 1
				replace `matchIDname' = `idvar' 					if `newMatch' == 1 & `grpdummy' == 1
				replace `matchIDname' = `pref' 						if `newMatch' == 1 & `grpdummy' == 0

				count if `grpdummy' == 1 & `match' == 0
				local left2Match = `r(N)'
				noi di "`left2Match' " _c

			}

			replace `noMatchReasonName' = 0 if `match' == 0 & `grpdummy' == 0

			keep `matchIDname' `matchDiffName' `idvar' `originalSort' `noMatchReasonName'

			tempfile mergefile

			save `mergefile'

		restore

		tempvar mergevar

		merge 1:1  `originalSort' using `mergefile', gen(`mergevar')

		replace `noMatchReasonName' = .n if `mergevar' == 1

		replace `noMatchReasonName' = 1 if `noMatchReasonName' == .

		noi di "{p_end}" _c
		noi tab `noMatchReasonName' `grpdummy', m

		sort `originalSort'

	}

	end

	*Cheack matchVar input
	cap program drop 	iematchMatchVarCheck
		program define 	iematchMatchVarCheck  , rclass

		args defaultName userName

		if "`defaultName'" == "_matchID" 		local optionName matchidname
		if "`defaultName'" == "_matchDiff" 		local optionName matchdiffname
		if "`defaultName'" == "_matchOrReason" 	local optionName matchnoname

		*All the default names
		local dfltNms _noMatchReason _matchDiff _matchOrReason

		*The other two default names
		local othDfltNms : list dfltNms - defaultName

		*Creating two locals with the deafult name in each local
		local Oth1 : word 1 of `othDfltNms'
		local Oth2 : word 2 of `othDfltNms'

		*Testing that the option name is not the same as the other variable's default name.
		if ("`defaultName'" == "_matchID" & ("`userName'" == "`Oth1'" | "`userName'" == "`Oth2'" | "`userName'" == "_ID"))  {

			noi di as error `" ("`defaultName'" == "_matchID" & ("`userName'" == "`Oth1'" | "`userName'" == "`Oth2'" | "`userName'" == "_ID")) "'

			noi di as error "The new name specified in `optionName'() is not allowed to be _ID, `Oth1', nor `Oth2'"
			error 198
		}

		*Checking set-up for the ID var used for _matchID
		if "`userName'" == "" {

			*CSet the default name
			local validMatchVarname `defaultName'

			*Make sure that the default name _ID is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				noi di as error "A variable with name `validMatchVarname' is already defined. Either drop this variable or specify a new variable name using `optionName'()."
				error 110

			}

		}

		else {

			*CSet the default name
			local validMatchVarname `userName'

			*Make sure that the default name _ID is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				noi di as error "The variable name specified in `optionName'(`userName') is already defined in the data set. Either drop this variable or specify a another variable name using `optionName'()."
				error 110

			}

		}

		*Testing that the new name is valid by creating a variable that is dropped immedeatly afterwards
		cap gen `validMatchVarname' = .

		if _rc != 0 {

			noi di as error "The variable name specified in `optionName'(`userName') is not a valid variable name."
			error _rc

		}
		drop `validMatchVarname'

		return local validVarName 	"`validMatchVarname'"

	end

	clear

*	set seed 1235324

	set obs 40000

	gen id = _n

	gen rand1 = uniform()

	gen tmt = (rand1 < .40)

	replace tmt = . if (rand1 < .05)

	drop rand1
	tab tmt

	gen p_hat = uniform()

	replace p_hat = p_hat + .01 if tmt == 1

	*replace p_hat = . if p_hat < .01

	iematch  , grp(tmt) match(p_hat) idvar(id)  maxdiff(.01) // matchidname(Kallefille)

	clear all

	*command starts here

	cap program drop 	iematch
	    program define	iematch

		syntax  [if] [in]  , ///
			GRPdummy(varname) 		///
			MATCHcont(varname) 		///
			[						///
			idvar(varname)			///
			matchidname(string)		///
			matchdiffname(string)	///
			matchnoname(string)		///
			]

		*****
		*1 format errors as smcl
		*2 make match only on obs with value in group dummy
		*3 m:1 match

		noi di "Syntax OK!"

		pause on

		qui {

		********************************
		*
		*	Gen temp sort and merge var
		*
		********************************

		tempvar originalSort

		gen `originalSort' = _n

		preserve

			if "`if'`in'" != "" keep `if'`in'

			********************************
			*
			*	Checking group dummy
			*
			********************************

			*Test that the group dummy is 1, 0 or missing
			cap assert inlist(`grpdummy', 1, 0, .) | `grpdummy' > .
			if _rc != 0 {

				di as error "The variable in grpdummy(`grpdummy') is not a dummy variable. The variable is only allowed to have the values 1, 0 or missing. Observations with missing varaibles in the grpdummy are ignored by this command."
				error _rc
			}

			**Exclude obs with missing value in groupdummy

			*Count number of obs to be dropped and and output that number if more than zero
			count if missing(`grpdummy')
			if `r(N)' > 0 {
				*This is not an error just outputting the number
				noi di "`r(N)' observation(s) were excluded due to missing value in group dummy"
			}
			*Drop obs with missing
			drop if missing(`grpdummy')

			********************************
			*
			*	Checking match continous var
			*
			********************************

			*Test that the continous is numeric
			cap confirm numeric variable `matchcont'
			if _rc != 0 {

				di as error "The variable to match on specified in matchcont(`matchcont') is not a numeric variable."
				error 109

			}

			**Exclude obs with missing value in groupdummy

			*Count number of obs to be dropped and and output that number if more than zero
			count if missing(`matchcont')
			if `r(N)' > 0 {
				*This is not an error just outputting the number
				noi di "`r(N)' observation(s) were excluded due to missing value in match varaible"
			}
			*Drop obs with missing
			drop if missing(`matchcont')

			********************************
			*
			*	Checking duplicates in varaible names in options
			*
			********************************

			*Create a local of all varnames used in any option
			local allnewvars `idvar' `matchidname' `matchdiffname' `matchnoname'

			*Check if there are any duplicates in the local of all varnames in options
			local dupnewvars : list dups allnewvars

			*Throw error if there were any duplicates
			if "`dupnewvars'" != "" {

				di as error "The same variable name was used twice or more in the options. Go back and check syntax"
				error 198

			}

			********************************
			*
			*	Checking ID var
			*
			********************************

			*Checking set-up for the ID var used for _matchID
			if "`idvar'" == "" {

				*Copy this name
				local idvar _ID

				*Make sure that the default name _ID is not already used
				cap confirm variable `idvar'
				if _rc == 0 {

					di as error "A variable with name `idvar' is already defined. Either drop this variable or specify a variable using idvar() that fully and uniquely identifies the data set."
					error 110

				}

				*Generate the ID var from row number
				gen `idvar' = _n

			}
			else {

				*Make sure that user specified ID is uniquely and fully identifying the observations
				cap isid `idvar'
				if _rc != 0 {

					di as error "Variable `idvar' specified in idvar() does not fully and uniquely identify the observations in the data set. Meaning that `idvar' is not allowed to have any duplicates or missing values. Make `idvar' uniquly and fully idnetifying or exclude the varaibleas using if or in."
					error 450

				}
			}

			********************************
			*
			*	Checking and generating the MatchID and the MatchDiff var
			*
			********************************

			*Test input from user or test if deafault value is valiable
			iematchMatchVarCheck _matchID `matchidname'
			local matchIDname "`r(validVarName)'"
			gen  `matchIDname' = .

			iematchMatchVarCheck _matchDiff `matchdiffname'
			local matchDiffName "`r(validVarName)'"
			gen `matchDiffName' = .

			iematchMatchVarCheck _matchOrReason `matchnoname'
			local noMatchReasonName "`r(validVarName)'"
			gen `noMatchReasonName' = .

			*Create a label for missing values that explains no match
			label define matchLabel 0 "Not Matched" 1 "Matched" .n "Obs not included in match"

			*Apply the label
			label value `noMatchReasonName' matchLabel

			********************************
			*
			*	Creating tempvar used
			*
			********************************

			*Initiate the temporary variables used by this command
			tempvar hi_diff lo_diff pref match newMatch rand

			gen `hi_diff' = .
			gen `lo_diff' = .

			gen `pref' = .
			gen byte `match' = 0
			gen byte `newMatch' = 0

			** Generate random variable that seperate two obs
			*  with the same value in the continuos variable
			gen `rand' = uniform()

			********************************
			*
			*	Start matching
			*
			********************************
			noi di ""
			noi di "{pstd}Observations left to match: {p_end}"
			count if `grpdummy' == 1 & `match' == 0
			local left2Match = `r(N)'
			noi di "{pstd}`left2Match' " _c

			while (`left2Match' > 0) {

				sort `match' `matchcont' `rand'

				replace `lo_diff' 	= .
				replace `hi_diff' 	= .
				replace `pref'		= . if `match' == 0
				replace `newMatch' 	= 0

				replace `lo_diff' = abs(`matchcont' - `matchcont'[_n-1]) if `match' == 0 & `grpdummy' != `grpdummy'[_n-1]
				replace `hi_diff' = abs(`matchcont' - `matchcont'[_n+1]) if `match' == 0 & `grpdummy' != `grpdummy'[_n+1]

				replace `pref' = `idvar'[_n-1] if `match' == 0 & `lo_diff' <  `hi_diff'
				replace `pref' = `idvar'[_n+1] if `match' == 0 & `lo_diff' >= `hi_diff'

				replace `newMatch' = 1 if `match' == 0 & `pref' == `idvar'[_n-1] & `pref'[_n-1] == `idvar'
				replace `newMatch' = 1 if `match' == 0 & `pref' == `idvar'[_n+1] & `pref'[_n+1] == `idvar'

				replace `match' = 1 							if `newMatch' == 1
				replace `matchDiffName' = min(`lo_diff', `hi_diff') 	if `newMatch' == 1
				replace `matchIDname' = `idvar' 						if `newMatch' == 1 & `grpdummy' == 1
				replace `matchIDname' = `pref' 						if `newMatch' == 1 & `grpdummy' == 0

				*order _match* , last

				count if `grpdummy' == 1 & `match' == 0
				local left2Match = `r(N)'
				noi di "`left2Match' " _c
			}

			replace `noMatchReasonName' = 0 if `match' == 0 & `grpdummy' == 0

			keep `matchIDname' `matchDiffName' `idvar' `originalSort' `noMatchReasonName'

			tempfile mergefile

			save `mergefile'

		restore

		tempvar mergevar

		merge 1:1  `originalSort' using `mergefile', gen(`mergevar')

		replace `noMatchReasonName' = .n if `mergevar' == 1

		replace `noMatchReasonName' = 1 if `noMatchReasonName' == .

		noi di "{p_end}" _c
		noi tab `noMatchReasonName' `grpdummy', m

		sort `originalSort'

	}

	end

	*Cheack matchVar input
	cap program drop 	iematchMatchVarCheck
		program define 	iematchMatchVarCheck  , rclass

		args defaultName userName

		if "`defaultName'" == "_matchID" 		local optionName matchidname
		if "`defaultName'" == "_matchDiff" 		local optionName matchdiffname
		if "`defaultName'" == "_matchOrReason" 	local optionName matchnoname

		*All the default names
		local dfltNms _noMatchReason _matchDiff _matchOrReason

		*The other two default names
		local othDfltNms : list dfltNms - defaultName

		*Creating two locals with the deafult name in each local
		local Oth1 : word 1 of `othDfltNms'
		local Oth2 : word 2 of `othDfltNms'

		*Testing that the option name is not the same as the other variable's default name.
		if ("`defaultName'" == "_matchID" & ("`userName'" == "`Oth1'" | "`userName'" == "`Oth2'" | "`userName'" == "_ID"))  {

			noi di as error `" ("`defaultName'" == "_matchID" & ("`userName'" == "`Oth1'" | "`userName'" == "`Oth2'" | "`userName'" == "_ID")) "'

			noi di as error "The new name specified in `optionName'() is not allowed to be _ID, `Oth1', nor `Oth2'"
			error 198
		}

		*Checking set-up for the ID var used for _matchID
		if "`userName'" == "" {

			*CSet the default name
			local validMatchVarname `defaultName'

			*Make sure that the default name _ID is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				noi di as error "A variable with name `validMatchVarname' is already defined. Either drop this variable or specify a new variable name using `optionName'()."
				error 110

			}

		}

		else {

			*CSet the default name
			local validMatchVarname `userName'

			*Make sure that the default name _ID is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				noi di as error "The variable name specified in `optionName'(`userName') is already defined in the data set. Either drop this variable or specify a another variable name using `optionName'()."
				error 110

			}

		}

		*Testing that the new name is valid by creating a variable that is dropped immedeatly afterwards
		cap gen `validMatchVarname' = .

		if _rc != 0 {

			noi di as error "The variable name specified in `optionName'(`userName') is not a valid variable name."
			error _rc

		}
		drop `validMatchVarname'

		return local validVarName 	"`validMatchVarname'"

	end

	clear

*	set seed 1235324

	set obs 400000

	gen id = _n

	gen rand1 = uniform()

	gen tmt = (rand1 < .40)

	*replace tmt = . if (rand1 < .05)

	drop rand1
	tab tmt

	gen p_hat = uniform()

	*replace p_hat = . if p_hat < .01

	iematch  , grp(tmt) match(p_hat) idvar(id) // matchidname(Kallefille)


Update census_compile.do

Update census_compile.do

 minor text fixes

/*
Author: Tanner Eastmond
Date: 1/12/17
Data: *.txt
*/

local filename = "`1'"

cd "path/to/data"
set more off, perm
insheet using `filename', clear delim($)

*generate filename so we can know what gz archives are giving us problems
gen filename = "`filename'"
*drop first row
drop if _n == 1
gen arkid = regexs(0) if regexm(v1,"tenthttps.*ark.*1:1:([A-Z0-9]+-[A-Z0-9]+)")
drop v1
foreach x of varlist _all{
	gen temp = substr(`x',1,strpos(`x'," "))
	levelsof temp, local(names)
	foreach y of local names{
		cap gen `y' = ""
	}
	drop temp
}

ds v*, not
foreach x of varlist `r(varlist)'{
	ds v*
	foreach y of varlist `r(varlist)'{
		replace `x' = `y' if regexm(`y',"`x'")
		replace `x' = regexs(0) if regexm(`x',"text.*/text")
	}
	replace `x' = subinstr(`x',"text","",.)
	replace `x' = subinstr(`x',"/","",.)
}
drop v*

compress
cap append using temp
save temp, replace

exit, STATA clear

/*
Author: Tanner Eastmond
Date: 1/12/17
Data: *.txt
*/

local filename = "`1'"

cd "R:\JoePriceResearch\record_linking\data\census_1910\data"
set more off, perm
insheet using `filename', clear delim($)

gen filename = "`filename'"
drop if _n == 1
gen arkid = regexs(0) if regexm(v1,"tenthttps.*ark.*1:1:[A-Z0-9]+-[A-Z0-9]+")
replace arkid = substr(arkid,5,.)
drop v1
foreach x of varlist _all{
	gen temp = substr(`x',1,strpos(`x'," "))
	levelsof temp, local(names)
	foreach y of local names{
		cap gen `y' = ""
	}
	drop temp
}

ds v*, not
foreach x of varlist `r(varlist)'{
	ds v*
	foreach y of varlist `r(varlist)'{
		replace `x' = `y' if regexm(`y',"`x'")
		replace `x' = regexs(0) if regexm(`x',"text.*/text")
	}
	replace `x' = subinstr(`x',"text","",.)
	replace `x' = subinstr(`x',"/","",.)
}
drop v*

compress
cap append using temp
save temp, replace

exit, STATA clear


remove whitespace

remove whitespace

clear all
set more off
set matsize 11000
program drop _all

qui adopath + "/mnt/nfs6/wikipedia.proj/wikibaseball/scripts/stata/ado"
declare_global

cd ${path}

// 1a. Produce Baseball / Basketball datasets
make_baseball

make_basketball

make_traf

make_rev

use ${lahman}bb_master, clear
append using ${basketball}bk_master
gen isbaseball = (minutesrank==.)
save ${stash}bbk_master, replace

// 1b. Merge data

make_merge

/////////// ANALYSIS //////////////////

// 1. Summary Stats
summary

t_test 1
t_test 0

// 2. Digitization
program drop _all
reg_all digit

// 3. Copy DD
reg_all copy

// 4. Copy DDD
reg_ddd

// 5. DD regressions -- Traf
reg_traf

// 5b. IV == Traf
ivest

// FIGURES

// 1. figure

// 2. Mean charts
meanline img Images(Baseball) 1
meanline img Images(Basketball) 0

program drop _all
meanline text Text(Baseball) 1
meanline text Text(Basketball) 0

meanline bd Citations(Baseball) 1
meanline bd Citations(Basketball) 0

// 6. Killer Pictures
killer_pic2 1
killer_pic2 0

// 7. Time dummies picture
ddpic bd 1
ddpic bd 0

ddpic img 1
ddpic img 0

ddpic text 1
ddpic text 0

// 8. Scatterplot
traf_scatter

// 9. Quality-wise impacts
chart_qual img 1
chart_qual traf 1
chart_qual text 1
chart_qual bd 1

// APPENDIX
reg_all digit ln

reg_all copy ln

reg_ddd ln

reg_traf ln

program drop _all
reg_robust copy


include second allocation rule

include second allocation rule

use "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie13-egen.dta", clear
gen runningwt = 0
gen prevwt = 0
save "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie5-1.dta", replace
keep if familyheadydandnotemp
bysort statefip puma: gen pumaid = _n
bysort statefip puma (invheadmaxyouthempp): replace runningwt = sum(perwt)
//use (headmaxyouthempp) or (invheadmaxyouthempp) above depending if you want min or max likelihood
replace prevwt = runningwt - perwt if runningwt > numjobspuma2
gen jobsleft = numjobspuma2 - prevwt
replace jobsleft = 0 if jobsleft < 0
replace jobsleft = 0 if runningwt < jobsleft
gen jobrecipient = 0
replace jobrecipient = 1 if runningwt < numjobspuma2 | jobsleft > 0
keep serial pernum jobrecipient jobsleft
joinby serial pernum using "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie5-1.dta", unmatched(both)

replace jobrecipient = 0 if missing(jobrecipient)
replace jobsleft = 0 if missing(jobsleft)

gen newperwt = perwt
egen maxrunningwt = max(runningwt), by(statefip puma)

expand 2 if jobsleft > 0, generate(duplicate)
replace newperwt = int(jobsleft) if duplicate == 1
replace newperwt = max(int(perwt-jobsleft),0) if duplicate == 0 & jobsleft > 0
replace jobrecipient = 0 if duplicate == 0 & jobsleft > 0

egen jobsreceived = sum(jobrecipient*newperwt)

gen job_money = 0
replace job_money = 10500 if jobrecipient

gen numjobsafter = numjobspuma2
replace numjobsafter = maxrunningwt if maxrunningwt < numjobspuma2

gen familyincome = ftotinc

gen newincome = familyincome+job_money

gen outofpov = (ftotinc <= povline) & (newincome > povline) & (newincome < 9999999)

egen outofpovfam = max(outofpov), by(serial famunit)

*sgini newincome if familyhead == 1 & familyincome < 9999999 [fweight=newperwt]
*sgini familyincome if familyhead == 1 & familyincome < 9999999 [fweight=newperwt]

gen povgap = povline - familyincome if familyincome < povline
gen newpovgap = povline - newincome if newincome < povline

gen poor = familyincome < povline
gen newpoor = newincome < povline

tab poor [fweight=newperwt]
tab newpoor [fweight=newperwt]

egen moneyspent = sum(job_money*newperwt) if jobrecipient
disp moneyspent[1]

tabulate job_money if familyhead [fweight=newperwt]

egen totalfam = sum(newperwt) if familyhead == 1
egen p0 = sum(newperwt) if poor & familyhead == 1
egen newp0 = sum(newperwt) if newpoor & familyhead == 1

egen p1 = sum((povgap*newperwt)/povline) if familyhead == 1
disp p1/totalfam
egen newp1 = sum((newpovgap*newperwt)/povline) if familyhead == 1
disp newp1/totalfam

egen p2 = sum((povgap/povline)^2*newperwt) if familyhead == 1
egen newp2 = sum((newpovgap/povline)^2*newperwt) if familyhead == 1

disp p2[7]/totalfam[7]
disp newp2[7]/totalfam[7]

bysort sex: tab jobrecipient [fweight=newperwt]
bysort racesing: tab jobrecipient [fweight=newperwt]
bysort finccut: tab jobrecipient [fweight=newperwt]
bysort educ: tab jobrecipient [fweight=newperwt]

use "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie13-egen.dta", clear
gen runningwt = 0
gen prevwt = 0
save "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie5-1.dta", replace
keep if familyheadydandnotemp
bysort statefip puma: gen pumaid = _n
bysort statefip puma (headmaxyouthempp): replace runningwt = sum(perwt)
//use (headmaxyouthempp) or (invheadmaxyouthempp) above depending if you want min or max likelihood
replace prevwt = runningwt - perwt if runningwt > numjobspuma
gen jobsleft = numjobspuma - prevwt
replace jobsleft = 0 if jobsleft < 0
replace jobsleft = 0 if runningwt < jobsleft
gen jobrecipient = 0
replace jobrecipient = 1 if runningwt < numjobspuma | jobsleft > 0
keep serial pernum jobrecipient jobsleft
joinby serial pernum using "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie5-1.dta", unmatched(both)

replace jobrecipient = 0 if missing(jobrecipient)
replace jobsleft = 0 if missing(jobsleft)

gen newperwt = perwt
egen maxrunningwt = max(runningwt), by(statefip puma)

expand 2 if jobsleft > 0, generate(duplicate)
replace newperwt = int(jobsleft) if duplicate == 1
replace newperwt = max(int(perwt-jobsleft),0) if duplicate == 0 & jobsleft > 0
replace jobrecipient = 0 if duplicate == 0 & jobsleft > 0

egen jobsreceived = sum(jobrecipient*newperwt)

gen job_money = 0
replace job_money = 10500 if jobrecipient

gen numjobsafter = numjobspuma
replace numjobsafter = maxrunningwt if maxrunningwt < numjobspuma

gen familyincome = ftotinc

gen newincome = familyincome+job_money

gen outofpov = (ftotinc <= povline) & (newincome > povline) & (newincome < 9999999)

egen outofpovfam = max(outofpov), by(serial famunit)

*sgini newincome if familyhead == 1 & familyincome < 9999999 [fweight=newperwt]
*sgini familyincome if familyhead == 1 & familyincome < 9999999 [fweight=newperwt]

gen povgap = povline - familyincome if familyincome < povline
gen newpovgap = povline - newincome if newincome < povline

gen poor = familyincome < povline
gen newpoor = newincome < povline

tab poor [fweight=newperwt]
tab newpoor [fweight=newperwt]

egen moneyspent = sum(job_money*newperwt) if jobrecipient
disp moneyspent[1]

tabulate job_money if familyhead [fweight=newperwt]

egen totalfam = sum(newperwt) if familyhead == 1
egen p0 = sum(newperwt) if poor & familyhead == 1
egen newp0 = sum(newperwt) if newpoor & familyhead == 1

egen p1 = sum((povgap*newperwt)/povline) if familyhead == 1
disp p1/totalfam
egen newp1 = sum((newpovgap*newperwt)/povline) if familyhead == 1
disp newp1/totalfam

egen p2 = sum((povgap/povline)^2*newperwt) if familyhead == 1
egen newp2 = sum((newpovgap/povline)^2*newperwt) if familyhead == 1

disp p2[7]/totalfam[7]
disp newp2[7]/totalfam[7]

bysort sex: tab jobrecipient [fweight=newperwt]
bysort racesing: tab jobrecipient [fweight=newperwt]
bysort finccut: tab jobrecipient [fweight=newperwt]
bysort educ: tab jobrecipient [fweight=newperwt]


Revert "Add iematch test file to branch"

	clear

	*command starts here

	cap program drop 	iematch
	    program define	iematch

		syntax  , GRPdummy(varname) MATCHcont(varname) [idvar(varname)]

		qui {

		if "`idvar'" == "" {

			gen _ID = _n

			local idvar _ID

		}

		tempvar hi_diff lo_diff pref match newMatch

		gen `hi_diff' = .
		gen `lo_diff' = .
		gen _matchDiff = .
		gen `pref' = .
		gen byte `match' = 0
		gen byte `newMatch' = 0
		gen _matchID = .

		*order `idvar' , after(`pref')

		**match loop

		count if `grpdummy' == 1 & `match' == 0
		local left2Match = `r(N)'

		pause on

			while (`left2Match' > 0) {

				sort `match' `matchcont'

				replace `lo_diff' 	= .
				replace `hi_diff' 	= .
				replace `pref'		= . if `match' == 0
				replace `newMatch' 	= 0

				replace `lo_diff' = abs(`matchcont' - `matchcont'[_n-1]) if `match' == 0 & `grpdummy' != `grpdummy'[_n-1]
				replace `hi_diff' = abs(`matchcont' - `matchcont'[_n+1]) if `match' == 0 & `grpdummy' != `grpdummy'[_n+1]

				replace `pref' = `idvar'[_n-1] if `match' == 0 & `lo_diff' <  `hi_diff'
				replace `pref' = `idvar'[_n+1] if `match' == 0 & `lo_diff' >= `hi_diff'

				replace `newMatch' = 1 if `match' == 0 & `pref' == `idvar'[_n-1] & `pref'[_n-1] == `idvar'
				replace `newMatch' = 1 if `match' == 0 & `pref' == `idvar'[_n+1] & `pref'[_n+1] == `idvar'

				replace `match' = 1 							if `newMatch' == 1
				replace _matchDiff = min(`lo_diff', `hi_diff') 	if `newMatch' == 1
				replace _matchID = `idvar' 						if `newMatch' == 1 & `grpdummy' == 1
				replace _matchID = `pref' 						if `newMatch' == 1 & `grpdummy' == 0

				*order _match* , last

			noi	count if `grpdummy' == 1 & `match' == 0
				local left2Match = `r(N)'

				*pause

			}
		}

	end

	set seed 1235324

	set obs 10000

	gen id = _n

	gen rand1 = uniform()

	gen tmt = (rand1 < .40)

	drop rand1
	tab tmt

	gen p_hat = uniform()

	iematch   , grp(tmt) match(p_hat) idvar(id)