module LegacyMeansTestedBenefits
#
# This module implements the pre-universal credit means-tested benefit system.
#
using ArgCheck
using Base: Bool
using Dates
using ScottishTaxBenefitModel
using .Definitions

using .STBIncomes

using .Utils: to_md_table

using .ModelHousehold:
    BenefitUnit,
    Household,
    Person,
    between_ages,
    count,
    empl_status_in,
    ge_age,
    get_benefit_units,
    has_children,
    is_lone_parent,
    is_severe_disability,
    is_single,
    le_age,
    num_adults,
    num_carers,
    pers_is_disabled,
    search

using .STBParameters:
    AgeLimits,
    ChildTaxCredit,
    HoursLimits,
    HousingBenefits,
    HousingRestrictions,
    IncomeRules,
    IncomeRules,
    LegacyMeansTestedBenefitSystem,
    MinimumWage,
    NonMeansTestedSys,
    PersonalAllowances,
    Premia,
    SavingsCredit,
    WorkingTaxCredit,
    reached_state_pension_age,
    state_pension_age

using .GeneralTaxComponents:
    RateBands,
    TaxResult,
    calctaxdue

using .Results:
    BenefitUnitResult,
    HouseholdResult,
    IndividualResult,
    LMTIncomes,
    LMTResults,
    LMTCanApplyFor,
    aggregate_tax,
    gross_pension_contributions,
    has_any,
    to_string,
    total

using .Intermediate:
    MTIntermediate,
    HHIntermed,
    apply_2_child_policy,
    born_before,
    is_working_hours,
    make_recipient,
    num_born_before,
    working_disabled

using .LocalLevelCalculations:
    apply_rent_restrictions

export
    calc_legacy_means_tested_benefits,
    calc_allowances,
    calc_incomes,
    calc_NDDs,
    calc_premia,
    calculateHB_CTR!,
    calcWTC_CTC!,
    make_lmt_benefit_applicability,
    num_qualifying_for_severe_disability,
    tariff_income

"""
Incomes for old style mt benefits
The CPAG guide ch 21/21 has over 100 pages on this stuff
this can no more than catch the gist.
"""
function calc_incomes(
    which_ben :: LMTBenefitType, # esa hb is jsa pc wtc ctr pc sc
    bu        :: BenefitUnit,
    bur       :: BenefitUnitResult,
    intermed  :: MTIntermediate,
    incrules  :: IncomeRules,
    hours     :: HoursLimits ) :: LMTIncomes
    T = typeof( incrules.permitted_work )
    mntr = bur.legacy_mtbens # shortcut
    inc = LMTIncomes{T}()
    gross_earn = zero(T)
    net_earn = zero(T)
    other = zero(T)

    if which_ben in [hb,ctr]
        inclist = incrules.hb_incomes
    elseif which_ben == pc
        inclist = incrules.pc_incomes
    elseif which_ben == sc
        inclist = incrules.sc_incomes
    else
        inclist = incrules.incomes
    end
    # children's income doesn't count see cpag p421, so:
    for pid in bu.adults
        pers = bu.people[pid]
        pres = bur.pers[pid]
        gpc = gross_pension_contributions(pres)
        gross =
            pres.income[WAGES] +
            pres.income[SELF_EMPLOYMENT_INCOME] # this includes losses
        if which_ben in [pc,is,jsa,esa,hb,ctr]
            net =
                gross - ## FIXME parameterise this so we can use gross/net
                pres.it.non_savings_tax - ## FIXME?? income[INCOME_TAX] ??
                pres.income[NATIONAL_INSURANCE] -
                0.5*gpc # .income[PENSION_CONTRIBUTIONS_EMPLOYEE]
            # println( "net=$net; gross=$gross pres.it.non_savings_tax = $(pres.it.non_savings_tax) gpc=$gpc ni=$(pres.income[NATIONAL_INSURANCE])")
        else
            # wtc,ctr all pension contributions but not IT/NI
            gross = max( 0.0, gross - gross_pension_contributions(pres)) # .income[PENSION_CONTRIBUTIONS_EMPLOYEE])
            net = gross
        end
        gross_earn += gross
        net_earn += max( 0.0, net )
        other += isum( pres.income, inclist )
        #    data=pers.income,
        #    calculated=pres.incomes,
        #    included=inclist )
    end
    # disregards
    # if which_ben in [hb,jsa,is,]
    # FIXME this is not quite right for ESA
    disreg = intermed.is_sing ?  incrules.low_single : incrules.low_couple

    if( which_ben == esa )
        if ! search( bu, is_working_hours, hours.lower )
            disreg = incrules.high
            # and some others ... see CPAG
        end
    elseif which_ben in [hb,ctr,jsa,is,pc]
        if intermed.is_sparent
            disreg = which_ben == hb ? incrules.lone_parent_hb : incrules.high
        elseif ! isdisjoint( mntr.premia, [
                carer_single,
                carer_couple,
                disability_couple,
                disability_single,
                severe_disability_couple,
                severe_disability_single,
                enhanced_disability_couple,
                enhanced_disability_single] )
            disreg = incrules.high
        end
    end

    if( which_ben in [hb,ctr] )
        # fixme do this above
        if has_any( bur, CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE, NON_CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE )
            disreg = incrules.high
        end
        # HB disregard CPAG p432 this, too, is very approximate
        # work 30+ hours - should really check premia if haskeys( mtr.premia )
        extra = 0.0
        # println( "hours.higher $(hours.higher)")
        if search( bu, is_working_hours, hours.higher )
            extra = incrules.hb_additional
            # println( "extra=$extra")
        elseif search(  bu, is_working_hours, hours.lower )
            if intermed.is_sparent || (intermed.num_children > 0) || intermed.is_disabled
                extra = incrules.hb_additional
            end
        end
        disreg += extra
        # childcare in HB - costs are assigned in frs to the children
        if ( intermed.num_children > 0 )
            maxcc = intermed.num_children == 1 ? incrules.childcare_max_1 : incrules.childcare_max_2
            cost_of_childcare = 0.0
            for pid in bu.children
                cost_of_childcare += bu.people[pid].cost_of_childcare
            end
            inc.childcare = min(cost_of_childcare, maxcc )
        end
    end

    """
    not even remotely right ... cpag 21
    """

    cap = 0.0
    for pid in bu.adults
        if bu.people[pid].over_20_k_saving
            cap = 100_000 # some huge number bigger than any reasonable capmax
        else
            for (at,val) in bu.people[pid].assets
                cap += val
            end
        end
    end
    inc.other_income = other
    inc.capital = cap
    inc.gross_earnings = gross_earn
    inc.net_earnings = max(0.0, net_earn - disreg ) - inc.childcare

    capmin = incrules.capital_min
    capmax = incrules.capital_max
    tariff = incrules.capital_tariff
    if intermed.someone_pension_age
        tariff = incrules.pensioner_tariff
        if which_ben == pc
            capmin = incrules.pc_capital_min
            capmax = incrules.pc_capital_max
        else
            capmin = incrules.pensioner_capital_min
            capmax = incrules.pensioner_capital_max
        end
    end

    inc.tariff_income = tariff_income(cap, capmin, tariff )
    inc.disqualified_on_capital = cap > capmax
    inc.total_income = inc.net_earnings + inc.other_income + inc.tariff_income
    inc.disregard = disreg
    return inc
end

"""
The strategy here is to include *all* benefits the BU is entitled to
and then decide later on which ones to route to. Source: CPAG chs 9-15
'Who can get XX' sections.
"""
function make_lmt_benefit_applicability(
    mt_ben_sys   :: LegacyMeansTestedBenefitSystem,
    intermed :: MTIntermediate,
    hrs      :: HoursLimits ) :: LMTCanApplyFor
    whichb = LMTCanApplyFor()
    if intermed.someone_pension_age #!! fixme both pension age for new claims 2019=>
        whichb.pc = true
    end
    if intermed.someone_pension_age_2016
        whichb.sc = true
    end
    # println( "intermed: intermed.num_working_16_or_less=$(intermed.num_working_16_or_less) intermed.economically_active=$(intermed.economically_active) intermed.num_working_16_or_less=$(intermed.num_working_16_or_less)")
    if (! intermed.all_pension_age ) && (intermed.num_working_16_or_less >= 1)
        if ((intermed.num_adults == 1) ||
           ((intermed.num_adults == 2) && (intermed.num_working_24_plus == 0)))
            if intermed.limited_capacity_for_work
                whichb.esa = true
            # CPAG 21/2 p236/7
            elseif( intermed.num_adults == 1) && (intermed.num_allowed_children>0) &&
                ((intermed.age_youngest_child <= 5) || (intermed.age_oldest_adult < 18))
                    whichb.is = true
            elseif intermed.economically_active
                whichb.jsa = true
            else
                whichb.is = true
            end
        end
    end # all pens age

    #
    # Tax credits
    # CTC - easy - but route pensioners to pension credit even
    # if they have children - this is how Age Concern do it, anyway.
    #
    if intermed.has_children && ( ! whichb.pc )
        whichb.ctc = true
    end
    #
    # WTC - not quite so easy
    #
    if intermed.someone_working_ft_and_25_plus
        whichb.wtc = true
    elseif (intermed.total_hours_worked >= hrs.med) && (intermed.num_working_pt>0) && intermed.has_children
        # ie. 24 hrs worked total and one person  >= 16 hrs and has children
        whichb.wtc = true
    elseif (intermed.num_working_pt>0) && intermed.someone_pension_age
        whichb.wtc = true
    elseif (intermed.num_working_pt>0) && intermed.is_sparent
        whichb.wtc = true
    elseif intermed.working_disabled
        whichb.wtc = true
    end
    # FIXME not really true
    if intermed.benefit_unit_number == 1
        whichb.hb = true
        whichb.ctr = true
    end
    # check!!! is this correct? is, etc. or wtc over pc/sc
    if( whichb.esa || whichb.jsa || whichb.is ) ## || whichb.wtc)
        whichb.pc = false
        whichb.sc = false
    end
    # FIXME we have abolished code twice!
    # override these if something abolished
    if mt_ben_sys.isa_jsa_esa_abolished
        whichb.jsa = false
        whichb.is = false
        whichb.esa = false
        # FIXME problem with pension credit here
        whichb.pc = false
    end
    whichb.ctc &= (! mt_ben_sys.child_tax_credit.abolished)
    whichb.wtc &= (! mt_ben_sys.working_tax_credit.abolished)
    whichb.hb &= (! mt_ben_sys.hb.abolished)
    whichb.ctr &= (! mt_ben_sys.ctr.abolished)
    whichb.sc &= (! mt_ben_sys.savings_credit.abolished)
    return whichb
end # make_lmt_benefit_applicability

# CTC disabled child definition:
#
# Getting Disability Living Allowance (DLA), Personal Independence Payment (PIP) or Armed Forces Independence Payment (AFIP)
# In hospital but otherwise would get DLA, PIP or AFIP
# Certified blind (or were until less than 28 weeks before you made the claim)

#
#

# Disabled adult for wtc
# Answer yes if you work at least 16 hours a week and any of the following apply to you:
#
#    you get a disability benefit, eg Disability Living Allowance, Attendance Allowance or Personal Independence Payment
#    you have recently been getting disability benefit, eg Incapacity Benefit or Employment and Support Allowance
#    you have a disability that makes it hard for you to get a job, eg you’re deaf or blind
#

"""
tariff income from capital.
See CPAG p488 £1 pw for every £250, or part of £250 above 6,000
"""
function tariff_income( cap :: Real, capital_min::Real, tariff :: Real )::Real
    return ceil( max(0.0, cap-capital_min)/tariff)
end

"""
WRONG AND NOT USED!
CPAG 2019/20 p347. I *thik* this is what it means ...
Moved to own function since it's convoluted.
FIXME move this as it's being used in the UC module
"""
function num_qualifying_for_severe_disability(
    bu    :: BenefitUnit,
    bures :: BenefitUnitResult,
    num_bus :: Int ) :: Int
    if num_bus > 1
        return 0;
    end
    n = 0
    for pid in bu.adults
        pers = bu.people[pid]
        if (pers.dla_self_care_type in [high,mid] ||
              pers.attendance_allowance_type != missing_lmh ||
              pers.registered_blind ||
              pers.pip_daily_living_type == enhanced_pip )
              n += 1
        end
        if bures.pers[pid].income[CARERS_ALLOWANCE] > 0
            n -= 1
        end
    end
    return max( 0, n )
end

"""
See: https://www.gov.uk/disability-premiums/eligibility
"""
function qualifies_for_disability_premium(
    pers :: Person,
    pres :: IndividualResult,
    prem_sys :: Premia ) :: Bool
    if pers.registered_blind
        return true
    end
    if any_positive( pres.income, prem_sys.disability_premium_qualifying_benefits)
        return true
    end
    return false
end

function qualifies_for_enhanced_disability(
    pers     :: Person,
    pres     :: IndividualResult,
    prem_sys :: Premia,
    nmt      :: NonMeansTestedSys,
    age_limits :: AgeLimits ) :: Bool
    if reached_state_pension_age(
        age_limits,
        pers.age,
        pers.sex )
        return false
    end
    if any_positive( pres.income, [NON_CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE,CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE])
        return true
    end
    for p1 in [
        PERSONAL_INDEPENDENCE_PAYMENT_DAILY_LIVING,
        SCOTTISH_DISABILITY_ASSISTANCE_WORKING_AGE_DAILY_LIVING]
        if p1 in prem_sys.enhanced_disability_premium_qualifying_benefits
            if pres.income[p1] >= nmt.pip.dl_enhanced
                return true
            end
        end
    end
    for p1 in [DLA_SELF_CARE, SCOTTISH_DISABILITY_ASSISTANCE_CHILDREN_DAILY_LIVING]
        if p1 in prem_sys.enhanced_disability_premium_qualifying_benefits
            if pres.income[p1] >= nmt.dla.care_high
                return true
            end
        end
    end
    ## FIXME support group for ESA
    return false
end

"""
FIXME this should be over the HOUSEHOLD not the benefit unit,
I think
"""
function num_adults_qualifiying_for_disability_premium(
    bu    :: BenefitUnit,
    bures :: BenefitUnitResult,
    prem_sys :: Premia ) :: Integer
    n = 0
    for pid in bu.adults
        if qualifies_for_disability_premium( bu.people[pid], bures.pers[pid], prem_sys )
            n += 1
        end
    end
    return n
end

function qualifies_for_severe_disability_premium(
    pers     :: Person,
    pres     :: IndividualResult,
    prem_sys :: Premia,
    nmt      :: NonMeansTestedSys ) :: Bool
    return qualifies_for_disability_premium( pers, pres, prem_sys ) &&
        any_positive( pres.income,
            [PERSONAL_INDEPENDENCE_PAYMENT_DAILY_LIVING,
            SCOTTISH_DISABILITY_ASSISTANCE_WORKING_AGE_DAILY_LIVING,
            DLA_SELF_CARE,
            SCOTTISH_DISABILITY_ASSISTANCE_CHILDREN_DAILY_LIVING,
            ATTENDANCE_ALLOWANCE,
            SCOTTISH_DISABILITY_ASSISTANCE_OLDER_PEOPLE] )
end

function calc_premia(
    which_ben :: LMTBenefitType,
    bu        :: BenefitUnit,
    bures     :: BenefitUnitResult,
    intermed  :: MTIntermediate,
    prem_sys  :: Premia,
    nmt       :: NonMeansTestedSys,
    age_limits:: AgeLimits ) :: Tuple
    premium = 0.0
    premset = LMTPremiaSet()
    # disabled child premium
    num_ads = num_adults( bu )
    if which_ben in [hb,ctr]
        if intermed.num_disabled_children > 0
            premium += intermed.num_disabled_children*prem_sys.disabled_child
            union!( premset, [disabled_child])
        end
    end
    # carer's premium - all benefits, I think.
    if intermed.num_carers == 1
        premium += prem_sys.carer_single
        union!( premset,[carer_single] )
    elseif intermed.num_carers == 2
        # FIXME what if 1 is a child?
        premium += prem_sys.carer_couple
        union!( premset, [carer_couple] )
    end
    # disability premiums
    ndis = num_adults_qualifiying_for_disability_premium(
        bu, bures, prem_sys )
    if which_ben in [is, jsa, hb, ctr] # FIXME not quite right but perhaps near enough - should be reached pension age? CPAG19 p344
        if ndis == 1
            premium += prem_sys.disability_single
            union!( premset,[disability_single])
        elseif ndis == 2
            premium += prem_sys.disability_couple
            union!( premset,[disability_couple])
        end
    end
    # enhanced disability - also for ESA
    if which_ben in [is, jsa, hb, ctr, esa]
        nenh = 0
        for pid in bu.adults
            nenh += qualifies_for_enhanced_disability(
                    bu.people[pid],
                    bures.pers[pid],
                    prem_sys,
                    nmt,
                    age_limits )
        end
        if nenh == 1
            premium += prem_sys.enhanced_disability_single
            union!( premset, [enhanced_disability_single] )
        elseif nenh == 2
            premium += prem_sys.enhanced_disability_couple
            union!( premset,[enhanced_disability_couple])
        end
    end
    # severe disability premium
    if ndis == num_ads # all adults disabled - check for severe dis
        nsev = 0
        for pid in bu.adults
            nsev += qualifies_for_severe_disability_premium(
                bu.people[pid],
                bures.pers[pid],
                prem_sys,
                nmt
            )
        end
        if nsev == 1
            premium += prem_sys.severe_disability_single
            union!( premset, [severe_disability_single])
        elseif nsev == 2
            premium += prem_sys.severe_disability_couple
            union!( premset, [severe_disability_couple])
        end
    end
    # FIXME this is not the right test for sev disabled child
    if which_ben in [hb,ctr,is,jsa,esa,pc]
        premium += intermed.num_severely_disabled_children*prem_sys.enhanced_disability_child
        if intermed.num_severely_disabled_children > 0
            union!( premset,[enhanced_disability_child] )
        end
    end
    #=
    if which_ben in [ hb, ctr, is, jsa, esa, pc, sc ]
        # CPAG 19/20 - I *think* this is what it means..
        nsd = num_qualifying_for_severe_disability( bu, bures, intermed.num_benefit_units ) > 0
        if nsd == 1
            premium += prem_sys.severe_disability_single
            union!( premset, [severe_disability_single])
        elseif nsd == 2
            premium += prem_sys.severe_disability_couple
            union!( premset, [severe_disability_couple])
        end
    end
    =#
    if which_ben in [ is, jsa, esa ] # this should almost never happen given our routing; cpag p345
        if intermed.someone_pension_age
            premium += prem_sys.pensioner_is
            union!( premset, [pensioner_is] )
        end
    end
    # end
    #
    # we're ignoring support components (p355-) for now.
    #
    return (premium, premset)
end

function calc_allowances(
    which_ben :: LMTBenefitType,
    intermed :: MTIntermediate,
    pas :: PersonalAllowances,
    ages :: AgeLimits,
    oo_housing_costs :: Real  ) :: Real
    pers_allow = 0.0
    @assert intermed.num_adults in [1,2]
    if which_ben == pc
        if intermed.num_adults == 1
            pers_allow = pas.pc_mig_single
        elseif intermed.num_adults == 2
            pers_allow = pas.pc_mig_couple
        end
        # children - cpag p 272 says there's an allowance for children
        # if not claiming CTC, but so far as I can see there's no
        # upper limit on CTC
        # .. anyway, the AC calculator suggests this:
        pers_allow += pas.pc_child * intermed.num_allowed_children
        ## FIXME also an allowance for 1st child born before 6th april 2018
    else
        if intermed.age_oldest_adult < 18
            # argh .. not there's a conditional on ESA that we don't cover here
            # seems to be it - no change for sps, marriage..
            # but some change
            if intermed.num_adults == 2
                pers_allow = pas.couple_both_under_18
            else
                if which_ben != esa
                    pers_allow = pas.age_18_24
                else # FIXME rename this allowance !!
                    pers_allow = pas.age_25_and_over
                end
            end
            # should be somethinh like ...
            #if which_ben in [is,jsa,esa]
            #
            #elseif which_ben == hb
            #    pers_allow = pas.age_18_24
            #end
        elseif intermed.age_youngest_adult < 18 && intermed.num_adults == 2
            pers_allow = pas.couple_one_over_18_high
            ## FIXME some cases lower than this see p 335 CPAG
        else # all over 17
            if intermed.num_adults == 1
                if intermed.num_children > 0 # single parent
                    if intermed.someone_pension_age
                        pers_allow = pas.lone_parent_over_pension_age
                    else
                        pers_allow = pas.lone_parent
                    end
                else
                    if intermed.someone_pension_age
                        pers_allow = pas.over_pension_age
                    elseif intermed.age_oldest_adult < 25
                        pers_allow = pas.age_18_24
                    else
                        pers_allow = pas.age_25_and_over
                    end
                end
            else # 2 adults, both at least 18
                if intermed.someone_pension_age
                    pers_allow = pas.couple_over_pension_age
                else
                    pers_allow = pas.couple_both_over_18
                end
            end # 2 adults
        end # no 16-17 yos
        if which_ben in [hb,ctr] #
            pers_allow += pas.child * intermed.num_allowed_children
        end
    end
    pers_allow += oo_housing_costs
    @assert pers_allow > 0
    return pers_allow
end

function calc_full_ctc(
    bu :: BenefitUnit,
    intermed :: MTIntermediate,
    ctc :: ChildTaxCredit )
    ctc_elements = 0.0
    # p282
    if num_born_before( bu, Date( 2017, 4, 6 )) > 0
        ctc_elements += ctc.family
    end
    ctc_elements += (intermed.num_allowed_children*ctc.child)
    ctc_elements += intermed.num_disabled_children*ctc.disability
    ctc_elements += intermed.num_severely_disabled_children*ctc.severe_disability
    return ctc_elements
end

function calcWTC_CTC!(
    benefit_unit_result :: BenefitUnitResult,
    benefit_unit :: BenefitUnit,
    intermed :: MTIntermediate,
    wtc :: WorkingTaxCredit,
    ctc :: ChildTaxCredit )

    bu = benefit_unit # aliases
    bures = benefit_unit_result
    bu_lmt = bures.legacy_mtbens
    can_apply_for = bu_lmt.can_apply_for
    it, ni = aggregate_tax( bures )

    other_income = it.savings_income + it.dividends_income
    # FIXME does this tread pensions correctly - they're disgregarded
    if other_income < wtc.non_earnings_minima
        other_income = 0.0
    end
    nsi = it.non_savings_income - (total( bures, PENSION_CONTRIBUTIONS_EMPLOYEE )+it.pension_relief_at_source )
    income = other_income + nsi
    wtc_elements = 0.0
    ctc_elements = 0.0
    threshold = wtc.threshold
    cost_of_childcare = 0.0
    if can_apply_for.wtc && ( ! wtc.abolished )
        wtc_elements = wtc.basic
        if intermed.is_sparent
            wtc_elements += wtc.lone_parent
        elseif intermed.num_adults > 1
            wtc_elements += wtc.couple
        end
        if intermed.someone_working_ft > 0
            wtc_elements += wtc.hours_ge_30
        end
        if intermed.working_disabled
            wtc_elements += wtc.disability
        end
        # FIXME do servere_working disabled
        cost_of_childcare = 0.0
        for pid in bu.children
            cost_of_childcare += bu.people[pid].cost_of_childcare
        end

        cost_of_childcare *= wtc.childcare_proportion

        if intermed.num_children > 1
            cost_of_childcare = min( wtc.childcare_max_2_plus_children, cost_of_childcare )
        else
            cost_of_childcare = min( wtc.childcare_max_1_child, cost_of_childcare )
        end
        wtc_elements += cost_of_childcare
    end
    if can_apply_for.ctc && ( ! ctc.abolished )
        ctc_elements = calc_full_ctc( bu, intermed, ctc)
        if ! can_apply_for.wtc
            threshold = ctc.threshold
        end
    end

    elements = ctc_elements + wtc_elements
    excess = wtc.taper * max( 0.0, income - threshold )
    wtc_ctc = max( 0.0, elements - excess )
    # allocate
    ctc_amt = min( wtc_ctc, ctc_elements )
    wtc_amt = wtc_ctc - ctc_amt
    ## assign to an individual

    bu_lmt.wtc_ctc_tapered_excess = excess
    bu_lmt.wtc_income = income
    bu_lmt.ctc_elements = ctc_elements
    bu_lmt.wtc_elements = wtc_elements
    bu_lmt.cost_of_childcare = cost_of_childcare
    bu_lmt.wtc_ctc_threshold = threshold

    # to spouse if has one
    recipient = make_recipient( bu, WORKING_TAX_CREDIT )
    bures.legacy_mtbens.wtc_recipient = recipient
    bures.pers[recipient].income[WORKING_TAX_CREDIT] = wtc_amt

    recipient = make_recipient( bu, CHILD_TAX_CREDIT )
    bures.pers[recipient].income[CHILD_TAX_CREDIT] = ctc_amt
end

function calc_NDDs(
    bu       :: BenefitUnit,
    bur      :: BenefitUnitResult,
    intermed :: MTIntermediate,
    incomes  :: LMTIncomes,
    hb       :: HousingBenefits )::Real
    ndd = 0.0
    # income based on the couples income if it is a couple
    any_pay_ndds = false
    for pid in bu.adults
        pays_ndd = true
        pers = bu.people[pid]
        persr = bur.pers[pid]
        # fixme check this list & reference
        if any_positive( persr.income,
            [ATTENDANCE_ALLOWANCE,
             DLA_SELF_CARE,
             PERSONAL_INDEPENDENCE_PAYMENT_DAILY_LIVING,
             SCOTTISH_DISABILITY_ASSISTANCE_WORKING_AGE_DAILY_LIVING,
             PENSION_CREDIT] )
            pays_ndd = false
        elseif pers.registered_blind || pers.registered_partially_sighted
            pays_ndd = false
        elseif pers.age < 18
            pays_ndd = false
        elseif pers.age < 25
            if any_positive(
                    persr.income,
                    [INCOME_SUPPORT,
                    NON_CONTRIB_JOBSEEKERS_ALLOWANCE,
                    NON_CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE] )
                pays_ndd = false
            end
        end
        any_pay_ndds = any_pay_ndds || pays_ndd
    end # pids loop
    if any_pay_ndds
        if intermed.someone_working_ft
            n = size(hb.ndd_incomes)[1]
            w = n
            for i in 1:n
                if hb.ndd_incomes[i] > incomes.gross_earnings
                    w = i
                    break
                end
            end
            ndd = hb.ndd_deductions[w]
        else
            ndd = hb.ndd_deductions[1]
        end
    end # any ndd payable
    return ndd
end

function calculateHB_CTR!(
    household_result :: HouseholdResult,
    which_ben        :: LMTBenefitType,
    hh               :: Household,
    intermed         :: HHIntermed,
    lmt_ben_sys      :: LegacyMeansTestedBenefitSystem,
    age_limits       :: AgeLimits,
    nmt_bens         :: NonMeansTestedSys )
    @argcheck which_ben in [ctr, hb]

    eligible_amount = which_ben == ctr ?
        total(household_result, LOCAL_TAXES ) :
        household_result.housing.allowed_rent

    bus = get_benefit_units( hh )
    nbus = size(bus)[1]
    ndds = 0.0
    for bn in nbus:-1:1 # fixme bn=>buno for consistency
        bu = bus[bn]
        bures = household_result.bus[bn]
        incomes = calc_incomes(
            hb,
            bu,
            bures,
            intermed.buint[bn],
            lmt_ben_sys.income_rules,
            lmt_ben_sys.hours_limits )
        if bn == 1
            benefit = max(0.0, eligible_amount - ndds ) # ndds deducted from eligible rent
            premium = 0.0
            allowances = 0.0
            passported = false
            # FIXME we're doing passpored twice
            if has_any( bures, lmt_ben_sys.hb.passported_bens... )
                # no need to do anything
                passported = true
            else
                ## FIXME pass this in
                premium, premset = calc_premia(
                    hb,
                    bu,
                    bures,
                    intermed.buint[bn],
                    lmt_ben_sys.premia,
                    nmt_bens,
                    age_limits )
                union!(bures.legacy_mtbens.premia, premset)
                allowances = calc_allowances(
                    hb,
                    intermed.buint[bn],
                    lmt_ben_sys.allowances,
                    age_limits,
                    0.0  )
                excess = max( 0.0, incomes.total_income - (premium+allowances))
                if excess > 0
                    taper = which_ben == ctr ? lmt_ben_sys.ctr.taper : lmt_ben_sys.hb.taper
                    benefit = max( 0.0, benefit - taper*excess )
                end

            end

            recipient = make_recipient( bu, HOUSING_BENEFIT )
            bures.legacy_mtbens.hb_recipient = recipient
            bures.legacy_mtbens.ctr_recipient = recipient

            if which_ben == hb
                bures.pers[recipient].income[HOUSING_BENEFIT] = benefit
                bures.legacy_mtbens.hb_passported = passported
                bures.legacy_mtbens.hb_eligible_rent = eligible_amount
                if ! passported
                    bures.legacy_mtbens.hb_premia = premium
                    bures.legacy_mtbens.hb_allowances = allowances
                    bures.legacy_mtbens.hb_incomes = incomes
                end
            elseif which_ben == ctr
                bures.pers[recipient].income[COUNCIL_TAX_BENEFIT] = benefit
                bures.legacy_mtbens.ctr = benefit
                bures.legacy_mtbens.ctr_passported = passported
                bures.legacy_mtbens.ctr_eligible_amount = eligible_amount
                if ! passported
                    bures.legacy_mtbens.ctr_premia = premium
                    bures.legacy_mtbens.ctr_allowances = allowances
                    bures.legacy_mtbens.ctr_incomes = incomes
                end
            end
        else # ndds for hb, not ctr
            if which_ben == hb
                ndds += calc_NDDs(
                    bu,
                    bures,
                    intermed.buint[bn],
                    incomes,
                    lmt_ben_sys.hb )
            end
        end
    end
end # calculateHB_CTR

function calc_legacy_means_tested_benefits!(
    benefit_unit_result :: BenefitUnitResult,
    benefit_unit :: BenefitUnit,
    intermed     :: MTIntermediate,
    mt_ben_sys   :: LegacyMeansTestedBenefitSystem,
    age_limits   :: AgeLimits,
    hours        :: HoursLimits,
    nmt_bens     :: NonMeansTestedSys,
    hh :: Union{Nothing,Household} ) # passed on 1st BU only
    # aliases
    bures = benefit_unit_result
    bu = benefit_unit
    premium = 0.0
    bures.legacy_mtbens.can_apply_for = make_lmt_benefit_applicability(
        mt_ben_sys,
        intermed,
        mt_ben_sys.hours_limits
    )
    # FIXME we need to check the fuck out of this charge
    oo_housing_costs = 0.0
    if hh !== nothing
        if is_owner_occupier( hh.tenure )
            oo_housing_costs = hh.other_housing_charges
        end
    end
    # alias
    can_apply_for = bures.legacy_mtbens.can_apply_for
    # FIXME MIG not really the right name for is,jsa,esa
    which_mig = nothing
    if can_apply_for.esa
        which_mig = esa
    elseif can_apply_for.jsa
        which_mig = jsa
    elseif can_apply_for.is
        which_mig = is
    end
    mig_entitlement = 0.0 # we need to record this for wtc/ctc routing below
    if which_mig !== nothing
        premium,premset = calc_premia(
            which_mig,
            bu,
            bures,
            intermed,
            mt_ben_sys.premia,
            nmt_bens,
            age_limits )
        union!(bures.legacy_mtbens.premia, premset)
        incomes = calc_incomes(
            which_mig,
            bu,
            bures,
            intermed,
            mt_ben_sys.income_rules,
            hours )
        allowances = calc_allowances(
            which_mig,
            intermed,
            mt_ben_sys.allowances,
            age_limits,
            oo_housing_costs )
        bures.legacy_mtbens.mig = premium+allowances
        bures.legacy_mtbens.mig_incomes = incomes
        bures.legacy_mtbens.mig_allowances = allowances
        bures.legacy_mtbens.mig_premia = premium
        mig_entitlement = max( 0.0, bures.legacy_mtbens.mig - incomes.total_income );
        if (! incomes.disqualified_on_capital) && (mig_entitlement > 0)
            # FIXME we just allocate payment to the head of
            # the BU - make this a function or make can_apply_for
            # specify the payee as well as whether the BU qualifies
            if can_apply_for.ctc
                recipient = make_recipient( bu, CHILD_TAX_CREDIT )
                bures.pers[recipient].income[CHILD_TAX_CREDIT] =
                    calc_full_ctc( bu, intermed, mt_ben_sys.child_tax_credit )
            end
            if can_apply_for.esa
                recipient = make_recipient( bu, NON_CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE )
                bures.pers[recipient].income[NON_CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE] = mig_entitlement
                # bures.legacy_mtbens.esa = entitlement
            elseif can_apply_for.jsa
                recipient = make_recipient( bu, NON_CONTRIB_JOBSEEKERS_ALLOWANCE )
                bures.pers[recipient].income[NON_CONTRIB_JOBSEEKERS_ALLOWANCE] = mig_entitlement
                # bures.legacy_mtbens.jsa = entitlement
            elseif can_apply_for.is
                recipient = make_recipient( bu, INCOME_SUPPORT )
                bures.pers[recipient].income[INCOME_SUPPORT] = mig_entitlement
                # bures.legacy_mtbens.is = entitlement
            end
            bures.legacy_mtbens.mig_recipient = recipient
            can_apply_for.wtc = false # no overlapping
        end
    end

    if can_apply_for.pc && ( ! mt_ben_sys.pen_credit_abolished)
        premium, premset = calc_premia(
            pc,
            bu,
            bures,
            intermed,
            mt_ben_sys.premia,
            nmt_bens,
            age_limits )
        union!( bures.legacy_mtbens.premia, premset )
        incomes = calc_incomes(
            pc,
            bu,
            bures,
            intermed,
            mt_ben_sys.income_rules,
            hours )
        allowances = calc_allowances(
            pc,
            intermed,
            mt_ben_sys.allowances,
            age_limits,
            oo_housing_costs
        )
        # NOTE - there can be 'qualifying housing costs' in the MIG;
        # see: CPAG 2122
        bures.legacy_mtbens.mig = premium+allowances # = max( 0.0, entitlement - incomes.total_income );
        bures.legacy_mtbens.pc_incomes = incomes
        bures.legacy_mtbens.pc_allowances = allowances
        bures.legacy_mtbens.pc_premia = premium
        pc_entitlement = max( 0.0,
            bures.legacy_mtbens.mig -
            bures.legacy_mtbens.pc_incomes.total_income )

        recipient = make_recipient( bu, SAVINGS_CREDIT )

        if can_apply_for.sc && ( ! incomes.disqualified_on_capital ) && ( ! mt_ben_sys.savings_credit.abolished )
            scsys = mt_ben_sys.savings_credit #  shortcut
            sc_incomes = calc_incomes(
                sc,
                bu,
                bures,
                intermed,
                mt_ben_sys.income_rules,
                hours
                 )
            thresh = scsys.threshold_single
            maxpay = scsys.max_single
            if intermed.num_adults > 1
                thresh = scsys.threshold_couple
                maxpay = scsys.max_couple
            end
            # see: CPAG 2011/12 edition ch 19
            scent = 0.0
            income_over_thresh = max(0.0, sc_incomes.total_income - thresh )
            if income_over_thresh > 0
                sc_maximum = min( maxpay, scsys.withdrawal_rate * income_over_thresh)
                income_over_mig = incomes.total_income - bures.legacy_mtbens.mig
                if income_over_mig <= 0.0
                    scent = sc_maximum
                else
                    scent = max( 0.0, sc_maximum - (1-scsys.withdrawal_rate)*income_over_mig)
                end
                bures.pers[recipient].income[SAVINGS_CREDIT] = scent
                bures.legacy_mtbens.sc_incomes = sc_incomes
            end
        end

        bures.pers[recipient].income[PENSION_CREDIT] = pc_entitlement
    end

    #
    # Do WTC iff you can apply for it or if you qualify for CTC
    # but not the automatic full amount
    #
    if can_apply_for.wtc || (can_apply_for.ctc && ( mig_entitlement == 0.0 ))
        calcWTC_CTC!(
                bures,
                bu,
                intermed,
                mt_ben_sys.working_tax_credit,
                mt_ben_sys.child_tax_credit )

    end

    if has_any( bures,
            PENSION_CREDIT,
            NON_CONTRIB_JOBSEEKERS_ALLOWANCE,
            NON_CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE,
            INCOME_SUPPORT )
        bures.legacy_mtbens.hb_passported = true
        bures.legacy_mtbens.ctr_passported = true

    end
end

"""
 TODO FIXME housing costs for oo if qualifies for PC,JSA etc.
    see CPAG ch 18
"""
function calc_legacy_means_tested_benefits!(
            household_result :: HouseholdResult,
            household        :: Household,
            intermed         :: HHIntermed,
            lmt_ben_sys      :: LegacyMeansTestedBenefitSystem,
            age_limits       :: AgeLimits,
            hours            :: HoursLimits,
            nmt_bens         :: NonMeansTestedSys,
            hr               :: HousingRestrictions )
    # fixme not just for renters? fixme do this earlier
    household_result.housing = apply_rent_restrictions(
        household, intermed.hhint, hr )
    bus = get_benefit_units(household)
    nbus = size( bus )[1]
    for buno in 1:nbus
        calc_legacy_means_tested_benefits!(
            household_result.bus[buno],
            bus[buno],
            intermed.buint[buno],
            lmt_ben_sys,
            age_limits,
            hours,
            nmt_bens,
            buno == 1 ? household : nothing )
    end
    # hb using the whole hhls but assigned to 1st bu
    if ! lmt_ben_sys.hb.abolished
        calculateHB_CTR!(
            household_result,
            hb,
            household,
            intermed,
            lmt_ben_sys,
            age_limits,
            nmt_bens )
    end

    if ! lmt_ben_sys.ctr.abolished
        calculateHB_CTR!(
            household_result,
            ctr,
            household,
            intermed,
            lmt_ben_sys,
            age_limits,
            nmt_bens )
    end
    #
end

end # module LegacyMeansTestedBenefits


#=

This file is the entry-point for the Absorption module.

It includes this module's source files and exports the relevant keywords.

=#

module Absorption

using SpecialFunctions          # For complex error functions
using Parameters                # For constructing HitranTable with keywords
using DocStringExtensions       # For simplifying docstring
using Interpolations            # For interpolating in lookup tables and interpolating CS
using JLD2                      # For saving and loading the interpolator
using ProgressMeter             # For showing progress, especially in creating interpolator
using KernelAbstractions        # For heterogeneous (GPU+CPU) programming
using CUDAKernels               # Access to CUDADevice
using CUDA                      # For GPU programming
using ForwardDiff, DiffResults  # For auto-differentiation
using NetCDF                    # For loading NetCDF files with constants
using ..Architectures           # For GPU/CPU convenience
using ..Architectures: CPU, GPU # Again for GPU/CPU convenience
import DataInterpolations: CubicSpline as DI_CS # For use in qoft

include("constants/constants.jl")                   # Scientific and mathematical constants
include("constants/mol_weights.jl")                 # Molecular weights
include("constants/TIPS_2017.jl")                   # Partition sums data
include("types.jl")                                 # All types used in this module
include("read_hitran.jl")                                # HITRAN file-related functions
include("complex_error_functions.jl")               # CEFs used in line broadening
include("make_model_helpers.jl")                  # CS interpolator functions
include("compute_absorption_cross_section.jl")      # Cross-section from HITRAN
include("autodiff_helper.jl")                       # Auto-differentiation

# Export the Cross Section models
export AbstractCrossSectionModel, HitranModel, InterpolationModel

# Export the absorption_cross_section functions
export compute_absorption_cross_section, absorption_cross_section

# Export the hitran functions from hitran.jl
export read_hitran, make_hitran_model

# Export the broadening function types
export Doppler, Lorentz, Voigt, AbstractBroadeningFunction

# Export the complex error functions
export AbstractComplexErrorFunction, HumlicekErrorFunction,
       HumlicekWeidemann32VoigtErrorFunction, HumlicekWeidemann32SDErrorFunction,
       CPF12ErrorFunction, ErfcHumliErrorFunctionVoigt, ErfcHumliErrorFunctionSD,
       ErfcErrorFunction

# Export the hitran table struct type
export HitranTable

# Export the interpolator functions
export make_interpolation_model, save_interpolation_model, load_interpolation_model

end


module Lasso
using Compat
module Util
    # Extract fields from object into function locals
    # See https://github.com/JuliaLang/julia/issues/9755
    macro extractfields(from, fields...)
        esc(Expr(:block, [:($(fields[i]) = $(from).$(fields[i])) for i = 1:length(fields)]...))
    end
    export @extractfields
end

import Base.LinAlg.BlasReal
include("FusedLasso.jl")
include("TrendFiltering.jl")

using Reexport, StatsBase, .Util
@reexport using GLM, Distributions, .FusedLassoMod, .TrendFiltering
using GLM.FPVector, GLM.wrkwt!
export LassoPath, fit, fit!, coef

## HELPERS FOR SPARSE COEFFICIENTS

immutable SparseCoefficients{T} <: AbstractVector{T}
    coef::Vector{T}              # Individual coefficient values
    coef2predictor::Vector{Int}  # Mapping from indices in coef to indices in original X
    predictor2coef::Vector{Int}  # Mapping from indices in original X to indices in coef

    SparseCoefficients(n::Int) = new(T[], Int[], zeros(Int, n))
end

function Base.A_mul_B!{T}(out::Vector, X::Matrix, coef::SparseCoefficients{T})
    fill!(out, zero(eltype(out)))
    @inbounds for icoef = 1:nnz(coef)
        ipred = coef.coef2predictor[icoef]
        c = coef.coef[icoef]
        @simd for i = 1:size(X, 1)
            out[i] += c*X[i, ipred]
        end
    end
    out
end

function Base.A_mul_B!{T}(out::Vector, X::SparseMatrixCSC, coef::SparseCoefficients{T})
    @extractfields X colptr rowval nzval
    fill!(out, zero(eltype(out)))
    @inbounds for icoef = 1:nnz(coef)
        ipred = coef.coef2predictor[icoef]
        c = coef.coef[icoef]
        @simd for i = colptr[ipred]:colptr[ipred+1]-1
            out[rowval[i]] += c*nzval[i]
        end
    end
    out
end

function Base.dot{T}(x::Vector{T}, coef::SparseCoefficients{T})
    v = 0.0
    @inbounds @simd for icoef = 1:nnz(coef)
        v += x[coef.coef2predictor[icoef]]*coef.coef[icoef]
    end
    v
end

Base.size(x::SparseCoefficients) = (length(x.predictor2coef),)
Base.nnz(x::SparseCoefficients) = length(x.coef)
Base.getindex{T}(x::SparseCoefficients{T}, ipred::Int) =
    x.predictor2coef[ipred] == 0 ? zero(T) : x.coef[x.predictor2coef[ipred]]

function Base.setindex!{T}(A::Matrix{T}, coef::SparseCoefficients, rg::UnitRange{Int}, i::Int)
    A[:, i] = zero(T)
    for icoef = 1:nnz(coef)
        A[rg[coef.coef2predictor[icoef]], i] = coef.coef[icoef]
    end
    A
end

function Base.copy!(x::SparseCoefficients, y::SparseCoefficients)
    length(x) == length(y) || throw(DimensionMismatch())
    n = length(y.coef)
    resize!(x.coef, n)
    resize!(x.coef2predictor, n)
    copy!(x.coef, y.coef)
    copy!(x.coef2predictor, y.coef2predictor)
    copy!(x.predictor2coef, y.predictor2coef)
    x
end

# Add a new coefficient to x, returning its index in x.coef
function addcoef!{T}(x::SparseCoefficients{T}, ipred::Int)
    push!(x.coef, zero(T))
    push!(x.coef2predictor, ipred)
    coefindex = nnz(x)
    x.predictor2coef[ipred] = coefindex
end

# Add newcoef to column i of coefs
function addcoefs!(coefs::SparseMatrixCSC, newcoef::SparseCoefficients, i::Int)
    n = nnz(coefs)
    nzval = coefs.nzval
    rowval = coefs.rowval
    resize!(nzval, n+nnz(newcoef))
    resize!(rowval, n+nnz(newcoef))
    @inbounds for ipred = 1:length(newcoef.predictor2coef)
        icoef = newcoef.predictor2coef[ipred]
        if icoef != 0
            cval = newcoef.coef[icoef]
            if cval != 0
                n += 1
                nzval[n] = cval
                rowval[n] = ipred
            end
        end
    end
    resize!(nzval, n)
    resize!(rowval, n)
    coefs.colptr[i+1:end] = n+1
end

## COEFFICIENT ITERATION IN SEQUENTIAL OR RANDOM ORDER

if VERSION >= v"0.4-dev+1915"
    # Julia 0.4 has a nice interface that lets us do random coefficient
    # iteration quickly.
    immutable RandomCoefficientIterator
        rng::MersenneTwister
        rg::Base.Random.RangeGeneratorInt{Int,@compat UInt}
        coeforder::Vector{Int}
    end
    const RANDOMIZE_DEFAULT = true

    RandomCoefficientIterator() =
        RandomCoefficientIterator(MersenneTwister(1337), Base.Random.RangeGenerator(1:2), Int[])
else
    immutable RandomCoefficientIterator
        RandomCoefficientIterator() = error("randomization not supported on Julia 0.3")
    end
    const RANDOMIZE_DEFAULT = false
end

typealias CoefficientIterator @compat Union{UnitRange{Int},RandomCoefficientIterator}

# Iterate over coefficients in random order
function Base.start(x::RandomCoefficientIterator)
    if !isempty(x.coeforder)
        @inbounds for i = length(x.coeforder):-1:2
            j = rand(x.rng, x.rg)
            x.coeforder[i], x.coeforder[j] = x.coeforder[j], x.coeforder[i]
        end
    end
    return 1
end
Base.next(x::RandomCoefficientIterator, i) = (x.coeforder[i], i += 1)
Base.done(x::RandomCoefficientIterator, i) = i > length(x.coeforder)

# Add an additional coefficient and return a new CoefficientIterator
function addcoef(x::RandomCoefficientIterator, icoef::Int)
    push!(x.coeforder, icoef)
    RandomCoefficientIterator(x.rng, Base.Random.RangeGenerator(1:length(x.coeforder)), x.coeforder)
end
addcoef(x::UnitRange{Int}, icoef::Int) = 1:length(x)+1

## LASSO PATH

type LassoPath{S<:@compat(Union{LinearModel,GeneralizedLinearModel}),T} <: RegressionModel
    m::S
    nulldev::T                    # null deviance
    nullb0::T                     # intercept of null model, if one was fit
    λ::Vector{T}                  # shrinkage parameters
    autoλ::Bool                   # whether λ is automatically determined
    Xnorm::Vector{T}              # original squared norms of columns of X before standardization
    pct_dev::Vector{T}            # percent deviance explained by each model
    coefs::SparseMatrixCSC{T,Int} # model coefficients
    b0::Vector{T}                 # model intercepts
    niter::Int                    # number of coordinate descent iterations

    LassoPath(m, nulldev::T, nullb0::T, λ::Vector{T}, autoλ::Bool, Xnorm::Vector{T}) =
        new(m, nulldev, nullb0, λ, autoλ, Xnorm)
end

function Base.show(io::IO, path::LassoPath)
    prefix = isa(path.m, GeneralizedLinearModel) ? string(typeof(path.m.rr.d).name.name, " ") : ""
    println(io, prefix*"Lasso Solution Path ($(size(path.coefs, 2)) solutions for $(size(path.coefs, 1)) predictors in $(path.niter) iterations):")

    coefs = path.coefs
    ncoefs = zeros(Int, size(coefs, 2))
    for i = 1:size(coefs, 2)-1
        ncoefs[i] = coefs.colptr[i+1] - coefs.colptr[i]
    end
    ncoefs[end] = nnz(coefs) - coefs.colptr[size(coefs, 2)] + 1
    Base.showarray(io, [path.λ path.pct_dev ncoefs]; header=false)
end

StatsBase.coef(path::LassoPath) = path.coefs

## MODEL CONSTRUCTION

# Controls early stopping criteria with automatic λ
const MIN_DEV_FRAC_DIFF = 1e-5
const MAX_DEV_FRAC = 0.999

# Compute automatic λ values based on X'y and λminratio
function computeλ(Xy, λminratio, α, nλ)
    λmax = abs(Xy[1])
    for i = 1:length(Xy)
        x = abs(Xy[i])
        λmax = ifelse(x > λmax, x, λmax)
    end
    λmax /= α
    logλmax = log(λmax)
    λ = exp(linspace(logλmax, logλmax + log(λminratio), nλ))
end

function StatsBase.fit{T<:AbstractFloat,V<:FPVector}(::Type{LassoPath},
                                                     X::AbstractMatrix{T}, y::V, d::UnivariateDistribution=Normal(),
                                                     l::Link=canonicallink(d);
                                                     wts::@compat(Union{FPVector,Void})=ones(T, length(y)),
                                                     offset::V=similar(y, 0),
                                                     α::Number=one(eltype(y)), nλ::Int=100,
                                                     λminratio::Number=ifelse(size(X, 1) < size(X, 2), 0.01, 1e-4),
                                                     λ::@compat(Union{Vector,Void})=nothing, standardize::Bool=true,
                                                     intercept::Bool=true,
                                                     naivealgorithm::Bool=(!isa(d, Normal) || !isa(l, IdentityLink) || size(X, 2) > 5*size(X, 1)),
                                                     dofit::Bool=true,
                                                     irls_tol::Real=1e-7, randomize::Bool=RANDOMIZE_DEFAULT,
                                                     maxncoef::Int=min(size(X, 2), 2*size(X, 1)), fitargs...)
    size(X, 1) == size(y, 1) || DimensionMismatch("number of rows in X and y must match")
    n = length(y)
    length(wts) == n || error("length(wts) = $(length(wts)) should be 0 or $n")

    # Standardize predictors if requested
    if standardize
        Xnorm = vec(full(std(X, 1, corrected=false)))
        for i = 1:length(Xnorm)
            @inbounds Xnorm[i] = 1/Xnorm[i]
        end
        X = scale(X, Xnorm)
    else
        Xnorm = T[]
    end

    # Lasso initialization
    α = convert(T, α)
    λminratio = convert(T, λminratio)
    coefitr = randomize ? RandomCoefficientIterator() : (1:0)
    cd = naivealgorithm ? NaiveCoordinateDescent{T,intercept,typeof(X),typeof(coefitr)}(X, α, maxncoef, 1e-7, coefitr) :
                          CovarianceCoordinateDescent{T,intercept,typeof(X),typeof(coefitr)}(X, α, maxncoef, 1e-7, coefitr)

    # GLM response initialization
    autoλ = λ == nothing
    wts .*= convert(T, 1/sum(wts))
    off = convert(Vector{T}, offset)

    if isa(d, Normal) && isa(l, IdentityLink)
        # Special no-IRLS case
        mu = isempty(offset) ? y : y + off
        nullb0 = intercept ? mean(mu, weights(wts)) : zero(T)
        nulldev = 0.0
        @simd for i = 1:length(mu)
            @inbounds nulldev += abs2(mu[i] - nullb0)*wts[i]
        end

        if autoλ
            # Find max λ
            if intercept
                muscratch = Array(T, length(mu))
                @simd for i = 1:length(mu)
                    @inbounds muscratch[i] = (mu[i] - nullb0)*wts[i]
                end
            else
                muscratch = mu.*wts
            end
            Xy = X'muscratch
            λ = computeλ(Xy, λminratio, α, nλ)
        else
            λ = convert(Vector{T}, λ)
        end

        # First y is just a placeholder here
        model = LinearModel(LmResp{typeof(y)}(mu, off, wts, y), cd)
    else
        # Fit to find null deviance
        # Maybe we should reuse this GlmResp object?
        nullmodel = fit(GeneralizedLinearModel, ones(T, n, ifelse(intercept, 1, 0)), y, d, l;
                        wts=wts, offset=offset, convTol=irls_tol)
        nulldev = deviance(nullmodel)

        if autoλ
            # Find max λ
            Xy = X'*broadcast!(*, nullmodel.rr.wrkresid, nullmodel.rr.wrkresid, nullmodel.rr.wrkwts)
            λ = computeλ(Xy, λminratio, α, nλ)
            nullb0 = intercept ? coef(nullmodel)[1] : zero(T)
        else
            λ = convert(Vector{T}, λ)
            nullb0 = zero(T)
        end

        eta = GLM.initialeta!(d, l, similar(y), y, wts, off)
        rr = GlmResp{typeof(y),typeof(d),typeof(l)}(y, d, l, eta, similar(eta), offset, wts)
        model = GeneralizedLinearModel(rr, cd, false)
    end

    # Fit path
    path = LassoPath{typeof(model),T}(model, nulldev, nullb0, λ, autoλ, Xnorm)
    dofit && fit!(path; irls_tol=irls_tol, fitargs...)
    path
end

include("coordinate_descent.jl")

end


# Parts of this code was written by @jpfairbanks.

# Parallel Breadth-first search / traversal using a frontier based parallelized
# approach.

#################################################
#
# Parallel frontier based Breadth-first search approach
#
#################################################

using Base.Threads

import Base: push!, shift!, isempty, getindex

export bfs_tree, LevelSynchronousBFS

mutable struct LevelSynchronousBFS <: AbstractGraphVisitAlgorithm end

"""
    ThreadQueue

A thread safe queue implementation for using as the queue for BFS.
"""

immutable ThreadQueue{T, N<:Integer}
    data::Vector{T}
    head::Atomic{N} #Index of the head
    tail::Atomic{N} #Index of the tail
end

function ThreadQueue(T::Type, maxlength::N) where N <: Integer
    q = ThreadQueue(Vector{T}(maxlength), Atomic{N}(1), Atomic{N}(1))
    return q
end

function push!{T, N}(q::ThreadQueue{T, N}, val::T)
    # TODO: check that head > tail
    offset = atomic_add!(q.tail, one(N))
    q.data[offset] = val
    return offset
end

function shift!{T, N}(q::ThreadQueue{T, N})
    # TODO: check that head < tail
    offset = atomic_add!(q.head, one(N))
    return q.data[offset]
end

function isempty{T, N}(q::ThreadQueue{T, N})
    return ( q.head[] == q.tail[] ) && q.head != one(N)
end

function getindex{T}(q::ThreadQueue{T}, iter)
    return q.data[iter]
end

# Traverses the vertices in the queue and adds newly found successors to the queue.
function bfskernel{T <: Integer}(
        alg::LevelSynchronousBFS,
        next::ThreadQueue, # Thread safe queue to add vertices to
        g::AbstractGraph, # The graph
        parents::Array{Atomic{T}}, # Parents array
        level::Array{T} # Vertices in the current frontier
    )
    @threads for src in level
        vertexneighbors = neighbors(g, src) # Get the neighbors of the vertex
        for vertex in vertexneighbors
            # Atomically check and set parent value if not set yet.
            parent = atomic_cas!(parents[vertex], zero(T), src)
            if parent==0
                push!(next, vertex) #Push onto queue if newly found
            end
        end
    end
end

"""
    bfs_tree!(LevelSynchronousBFS(), g, src, parents)

Provide a parallel breadth-first traversal of the graph `g` starting with source vertex `s`,
and return a parents array. The returned array is an Array of `Atomic` integers.

### Implementation Notes
This function uses `@threads` for parallelism which depends on the `JULIA_NUM_THREADS`
environment variable to decide the number of threads to use. Refer `@threads` documentation
for more details.
"""
function bfs_tree!(
        alg::LevelSynchronousBFS,
        next::ThreadQueue, # Thread safe queue to add vertices to
        g::AbstractGraph, # The graph
        source::T, # Source vertex
        parents::Array{Atomic{T}} # Parents array
    ) where T<:Integer
    parents[source][]=source # Set source to source
    push!(next, source) # Add source to the queue
    while !isempty(next)
        level = next[next.head[]:next.tail[]-1] # Get vertices in the frontier
        next.head[] = next.tail[] # reset the queue
        bfskernel(alg, next, g, parents, level) # Find new frontier
    end
    return parents
end

"""
    bfs_tree(LevelSynchronousBFS(), g, s, nv)

Provide a parallel breadth-first traversal of the graph `g` starting with source vertex `s`,
and return a directed acyclic graph of vertices in the order they were discovered
using a frontier based parallel approach.

### Implementation Notes
This function uses `@threads` for parallelism which depends on the `JULIA_NUM_THREADS`
environment variable to decide the number of threads to use. Refer `@threads` documentation
for more details.
This function is a high level wrapper around [`bfs_tree!`](@ref); use that function for more performance.
"""
function bfs_tree{T <: Integer}(alg::LevelSynchronousBFS, g::AbstractGraph, source::T, nv::T)
    next = ThreadQueue(T, nv) # Initialize threadqueue
    parents = [Atomic{T}(0) for i=1:nv] # Create parents array
    bfs_tree!(alg, next, g, source, parents)
    tree([i[] for i in parents])
end

function bfs_tree{T <: Integer}(alg::LevelSynchronousBFS, g::AbstractGraph, source::T)
    nvg = nv(g)
    bfs_tree(alg, g, source, nvg)
end


using MathOptInterface, Test

const CleverDicts = MathOptInterface.Utilities.CleverDicts

# Note: `MyKey` is just for testing. You wouldn't want to use it in practice
# because the key type of the dictionary isn't a concrete type.
struct MyKey{X} end
CleverDicts.key_to_index(::MyKey{X}) where {X} = X
CleverDicts.index_to_key(::Type{MyKey}, index::Int) = MyKey{index}()

@testset "CleverDict" begin
    @testset "MyKey type" begin
        d = CleverDicts.CleverDict{MyKey, String}()
        key = CleverDicts.add_item(d, "first")
        @test key == MyKey{1}()
        @test d[MyKey{1}()] == "first"
    end

    @testset "get/set" begin
        d = CleverDicts.CleverDict{MathOptInterface.VariableIndex, String}()
        key = CleverDicts.add_item(d, "first")
        @test key == MathOptInterface.VariableIndex(1)
        sizehint!(d, 1)
        @test d[key] == "first"
        @test haskey(d, key) == true
        @test_throws KeyError d[MathOptInterface.VariableIndex(2)]
        delete!(d, key)
        sizehint!(d, 2)
        @test_throws KeyError d[key]
        # set index is valid now
        # @test_throws KeyError d[key] = "key"
        @test haskey(d, key) == false
        key2 = CleverDicts.add_item(d, "second")
        @test key2 == MathOptInterface.VariableIndex(2)
        @test d[key2] == "second"
        d[key2] = "third"
        @test d[key2] == "third"

        empty!(d)

        key = CleverDicts.add_item(d, "first")
        @test key == MathOptInterface.VariableIndex(1)
        @test d[key] == "first"
        d[key] = "zeroth"
        @test d[key] == "zeroth"
        @test haskey(d, key) == true
        @test_throws KeyError d[MathOptInterface.VariableIndex(2)]
        delete!(d, key)
        @test_throws KeyError d[key]
        # set index is valid now
        # @test_throws KeyError d[key] = "key"
        @test haskey(d, key) == false
        key2 = CleverDicts.add_item(d, "second")
        @test key2 == MathOptInterface.VariableIndex(2)
        @test d[key2] == "second"
    end

    @testset "LinearIndex" begin
        d = CleverDicts.CleverDict{MathOptInterface.VariableIndex, String}()
        key = CleverDicts.add_item(d, "first")
        @test d[CleverDicts.LinearIndex(1)] == "first"
        key2 = CleverDicts.add_item(d, "second")
        @test d[CleverDicts.LinearIndex(2)] == "second"
        @test length(d) == 2
        delete!(d, key)
        @test d[CleverDicts.LinearIndex(1)] == "second"
        @test_throws KeyError d[CleverDicts.LinearIndex(2)]
        @test length(d) == 1
    end

    @testset "keys/values" begin
        d = CleverDicts.CleverDict{MathOptInterface.VariableIndex, String}()
        key = CleverDicts.add_item(d, "first")
        key2 = CleverDicts.add_item(d, "second")
        @test collect(keys(d)) == [MathOptInterface.VariableIndex(1), MathOptInterface.VariableIndex(2)]
        @test collect(values(d)) == ["first", "second"]
        delete!(d, key)
        key3 = CleverDicts.add_item(d, "third")
        @test collect(keys(d)) == [MathOptInterface.VariableIndex(2), MathOptInterface.VariableIndex(3)]
        @test collect(values(d)) == ["second", "third"]
    end

    @testset "iterate" begin
        d = CleverDicts.CleverDict{MathOptInterface.VariableIndex, String}()
        key = CleverDicts.add_item(d, "first")
        key2 = CleverDicts.add_item(d, "second")
        my_keys = MathOptInterface.VariableIndex[]
        my_values = String[]
        for (k, v) in d
           push!(my_keys, k)
           push!(my_values, v)
        end
        @test my_keys == [MathOptInterface.VariableIndex(1), MathOptInterface.VariableIndex(2)]
        @test my_values == ["first", "second"]
        delete!(d, key)
        key3 = CleverDicts.add_item(d, "third")
        my_keys = MathOptInterface.VariableIndex[]
        my_values = String[]
        for (k, v) in d
           push!(my_keys, k)
           push!(my_values, v)
        end
        @test my_keys == [MathOptInterface.VariableIndex(2), MathOptInterface.VariableIndex(3)]
        @test my_values == ["second", "third"]
    end

    @testset "iterate ii" begin
        d = CleverDicts.CleverDict{MathOptInterface.VariableIndex, String}()
        key = CleverDicts.add_item(d, "first")
        key2 = CleverDicts.add_item(d, "second")
        my_keys = MathOptInterface.VariableIndex[]
        my_values = String[]
        for (k, v) in d
            push!(my_keys, k)
            push!(my_values, v)
        end
        @test my_keys == [MathOptInterface.VariableIndex(1), MathOptInterface.VariableIndex(2)]
        @test my_values == ["first", "second"]
        delete!(d, key)
        @test d[CleverDicts.LinearIndex(1)] == "second"
        key3 = CleverDicts.add_item(d, "third")
        my_keys = MathOptInterface.VariableIndex[]
        my_values = String[]
        for (k, v) in d
            push!(my_keys, k)
            push!(my_values, v)
        end
        @test my_keys == [MathOptInterface.VariableIndex(2), MathOptInterface.VariableIndex(3)]
        @test my_values == ["second", "third"]
    end

    @testset "iterate iii" begin
        d = CleverDicts.CleverDict{MathOptInterface.VariableIndex, String}()
        y = 0
        for (k, v) in d
            y += 1
        end
        @test y == 0
    end

    @testset "haskey" begin
        d = CleverDicts.CleverDict{MathOptInterface.VariableIndex, String}()
        @test !haskey(d, 1)
        k = CleverDicts.add_item(d, "a")
        @test haskey(d, k)
        j = CleverDicts.add_item(d, "b")
        @test haskey(d, j)
        delete!(d, k)
        @test !haskey(d, k)
        @test haskey(d, j)
    end

    @testset "haskey" begin
        d = CleverDicts.CleverDict{MathOptInterface.VariableIndex, String}()
        @test isempty(d) == true
        k = CleverDicts.add_item(d, "a")
        @test isempty(d) == false
        delete!(d, k)
        @test isempty(d) == true
        j = CleverDicts.add_item(d, "b")
        @test isempty(d) == false
    end

    @testset "delete!" begin
        d = CleverDicts.CleverDict{MathOptInterface.VariableIndex, String}()
        @test length(d) == 0
        @test delete!(d, MathOptInterface.VariableIndex(0)) == nothing
        k1 = CleverDicts.add_item(d, "a")
        k2 = CleverDicts.add_item(d, "b")
        d[CleverDicts.LinearIndex(2)] == "b"
        delete!(d, k1)
        d[CleverDicts.LinearIndex(1)] == "b"
        k3 = CleverDicts.add_item(d, "c")
        @test d[k3] == "c"
        @test d[CleverDicts.LinearIndex(1)] == "b"
        @test d[CleverDicts.LinearIndex(2)] == "c"
    end

    @testset "Dense Operations" begin
        mul2(x) = x * 2
        div2(x) = div(x, 2)
        d = CleverDicts.CleverDict{Int, Float64}(div2, mul2, 3)

        d[4] = 0.25
        @test !haskey(d, 2)
        @test haskey(d, 4)
        @test !haskey(d, 6)
        @test length(d) == 1
        @test collect(d) == [4 => 0.25]
        @test d[4] == 0.25

        d[2] = 1.5
        @test haskey(d, 2)
        @test haskey(d, 4)
        @test !haskey(d, 6)
        @test length(d) == 2
        @test collect(d) == [2 => 1.5, 4 => 0.25]
        @test d[2] == 1.5
        @test d[4] == 0.25

        d[6] = 0.75
        @test haskey(d, 2)
        @test haskey(d, 4)
        @test haskey(d, 6)
        @test length(d) == 3
        @test collect(d) == [2 => 1.5, 4 => 0.25, 6 => 0.75]
        @test d[2] == 1.5
        @test d[4] == 0.25
        @test d[6] == 0.75
    end
end


using Test

@info "Testing TimeSeriesEcon:"
using TimeSeriesEcon
cd(joinpath(dirname(pathof(TimeSeriesEcon)), "..", "test")) do
    include(joinpath(pwd(), "runtests.jl"))
end

@info "Testing ModelBaseEcon:"
using ModelBaseEcon
cd(joinpath(dirname(pathof(ModelBaseEcon)), "..", "test")) do
    include(joinpath(pwd(), "runtests.jl"))
end

@info "Testing StateSpaceEcon:"
using StateSpaceEcon
cd(joinpath(dirname(pathof(StateSpaceEcon)), "..", "test")) do
    include(joinpath(pwd(), "runtests.jl"))
end

@info "Testing Tutorials"
@testset "US_SW07" begin
    include(joinpath("..", "src", "Tutorials", "US_SW07", "main.jl"))
end

@info "All done."


# Abstract types

## Spatial sampling types

abstract type AbstractSpatialSampling{T} end
abstract type AbstractCartesianSpatialSampling{T}<:AbstractSpatialSampling{T} end

## k-space sampling types

abstract type AbstractKSpaceSampling{T} end
abstract type AbstractKSpaceFixedSizeSampling{T}<:AbstractKSpaceSampling{T} end
abstract type AbstractKSpaceCartesianSampling{T}<:AbstractKSpaceFixedSizeSampling{T} end


using HDF5

d = h5read(ARGS[1], "/")
x = d["eigen_vector_0"]
for i in 1:(length(d["eigen_values"])-1)
    x += d["eigen_vector_$i"]
end

println(length(x))
for i in x
    println(i)
end


"Hacked Parameter Space"
module Parameters

export choice, Phi

struct Phi
end

choice(::Phi, xs::AbstractVector) = rand(xs)
choice(::Phi, ::Type{T}) where T <:Number = rand(T)

end


# easier creation of atoms

function (p::Predicate)(args::Vararg{Union{Term,Int64,Float64}})
    Literal(p, [x for x in args])
end

function (f::Functor)(args::Vararg{Union{Term,Int64,Float64}})
    Structure(f, [x for x in args])
end

# \+(literal::Literal) = Negation(literal)

Base.:!(literal::Literal) = Negation(literal)

"""
    Easier syntax for constructing clauses
"""
function Base.:&(left::Union{Literal,Negation,Proposition}, right::Union{Negation,Literal,Proposition})
    Conj([left, right])
end

function Base.:&(left::Conj, right::Union{Literal,Negation,Proposition})
    Conj(vcat(left, right))
end

function Base.:(<=)(left::Union{Literal,Proposition}, right::Conj)
    Clause(left, right)
end

function Base.:(<=)(left::Union{Literal,Proposition}, right::Union{Literal,Negation})
    Clause(left, Conj([right]))
end


# read.jl: Silo variable reading routines

export siloread

function siloread{T1<:AbstractString}(file::silofile, name::T1)
  const curve_int = unsafe_load(DBGetCurve(file.dbfile, name))
#   dump(curve_int)
  const npts = curve_int.npts
#   println("Importing $npts points from ", bytestring(curve_int.title))
#   dump(curve_int.x)
  const y_values_eltype = curve_int.datatype
#   println(y_values_eltype)
#   datatype = Float32
  if haskey(DBdatatype.SiloJuliatypemap, y_values_eltype)
    datatype = DBdatatype.SiloJuliatypemap[y_values_eltype]
  else
    error("Datatype $y_values_eltype currently unsupported.")
  end
  xv=pointer_to_array(reinterpret(Ptr{datatype}, curve_int.x), npts)
  yv=pointer_to_array(reinterpret(Ptr{datatype}, curve_int.y), npts)
  return xv, yv
end


using CairoMakie
using Pollen
using FastAI
using FluxTraining
using DLPipelines
using DataAugmentation
using FilePathsBase

ENV["DATADEPS_ALWAYS_ACCEPT"] = "true"

refmodules = [FastAI, FluxTraining, DLPipelines, DataAugmentation, FastAI.Datasets, FastAI.Models]
project = Pollen.documentationproject(FastAI; refmodules = refmodules)
Pollen.fullbuild(project, Pollen.FileBuilder(Pollen.HTML(), p"dev/"))


using DataFrames: DataFrames

@testset "stats" begin
    idata = load_arviz_data("centered_eight")

    @testset "compare" begin
        idata2 = load_arviz_data("non_centered_eight")
        df = compare(Dict("a" => idata, "b" => idata2))
        @test df isa DataFrames.DataFrame
    end

    @testset "hdi" begin
        rng = Random.MersenneTwister(42)
        x = randn(rng, 100)
        @test hdi(x) == ArviZ.arviz.hdi(x)
    end

    @testset "r2_score" begin
        rng = Random.MersenneTwister(42)
        ytrue = randn(rng, 100)
        ypred = randn(rng, 100)
        df = r2_score(ytrue, ypred)
        @test df isa DataFrames.DataFrame
        @test all(df == ArviZ.todataframes(ArviZ.arviz.r2_score(ytrue, ypred)))
    end

    @testset "loo" begin
        df = loo(idata)
        @test df isa DataFrames.DataFrame
        @test all(df == ArviZ.todataframes(ArviZ.arviz.loo(idata)))
    end

    @testset "waic" begin
        df = waic(idata)
        @test df isa DataFrames.DataFrame
        @test all(df == ArviZ.todataframes(ArviZ.arviz.waic(idata)))
    end

    @testset "loo_pit" begin
        ret = loo_pit(idata; y="obs")
        @test ret == ArviZ.arviz.loo_pit(idata; y="obs")
    end

    @testset "summarystats" begin
        rng = MersenneTwister(42)
        nchains, ndraws = 4, 10
        idata = convert_to_inference_data(
            Dict(
                "a" => randn(rng, nchains, ndraws), "b" => randn(rng, nchains, ndraws, 3, 4)
            ),
        )

        s = summarystats(idata)
        @test s isa DataFrames.DataFrame
        @test string(first(names(summarystats(idata)))) == "variable"
        @test string(first(names(summarystats(idata; fmt="wide")))) == "variable"
        @test :variable in propertynames(summarystats(idata; fmt="wide"))
        @test "a" ∈ s.variable
        @test "b" ∉ s.variable
        @test "b[0, 0]" ∉ s.variable
        @test "b[1, 1]" ∈ s.variable

        s2 = summarystats(idata; fmt="long")
        @test s2 isa DataFrames.DataFrame
        @test string(first(names(s2))) == "statistic"
        @test "mean" ∈ s2.statistic

        s3 = summarystats(idata; fmt="xarray")
        @test s3 isa ArviZ.Dataset
    end

    @testset "ArviZ.summary" begin
        rng = MersenneTwister(42)
        nchains, ndraws = 4, 10
        data = Dict(
            "a" => randn(rng, nchains, ndraws), "b" => randn(rng, nchains, ndraws, 3, 4)
        )

        @test ArviZ.summary(data) isa DataFrames.DataFrame
    end
end


__precompile__(true)

module DistributedArrays

using Primes
using Primes: factor

importall Base
import Base.Callable
import Base.BLAS: axpy!

export (.+), (.-), (.*), (./), (.%), (.<<), (.>>), div, mod, rem, (&), (|), ($)
export DArray, SubDArray, SubOrDArray, @DArray
export dzeros, dones, dfill, drand, drandn, distribute, localpart, localindexes, ppeval, samedist
export close, darray_closeall

include("core.jl")
include("serialize.jl")
include("mapreduce.jl")
include("linalg.jl")
include("sort.jl")

end # module


using ClearswiApp
@time msg = clearswi_main(ARGS)
println(msg)


module CuckooFilters
using SHA: sha256

include("bucket.jl")
include("filter.jl")

fingerprint(x::AbstractVector{UInt8}) = sha256(x)[1]
fingerprint(x::Integer) = UInt8(x & 0xff)
fingerprint_type(::Type{<:AbstractVector{UInt8}}) = UInt8
fingerprint_type(::Type{<:Integer}) = UInt8

# Exports
export CuckooFilter
export insert_unique!

end # module


"""
    reproject(source::GeoFormat, target::GeoFormat, dim::Dimension, val)

`reproject` uses ArchGDAL.reproject, but implemented for a reprojecting
a value array of values, a single dimension at a time.
"""
reproject(source, target, dim, val) = val
function reproject(source::GeoFormat, target::GeoFormat, dim::Union{XDim,YDim}, val::Number)
    # This is a dumb way to do this. But it save having to inspect crs,
    # and prevents reprojections that dont make sense from working.
    # A better method for this should be implemented in future.
    return first(reproject(source, target, dim, [val]))
end
function reproject(source::GeoFormat, target::GeoFormat, dim, vals::NTuple{N}) where N
    reps = reproject(source, target, dim, [vals...])
    return ntuple(x -> reps[x], N)
end
function reproject(source::GeoFormat, target::GeoFormat, dim::Union{XDim,YDim}, vals::AbstractArray)
    reshape(reproject(source, target, dim, vec(vals)), size(vals))
end
function reproject(source::GeoFormat, target::GeoFormat, ::XDim, vals::AbstractVector)
    paired_vals = map(v -> (v, zero(v)), vals)
    push!(paired_vals, (first(vals), one(first(vals))))
    reps = AG.reproject(paired_vals, source, target; order=:trad)
    at_one = pop!(reps)
    first(reps)[1] == at_one[1] || _rep_error(source, target)
    return map(r -> r[1], reps)
end
function reproject(source::GeoFormat, target::GeoFormat, dim::YDim, vals::AbstractVector)
    paired_vals = map(v -> (zero(v), v), vals)
    push!(paired_vals, (one(first(vals)), first(vals)))
    reps = AG.reproject(paired_vals, source, target; order=:trad)
    at_one = pop!(reps)
    first(reps)[2] == at_one[2] || _rep_error(source, target)
    return map(r -> r[2], reps)
end

_rep_error(source, target) = throw(ArgumentError("Cannot reproject from: \n $source \nto: \n $target"))


using SeaPearl: DefaultStateRepresentation, VariableVertex, ValueVertex

function plottripartite(sr::DefaultStateRepresentation)
    cpmodel = sr.cplayergraph
    n = cpmodel.totalLength
    nodefillc = []
    for id in 1:n
        v = cpmodel.idToNode[id]
        if isa(v, VariableVertex) push!(nodefillc,"red")
        elseif isa(v, ValueVertex) push!(nodefillc,"blue")
        else  push!(nodefillc,"black") end
    end
    gplot(cpmodel;nodefillc=nodefillc)
end

export plottripartite


function filter_desc{T}(a::CuArray{T}, format=CUDNN_TENSOR_NCHW)
  csize = Cint[size(a,i) for i=ndims(a):-1:1]
  #csize = Cint[size(a,i) for i=1:ndims(a)]
  p = Ptr{Void}[0]
  cudnnCreateFilterDescriptor(p)
  cudnnSetFilterNdDescriptor(p[1], datatype(T), format, ndims(a), csize)
  fd = new(p[1])
  finalizer(fd, cudnnDestroyFilterDescriptor)
  fd
end


@testset "Peters' Finite State" begin

    submodel = Submodel(Peters{4}())

    # test provided jacobians
    submodel_jacobian_tests(submodel)

    # test consistency of input/output functions
    submodel_io_tests(submodel)
end


using ShortStrings, BenchmarkTools

# these two should be on par
@benchmark hash($(ShortString{UInt128}("this is a test")), $(zero(UInt)))
@benchmark hash($(rand(UInt128)), $(zero(UInt)))

# faster for some reason
@benchmark hash($"this is a test", $(zero(UInt)))

# these two should be on par
@benchmark hash($(ShortString{UInt128}("this is a test")))
@benchmark hash($(rand(UInt128)))

# faster
@benchmark hash($"this is a test")


"""
Base type for objects wrapping measurement data.
"""
abstract AbstractData

"""
    data(s::Symbol,args...)

Create data objects.

Currently implemented variants:
---

"""
data(s::Symbol,args...) = data(Val{s},args...)

"""
    datatypename(d)
Return the name of the data type (used by `show()`).
"""
function datatypename(d::AbstractData) end

"""
    numvalues(d)
Return the number of data values.
"""
function numvalues(d::AbstractData) end

"""
    numvars(d)
Return the number of variables.
"""
function numvars(d::AbstractData) end

"""
    eltype(d)
Return the type of the data values.
"""
function eltype(d::AbstractData) end

"""
    generate!(d)
Generate a set of data values. The function should return its argument d.
"""
function generate!(d::AbstractData) end

"""
    dataindex(d)
Retrieve the data index.
"""
function dataindex(d::AbstractData) end

"""
    datavalues(d)
Retrieve the data values.
"""
function datavalues(d::AbstractData) end

function show(io::IO,d::AbstractData)
    nvals = (n = numvalues(d) ; n>1?string(n," values"):string(n," value"))
    nvars = (n = numvars(d) ; n>1?string(n," variables"):string(n," variable"))
    println(io,"$(datatypename(d)) with $(nvars) and $(nvals)")
    println(io," index: ",dataindex(d))
    println(io," values: ",datavalues(d))
    nothing
end


module CyclicArrays

 include("utils.jl")
 export CyclicArray,shiftc,stagger,diff

 """
     CyclicArray
 CyclicArray data structure. Available constructors:
 ```
 CyclicArray(data::AbstractArray{T,N}, connections::AbstractArray)
 ```
 """
 struct CyclicArray{T,N} <: AbstractArray{T,N} #inherits from AbstractArray
     data::AbstractArray{T,N}
     connections
 end

 #Base.show(io::IO, A::CyclicArray{T,1}) where T = Base.show(io, A.data)
 Base.show(io::IO, ::MIME"text/plain", A::CyclicArray{T,1}) where{T} =
           print(io, length(A),"-element CyclicArray{$T,1}:\n   ", A.data)
 Base.view(A::CyclicArray) = Base.view(A.data)

 CyclicArray(x::CyclicArray)=CyclicArray([],x.connections)
 CyclicArray(x::AbstractArray,y::CyclicArray)=CyclicArray(x,y.connections)
 CyclicArray(connections)=CyclicArray([],connections)

 Base.ndims(A::CyclicArray) = ndims(A.data)
 Base.Dims(A::CyclicArray) = Dims(A.data)
 Base.size(A::CyclicArray) = size(A.data)
 Base.length(A::CyclicArray)=length(A.data)
 Base.axes(A::CyclicArray) = axes(A.data)
 Base.findall(A::CyclicArray) = findall(A.data)
 Base.findfirst(A::CyclicArray) = findfirst(A.data)
 Base.findlast(A::CyclicArray) = findlast(A.data)
 Base.findprev(A::CyclicArray, I...) = findprev(A.data, I...)
 Base.findnext(A::CyclicArray, I...) = findnext(A.data, I...)
 Base.checkbounds(A::CyclicArray, I...) = nothing
 Base.CartesianIndices(A::CyclicArray) = CartesianIndices(A.data)
 Base.maximum(A::CyclicArray,dims) = maximum(A.data, dims=dims)
 Base.minimum(A::CyclicArray,dims) = minimum(A.data, dims=dims)

 function Base.diff(A::CyclicArray; dims=1::Integer)
  I=size(A.data)
  I1=[UnitRange(1:I[i]) for i in 1:length(I)]
  I2=copy(I1)
  I2[dims]=I1[dims].+1
  B=CyclicArray(zeros(size(A.data)),A.connections)
  CI1=CartesianIndices(Tuple(I1))
  CI2=CartesianIndices(Tuple(I2))
  N=length(CI1[1])
  for n in 1:length(CI1)
    I1b=[CI1[n][i]  for i in 1:N]
    I2b=[CI2[n][i]  for i in 1:N]
    B[I1b...]=A[I2b...]-A[I1b...]
  end
  return B
 end

 function stagger(A::CyclicArray; dims=1::Integer, frac=0.5::Real)
  I=size(A.data)
  I1=[UnitRange(1:I[i]) for i in 1:length(I)]
  I2=copy(I1)
  I2[dims]=I1[dims].+Int(sign(frac))
  B=CyclicArray(zeros(size(A.data)),A.connections)
  CI1=CartesianIndices(Tuple(I1))
  CI2=CartesianIndices(Tuple(I2))
  N=length(CI1[1])
  for n in 1:length(CI1)
    I1b=[CI1[n][i]  for i in 1:N]
    I2b=[CI2[n][i]  for i in 1:N]
    B[I1b...]=(1-frac)*A[I1b...]+frac*A[I2b...]
  end
  return B
 end

 function shiftc(A::CyclicArray; dims=1::Integer, shift=1::Real)
   I=size(A.data)
   I1=[UnitRange(1:I[i]) for i in 1:length(I)]
   I2=copy(I1)
   I2[dims]=I1[dims].+shift
   B=CyclicArray(zeros(size(A.data)),A.connections)
   CI1=CartesianIndices(Tuple(I1))
   CI2=CartesianIndices(Tuple(I2))
   N=length(CI1[1])
   for n in 1:length(CI1)
     I1b=[CI1[n][i]  for i in 1:N]
     I2b=[CI2[n][i]  for i in 1:N]
     B[I1b...]=A[I2b...]
   end
   return B
  end

 Base.:-(A::CyclicArray)=CyclicArray(-A.data,A.connections)
 Base.:*(A::CyclicArray, B::CyclicArray)=CyclicArray(.*(A.data,B.data),A.connections)
 Base.:*(A::Number, B::CyclicArray)=CyclicArray(.*(A,B.data),B.connections)
 Base.:*(A::CyclicArray, B::Number)=CyclicArray(.*(A.data,B),A.connections)
 Base.:-(A::CyclicArray, B::CyclicArray)=CyclicArray(.-(A.data,B.data),A.connections)
 Base.:-(A::Number, B::CyclicArray)=CyclicArray(.-(A,B.data),B.connections)
 Base.:-(A::CyclicArray, B::Number)=CyclicArray(.-(A.data,B),A.connections)
 Base.:+(A::CyclicArray, B::CyclicArray)=CyclicArray(.+(A.data,B.data),A.connections)
 Base.:+(A::Number, B::CyclicArray)=CyclicArray(.+(A,B.data),B.connections)
 Base.:+(A::CyclicArray, B::Number)=CyclicArray(.+(A.data,B),A.connections)
 Base.:^(A::CyclicArray, B::CyclicArray)=CyclicArray(.^(A.data,B.data),A.connections)
 Base.:^(A::Number, B::CyclicArray)=CyclicArray(.^(A,B.data),B.connections)
 Base.:^(A::CyclicArray, B::Number)=CyclicArray(.^(A.data,B),A.connections)
 Base.:/(A::CyclicArray, B::CyclicArray)=CyclicArray(./(A.data,B.data),A.connections)
 Base.:/(A::Number, B::CyclicArray)=CyclicArray(./(A,B.data),B.connections)
 Base.:/(A::CyclicArray, B::Number)=CyclicArray(./(A.data,B),A.connections)
 Base.:\(A::CyclicArray, B::CyclicArray)=CyclicArray(.\(A.data,B.data),A.connections)
 Base.:\(A::Number, B::CyclicArray)=CyclicArray(.\(A,B.data),B.connections)
 Base.:\(A::CyclicArray, B::Number)=CyclicArray(.\(A.data,B),A.connections)

 function Base.getindex(A::CyclicArray, I::Vararg{Int, N}) where N # implements A[I]
   connections=A.connections
   nfaces=size(A.connections)[1]
   nspatial=size(A.connections)[2]
   I1=[i for i in I]
   N0=length(I1)
   S=size(A)
   if nspatial==3
     nx=S[N0];ny=S[N0-1];nz=S[N0-2];
     while (I1[N0]<1 || I1[N0]>nx) || (I1[N0-1]<1 || I1[N0-1]>ny) || (I1[N0-2]<1 || I1[N0-2]>nz)
       I2=I1
       i=I1[N0]
       j=I1[N0-1]
       k=I1[N0-2]
       if nfaces>1
         f=I1[N0-2]
       else
         f=1
       end
       if I1[N0]<1
         if connections[f,1,1,1]==-1
           return NaN
         else
           f1=connections[f,1,1,1]
           axis=connections[f,1,1,2];
           side=connections[f,1,1,3];
           flip=connections[f,1,1,4];
           I2[N0] = kd(axis,1) * kd(side,1) * (1-i)   +
                   kd(axis,1) * kd(side,2) * (nx+i) +
                   kd(axis,2) * kd(flip,0) *  j     +
                   kd(axis,2) * kd(flip,1) * (nx+1-j)
           I2[N0-1] = kd(axis,1) * kd(flip,0) *  j +
                     kd(axis,1) * kd(flip,1) * (ny+1+j) +
                     kd(axis,2) * kd(side,1) * (1-i) +
                     kd(axis,2) * kd(side,2) * (ny+i)
           I2[N0-2] = k
           I1=I2
         end
       end
       if I1[N0]>nx
         if connections[f,1,2,1]==-1
           return NaN
         else
           f1=connections[f,1,2,1];
           axis=connections[f,1,2,2];
           side=connections[f,1,2,3];
           flip=connections[f,1,2,4];
           I2[N0] = kd(axis,1) * kd(side,1) * (i-nx)   +
                   kd(axis,1) * kd(side,2) * ((nx+1)-(i-nx)) +
                   kd(axis,2) * kd(flip,0) *  j     +
                   kd(axis,2) * kd(flip,1) * (nx+1-j)
           I2[N0-1] = kd(axis,1) * kd(flip,0) *  j +
                     kd(axis,1) * kd(flip,1) * (ny+1+j) +
                     kd(axis,2) * kd(side,1) * (1-ny) +
                     kd(axis,2) * kd(side,2) * ((ny+1)-(i-ny))
           I2[N0-2] = k
           I1=I2
         end
       end

       if I1[N0-1]<1
         if connections[f,2,1,1]==-1
           return NaN
         else
           f1=connections[f,2,1,1]
           axis=connections[f,2,1,2];
           side=connections[f,2,1,3];
           flip=connections[f,2,1,4];
           I2[N0] = kd(axis,1) * kd(side,1) * (1-i)   +
                   kd(axis,1) * kd(side,2) * (nx-j) +
                   kd(axis,2) * kd(flip,0) *  i     +
                   kd(axis,2) * kd(flip,1) * (nx+1-i)
           I2[N0-1] = kd(axis,1) * kd(flip,0) *  i +
                     kd(axis,1) * kd(flip,1) * (ny+1-i) +
                     kd(axis,2) * kd(side,1) * (1-j) +
                     kd(axis,2) * kd(side,2) * (ny+j)
           I2[N0-2] = k
           I1=I2
         end
       end
       if I1[N0-1]>ny
         if connections[f,2,2,1]==-1
           return NaN
         else
           f1=connections[f,2,2,1]
           axis=connections[f,2,2,2];
           side=connections[f,2,2,3];
           flip=connections[f,2,2,4];
           I2[N0] = kd(axis,1) * kd(side,1) * (j-ny)   +
                   kd(axis,1) * kd(side,2) * (ny+1-(j-ny)) +
                   kd(axis,2) * kd(flip,0) *  i     +
                   kd(axis,2) * kd(flip,1) * (nx+1-i)
           I2[N0-1] = kd(axis,1) * kd(flip,0) *  i +
                     kd(axis,1) * kd(flip,1) * (ny+1-i) +
                     kd(axis,2) * kd(side,1) * (j-ny) +
                     kd(axis,2) * kd(side,2) * (ny+1-(ny-j))
           I2[N0-2] = k
           I1=I2
         end
       end

       if I1[N0-2]<1
         if connections[f,3,1,1]==-1
           return NaN
         else
           f1=connections[f,3,1,1]
           axis=connections[f,3,1,2];
           side=connections[f,3,1,3];
           flip=connections[f,3,1,4];
           I2[N0] = i
           I2[N0-1] = k
           I2[N0-2] = kd(axis,3) * kd(side,1) * (k-nz)   +
                     kd(axis,3) * kd(side,2) * (nz+1-(k-nz))
           I1=I2
         end
       end
       if I1[N0-2]>nz
         if connections[f,3,2,1]==-1
           return NaN
         else
           f1=connections[f,3,2,1]
           axis=connections[f,3,2,2];
           side=connections[f,3,2,3];
           flip=connections[f,3,2,4];
           I2[N0] = i
           I2[N0-1] = j
           I2[N0-2] = kd(axis,3) * kd(side,1) * (k-nz)   +
                     kd(axis,3) * kd(side,2) * (nz+1-(k-nz))
           I1=I2
         end
       end
     end
   # 2d array
   elseif nspatial==2
     nx=S[N0];ny=S[N0-1]
     while (I1[N0]<1 || I1[N0]>nx) || (I1[N0-1]<1 || I1[N0-1]>ny)
       I2=I1
       i=I1[N0]
       j=I1[N0-1]
       if nfaces>1
         f=I1[N0-2]
       else
         f=1
       end
       if I1[N0]<1
         if connections[f,1,1,1]==-1
           return NaN
         else
           f1=connections[f,1,1,1]
           axis=connections[f,1,1,2];
           side=connections[f,1,1,3];
           flip=connections[f,1,1,4];
           I2[N0] = kd(axis,1) * kd(side,1) * (1-i)   +
                   kd(axis,1) * kd(side,2) * (nx+i) +
                   kd(axis,2) * kd(flip,0) *  j     +
                   kd(axis,2) * kd(flip,1) * (nx+1-j)
           I2[N0-1] = kd(axis,1) * kd(flip,0) *  j +
                     kd(axis,1) * kd(flip,1) * (ny+1+j) +
                     kd(axis,2) * kd(side,1) * (1-i) +
                     kd(axis,2) * kd(side,2) * (ny+i)
           I1=I2
         end
       end
       if I1[N0]>nx
         if connections[f,1,2,1]==-1
           return NaN
         else
           f1=connections[f,1,2,1];
           axis=connections[f,1,2,2];
           side=connections[f,1,2,3];
           flip=connections[f,1,2,4];
           I2[N0] = kd(axis,1) * kd(side,1) * (i-nx)   +
                   kd(axis,1) * kd(side,2) * ((nx+1)-(i-nx)) +
                   kd(axis,2) * kd(flip,0) *  j     +
                   kd(axis,2) * kd(flip,1) * (nx+1-j)
           I2[N0-1] = kd(axis,1) * kd(flip,0) *  j +
                     kd(axis,1) * kd(flip,1) * (ny+1+j) +
                     kd(axis,2) * kd(side,1) * (1-ny) +
                     kd(axis,2) * kd(side,2) * ((ny+1)-(i-ny))
           I1=I2
         end
       end

       if I1[N0-1]<1
         if connections[f,2,1,1]==-1
           return NaN
         else
           f1=connections[f,2,1,1]
           axis=connections[f,2,1,2];
           side=connections[f,2,1,3];
           flip=connections[f,2,1,4];
           I2[N0] = kd(axis,1) * kd(side,1) * (1-i)   +
                   kd(axis,1) * kd(side,2) * (nx-j) +
                   kd(axis,2) * kd(flip,0) *  i     +
                   kd(axis,2) * kd(flip,1) * (nx+1-i)
           I2[N0-1] = kd(axis,1) * kd(flip,0) *  i +
                     kd(axis,1) * kd(flip,1) * (ny+1-i) +
                     kd(axis,2) * kd(side,1) * (1-j) +
                     kd(axis,2) * kd(side,2) * (ny+j)
           I1=I2
         end
       end
       if I1[N0-1]>ny
         if connections[f,2,2,1]==-1
           return NaN
         else
           f1=connections[f,2,2,1]
           axis=connections[f,2,2,2];
           side=connections[f,2,2,3];
           flip=connections[f,2,2,4];
           I2[N0] = kd(axis,1) * kd(side,1) * (j-ny)   +
                   kd(axis,1) * kd(side,2) * (ny+1-(j-ny)) +
                   kd(axis,2) * kd(flip,0) *  i     +
                   kd(axis,2) * kd(flip,1) * (nx+1-i)
           I2[N0-1] = kd(axis,1) * kd(flip,0) *  i +
                     kd(axis,1) * kd(flip,1) * (ny+1-i) +
                     kd(axis,2) * kd(side,1) * (j-ny) +
                     kd(axis,2) * kd(side,2) * (ny+1-(ny-j))
           I1=I2
         end
       end
     end
 # 1d array
   elseif nspatial==1
     nx=size(A)[N0];
     while (I1[N0]<1 || I1[N0]>nx)
       I2=I1
       i=I1[N0]
       if nfaces>1
         f=I1[N0-1]
       else
         f=1
       end
       if I1[N0]<1
         if connections[f,1,1,1]==-1
           return NaN
         else
           f1=connections[f,1,1,1]
           axis=connections[f,1,1,2];
           side=connections[f,1,1,3];
           flip=connections[f,1,1,4];
           I2[N0] = kd(axis,1) * kd(side,1) * (1-i) +
                    kd(axis,1) * kd(side,2) * (nx+i)
           I1=I2
         end
       end
       if I1[N0]>nx
         if connections[f,1,2,1]==-1
           return NaN
         else
           f1=connections[f,1,2,1];
           axis=connections[f,1,2,2];
           side=connections[f,1,2,3];
           flip=connections[f,1,2,4];
           I2[N0] = kd(axis,1) * kd(side,1) * (i-nx)   +
                   kd(axis,1) * kd(side,2) * ((nx+1)-(i-nx))
           I1=I2
         end
       end
     end
   end
   return Base.getindex(A.data,I1[N0-length(S)+1:N0]...)
 end

 Base.getindex(A::CyclicArray, I) = [ Base.getindex(A::CyclicArray, i) for i in I]

 function Base.setindex!(A::CyclicArray,value,I::Vararg{Int, N}) where N # A[I] = value
   connections=A.connections
   nfaces=size(A.connections)[1]
   nspatial=size(A.connections)[2]
   I1=[i for i in I]
   N0=length(I1)
   S=size(A)
   if nspatial==3
     nx=S[N0];ny=S[N0-1];nz=S[N0-2];
     while (I1[N0]<1 || I1[N0]>nx) || (I1[N0-1]<1 || I1[N0-1]>ny) || (I1[N0-2]<1 || I1[N0-2]>nz)
       I2=I1
       i=I1[N0]
       j=I1[N0-1]
       k=I1[N0-2]
       if nfaces>1
         f=I1[N0-2]
       else
         f=1
       end
       if I1[N0]<1
         if connections[f,1,1,1]==-1
           return NaN
         else
           f1=connections[f,1,1,1]
           axis=connections[f,1,1,2];
           side=connections[f,1,1,3];
           flip=connections[f,1,1,4];
           I2[N0] = kd(axis,1) * kd(side,1) * (1-i)   +
                   kd(axis,1) * kd(side,2) * (nx+i) +
                   kd(axis,2) * kd(flip,0) *  j     +
                   kd(axis,2) * kd(flip,1) * (nx+1-j)
           I2[N0-1] = kd(axis,1) * kd(flip,0) *  j +
                     kd(axis,1) * kd(flip,1) * (ny+1+j) +
                     kd(axis,2) * kd(side,1) * (1-i) +
                     kd(axis,2) * kd(side,2) * (ny+i)
           I2[N0-2] = k
           I1=I2
         end
       end
       if I1[N0]>nx
         if connections[f,1,2,1]==-1
           return NaN
         else
           f1=connections[f,1,2,1];
           axis=connections[f,1,2,2];
           side=connections[f,1,2,3];
           flip=connections[f,1,2,4];
           I2[N0] = kd(axis,1) * kd(side,1) * (i-nx)   +
                   kd(axis,1) * kd(side,2) * ((nx+1)-(i-nx)) +
                   kd(axis,2) * kd(flip,0) *  j     +
                   kd(axis,2) * kd(flip,1) * (nx+1-j)
           I2[N0-1] = kd(axis,1) * kd(flip,0) *  j +
                     kd(axis,1) * kd(flip,1) * (ny+1+j) +
                     kd(axis,2) * kd(side,1) * (1-ny) +
                     kd(axis,2) * kd(side,2) * ((ny+1)-(i-ny))
           I2[N0-2] = k
           I1=I2
         end
       end

       if I1[N0-1]<1
         if connections[f,2,1,1]==-1
           return NaN
         else
           f1=connections[f,2,1,1]
           axis=connections[f,2,1,2];
           side=connections[f,2,1,3];
           flip=connections[f,2,1,4];
           I2[N0] = kd(axis,1) * kd(side,1) * (1-i)   +
                   kd(axis,1) * kd(side,2) * (nx-j) +
                   kd(axis,2) * kd(flip,0) *  i     +
                   kd(axis,2) * kd(flip,1) * (nx+1-i)
           I2[N0-1] = kd(axis,1) * kd(flip,0) *  i +
                     kd(axis,1) * kd(flip,1) * (ny+1-i) +
                     kd(axis,2) * kd(side,1) * (1-j) +
                     kd(axis,2) * kd(side,2) * (ny+j)
           I2[N0-2] = k
           I1=I2
         end
       end
       if I1[N0-1]>ny
         if connections[f,2,2,1]==-1
           return NaN
         else
           f1=connections[f,2,2,1]
           axis=connections[f,2,2,2];
           side=connections[f,2,2,3];
           flip=connections[f,2,2,4];
           I2[N0] = kd(axis,1) * kd(side,1) * (j-ny)   +
                   kd(axis,1) * kd(side,2) * (ny+1-(j-ny)) +
                   kd(axis,2) * kd(flip,0) *  i     +
                   kd(axis,2) * kd(flip,1) * (nx+1-i)
           I2[N0-1] = kd(axis,1) * kd(flip,0) *  i +
                     kd(axis,1) * kd(flip,1) * (ny+1-i) +
                     kd(axis,2) * kd(side,1) * (j-ny) +
                     kd(axis,2) * kd(side,2) * (ny+1-(ny-j))
           I2[N0-2] = k
           I1=I2
         end
       end

       if I1[N0-2]<1
         if connections[f,3,1,1]==-1
           return NaN
         else
           f1=connections[f,3,1,1]
           axis=connections[f,3,1,2];
           side=connections[f,3,1,3];
           flip=connections[f,3,1,4];
           I2[N0] = i
           I2[N0-1] = k
           I2[N0-2] = kd(axis,3) * kd(side,1) * (k-nz)   +
                     kd(axis,3) * kd(side,2) * (nz+1-(k-nz))
           I1=I2
         end
       end
       if I1[N0-2]>nz
         if connections[f,3,2,1]==-1
           return NaN
         else
           f1=connections[f,3,2,1]
           axis=connections[f,3,2,2];
           side=connections[f,3,2,3];
           flip=connections[f,3,2,4];
           I2[N0] = i
           I2[N0-1] = j
           I2[N0-2] = kd(axis,3) * kd(side,1) * (k-nz)   +
                     kd(axis,3) * kd(side,2) * (nz+1-(k-nz))
           I1=I2
         end
       end
     end
   # 2d array
   elseif nspatial==2
     nx=S[N0];ny=S[N0-1]
     while (I1[N0]<1 || I1[N0]>nx) || (I1[N0-1]<1 || I1[N0-1]>ny)
       I2=I1
       i=I1[N0]
       j=I1[N0-1]
       if nfaces>1
         f=I1[N0-2]
       else
         f=1
       end
       if I1[N0]<1
         if connections[f,1,1,1]==-1
           return NaN
         else
           f1=connections[f,1,1,1]
           axis=connections[f,1,1,2];
           side=connections[f,1,1,3];
           flip=connections[f,1,1,4];
           I2[N0] = kd(axis,1) * kd(side,1) * (1-i)   +
                   kd(axis,1) * kd(side,2) * (nx+i) +
                   kd(axis,2) * kd(flip,0) *  j     +
                   kd(axis,2) * kd(flip,1) * (nx+1-j)
           I2[N0-1] = kd(axis,1) * kd(flip,0) *  j +
                     kd(axis,1) * kd(flip,1) * (ny+1+j) +
                     kd(axis,2) * kd(side,1) * (1-i) +
                     kd(axis,2) * kd(side,2) * (ny+i)
           I1=I2
         end
       end
       if I1[N0]>nx
         if connections[f,1,2,1]==-1
           return NaN
         else
           f1=connections[f,1,2,1];
           axis=connections[f,1,2,2];
           side=connections[f,1,2,3];
           flip=connections[f,1,2,4];
           I2[N0] = kd(axis,1) * kd(side,1) * (i-nx)   +
                   kd(axis,1) * kd(side,2) * ((nx+1)-(i-nx)) +
                   kd(axis,2) * kd(flip,0) *  j     +
                   kd(axis,2) * kd(flip,1) * (nx+1-j)
           I2[N0-1] = kd(axis,1) * kd(flip,0) *  j +
                     kd(axis,1) * kd(flip,1) * (ny+1+j) +
                     kd(axis,2) * kd(side,1) * (1-ny) +
                     kd(axis,2) * kd(side,2) * ((ny+1)-(i-ny))
           I1=I2
         end
       end

       if I1[N0-1]<1
         if connections[f,2,1,1]==-1
           return NaN
         else
           f1=connections[f,2,1,1]
           axis=connections[f,2,1,2];
           side=connections[f,2,1,3];
           flip=connections[f,2,1,4];
           I2[N0] = kd(axis,1) * kd(side,1) * (1-i)   +
                   kd(axis,1) * kd(side,2) * (nx-j) +
                   kd(axis,2) * kd(flip,0) *  i     +
                   kd(axis,2) * kd(flip,1) * (nx+1-i)
           I2[N0-1] = kd(axis,1) * kd(flip,0) *  i +
                     kd(axis,1) * kd(flip,1) * (ny+1-i) +
                     kd(axis,2) * kd(side,1) * (1-j) +
                     kd(axis,2) * kd(side,2) * (ny+j)
           I1=I2
         end
       end
       if I1[N0-1]>ny
         if connections[f,2,2,1]==-1
           return NaN
         else
           f1=connections[f,2,2,1]
           axis=connections[f,2,2,2];
           side=connections[f,2,2,3];
           flip=connections[f,2,2,4];
           I2[N0] = kd(axis,1) * kd(side,1) * (j-ny)   +
                   kd(axis,1) * kd(side,2) * (ny+1-(j-ny)) +
                   kd(axis,2) * kd(flip,0) *  i     +
                   kd(axis,2) * kd(flip,1) * (nx+1-i)
           I2[N0-1] = kd(axis,1) * kd(flip,0) *  i +
                     kd(axis,1) * kd(flip,1) * (ny+1-i) +
                     kd(axis,2) * kd(side,1) * (j-ny) +
                     kd(axis,2) * kd(side,2) * (ny+1-(ny-j))
           I1=I2
         end
       end
     end
 # 1d array
   elseif nspatial==1
     nx=S[N0];
     while (I1[N0]<1 || I1[N0]>nx)
       I2=I1
       i=I1[N0]
       if nfaces>1
         f=I1[N0-1]
       else
         f=1
       end
       if I1[N0]<1
         if connections[f,1,1,1]==-1
           return NaN
         else
           f1=connections[f,1,1,1]
           axis=connections[f,1,1,2];
           side=connections[f,1,1,3];
           flip=connections[f,1,1,4];
           I2[N0] = kd(axis,1) * kd(side,1) * (1-i) +
                    kd(axis,1) * kd(side,2) * (nx+i)
           I1=I2
         end
       end
       if I1[N0]>nx
         if connections[f,1,2,1]==-1
           return NaN
         else
           f1=connections[f,1,2,1];
           axis=connections[f,1,2,2];
           side=connections[f,1,2,3];
           flip=connections[f,1,2,4];
           I2[N0] = kd(axis,1) * kd(side,1) * (i-nx)   +
                   kd(axis,1) * kd(side,2) * ((nx+1)-(i-nx))
         end
       end
     end
   end
   return Base.setindex!(A.data,value,I1[N0-nspatial+1:N0]...)
 end
 Base.setindex(A::CyclicArray,value, I) = [Base.setindex(A::CyclicArray,value, I) for i in I]
 Base.IndexStyle(::Type{CyclicArray}) = IndexCartesian()

# function Base.getindex(A::CyclicArray,value,I::Vararg{CartesianIndices, N}) where N
#  I1=[i for i in I]
# end

end # module


function foo()
    lines = readlines(open("data/2.txt", "r"))
    for i in 1:26
        x = Set{String}()
        for line in lines
            if i == 1
                str = line[2:end]
            elseif i == 26
                str = line[1:end-1]
            else
                str = line[1:i-1] * line[i+1:end]
            end
            if str in x
                return str
            end
            push!(x, str)
        end
    end
end

println(foo())