d=pd.read_csv('iat_data.csv',index_col=0)

d.head()

#Number of trials per subject
#Note that Subject 1 has too few trials
d.groupby('subjnum').subjnum.count().head()

#Number of subjects in this data set
d.subjnum.unique()

#Conditions
d.condition.unique()

#Blocks
d.block.unique()

#Correct coded as 1, errors coded as 0 in correct column
d.correct.unique()

"""
Explanation: Example data from the Death Implicit Association Test
Nock, M.K., Park, J.M., Finn, C.T., Deliberto, T.L., Dour, H.J., & Banaji, M.R. (2010). Measuring the suicidal mind: Implicit cognition predicts suicidal behavior. Psychological Science, 21(4), 511–517. https://doi.org/10.1177/0956797610364762
pyiat will work with any IAT data.
import data
End of explanation
"""


d1,fs1=analyze_iat(d,subject='subjnum',rt='latency',condition='condition',correct='correct'\
      ,cond1='Death/Not Me,Life/Me',cond2='Life/Not Me,Death/Me'\
      ,block='block',blocks=[2,3,5,6],fastslow_stats=True)

"""
Explanation: <blockquote> 
Blocks 0,1 & 4 - which contain conditions 'Death,Life', 'Not Me,Me', 'Life,Death' are practice blocks, meaning they do not contain relevant data because they do not contrast the different categories.
</blockquote>
<blockquote> 
Therefore, we will enter blocks 2,3,5,6 and contditions 'Life/Not Me,Death/Me', 'Death/Not Me,Life/Me' into analyze_iat.
</blockquote>
<blockquote> 
We are entering the "correct" column, which contains 1 for correct and 0 for errors. We could enter the "errors" column and then just set the error_or_correct argument to 'error.' 
</blockquote>
<blockquote> 
Finally, we have the option to return the total number and percentage of trials that are removed because they are either too fast (default : 400ms) or too slow (default : 10000ms). This will return the number and percentage across  all subjects and across just subjects that do not receive a flag indicating they had poor performance on some metric. 
</blockquote>

pyiat
Return a weighted d-scores. It will also return all error and too fast/too slow trial information and flags indicating poor performance as well as the number of blocks
End of explanation
"""


d1.iloc[:,0:14].head()

"""
Explanation: output
First 14 columns contain the number of trials, - overall, for each condition and for each block - both before and after excluding fast\slow trials
End of explanation
"""


d1.iloc[:,14:21].head()

"""
Explanation: Next 7 columns contain error the number of trials - overall, within each condition and within each block
Error rates are calculated prior to excluding fast\slow trials but there is an option - errors_after_fastslow_rmvd - that if set to True will remove fast/slow trials prior to calculating error rates
End of explanation
"""


d1.iloc[:,21:28].head()

"""
Explanation: Next 7 columns contain pct of too fast trials - overall, within each condition and within each block
End of explanation
"""


d1.iloc[:,28:35].head()

"""
Explanation: Next 7 columns contain pct of too slow trials - overall, within each condition and within each block
End of explanation
"""


d1.iloc[:,35].to_frame().head()

"""
Explanation: Column 35 contains the number of blocks
End of explanation
"""


d1.iloc[:,36:58].head()

"""
Explanation: Next 22 columns contain whether a poor performance criterion\cutoff was flagged - across error rates, too fast rates, too slow rates, and number of blocks
End of explanation
"""


d1.iloc[:,58].to_frame().head()

"""
Explanation: Column 58 contains a 1 if subject passed any poor performance crierion\cuttoff
End of explanation
"""


d1.iloc[:,59:62].head()

"""
Explanation: Columns 59-62 contain D scores for early and late trials and a final overall weighted D score
End of explanation
"""


#Prepare data to enter into r package - need to have blocks be a string and need to divide data into 2 separate
#dataframes for people that received "Death,Me" first and for those that received "Life,Me" first. 
d['block_str']=d.block.astype(str)

d1_r_subn=d[(d.condition=='Death/Not Me,Life/Me')&(d.block>4)].subjnum.unique()
d1_r=d[d.subjnum.isin(d1_r_subn)]
d2_r_subn=d[(d.condition=='Life/Not Me,Death/Me')&(d.block>4)].subjnum.unique()
d2_r=d[d.subjnum.isin(d2_r_subn)]

%R -i d1_r
%R -i d2_r

%%R
dscore_first <- cleanIAT(my_data = d1_r,
                         block_name = "block_str",
                         trial_blocks = c("2","3", "5", "6"),
                         session_id = "subjnum",
                         trial_latency = "latency",
                         trial_error = "errors",
                         v_error = 1, v_extreme = 2, v_std = 1)

dscore_second <- cleanIAT(my_data = d2_r,
                         block_name = "block_str",
                         trial_blocks = c("2","3", "5", "6"),
                         session_id = "subjnum",
                         trial_latency = "latency",
                         trial_error = "errors",
                         v_error = 1, v_extreme = 2, v_std = 1)

r_dsc <- rbind(dscore_first, dscore_second)

%R -o dscore_first
%R -o dscore_second
#Then we need to combine the separate dataframes
#One of these the scores are flipped so need to flip back
dscore_second.IAT=dscore_second.IAT*-1
iat_r_dsc=pd.concat([dscore_first,dscore_second])
iat_r_dsc.index=iat_r_dsc.subjnum
iat_r_dsc=iat_r_dsc.sort_index()

py_r_iat=pd.concat([d1.dscore,iat_r_dsc.IAT],axis=1)
py_r_iat.head()

#Correlation between pyiat (dscore) and R package (IAT) = 1
py_r_iat.corr()

"""
Explanation: Compare D scores with R package "iat"
https://cran.r-project.org/web/packages/IAT/
End of explanation
"""


fs1

"""
Explanation: In the pyiat command above, we entered an argument to return fast-slow stats
This returns total perecentage of too fast and too slow trials across all subjects and only unflagged, presumably included subjects
End of explanation
"""


d2,fs2=analyze_iat(d,subject='subjnum',rt='latency',condition='condition',correct='correct'\
      ,cond1='Death/Not Me,Life/Me',cond2='Life/Not Me,Death/Me'\
      ,block='block',blocks=[2,3,5,6],fastslow_stats=True,each_stim=True,stimulus='trial_word')

"""
Explanation: Other options
D scores for each stimulus (i.e. each word)
Requires each_stim=True and name of the column containing the stimuli in the stimulus column
End of explanation
"""


d2.iloc[:,59:].head()

"""
Explanation: D scores for each word as well as all error and fast\slow trial output
End of explanation
"""


d3,fs3=analyze_iat(d,subject='subjnum',rt='latency',condition='condition',correct='correct'\
      ,cond1='Death/Not Me,Life/Me',cond2='Life/Not Me,Death/Me'\
      ,block='block',blocks=[2,3,5,6],fastslow_stats=True,weighted=False)

"""
Explanation: Unweighted D scores
End of explanation
"""


d3.iloc[:,24:].head()

"""
Explanation: This produces less output as it does not report any information on a block basis
End of explanation
"""


d4,fs4=analyze_iat(d,subject='subjnum',rt='latency',condition='condition',correct='correct'\
      ,cond1='Death/Not Me,Life/Me',cond2='Life/Not Me,Death/Me'\
      ,block='block',blocks=[2,3,5,6],fastslow_stats=True,each_stim=True,stimulus='trial_word',weighted=False)

d4.iloc[:,26:].head()

"""
Explanation: Unweighted D scores for each stimulus
End of explanation
"""


dbt=pd.read_csv('_orig_biat.csv',index_col=0)

dbt.head()

db=pd.read_csv('_orig_biat_scored.csv',index_col=0)

"""
Explanation: There are a few more options, including (1) setting the too fast\too slow threshold, (2) setting the cutoffs for flags, (3) reporting errors and too fast\slow trial counts instead of percentages (4) printing the output to an excel spreadsheet.
End of explanation
"""


biatd1=analyze_iat(dbt,subject='session_id',rt='trial_latency',condition='block_pairing_definition',\
                   correct='trial_error',error_or_correct='error'\
      ,cond2='(unnamed)/Death,Me/Life',cond1='(unnamed)/Life,Me/Death'\
      ,block='block_number',blocks=[0, 1, 2, 3,4,5],biat=True,rmv_1st_4trls=True,trl_num='trial_number',\
       each_stim=True,stimulus='trial_name')

biatd1

s=['session_id']
s.extend(list(db.filter(like='DS').columns.values))

db[s].head(10)

dscore2=biatd1[biatd1.dscore2.notnull()].index

incld=db[db.session_id.isin(dscore2)][db[db.session_id.isin(dscore2)].DScore1.notnull()].session_id

biatd1[biatd1.index.isin(incld)].dscore2.corr(db[db.session_id.isin(incld)].DScore1)

db.index=db.session_id

t=pd.concat([biatd1.dscore2.apply(lambda x: np.round(x,4)),db.DScore1.apply(lambda x: np.round(x,4))],axis=1)

t.corr()

t[((t.dscore2.notnull())&(t.DScore1.notnull()))].corr()

t.head()

df=dbt[dbt.session_id==2618881218].copy(deep=True)
rt='trial_latency'

df.loc[df[rt]>2000,rt]=2000

df[df[rt]>2000]

df[df[rt]<400]

t[(t.dscore2-t.DScore1)<0]

t[(t.dscore2-t.DScore1)>0]

biatd1.filter(like='dsc')

biatd=pd.read_csv('biat.csv',index_col=0)

biatd1

overall_err_cut=.3
cond_err_cut=.4
block_err_cut=.4

cutoffs=[overall_err_cut,cond_err_cut,cond_err_cut]

cutoffs.extend(list(np.repeat(block_err_cut,len(blocks))))

cutoffs

biatd1

biat

df=biatd
correct='trial_error'
subject='session_id'
condition='block_pairing_definition'
block='block_number'
cond1='(unnamed)/Death,Me/Life'
cond2='(unnamed)/Life,Me/Death'
blocks=[0, 1, 2, 3]
include_blocks=True
flag_outformat='pct'
rt='trial_latency'
fast_rt=400
slow_rt=10000
error_or_correct='error'
weighted=True
errors_after_fastslow_rmvd=False
df_fastslow_rts_rmvd=False
biat=True
var='trial_error'

idx=pd.IndexSlice

    outcms=get_error_fastslow_rates(df,correct,subject,condition,block,cond1,cond2,blocks,flag_outformat,include_blocks,\
    rt,fast_rt,slow_rt,error_or_correct,weighted,errors_after_fastslow_rmvd,df_fastslow_rts_rmvd,biat)


    if flag_outformat=='pct':
        all_df=df.groupby(subject)[var].mean()
        ##By condition
        cond1_df=df[(df[condition]==cond1)].groupby(subject)[var].mean()
        cond2_df=df[(df[condition]==cond2)].groupby(subject)[var].mean()
        ##By condition and block
        if include_blocks == True:
            blcnd=df.groupby([subject,condition,block])[var].mean()
    elif flag_outformat=='sum':
        all_df=df.groupby(subject)[var].sum()
        ##By condition
        cond1_df=df[(df[condition]==cond1)].groupby(subject)[var].sum()
        cond2_df=df[(df[condition]==cond2)].groupby(subject)[var].sum()
        ##By condition and block
        if include_blocks == True:
            blcnd=df.groupby([subject,condition,block])[var].sum()
    elif flag_outformat=='count':
        all_df=df.groupby(subject)[var].count()
        ##By condition
        cond1_df=df[(df[condition]==cond1)].groupby(subject)[var].count()
        cond2_df=df[(df[condition]==cond2)].groupby(subject)[var].count()
        ##By condition and block
        if include_blocks == True:
            blcnd=df.groupby([subject,condition,block])[var].count()

    if (include_blocks == True) and (biat==False):
        cond1_bl1=blcnd.loc[idx[:,cond1,[blocks[0],blocks[2]]]]
        cond1_bl2=blcnd.loc[idx[:,cond1,[blocks[1],blocks[3]]]]
        cond2_bl1=blcnd.loc[idx[:,cond2,[blocks[0],blocks[2]]]]
        cond2_bl2=blcnd.loc[idx[:,cond2,[blocks[1],blocks[3]]]]
        #Drop block and condidition levels to subtract means
        for df_tmp in [cond1_bl1,cond1_bl2,cond2_bl1,cond2_bl2]:
            df_tmp.index=df_tmp.index.droplevel([1,2])
        out=pd.concat([all_df,cond1_df,cond2_df,cond1_bl1,cond1_bl2,cond2_bl1,cond2_bl2],axis=1)

    elif (include_blocks == True) and (biat==True):
        if len(blocks)>=2:
            cond1_bl1=blcnd.loc[idx[:,cond1,[blocks[0],blocks[1]]]]
            cond2_bl1=blcnd.loc[idx[:,cond2,[blocks[0],blocks[1]]]]
            for df_tmp in [cond1_bl1,cond2_bl1]:
                df_tmp.index=df_tmp.index.droplevel([1,2])
            out=pd.concat([all_df,cond1_df,cond2_df,cond1_bl1,cond2_bl1],axis=1)
        if len(blocks)>=4:
            cond1_bl2=blcnd.loc[idx[:,cond1,[blocks[2],blocks[3]]]]
            cond2_bl2=blcnd.loc[idx[:,cond2,[blocks[2],blocks[3]]]]
            for df_tmp in [cond1_bl2,cond2_bl2]:
                df_tmp.index=df_tmp.index.droplevel([1,2])
            out=pd.concat([out,cond1_bl2,cond2_bl2],axis=1)
        if len(blocks)==6:
            cond1_bl3=blcnd.loc[idx[:,cond1,[blocks[4],blocks[5]]]]
            cond2_bl3=blcnd.loc[idx[:,cond2,[blocks[4],blocks[5]]]]
            for df_tmp in [cond1_bl3,cond2_bl3]:
                df_tmp.index=df_tmp.index.droplevel([1,2])
            out=pd.concat([out,cond1_bl3,cond2_bl3],axis=1)
    elif include_blocks == False:
        out=pd.concat([all_df,cond1_df,cond2_df],axis=1)

out

pd.concat(outcms,axis=1)

biat['correct']=np.abs(1-biat.trial_error)

blcnd=biat.groupby(['session_id','block_pairing_definition','block_number'])['trial_error'].count()


blocks=[0, 1, 2, 3]
cond1='(unnamed)/Death,Me/Life'
cond2='(unnamed)/Life,Me/Death'

        cond1_bl1=blcnd.loc[idx[:,cond1,[blocks[0],blocks[2]]]]
        cond1_bl2=blcnd.loc[idx[:,cond1,[blocks[1],blocks[3]]]]
        cond2_bl1=blcnd.loc[idx[:,cond2,[blocks[0],blocks[2]]]]
        cond2_bl2=blcnd.loc[idx[:,cond2,[blocks[1],blocks[3]]]]

cond2_bl2

        for df_tmp in [cond1_bl1,cond1_bl2,cond2_bl1,cond2_bl2]:
            df_tmp.index=df_tmp.index.droplevel([1,2])


all_df=biat.groupby('session_id')['trial_error'].mean()

"""
Explanation: biatd1=analyze_iat(biatd,subject='session_id',rt='trial_latency',condition='block_pairing_definition',\
                   correct='trial_error',error_or_correct='error'\
      ,cond1='(unnamed)/Death,Me/Life',cond2='(unnamed)/Life,Me/Death'\
      ,block='block_number',blocks=[0, 1, 2, 3,4,5],biat=True)
End of explanation
"""


