updated html output

updated html output

/*------------------------------------------------------------------------------
# Name:		02_hhchar
# Purpose:	Process household characteristics and education characteristics
# Author:	Tim Essam
# Created:	2015/2/26
# License:	MIT License
# Ado(s):	labutil, labutil2 (ssc install labutil, labutil2)
# Dependencies: copylables, attachlabels, 00_SetupFoldersGlobals.do
#-------------------------------------------------------------------------------
*/

clear
capture log close
log using "$pathlog/02_hhchar", replace
di in yellow "`c(current_date)' `c(current_time)'"

use "$pathraw/GSEC2.dta", replace

* Merge education data to household roster using the force command
merge 1:1 HHID PID using "$pathraw/GSEC4.dta", force
ren _merge gsecMerge
merge 1:1 HHID PID using "$pathraw/GSEC3.dta", force

/* Demographic list to calculate
1. Head of Household Sex
2. Relationship Status
*/

* Create head of household variable based on primary respondent and sex
g byte hoh = h2q4 == 1
la var hoh "Head of household"

g byte femhead = h2q3 == 2 & h2q4 == 1
la var femhead "Female head of household"

g agehead = h2q8 if hoh == 1
la var agehead "Age of head of household"

g ageheadsq = agehead^2
la var ageheadsq "Squared age of the head (for non-linear effects)"

* Relationship status variables
g byte marriedHohm = h2q10 == 1 & hoh==1
la var marriedHohm "married HOH monogamous"

g byte marriedHoh = hoh ==1 & inlist(h2q10, 1, 2)
la var marriedHoh "married head of household (any type)"

g byte marriedHohp = h2q10 == 2 & hoh==1
la var marriedHohp "married HOH polygamous"

g byte divorcedHead = (h2q10 == 3 & hoh==1)
la var divorcedHead "divorced HoH"

g byte divorcedFemhead = (h2q10 == 3 & femhead == 1)
la var divorcedFemhead "divorced Female head of household"

g byte widowHead = (h2q10 == 4 & hoh==1)
la var widowHead "widowed HoH"

g byte widowFemhead = (h2q10 == 4 & femhead == 1)
la var widowFemhead "Widowed Female head of household"

g byte singleHead = (h2q10==5 & hoh==1)
la var singleHead "single HoH"

g byte singleFemhead = (h2q10==5 & femhead)
la var singleFemhead "single HoH"

* Calculate household demographics (size, adult equivalent units, dep ratio, etc).

/* Household size - Household size refers to the number of usual members in a
household. Usual members are defined as those who have lived in the household
for at least 6 months in the past 12 months. However, it includes persons who
may have spent less than 6 months during the last 12 months in the household
but have joined the household with intention to live permanently or for an
extended period of time.
* http://www.ubos.org/UNHS0910/chapter2_householdcharacteristics.html */

*Create a flag variable for determining who is a usual member from above.
g byte hhmemb = (inlist(h2q7, 1, 2) & h2q5 >= 6)
egen hhsize = total(hhmemb), by(HHID)
la var hhsize "household size"

* Create sex ratio for households
g byte male = h2q3 == 1 & hhmemb == 1
g byte female = h2q3 == 2 & hhmemb == 1
la var male "male hh members"
la var female "female hh members"

egen msize = total(male), by(HHID)
la var msize "number of males in hh"

egen fsize = total(female), by(HHID)
la var fsize "number of females in hh"

* Create a gender ratio variable
g sexRatio = msize/fsize
recode sexRatio (. = 0) if fsize==0
la var sexRatio "Ratio of males to females in HH"

* Calculate age demographics
g byte under15tmp = h2q8 <= 15 & hhmemb ==1
g byte under24tmp = h2q8 <= 24 &  hhmemb==1

egen under15 = total(under15tmp), by(HHID)
la var under15 "number of hh members under 15 years old"

egen under24 = total(under24tmp), by(HHID)
la var under24 "number of hh members under 24 years old"

egen under15male = total(under15tmp) if male==1, by(HHID)
la var under15male "number of male hh members under 15"

egen under15female = total(under15tmp) if female==1, by(HHID)
la var under15female "number of female hh members under 15"

egen under24male = total(under24tmp) if male==1, by(HHID)
la var under24male "number of male hh members under 24"

egen under24female = total(under24tmp) if female==1, by(HHID)
la var under24female "number of female hh members under 24"

/* Create intl. HH dependency ratio (age ranges appropriate for Bangladesh)
# HH Dependecy Ratio = [(# people 0-14 + those 65+) / # people aged 15-64 ] * 100 #
The dependency ratio is defined as the ratio of the number of members in the age groups
of 0–14 years and above 60 years to the number of members of working age (15–60 years).
The ratio is normally expressed as a percentage (data below are multiplied by 100 for pcts.*/
g byte numDepRatio = (h2q8 < 15 | h2q8 > 64) & hhmemb == 1
g byte demonDepRatio = numDepRatio != 1 & hhmemb == 1
egen totNumDepRatio = total(numDepRatio), by(HHID)
egen totDenomDepRatio = total(demonDepRatio), by(HHID)

* Check that numbers add to hhsize
assert hhsize == totNumDepRatio+totDenomDepRatio if hhmemb==1
g depRatio = (totNumDepRatio/totDenomDepRatio)*100 if totDenomDepRatio!=.
recode depRatio (. = 0) if totDenomDepRatio==0
la var depRatio "Dependency Ratio"

* Calculate household labor shares (ages 12 - 60)
/* Household Labor Shares */
g byte hhLabort = (h2q8>= 12 & h2q8<60) & hhmemb == 1
egen hhlabor = total(hhLabort), by(HHID)
la var hhlabor "hh labor age>11 & < 60"

g byte mlabort = (h2q8>= 12 & h2q8<60 & male == 1)
egen mlabor = total(mlabort), by(HHID)
la var mlabor "hh male labor age>11 & <60"

g byte flabort = (h2q8>= 12 & h2q8<60 & female == 1)
egen flabor = total(flabort), by(HHID)
la var flabor "hh female labor age>11 & <60"
drop hhLabort mlabort flabort

* Male/Female labor share in hh
g mlaborShare = mlabor/hhlabor
recode mlaborShare (. = 0) if hhlabor == 0
la var mlaborShare "share of working age males in hh"

g flaborShare = flabor/hhlabor
recode flaborShare (. = 0) if hhlabor == 0
la var flaborShare "share of working age females in hh"

* Generate adult equivalents in household
g male10 	= 1
g fem10_19 	= 0.84
g fem20		= 0.72
g child10	= 0.60

g ae = .
replace ae = male10 if (h2q8 >=10 ) & male == 1
replace ae = fem10_19 if (h2q8 >= 10 & h2q8 < 20) & female == 1
replace ae = fem20 if (h2q8 >= 20) & female == 1
replace ae = child10 if (h2q8) < 10 & hhmemb == 1
la var ae "Adult equivalents in household"

egen adultEquiv = total(ae), by(HHID)
la var adultEquiv "Total adult equivalent units"

**********************
* Education outcomes *
**********************
/* Literacy is defined as one’s ability to read with understanding and to
 write meaningfully in any language. */
g byte literateHoh = h4q4 == 4 & hoh == 1
g byte literateSpouse = h4q4 == 4 & h2q4 == 2 & hhmemb == 1

la var literateHoh "Hoh is literate"
la var literateSpouse "Spouse is literate"

/* Education level values found in h4q7 defined using the following:
http://microdata.worldbank.org/index.php/catalog/565/datafile/F2/V110
http://www.classbase.com/countries/Uganda/Education-System
	No Education (0)
	Pre-Primary (Less than Primary Year 1)
	Primary Level (Years 1 - 7)
	Post-Primary Specialized Training or Certificate
	Junior Vocational/Technical (Years 8 - 10)
	Lower Secondary (Years 8 - 11)
	Upper Secondary (Years 11 - 13)
	Post-Secondary Specialized Training or Certificate
	Tertiary (Above Secondary other than Post-Secondary Specialized Training or Cer
tificate)
*/
g educ = .
la var educ "Education levels"
* No education (This includes:"Don't Know" and "2" Responses))
replace educ = 0 if inlist(h4q7, 2, 99)
* Pre-primary
replace educ = 1 if inlist(h4q7, 10)
* Primary
replace educ = 2 if inlist(h4q7, 11, 12, 13, 14, 15, 16, 17)
* Post-Primary Specialized Training or Certificate
replace educ = 3 if inlist(h4q7, 41)
* Junior Techincal/Vocational
replace educ = 4 if inlist(h4q7, 21, 22, 23)
* Lower Secondary
replace educ = 5 if inlist(h4q7, 31, 32, 33, 34)
* Upper Secondary
replace educ = 6 if inlist(h4q7, 35, 36)
* Post-Secondary Specialized Training or Certificate
replace educ = 7 if inlist(h4q7, 51)
* Tertiary
replace educ = 8 if inlist(h4q7, 61)

g educHoh = educ if hoh == 1
g educSpouse = educ if h2q4 == 2 & hhmemb == 1

lab def ed 0 "No education" 1 "Pre-primary" 2 "Primary" 3 "Post-Primary" /*
*/ 4 "Junior Techincal/Vocational " 5 "Lower Secondary" 6 "Upper Secondary" /*
*/ 7 "Post-Secondary Specialized" 8 "Tertiary"
la values educ ed
la values educHoh ed
la values educSpouse ed

* Create variable to reflect the maximum level of education in the household for those 25+
egen educAdult = max(educ) if h2q8>24 & hhmemb ==1, by(HHID)
egen educAdultM = max(educ) if h2q8>24 & hhmemb ==1 & male == 1, by(HHID)
egen educAdultF = max(educ) if h2q8>24 & hhmemb ==1 & female == 1, by(HHID)

la var educAdult "Highest adult education in household"
la var educAdultM "Highest male adult education in household"
la var educAdultF "Highest female adult education in household"
la var educHoh "Education of Hoh"
la var educSpouse "Education of spouse"

* Calculate school expenses for all regular household members
egen totSchoolExptmp = rsum2(h4q15a h4q15b h4q15c h4q15d h4q15e h4q15f)
replace totSchoolExptmp = h4q15g if totSchoolExp == 0
egen totSchoolExp = total(totSchoolExptmp), by(HHID)
la var totSchoolExp "Total school expenses for household"

g byte aidSchool = (h4q16 == 1)
la var aidSchool "HH member received school scholarship (all sources)"

* Household member slept under mosquito net
g byte mosqTreatnet = h3q10 == 2 & hhmemb == 1
g byte mosqNet = (inlist(h3q10, 1, 2) & hhmemb == 1)

* Occupation of mother and father (do not know codes yet, need to get them).
g byte occupFath = h3q4 if hhmemb == 1
g byte oocupMoth = h3q7 if hhmemb == 1

drop totNumDepRatio totDenomDepRatio demonDepRatio numDepRatio male10 fem10_19 fem20 child10 /*
*/ under15tmp under24tmp hhmemb totSchoolExptmp

* Retain only derived data for collapsing
ds(h2q* h4* T6* T2* LocID PID gsecMerge h3q* _merge educ), not
keep `r(varlist)'

* Collapse everything down to HH-level using max values for all vars
* Copy variable labels to reapply after collapse
include "$pathdo/copylabels.do"

ds(HHID), not
collapse (max) `r(varlist)', by(HHID)

* Reapply variable lables & value labels
include "$pathdo/attachlabels.do"

* Summarize collapsed data and review for potential coding errors
sum

foreach x of varlist  educHoh educSpouse educAdult educAdultM educAdultF educHoh educSpouse {
	la values `x' ed
	tab `x'
	}
*end

* Save
save "$pathout/hhchar.dta", replace

* Keep a master file of only household id's for missing var checks
use "$pathraw/GSEC2", replace
keep HHID PID
save "$pathout/hhid.dta", replace

* Create an html file of the log for internet sharability
log2html "$pathlog/02_hhchar", replace
log close

/*------------------------------------------------------------------------------
# Name:		02_hhchar
# Purpose:	Process household characteristics and education characteristics
# Author:	Tim Essam
# Created:	2015/2/26
# License:	MIT License
# Ado(s):	labutil, labutil2 (ssc install labutil, labutil2)
# Dependencies: copylables, attachlabels, 00_SetupFoldersGlobals.do
#-------------------------------------------------------------------------------
*/

clear
capture log close
log using "$pathlog/01_hhchar.log", replace

use "$pathraw/GSEC2.dta", replace

* Merge education data to household roster using the force command
merge 1:1 HHID PID using "$pathraw/GSEC4.dta", force
ren _merge gsecMerge
merge 1:1 HHID PID using "$pathraw/GSEC3.dta", force

/* Demographic list to calculate
1. Head of Household Sex
2. Relationship Status
*/

* Create head of household variable based on primary respondent and sex
g byte hoh = h2q4 == 1
la var hoh "Head of household"

g byte femhead = h2q3 == 2 & h2q4 == 1
la var femhead "Female head of household"

g agehead = h2q8 if hoh == 1
la var agehead "Age of head of household"

g ageheadsq = agehead^2
la var ageheadsq "Squared age of the head (for non-linear effects)"

* Relationship status variables
g byte marriedHohm = h2q10 == 1 & hoh==1
la var marriedHohm "married HOH monogamous"

g byte marriedHoh = hoh ==1 & inlist(h2q10, 1, 2)
la var marriedHoh "married head of household (any type)"

g byte marriedHohp = h2q10 == 2 & hoh==1
la var marriedHohp "married HOH polygamous"

g byte divorcedHead = (h2q10 == 3 & hoh==1)
la var divorcedHead "divorced HoH"

g byte divorcedFemhead = (h2q10 == 3 & femhead == 1)
la var divorcedFemhead "divorced Female head of household"

g byte widowHead = (h2q10 == 4 & hoh==1)
la var widowHead "widowed HoH"

g byte widowFemhead = (h2q10 == 4 & femhead == 1)
la var widowFemhead "Widowed Female head of household"

g byte singleHead = (h2q10==5 & hoh==1)
la var singleHead "single HoH"

g byte singleFemhead = (h2q10==5 & femhead)
la var singleFemhead "single HoH"

* Calculate household demographics (size, adult equivalent units, dep ratio, etc).

/* Household size - Household size refers to the number of usual members in a
household. Usual members are defined as those who have lived in the household
for at least 6 months in the past 12 months. However, it includes persons who
may have spent less than 6 months during the last 12 months in the household
but have joined the household with intention to live permanently or for an
extended period of time.
* http://www.ubos.org/UNHS0910/chapter2_householdcharacteristics.html */

*Create a flag variable for determining who is a usual member from above.
g byte hhmemb = (inlist(h2q7, 1, 2) & h2q5 >= 6)
egen hhsize = total(hhmemb), by(HHID)
la var hhsize "household size"

* Create sex ratio for households
g byte male = h2q3 == 1 & hhmemb == 1
g byte female = h2q3 == 2 & hhmemb == 1
la var male "male hh members"
la var female "female hh members"

egen msize = total(male), by(HHID)
la var msize "number of males in hh"

egen fsize = total(female), by(HHID)
la var fsize "number of females in hh"

* Create a gender ratio variable
g sexRatio = msize/fsize
recode sexRatio (. = 0) if fsize==0
la var sexRatio "Ratio of males to females in HH"

* Calculate age demographics
g byte under15tmp = h2q8 <= 15 & hhmemb ==1
g byte under24tmp = h2q8 <= 24 &  hhmemb==1

egen under15 = total(under15tmp), by(HHID)
la var under15 "number of hh members under 15 years old"

egen under24 = total(under24tmp), by(HHID)
la var under24 "number of hh members under 24 years old"

egen under15male = total(under15tmp) if male==1, by(HHID)
la var under15male "number of male hh members under 15"

egen under15female = total(under15tmp) if female==1, by(HHID)
la var under15female "number of female hh members under 15"

egen under24male = total(under24tmp) if male==1, by(HHID)
la var under24male "number of male hh members under 24"

egen under24female = total(under24tmp) if female==1, by(HHID)
la var under24female "number of female hh members under 24"

/* Create intl. HH dependency ratio (age ranges appropriate for Bangladesh)
# HH Dependecy Ratio = [(# people 0-14 + those 65+) / # people aged 15-64 ] * 100 #
The dependency ratio is defined as the ratio of the number of members in the age groups
of 0–14 years and above 60 years to the number of members of working age (15–60 years).
The ratio is normally expressed as a percentage (data below are multiplied by 100 for pcts.*/
g byte numDepRatio = (h2q8 < 15 | h2q8 > 64) & hhmemb == 1
g byte demonDepRatio = numDepRatio != 1 & hhmemb == 1
egen totNumDepRatio = total(numDepRatio), by(HHID)
egen totDenomDepRatio = total(demonDepRatio), by(HHID)

* Check that numbers add to hhsize
assert hhsize == totNumDepRatio+totDenomDepRatio if hhmemb==1
g depRatio = (totNumDepRatio/totDenomDepRatio)*100 if totDenomDepRatio!=.
recode depRatio (. = 0) if totDenomDepRatio==0
la var depRatio "Dependency Ratio"

* Calculate household labor shares (ages 12 - 60)
/* Household Labor Shares */
g byte hhLabort = (h2q8>= 12 & h2q8<60) & hhmemb == 1
egen hhlabor = total(hhLabort), by(HHID)
la var hhlabor "hh labor age>11 & < 60"

g byte mlabort = (h2q8>= 12 & h2q8<60 & male == 1)
egen mlabor = total(mlabort), by(HHID)
la var mlabor "hh male labor age>11 & <60"

g byte flabort = (h2q8>= 12 & h2q8<60 & female == 1)
egen flabor = total(flabort), by(HHID)
la var flabor "hh female labor age>11 & <60"
drop hhLabort mlabort flabort

* Male/Female labor share in hh
g mlaborShare = mlabor/hhlabor
recode mlaborShare (. = 0) if hhlabor == 0
la var mlaborShare "share of working age males in hh"

g flaborShare = flabor/hhlabor
recode flaborShare (. = 0) if hhlabor == 0
la var flaborShare "share of working age females in hh"

* Generate adult equivalents in household
g male10 	= 1
g fem10_19 	= 0.84
g fem20		= 0.72
g child10	= 0.60

g ae = .
replace ae = male10 if (h2q8 >=10 ) & male == 1
replace ae = fem10_19 if (h2q8 >= 10 & h2q8 < 20) & female == 1
replace ae = fem20 if (h2q8 >= 20) & female == 1
replace ae = child10 if (h2q8) < 10 & hhmemb == 1
la var ae "Adult equivalents in household"

egen adultEquiv = total(ae), by(HHID)
la var adultEquiv "Total adult equivalent units"

**********************
* Education outcomes *
**********************
/* Literacy is defined as one’s ability to read with understanding and to
 write meaningfully in any language. */
g byte literateHoh = h4q4 == 4 & hoh == 1
g byte literateSpouse = h4q4 == 4 & h2q4 == 2 & hhmemb == 1

la var literateHoh "Hoh is literate"
la var literateSpouse "Spouse is literate"

/* Education level values found in h4q7 defined using the following:
http://microdata.worldbank.org/index.php/catalog/565/datafile/F2/V110
http://www.classbase.com/countries/Uganda/Education-System
	No Education (0)
	Pre-Primary (Less than Primary Year 1)
	Primary Level (Years 1 - 7)
	Post-Primary Specialized Training or Certificate
	Junior Vocational/Technical (Years 8 - 10)
	Lower Secondary (Years 8 - 11)
	Upper Secondary (Years 11 - 13)
	Post-Secondary Specialized Training or Certificate
	Tertiary (Above Secondary other than Post-Secondary Specialized Training or Cer
tificate)
*/
g educ = .
la var educ "Education levels"
* No education (This includes:"Don't Know" and "2" Responses))
replace educ = 0 if inlist(h4q7, 2, 99)
* Pre-primary
replace educ = 1 if inlist(h4q7, 10)
* Primary
replace educ = 2 if inlist(h4q7, 11, 12, 13, 14, 15, 16, 17)
* Post-Primary Specialized Training or Certificate
replace educ = 3 if inlist(h4q7, 41)
* Junior Techincal/Vocational
replace educ = 4 if inlist(h4q7, 21, 22, 23)
* Lower Secondary
replace educ = 5 if inlist(h4q7, 31, 32, 33, 34)
* Upper Secondary
replace educ = 6 if inlist(h4q7, 35, 36)
* Post-Secondary Specialized Training or Certificate
replace educ = 7 if inlist(h4q7, 51)
* Tertiary
replace educ = 8 if inlist(h4q7, 61)

g educHoh = educ if hoh == 1
g educSpouse = educ if h2q4 == 2 & hhmemb == 1

lab def ed 0 "No education" 1 "Pre-primary" 2 "Primary" 3 "Post-Primary" /*
*/ 4 "Junior Techincal/Vocational " 5 "Lower Secondary" 6 "Upper Secondary" /*
*/ 7 "Post-Secondary Specialized" 8 "Tertiary"
la values educ ed
la values educHoh ed
la values educSpouse ed

* Create variable to reflect the maximum level of education in the household for those 25+
egen educAdult = max(educ) if h2q8>24 & hhmemb ==1, by(HHID)
egen educAdultM = max(educ) if h2q8>24 & hhmemb ==1 & male == 1, by(HHID)
egen educAdultF = max(educ) if h2q8>24 & hhmemb ==1 & female == 1, by(HHID)

la var educAdult "Highest adult education in household"
la var educAdultM "Highest male adult education in household"
la var educAdultF "Highest female adult education in household"
la var educHoh "Education of Hoh"
la var educSpouse "Education of spouse"

* Calculate school expenses for all regular household members
egen totSchoolExptmp = rsum2(h4q15a h4q15b h4q15c h4q15d h4q15e h4q15f)
replace totSchoolExptmp = h4q15g if totSchoolExp == 0
egen totSchoolExp = total(totSchoolExptmp), by(HHID)
la var totSchoolExp "Total school expenses for household"

g byte aidSchool = (h4q16 == 1)
la var aidSchool "HH member received school scholarship (all sources)"

* Household member slept under mosquito net
g byte mosqTreatnet = h3q10 == 2 & hhmemb == 1
g byte mosqNet = (inlist(h3q10, 1, 2) & hhmemb == 1)

* Occupation of mother and father (do not know codes yet, need to get them).
g byte occupFath = h3q4 if hhmemb == 1
g byte oocupMoth = h3q7 if hhmemb == 1

drop totNumDepRatio totDenomDepRatio demonDepRatio numDepRatio male10 fem10_19 fem20 child10 /*
*/ under15tmp under24tmp hhmemb totSchoolExptmp

* Retain only derived data for collapsing
ds(h2q* h4* T6* T2* LocID PID gsecMerge h3q* _merge educ), not
keep `r(varlist)'

* Collapse everything down to HH-level using max values for all vars
* Copy variable labels to reapply after collapse
include "$pathdo2/copylabels.do"

ds(HHID), not
collapse (max) `r(varlist)', by(HHID)

* Reapply variable lables & value labels
include "$pathdo2/attachlabels.do"

* Summarize collapsed data and review for potential coding errors
sum

foreach x of varlist  educHoh educSpouse educAdult educAdultM educAdultF educHoh educSpouse {
	la values `x' ed
	tab `x'
	}
*end

* Save
save "$pathout/hhchar.dta", replace

* Keep a master file of only household id's for missing var checks
use "$pathraw/GSEC2", replace
keep HHID PID
save "$pathout/hhid.dta", replace

* Create an html file of the log for internet sharability
log2html "$pathlog/02_hhchar", replace
log close


Updating version at parallel_version

Updating version at parallel_version

*! version 1.15.8.19  19agol2015
*! PARALLEL: Stata module for parallel computing
*! by George G. Vega [cre,aut], Brian Quistorff [ctb]
*!
*! Project website:
*!   https://github.com/gvegayon/parallel
*! Bug reports:
*!   https://github.com/gvegayon/parallel/issues
/*
Copyright (c) 2014  <George G. Vega>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

// Syntax parser
cap program drop parallel
program def parallel
    version 11.0

	// Checks wether if is parallel prefix or not
	if  (regexm(`"`0'"', "^(do|clean|setclusters|break|version|append|printlog|viewlog)")) {
	/* If not prefix */
		*If user initiated, clean the logs
		if regexs(0)=="clean" local add_on = cond(regexm(`"`0'"',","), "logs", ", logs")
		parallel_`0' `add_on'
	}
	else if (regexm(`"`0'"', "^(bs|sim)[,]?[\s ]?")) {
	/* Prefix bootstrap or simulate */
		local cmd = regexs(1)
		local 0   = regexr(`"`0'"', "^(bs|sim)", "")
		gettoken x 0 : 0, parse(":") bind
		local 0 = regexr(`"`0'"', "^[:]", "")
		gettoken x options : x, parse(",") bind

		gettoken 0 argopt : 0, parse(",") bind
		parallel_`cmd' `0', argopt(`argopt') `options'
	}
	else if (regexm(`"`0'"',"^([,]?.*[:])")) {
	/* if prefix */
		gettoken x 0 : 0, parse(":") bind
		local 0 = regexr(`"`0'"', "^[:]", "")
		// Gets the options (if these exists) of parallel
		gettoken x options : x, parse(",") bind

		gettoken 0 argopt : 0, parse(",") bind
		parallel_do `0', `options' prefix argopt(`argopt')
	}
	else {
		local val = regexm(`"`0'"', "^([a-zA-Z0-9_]*).*")
		local subcommand = regexs(1)
		di as result `"-`subcommand'-"' as error " invalid parallel subcommand" as text " ({stata h parallel:{it:help parallel}})"
		exit 198
	}
end

/* Returns the version of parallel */
program def parallel_version, rclass
	version 11.0
	di as result "parallel" as text " Stata module for parallel computing"
	di as result "vers" as text " 1.15.8.19  (19agol2015)"
	di as result "auth" as text " George G. Vega [cre,aut], Brian Quistorff [ctb]"

	return local pll_vers = "1.15.8.19"
end

/* Take a look to logfiles */
program def parallel_printlog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)]
	parallel_checklog `pll_instance', e(`event') action(print)
end

program def parallel_viewlog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)]
	parallel_checklog `pll_instance', e(`event') action(view)
end

program def parallel_checklog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)] action(string)

	if ("`event'"=="") local event = "$LAST_PLL_ID"

	if ("`event'"!=".") {
		/* By default uses 1 */
		if ("`pll_instance'" == "") local pll_instance 1

		local pll_instance : di %04.0f `pll_instance'

		/* Does de file exists */
		local logname = "__pll`event'_do`pll_instance'.log"

		if (c(os) == "Windows") local logname = "`c(tmpdir)'`logname'"
		else local logname = "`c(tmpdir)'/`logname'"

		/* If the logfile does not exists, do nothing*/
		cap confirm file "`logname'"
		if (_rc) {
			di as result "No logfile for instance -`pll_instance'- of parallel process -`event'- found"
			exit
		}

		/* Showing the log in screen */
		if ("`action'"=="print") {
			di as result "{hline 80}"
			di as result %~80s "beginning of file -`logname'-"
			di as result "{hline 80}"
			type `"`logname'"'
			di as result "{hline 80}"
			di as result %~80s "end of file -`logname'-"
			di as result "{hline 80}"
		}
		else view `"`logname'"'
	}
	else {
		di as error "It seems that you haven't use -parallel- yet."
		exit 601
	}

end

////////////////////////////////////////////////////////////////////////////////
// Splits the dataset into clusters
cap program drop parallel_spliter
program def parallel_spliter
	version 11.0
	syntax [namelist(name=xtstructure)] [,parallelid(string) sorting(integer 0) force(integer 0) keepusing(varlist)]
	//args xtstructure parallelid Sorting Force

	if length("$PLL_CLUSTERS") == 0 {
		di as err "Number of clusters not fixed." as text " Set the number of clusters with " _newline as err "-{cmd:parallel setclusters #}-"
		exit 198
	}

	quietly {
		gen _`parallelid'cut = .

		if (length("`xtstructure'")) {

			/* Checks wheather if the data is in the correct sorting */
			if !`sorting' & !`force' {
				error 5
			}

			/* Checking the data types */
			foreach var of varlist `xtstructure' {
				capture confirm string var `var'
				if (`=_rc') local numvars `numvars' `var'
				else local strvars `strvars' `var'
			}
		}

		/* Checking types of data */
		if (length("`numvars'")) local numvars st_data(.,"`numvars'")
		else local numvars J(0,0,.)

		if (length("`strvars'")) local strvars st_data(.,"`strvars'")
		else local strvars J(0,0,"")

		/* Processing in MATA */
		mata: st_store(., "_`parallelid'cut", parallel_divide_index(`numvars', `strvars'))

		if (length("`keepusing'")) {
			keep _`parallelid'cut `keepusing'
			gen __pllnobs`parallelid' = _n
		}

		save __pll`parallelid'_dataset, replace

		drop _all
	}

end

////////////////////////////////////////////////////////////////////////////////
// MAIN PROGRAM
*cap program drop parallel_do
program def parallel_do, rclass

	version 11.0

	#delimit ;
	syntax anything(name=dofile equalok everything)
		[, by(string)
		Keep
		KEEPLast
		prefix
		Force
		PROGrams(namelist)
		Mata
		NOGlobals
		KEEPTiming
		Seeds(string)
		NOData
		Randtype(string)
		Timeout(integer 60)
		PROCessors(integer 0)
		argopt(string)
		KEEPUsing(string)
		SETparallelid(string)
		];
	#delimit cr

	if length("$PLL_CLUSTERS") == 0 {
		di "{error:You haven't set the number of clusters}" _n "{error:Please set it with: {cmd:parallel setclusters} {it:#}}"
		exit 198
	}

	// Initial checks
	foreach opt in macrolist keep keeplast prefix force mata noglobals keeptiming nodata {
		local `opt' = length("``opt''") > 0
	}

	/* Randtype */
	if ("`randtype'" == "") local randtype = "datetime"

	/* If no data parsing has to be done (because of no data!) */
	if (!(c(N)*c(k))) local nodata 1

	if (!`keeptiming') {
		timer clear 98
		timer clear 99
	}

	timer on 98

	// Delets last parallel instance ran
	if (`keeplast' & length("`r(pll_id)'")) cap parallel_clean, e(`r(pll_id)')

	// Gets some global values
	local sfn = "$S_FN"

	// Gets the directory where to work at
	if (!`prefix') {

		/* First checks if the file exists */
		mata: parallel_normalizepath(`"`dofile'"',1)
		local pll_dir = "`filedir'"
		local dofile = "`filename'"
	}
	else local pll_dir = c(pwd)+"/"

	local initialdir = c(pwd)
	qui cd "`pll_dir'"

	if length("`by'") != 0 {
		local sortlist: sortedby
		local sorting = regexm("`sortlist'","^`by'")
	}
	else local sorting = 0

	/* Creates a unique ID for the process and secures it */
	if ("`setparallelid'"=="") mata: parallel_sandbox(5)
	else local parallelid = "`setparallelid'"

	global LAST_PLL_ID = "`parallelid'"
	global LAST_PLL_N = $PLL_CLUSTERS
	global LAST_PLL_DIR = "`pll_dir'"

	/* Generates database clusters */
	if (!`nodata') parallel_spliter `by' , parallelid(`parallelid') sorting(`sorting') force(`force') keepusing(`keepusing')

	/* Starts building the files */
	quietly {

		/* Saves mata objects */
		if (`mata') {
			mata: mata mlib create l__pll`parallelid'_mlib, replace
			cap mata: mata mlib add l__pll`parallelid'_mlib *()
			cap mata: mata matsave __pll`parallelid'_mata.mmat *, replace
			if (`=_rc') local matasave = 0
			else local matasave = 1
		}
		else local matasave = 0
	}

	/* Writing the dofile */
	mata: st_local("errornum", strofreal(parallel_write_do(strtrim(`"`dofile' `argopt'"'), "`parallelid'", $PLL_CLUSTERS, `prefix', `matasave', !`noglobals', "`seeds'", "`randtype'", `nodata', "`pll_dir'", "`programs'", `processors')))

	/* Checking if every thing is ok */
	if (`errornum') {
		if (!`nodata') {
			qui use __pll`parallelid'_dataset, clear

			// Restores original S_FN (file name) value
			global S_FN = "`sfn'"

			/* Removing the cut variable */
			qui drop _`parallelid'cut
		}

		/* Removes the sandbox file (unprotect the files) */
		mata: parallel_sandbox(2, "`parallelid'")

		error `errornum'
	}

	timer off 88
	cap timer list
	if (r(t88) == .) local pll_t_setu = 0
	else local pll_t_setu = r(t88)

	/* Running the dofiles */
	timer on 99
	mata: st_local("nerrors", strofreal(parallel_run("`parallelid'",$PLL_CLUSTERS,`"$PLL_STATA_PATH"',`=`timeout'*1000')))
	timer off 99

	/* If parallel finished with an error it restores the dataset */
	if (`nerrors' & !`nodata') {
		qui use __pll`parallelid'_dataset, clear
		global S_FN = "`sfn'"
		cap drop _`parallelid'cut
	}

	cap timer list
	if (r(t99) == .) local pll_t_calc = 0
	else local pll_t_calc = r(t99)
	local pll_t_reps = r(nt99)

	timer on 97
	// Paste the databases
	if (!`nodata' & !`nerrors') {
		parallel_fusion `parallelid', clusters($PLL_CLUSTERS) keepusing(`keepusing')

		// Restores original S_FN (file name) value
		global S_FN = "`sfn'"

		cap drop _`parallelid'cut
	}

	/* Removes the sandbox file (unprotect the files) */
	if ("`setparallelid'" == "") {
		mata: parallel_sandbox(2, "`parallelid'")
		if (!`keep' & !`keeplast') parallel_clean, e("`parallelid'")
	}

	timer off 97
	cap timer list
	if (r(t97) == .) local pll_t_fini = 0
	else local pll_t_fini = r(t97)

	qui timer list
	return local  pll_seeds="`pllseeds'"
	local pllseeds ""
	return scalar pll_errs = `nerrors'
	return local  pll_dir "`pll_dir'"
	return scalar pll_t_reps = `pll_t_reps'
	return scalar pll_t_setu = `pll_t_setu'
	return scalar pll_t_calc = `pll_t_calc'
	return scalar pll_t_fini = `pll_t_fini'
	return local pll_id = "`parallelid'"
	return scalar pll_n = $PLL_CLUSTERS

	qui cd "`initialdir'"
end

////////////////////////////////////////////////////////////////////////////////
// Cleans all files generated by parallel
*cap program drop parallel_clean
program def parallel_clean
	version 11.0
	syntax [, Event(string) All Force Logs]

	if (length("`event'") != 0 & length("`all'") != 0) {
		di as error `"invalid syntax: Using -pll_id- and -all- jointly is not allowed."'
		exit 198
	}

	mata: parallel_clean("`event'", `=length("`all'") > 0', `=length("`force'") > 0', `=length("`logs'") > 0')
end

////////////////////////////////////////////////////////////////////////////////
// Sets the number of clusters as a global macro
*cap program drop parallel_setclusters
program parallel_setclusters
	version 11.0
	syntax anything(name=nclusters)  [, Force Statapath(string asis) Gateway(string) Includefile(string)]

	local nclusters = real(`"`nclusters'"')
	if (`nclusters' == .) {
		di as error `"Not allow: "#" Should be a number"'
		exit 109
	}
	local force = length("`force'")>0
	mata: parallel_setclusters(`nclusters', `force')
	mata: st_local("error", strofreal(parallel_setstatapath(`"`statapath'"', `force')))
	if (`error'){
        di as error `"Can not set Stata directory, try using -statapath()- option"'
        exit `error'
	}

	if "`c(mode)'"=="batch" & "`c(os)'"=="Windows" {
		if `"`gateway'"'==""{
			local gateway "pll_gateway.sh"
		}
		cap confirm file `"`gateway'"'
		if _rc {
			di "path: `c(pwd)'"
			di as error `"On Windows in batch-mode, parallel requires a gateway file and no such file found."'
			exit _rc
		}
		global PLL_GATEWAY_FNAME `"`gateway'"'
	}

	if `"`includefile'"'!=""{
		cap confirm file `"`includefile'"'
		if _rc {
			di as error `"Can not find the include file (`includefile')."'
			exit _rc
		}
		global PLL_INCLUDE_FILE `"`includefile'"'
	}
end

////////////////////////////////////////////////////////////////////////////////
// Exports

// Exports a copy of programs
*cap program drop program_export
program def program_export
	version 11.0
	syntax using/ [,Programlist(string) Inname(string)]

	mata: program_export("`using'", "`programlist'", "`inname'")
end

////////////////////////////////////////////////////////////////////////////////
// Appends the clusterized dataset
*cap program drop parallel_fusion
program def parallel_fusion
	version 11.0
	syntax anything(name=parallelid) , clusters(integer) [keepusing(string)]

	capture {
		cap use "__pll`parallelid'_dta0001.dta", clear
		if (_rc) di "{error:No dataset for instance 0001.}"

		forval i = 2/`clusters' {
			cap append using `"__pll`parallelid'_dta`=string(`i',"%04.0f")'.dta"'
			if (_rc) di "{error:No dataset for instance `=string(`i',"%04.0f")'.}"
		}

		/* If it just used a set of variables */
		if (length("`keepusing'")) {
			merge 1:1 __pllnobs`parallelid' using __pll`parallelid'_dataset, keep(3) nogen
			drop __pllnobs`parallelid'
		}
	}

end

////////////////////////////////////////////////////////////////////////////////
// Checks whether the user pressed break inside a loop
*cap program drop parallel_break
program def parallel_break
	version 11.0
	mata: parallel_break()
end

*! version 1.15.8.19  19agol2015
*! PARALLEL: Stata module for parallel computing
*! by George G. Vega (gvegayon at caltech.edu)
*!
*! Project website:
*!   https://github.com/gvegayon/parallel
/*
Copyright (c) 2014  <George G. Vega>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

// Syntax parser
cap program drop parallel
program def parallel
    version 11.0

	// Checks wether if is parallel prefix or not
	if  (regexm(`"`0'"', "^(do|clean|setclusters|break|version|append|printlog|viewlog)")) {
	/* If not prefix */
		*If user initiated, clean the logs
		if regexs(0)=="clean" local add_on = cond(regexm(`"`0'"',","), "logs", ", logs")
		parallel_`0' `add_on'
	}
	else if (regexm(`"`0'"', "^(bs|sim)[,]?[\s ]?")) {
	/* Prefix bootstrap or simulate */
		local cmd = regexs(1)
		local 0   = regexr(`"`0'"', "^(bs|sim)", "")
		gettoken x 0 : 0, parse(":") bind
		local 0 = regexr(`"`0'"', "^[:]", "")
		gettoken x options : x, parse(",") bind

		gettoken 0 argopt : 0, parse(",") bind
		parallel_`cmd' `0', argopt(`argopt') `options'
	}
	else if (regexm(`"`0'"',"^([,]?.*[:])")) {
	/* if prefix */
		gettoken x 0 : 0, parse(":") bind
		local 0 = regexr(`"`0'"', "^[:]", "")
		// Gets the options (if these exists) of parallel
		gettoken x options : x, parse(",") bind

		gettoken 0 argopt : 0, parse(",") bind
		parallel_do `0', `options' prefix argopt(`argopt')
	}
	else {
		local val = regexm(`"`0'"', "^([a-zA-Z0-9_]*).*")
		local subcommand = regexs(1)
		di as result `"-`subcommand'-"' as error " invalid parallel subcommand" as text " ({stata h parallel:{it:help parallel}})"
		exit 198
	}
end

/* Returns the version of parallel */
program def parallel_version, rclass
	version 11.0
	di as result "parallel" as text " Stata module for parallel computing"
	di as result "vers" as text " 1.14.7.22 (22jul2014)"
	di as result "auth" as text " George G. Vega (gvegayon at caltech.edu)"

	return local pll_vers = "1.14.7.22"
end

/* Take a look to logfiles */
program def parallel_printlog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)]
	parallel_checklog `pll_instance', e(`event') action(print)
end

program def parallel_viewlog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)]
	parallel_checklog `pll_instance', e(`event') action(view)
end

program def parallel_checklog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)] action(string)

	if ("`event'"=="") local event = "$LAST_PLL_ID"

	if ("`event'"!=".") {
		/* By default uses 1 */
		if ("`pll_instance'" == "") local pll_instance 1

		local pll_instance : di %04.0f `pll_instance'

		/* Does de file exists */
		local logname = "__pll`event'_do`pll_instance'.log"

		if (c(os) == "Windows") local logname = "`c(tmpdir)'`logname'"
		else local logname = "`c(tmpdir)'/`logname'"

		/* If the logfile does not exists, do nothing*/
		cap confirm file "`logname'"
		if (_rc) {
			di as result "No logfile for instance -`pll_instance'- of parallel process -`event'- found"
			exit
		}

		/* Showing the log in screen */
		if ("`action'"=="print") {
			di as result "{hline 80}"
			di as result %~80s "beginning of file -`logname'-"
			di as result "{hline 80}"
			type `"`logname'"'
			di as result "{hline 80}"
			di as result %~80s "end of file -`logname'-"
			di as result "{hline 80}"
		}
		else view `"`logname'"'
	}
	else {
		di as error "It seems that you haven't use -parallel- yet."
		exit 601
	}

end

////////////////////////////////////////////////////////////////////////////////
// Splits the dataset into clusters
cap program drop parallel_spliter
program def parallel_spliter
	version 11.0
	syntax [namelist(name=xtstructure)] [,parallelid(string) sorting(integer 0) force(integer 0) keepusing(varlist)]
	//args xtstructure parallelid Sorting Force

	if length("$PLL_CLUSTERS") == 0 {
		di as err "Number of clusters not fixed." as text " Set the number of clusters with " _newline as err "-{cmd:parallel setclusters #}-"
		exit 198
	}

	quietly {
		gen _`parallelid'cut = .

		if (length("`xtstructure'")) {

			/* Checks wheather if the data is in the correct sorting */
			if !`sorting' & !`force' {
				error 5
			}

			/* Checking the data types */
			foreach var of varlist `xtstructure' {
				capture confirm string var `var'
				if (`=_rc') local numvars `numvars' `var'
				else local strvars `strvars' `var'
			}
		}

		/* Checking types of data */
		if (length("`numvars'")) local numvars st_data(.,"`numvars'")
		else local numvars J(0,0,.)

		if (length("`strvars'")) local strvars st_data(.,"`strvars'")
		else local strvars J(0,0,"")

		/* Processing in MATA */
		mata: st_store(., "_`parallelid'cut", parallel_divide_index(`numvars', `strvars'))

		if (length("`keepusing'")) {
			keep _`parallelid'cut `keepusing'
			gen __pllnobs`parallelid' = _n
		}

		save __pll`parallelid'_dataset, replace

		drop _all
	}

end

////////////////////////////////////////////////////////////////////////////////
// MAIN PROGRAM
*cap program drop parallel_do
program def parallel_do, rclass

	version 11.0

	#delimit ;
	syntax anything(name=dofile equalok everything)
		[, by(string)
		Keep
		KEEPLast
		prefix
		Force
		PROGrams(namelist)
		Mata
		NOGlobals
		KEEPTiming
		Seeds(string)
		NOData
		Randtype(string)
		Timeout(integer 60)
		PROCessors(integer 0)
		argopt(string)
		KEEPUsing(string)
		SETparallelid(string)
		];
	#delimit cr

	if length("$PLL_CLUSTERS") == 0 {
		di "{error:You haven't set the number of clusters}" _n "{error:Please set it with: {cmd:parallel setclusters} {it:#}}"
		exit 198
	}

	// Initial checks
	foreach opt in macrolist keep keeplast prefix force mata noglobals keeptiming nodata {
		local `opt' = length("``opt''") > 0
	}

	/* Randtype */
	if ("`randtype'" == "") local randtype = "datetime"

	/* If no data parsing has to be done (because of no data!) */
	if (!(c(N)*c(k))) local nodata 1

	if (!`keeptiming') {
		timer clear 98
		timer clear 99
	}

	timer on 98

	// Delets last parallel instance ran
	if (`keeplast' & length("`r(pll_id)'")) cap parallel_clean, e(`r(pll_id)')

	// Gets some global values
	local sfn = "$S_FN"

	// Gets the directory where to work at
	if (!`prefix') {

		/* First checks if the file exists */
		mata: parallel_normalizepath(`"`dofile'"',1)
		local pll_dir = "`filedir'"
		local dofile = "`filename'"
	}
	else local pll_dir = c(pwd)+"/"

	local initialdir = c(pwd)
	qui cd "`pll_dir'"

	if length("`by'") != 0 {
		local sortlist: sortedby
		local sorting = regexm("`sortlist'","^`by'")
	}
	else local sorting = 0

	/* Creates a unique ID for the process and secures it */
	if ("`setparallelid'"=="") mata: parallel_sandbox(5)
	else local parallelid = "`setparallelid'"

	global LAST_PLL_ID = "`parallelid'"
	global LAST_PLL_N = $PLL_CLUSTERS
	global LAST_PLL_DIR = "`pll_dir'"

	/* Generates database clusters */
	if (!`nodata') parallel_spliter `by' , parallelid(`parallelid') sorting(`sorting') force(`force') keepusing(`keepusing')

	/* Starts building the files */
	quietly {

		/* Saves mata objects */
		if (`mata') {
			mata: mata mlib create l__pll`parallelid'_mlib, replace
			cap mata: mata mlib add l__pll`parallelid'_mlib *()
			cap mata: mata matsave __pll`parallelid'_mata.mmat *, replace
			if (`=_rc') local matasave = 0
			else local matasave = 1
		}
		else local matasave = 0
	}

	/* Writing the dofile */
	mata: st_local("errornum", strofreal(parallel_write_do(strtrim(`"`dofile' `argopt'"'), "`parallelid'", $PLL_CLUSTERS, `prefix', `matasave', !`noglobals', "`seeds'", "`randtype'", `nodata', "`pll_dir'", "`programs'", `processors')))

	/* Checking if every thing is ok */
	if (`errornum') {
		if (!`nodata') {
			qui use __pll`parallelid'_dataset, clear

			// Restores original S_FN (file name) value
			global S_FN = "`sfn'"

			/* Removing the cut variable */
			qui drop _`parallelid'cut
		}

		/* Removes the sandbox file (unprotect the files) */
		mata: parallel_sandbox(2, "`parallelid'")

		error `errornum'
	}

	timer off 88
	cap timer list
	if (r(t88) == .) local pll_t_setu = 0
	else local pll_t_setu = r(t88)

	/* Running the dofiles */
	timer on 99
	mata: st_local("nerrors", strofreal(parallel_run("`parallelid'",$PLL_CLUSTERS,`"$PLL_STATA_PATH"',`=`timeout'*1000')))
	timer off 99

	/* If parallel finished with an error it restores the dataset */
	if (`nerrors' & !`nodata') {
		qui use __pll`parallelid'_dataset, clear
		global S_FN = "`sfn'"
		cap drop _`parallelid'cut
	}

	cap timer list
	if (r(t99) == .) local pll_t_calc = 0
	else local pll_t_calc = r(t99)
	local pll_t_reps = r(nt99)

	timer on 97
	// Paste the databases
	if (!`nodata' & !`nerrors') {
		parallel_fusion `parallelid', clusters($PLL_CLUSTERS) keepusing(`keepusing')

		// Restores original S_FN (file name) value
		global S_FN = "`sfn'"

		cap drop _`parallelid'cut
	}

	/* Removes the sandbox file (unprotect the files) */
	if ("`setparallelid'" == "") {
		mata: parallel_sandbox(2, "`parallelid'")
		if (!`keep' & !`keeplast') parallel_clean, e("`parallelid'")
	}

	timer off 97
	cap timer list
	if (r(t97) == .) local pll_t_fini = 0
	else local pll_t_fini = r(t97)

	qui timer list
	return local  pll_seeds="`pllseeds'"
	local pllseeds ""
	return scalar pll_errs = `nerrors'
	return local  pll_dir "`pll_dir'"
	return scalar pll_t_reps = `pll_t_reps'
	return scalar pll_t_setu = `pll_t_setu'
	return scalar pll_t_calc = `pll_t_calc'
	return scalar pll_t_fini = `pll_t_fini'
	return local pll_id = "`parallelid'"
	return scalar pll_n = $PLL_CLUSTERS

	qui cd "`initialdir'"
end

////////////////////////////////////////////////////////////////////////////////
// Cleans all files generated by parallel
*cap program drop parallel_clean
program def parallel_clean
	version 11.0
	syntax [, Event(string) All Force Logs]

	if (length("`event'") != 0 & length("`all'") != 0) {
		di as error `"invalid syntax: Using -pll_id- and -all- jointly is not allowed."'
		exit 198
	}

	mata: parallel_clean("`event'", `=length("`all'") > 0', `=length("`force'") > 0', `=length("`logs'") > 0')
end

////////////////////////////////////////////////////////////////////////////////
// Sets the number of clusters as a global macro
*cap program drop parallel_setclusters
program parallel_setclusters
	version 11.0
	syntax anything(name=nclusters)  [, Force Statapath(string asis) Gateway(string) Includefile(string)]

	local nclusters = real(`"`nclusters'"')
	if (`nclusters' == .) {
		di as error `"Not allow: "#" Should be a number"'
		exit 109
	}
	local force = length("`force'")>0
	mata: parallel_setclusters(`nclusters', `force')
	mata: st_local("error", strofreal(parallel_setstatapath(`"`statapath'"', `force')))
	if (`error'){
        di as error `"Can not set Stata directory, try using -statapath()- option"'
        exit `error'
	}

	if "`c(mode)'"=="batch" & "`c(os)'"=="Windows" {
		if `"`gateway'"'==""{
			local gateway "pll_gateway.sh"
		}
		cap confirm file `"`gateway'"'
		if _rc {
			di "path: `c(pwd)'"
			di as error `"On Windows in batch-mode, parallel requires a gateway file and no such file found."'
			exit _rc
		}
		global PLL_GATEWAY_FNAME `"`gateway'"'
	}

	if `"`includefile'"'!=""{
		cap confirm file `"`includefile'"'
		if _rc {
			di as error `"Can not find the include file (`includefile')."'
			exit _rc
		}
		global PLL_INCLUDE_FILE `"`includefile'"'
	}
end

////////////////////////////////////////////////////////////////////////////////
// Exports

// Exports a copy of programs
*cap program drop program_export
program def program_export
	version 11.0
	syntax using/ [,Programlist(string) Inname(string)]

	mata: program_export("`using'", "`programlist'", "`inname'")
end

////////////////////////////////////////////////////////////////////////////////
// Appends the clusterized dataset
*cap program drop parallel_fusion
program def parallel_fusion
	version 11.0
	syntax anything(name=parallelid) , clusters(integer) [keepusing(string)]

	capture {
		cap use "__pll`parallelid'_dta0001.dta", clear
		if (_rc) di "{error:No dataset for instance 0001.}"

		forval i = 2/`clusters' {
			cap append using `"__pll`parallelid'_dta`=string(`i',"%04.0f")'.dta"'
			if (_rc) di "{error:No dataset for instance `=string(`i',"%04.0f")'.}"
		}

		/* If it just used a set of variables */
		if (length("`keepusing'")) {
			merge 1:1 __pllnobs`parallelid' using __pll`parallelid'_dataset, keep(3) nogen
			drop __pllnobs`parallelid'
		}
	}

end

////////////////////////////////////////////////////////////////////////////////
// Checks whether the user pressed break inside a loop
*cap program drop parallel_break
program def parallel_break
	version 11.0
	mata: parallel_break()
end


comments

comments

cap pr drop discretize
pr def discretize, rclass
	/*
	Discretises specified variable by constructing bins based
	on provided cutpoints / N number equal sized bins based on provided
	NBins
	*/

	syntax varlist, GENerate(name) [CUTpoints(numlist) NBins(numlist)]
	if "`nbins'" != "" {
		loc increments = 100/`nbins'
		loc cutpoints ""
		forv i = `increments'(`increments')100 {
			if `i' < 100 {
				loc cutpoints `cutpoints' `i'
			}
		}
	}
	loc ncutpoints: list sizeof cutpoints
	loc resolved_list ""
	_pctile `varlist',p(`cutpoints')
	forv i = 1/`ncutpoints' {
		loc resolved_list `resolved_list', `r(r`i')'
 	}
 	loc resolved_list `=substr("`resolved_list'",2,.)'
 	g `generate'=irecode(`varlist',`resolved_list')+1
 	if "`nbins'" != "" {
 		qui su `generate'
 		qui replace `generate' = `nbins' if `generate' > `nbins'
 	}
 	return local r_cut "`resolved_list'"
end

cap pr drop discretize
pr def discretize, rclass
	syntax varlist, GENerate(name) [CUTpoints(numlist) NBins(numlist)]
	if "`nbins'" != "" {
		loc increments = 100/`nbins'
		loc cutpoints ""
		forv i = `increments'(`increments')100 {
			if `i' < 100 {
				loc cutpoints `cutpoints' `i'
			}
		}
	}
	loc ncutpoints: list sizeof cutpoints
	loc resolved_list ""
	_pctile `varlist',p(`cutpoints')
	forv i = 1/`ncutpoints' {
		loc resolved_list `resolved_list', `r(r`i')'
 	}
 	loc resolved_list `=substr("`resolved_list'",2,.)'
 	g `generate'=irecode(`varlist',`resolved_list')+1
 	if "`nbins'" != "" {
 		qui su `generate'
 		qui replace `generate' = `nbins' if `generate' > `nbins'
 	}
 	return local r_cut "`resolved_list'"
end


redo: swap -O1 for -flto

redo: swap -O1 for -flto

# jgfs2
# (c) 2013 Justin Gottula
# The source code of this project is distributed under the terms of the
# simplified BSD license. See the LICENSE file for details.

exec >&2

TARGET=$1
TARGET_BASE=$2
OUTPUT=$3

export CC="ccache gcc"
export CXX="ccache g++"
export AR=ar

export CFLAGS="-std=gnu11 -O0 -ggdb -Wall -Wextra -Wno-unused-parameter \
-Wno-unused-function -include stddef.h -include stdbool.h -include stdint.h \
-Isrc -flto"
export CXXFLAGS="-std=c++11 -O0 -ggdb -Wall -Wextra -Wno-unused-parameter \
-Wno-unused-function -include cstddef -include cstdint -Isrc -flto"

DEFINES="-D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -DFUSE_USE_VERSION=26 \
-DDEBUG_FATAL"

LIB_OUT="bin/libjgfs2.a"
LIB_SRC=$(find lib -type f -iname '*.c')
LIB_OBJS=${LIB_SRC[@]//.c/.o}
LIB_LIBS=(-lm -lbsd -luuid)

FUSE_OUT="bin/fuse.jgfs2"
FUSE_SRC=$(find src/fuse -type f -iname '*.c')
FUSE_OBJS=${FUSE_SRC[@]//.c/.o}
FUSE_LIBS=(-lfuse)

TEST_OUT="bin/test.jgfs2"
TEST_SRC=$(find src/test -type f -iname '*.c')
TEST_OBJS=${TEST_SRC[@]//.c/.o}
TEST_LIBS=()

VIEW_OUT="bin/view.jgfs2"
VIEW_SRC=$(find src/view -type f -iname '*.c')
VIEW_OBJS=${VIEW_SRC[@]//.c/.o}
VIEW_LIBS=()

MKFS_OUT="bin/mkfs.jgfs2"
MKFS_SRC=$(find src/mkfs -type f -iname '*.c')
MKFS_OBJS=${MKFS_SRC[@]//.c/.o}
MKFS_LIBS=()

FSCK_OUT="bin/fsck.jgfs2"
FSCK_SRC=$(find src/fsck -type f -iname '*.c')
FSCK_OBJS=${FSCK_SRC[@]//.c/.o}
FSCK_LIBS=()

DEFRAG_OUT="bin/defrag.jgfs2"
DEFRAG_SRC=$(find src/defrag -type f -iname '*.c')
DEFRAG_OBJS=${DEFRAG_SRC[@]//.c/.o}
DEFRAG_LIBS=()

FSCTL_OUT="bin/fsctl.jgfs2"
FSCTL_SRC=$(find src/fsctl -type f -iname '*.c')
FSCTL_OBJS=${FSCTL_SRC[@]//.c/.o}
FSCTL_LIBS=()

ATTR_OUT="bin/attr.jgfs2"
ATTR_SRC=$(find src/attr -type f -iname '*.c')
ATTR_OBJS=${ATTR_SRC[@]//.c/.o}
ATTR_LIBS=()

function target_cxx_dep {
	$CXX $CXXFLAGS $DEFINES -o${TARGET//.o/.dep} -MM -MG ${TARGET//.o/.cxx}
}

function target_cxx {
	target_cxx_dep
	read DEPS <${TARGET//.o/.dep}
	redo-ifchange ${DEPS#*:}

	$CXX $CXXFLAGS $DEFINES -o$OUTPUT -c ${TARGET//.o/.cxx}
}

function target_cxx_link {
	redo-ifchange $OBJS

	$CXX $CFXXLAGS $LIBS -o$OUTPUT $OBJS
}

function target_c_dep {
	$CC $CFLAGS $DEFINES -o${TARGET//.o/.dep} -MM -MG ${TARGET//.o/.c}
}

function target_c {
	target_c_dep
	read DEPS <${TARGET//.o/.dep}
	redo-ifchange ${DEPS#*:}

	$CC $CFLAGS $DEFINES -o$OUTPUT -c ${TARGET//.o/.c}
}

function target_c_link {
	redo-ifchange $OBJS

	$CC $CFLAGS $LIBS -o$OUTPUT $OBJS
}

function target_lib {
	redo-ifchange $OBJS

	$AR rcs $OUTPUT $OBJS
}

case "$TARGET" in
all)
	redo lib fuse test view mkfs fsck defrag fsctl attr ;;
lib)
	redo-ifchange $LIB_OUT ;;
fuse)
	redo-ifchange $FUSE_OUT ;;
test)
	redo-ifchange $TEST_OUT ;;
view)
	redo-ifchange $VIEW_OUT ;;
mkfs)
	redo-ifchange $MKFS_OUT ;;
fsck)
	redo-ifchange $FSCK_OUT ;;
defrag)
	redo-ifchange $DEFRAG_OUT ;;
fsctl)
	redo-ifchange $FSCTL_OUT ;;
attr)
	redo-ifchange $ATTR_OUT ;;
$LIB_OUT)
	LIBS=""
	OBJS="${LIB_OBJS[@]}"
	target_lib
	;;
$FUSE_OUT)
	LIBS="${FUSE_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${FUSE_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$TEST_OUT)
	LIBS="${TEST_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${TEST_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$VIEW_OUT)
	LIBS="${VIEW_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${VIEW_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$MKFS_OUT)
	LIBS="${MKFS_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${MKFS_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$FSCK_OUT)
	LIBS="${FSCK_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${FSCK_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$DEFRAG_OUT)
	LIBS="${DEFRAG_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${DEFRAG_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$FSCTL_OUT)
	LIBS="${FSCTL_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${FSCTL_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$ATTR_OUT)
	LIBS="${ATTR_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${ATTR_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
*.o)
	if [[ -e "${TARGET//.o/.cxx}" ]]; then
		target_cxx
	elif [[ -e "${TARGET//.o/.c}" ]]; then
		target_c
	else
		echo "cannot find source file for '$TARGET'"
		exit 1
	fi
	;;
clean)
	rm -rf $(find bin/ -type f)
	rm -rf $(find src/ lib/ -type f -iname *.o)
	rm -rf $(find src/ lib/ -type f -iname *.dep)
	;;
backup)
	cd .. && tar -acvf backup/jgfs2-$(date +'%Y%m%d-%H%M').tar.xz jgfs2/
	;;
*)
	echo "unknown target '$TARGET'"
	exit 1
	;;
esac

# jgfs2
# (c) 2013 Justin Gottula
# The source code of this project is distributed under the terms of the
# simplified BSD license. See the LICENSE file for details.

exec >&2

TARGET=$1
TARGET_BASE=$2
OUTPUT=$3

export CC="ccache gcc"
export CXX="ccache g++"
export AR=ar

export CFLAGS="-std=gnu11 -O1 -ggdb -Wall -Wextra -Wno-unused-parameter \
-Wno-unused-function -include stddef.h -include stdbool.h -include stdint.h \
-Isrc"
export CXXFLAGS="-std=c++11 -O1 -ggdb -Wall -Wextra -Wno-unused-parameter \
-Wno-unused-function -include cstddef -include cstdint -Isrc"

DEFINES="-D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -DFUSE_USE_VERSION=26 \
-DDEBUG_FATAL"

LIB_OUT="bin/libjgfs2.a"
LIB_SRC=$(find lib -type f -iname '*.c')
LIB_OBJS=${LIB_SRC[@]//.c/.o}
LIB_LIBS=(-lm -lbsd -luuid)

FUSE_OUT="bin/fuse.jgfs2"
FUSE_SRC=$(find src/fuse -type f -iname '*.c')
FUSE_OBJS=${FUSE_SRC[@]//.c/.o}
FUSE_LIBS=(-lfuse)

TEST_OUT="bin/test.jgfs2"
TEST_SRC=$(find src/test -type f -iname '*.c')
TEST_OBJS=${TEST_SRC[@]//.c/.o}
TEST_LIBS=()

VIEW_OUT="bin/view.jgfs2"
VIEW_SRC=$(find src/view -type f -iname '*.c')
VIEW_OBJS=${VIEW_SRC[@]//.c/.o}
VIEW_LIBS=()

MKFS_OUT="bin/mkfs.jgfs2"
MKFS_SRC=$(find src/mkfs -type f -iname '*.c')
MKFS_OBJS=${MKFS_SRC[@]//.c/.o}
MKFS_LIBS=()

FSCK_OUT="bin/fsck.jgfs2"
FSCK_SRC=$(find src/fsck -type f -iname '*.c')
FSCK_OBJS=${FSCK_SRC[@]//.c/.o}
FSCK_LIBS=()

DEFRAG_OUT="bin/defrag.jgfs2"
DEFRAG_SRC=$(find src/defrag -type f -iname '*.c')
DEFRAG_OBJS=${DEFRAG_SRC[@]//.c/.o}
DEFRAG_LIBS=()

FSCTL_OUT="bin/fsctl.jgfs2"
FSCTL_SRC=$(find src/fsctl -type f -iname '*.c')
FSCTL_OBJS=${FSCTL_SRC[@]//.c/.o}
FSCTL_LIBS=()

ATTR_OUT="bin/attr.jgfs2"
ATTR_SRC=$(find src/attr -type f -iname '*.c')
ATTR_OBJS=${ATTR_SRC[@]//.c/.o}
ATTR_LIBS=()

function target_cxx_dep {
	$CXX $CXXFLAGS $DEFINES -o${TARGET//.o/.dep} -MM -MG ${TARGET//.o/.cxx}
}

function target_cxx {
	target_cxx_dep
	read DEPS <${TARGET//.o/.dep}
	redo-ifchange ${DEPS#*:}

	$CXX $CXXFLAGS $DEFINES -o$OUTPUT -c ${TARGET//.o/.cxx}
}

function target_cxx_link {
	redo-ifchange $OBJS

	$CXX $CFXXLAGS $LIBS -o$OUTPUT $OBJS
}

function target_c_dep {
	$CC $CFLAGS $DEFINES -o${TARGET//.o/.dep} -MM -MG ${TARGET//.o/.c}
}

function target_c {
	target_c_dep
	read DEPS <${TARGET//.o/.dep}
	redo-ifchange ${DEPS#*:}

	$CC $CFLAGS $DEFINES -o$OUTPUT -c ${TARGET//.o/.c}
}

function target_c_link {
	redo-ifchange $OBJS

	$CC $CFLAGS $LIBS -o$OUTPUT $OBJS
}

function target_lib {
	redo-ifchange $OBJS

	$AR rcs $OUTPUT $OBJS
}

case "$TARGET" in
all)
	redo lib fuse test view mkfs fsck defrag fsctl attr ;;
lib)
	redo-ifchange $LIB_OUT ;;
fuse)
	redo-ifchange $FUSE_OUT ;;
test)
	redo-ifchange $TEST_OUT ;;
view)
	redo-ifchange $VIEW_OUT ;;
mkfs)
	redo-ifchange $MKFS_OUT ;;
fsck)
	redo-ifchange $FSCK_OUT ;;
defrag)
	redo-ifchange $DEFRAG_OUT ;;
fsctl)
	redo-ifchange $FSCTL_OUT ;;
attr)
	redo-ifchange $ATTR_OUT ;;
$LIB_OUT)
	LIBS=""
	OBJS="${LIB_OBJS[@]}"
	target_lib
	;;
$FUSE_OUT)
	LIBS="${FUSE_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${FUSE_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$TEST_OUT)
	LIBS="${TEST_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${TEST_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$VIEW_OUT)
	LIBS="${VIEW_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${VIEW_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$MKFS_OUT)
	LIBS="${MKFS_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${MKFS_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$FSCK_OUT)
	LIBS="${FSCK_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${FSCK_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$DEFRAG_OUT)
	LIBS="${DEFRAG_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${DEFRAG_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$FSCTL_OUT)
	LIBS="${FSCTL_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${FSCTL_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$ATTR_OUT)
	LIBS="${ATTR_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${ATTR_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
*.o)
	if [[ -e "${TARGET//.o/.cxx}" ]]; then
		target_cxx
	elif [[ -e "${TARGET//.o/.c}" ]]; then
		target_c
	else
		echo "cannot find source file for '$TARGET'"
		exit 1
	fi
	;;
clean)
	rm -rf $(find bin/ -type f)
	rm -rf $(find src/ lib/ -type f -iname *.o)
	rm -rf $(find src/ lib/ -type f -iname *.dep)
	;;
backup)
	cd .. && tar -acvf backup/jgfs2-$(date +'%Y%m%d-%H%M').tar.xz jgfs2/
	;;
*)
	echo "unknown target '$TARGET'"
	exit 1
	;;
esac


Fix small error that stifled error in append.

*! vers 0.14.6.24 24jun2014
*! auth George G. Vega

program def parallel_append

	vers 11.0

	#delimit ;
	syntax [anything(name=files)] , Do(string asis) [
		in(string asis)
		if(string asis)
		Expression(string) Keep KEEPLast *];
	#delimit cr

	if ("`in'" != "") local in in `in'
	if ("`if'" != "") local if if `if'

	/* Checking arguments */
	if (`"`files'"' == "" & `"`expression'"' == "") {
		di as error "One of -files- or -expr()- must be specified."
		error 198
	}
	else if (`"`anything'"' != "" & `"`expression'"' != "") {
		di as error "-files- and -expr()- cannot be specified at the same time"
		error 198
	}

	/* Expanding the expression */
	if (`"`expression'"' != "") {
		mata: st_local("files",parallel_expand_expr(`"`expression'"'))
	}
	else if (regexm(`"`files'"', "[*]")) local files : dir . files "`files'"

	/* Checking cmd/dofile */
	cap confirm file `"`do'"'
	if (_rc) cap confirm file `"`do'.do"'
	if (!_rc) local do = `"`do'.do"'
	else {
		cap `do'
		if (_rc == 199) {
			di as error `"Error: No file or cmd nammed -`do'-"'
			exit 199
		}
	}

	/* Checking out the number of files */
	tokenize `"`files'"'
	local n = 0
	local i = 0
	local nerr = 0
	while ("``++n''" != "") {

		/* Checking whether it exists*/
		local fn = `"``n''"'
		if (!regexm(`"`fn'"',"\.dta$")) cap confirm file `"``n''.dta"'
		else cap confirm file `"``n''"'

		if (!_rc) {
			if (!regexm(`"`fn'"',"\.dta$")) local ext = ".dta"
			else local ext = ""
			local file`++i' = "``n''`ext'"
		}
		else {
			di "{result:Warning:}{text:The file -``n''.dta- couldn't be found.}"
		}
	}
	local n = `i'

	/* If no files had been found */
	if (!`n') {
		di as error "No files found"
		error 601
	}

	/* Showing the files that will be used */
	di "{result:The following files will be processed:}"
	forval i=1/`n' {
		di " " as result %03.0f `i' as text " `file`i''"
	}

	/* Checking the groups clusters */
	local size = `n'/$PLL_CLUSTERS

	local oldclusters = $PLL_CLUSTERS
	local olddir = $PLL_STATA_PATH
	if (`size' < 1) {
		qui parallel setclusters `n', statapath(`olddir') f
		local g = 1
		forval i=1/`n' {
			local group`g' `group`g'' `file`i''
		}
	}
	else {
		/* Grouping files */
		local g = 1
		forval i=1/`n' {
			local group`g' `group`g'' `file`i''
			if (!mod(`i',$PLL_CLUSTERS)) local ++g
		}
	}

	local ng = ceil(`size')

	di "{result:The files will be processed in the following order:}"
	forval i=1/`ng' {
		di " " as result %03.0f `i' as text " `group`i''"
	}

	/* Getting a common id for the files */
	mata: parallel_sandbox(5)
	local parallelid0 = "`parallelid'"
	local tmpid = "__pll`parallelid'_append"
	mkdir `tmpid'

	local nsave = 0
	forval i=1/`ng' {

		/* Writing the file */
		local f `tmpid'.do
		tempname fh
		qui file open `fh' using `f', w replace

		tokenize `group`i''
		local j = 0
		local k = 0
		file write `fh' `"cd "`c(pwd)'""' _newline
		while (`"``++j''"' != "") {
			file write `fh' `"if (\`pll_instance' == `++k') {"' _newline
			file write `fh' `"    use ``j'' `if' `in'"' _newline
			file write `fh' `"    local filename = `"``j''"'"' _newline
			file write `fh' `"    local tmpn = string(`++nsave',"%04.0f")"'_newline _newline "}" _newline
		}

		cap findfile `do'
		if (_rc) {
			file write `fh' `"`do'"' _newline
		}

		file write `fh' `"gen dta_source = "\`filename'""' _newline
		file write `fh' "compress" _newline
		file write `fh' "save `tmpid'/`tmpid'\`tmpn', replace" _newline

		file close `fh'

		qui parallel setclusters `--j', s(`olddir') f

		mata: parallel_sandbox(5)
		local parallelid`i' = "`parallelid'"
		cap noi parallel do `f', `options' nodata setparallelid(`parallelid') `keep' `keeplast'

		/* Checking if an error has occurred */
		if (_rc) {
			local orig_rc = _rc
			mata: parallel_sandbox(2, "$LAST_PLL_ID")
			if ("`keep'"=="" & "`keeplast'"=="") qui parallel clean, e(${LAST_PLL_ID}) nologs
			forval j=0/`i' {
				mata: parallel_sandbox(2, "`parallelid`j''")
				if ("`keep'"=="" & "`keeplast'"=="") qui parallel clean, e(`parallelid`j'') nologs
			}
			qui parallel setclusters `oldclusters', s(`olddir') f
			di as error "An error -`=_rc'- has occured while running parallel"
			cap rm `f'
			exit `orig_rc'
		}

		rm `f'
	}

	qui clear

	/* Appending all the results */
	forval i=1/`nsave' {
		local tmpn = "`tmpid'"+string(`i',"%04.0f")+".dta"
		cap {
			append using `tmpid'/`tmpn'
			rm `tmpid'/`tmpn'
		}

		if (!c(N)) cap use `tmpid'/`tmpn'
		if (_rc) local err `err' `file`i''
	}

	/* Labeling */
	quietly {
		if (c(N)) {
			encode dta_source, gen(dta_source2)
			drop dta_source
			ren dta_source2 dta_source
			lab var dta_source "Original dataset of the observation"
		}
	}

	/* Removing the tmp dir and free id */
	forval i = 0/`ng' {
		mata: parallel_sandbox(2, "`parallelid`i''")
		qui parallel clean, e(`parallelid`i'') nologs
	}

	if (`"`err'"'!="") di "{result:Warning:}{text:The following files could't be found}" _newline as text `"`=regexr(`"`err'"',"^[0]","")'"'

	qui parallel setclusters `oldclusters', s(`olddir') f

end

*! vers 0.14.6.24 24jun2014
*! auth George G. Vega

program def parallel_append

	vers 11.0

	#delimit ;
	syntax [anything(name=files)] , Do(string asis) [
		in(string asis)
		if(string asis)
		Expression(string) Keep KEEPLast *];
	#delimit cr

	if ("`in'" != "") local in in `in'
	if ("`if'" != "") local if if `if'

	/* Checking arguments */
	if (`"`files'"' == "" & `"`expression'"' == "") {
		di as error "One of -files- or -expr()- must be specified."
		error 198
	}
	else if (`"`anything'"' != "" & `"`expression'"' != "") {
		di as error "-files- and -expr()- cannot be specified at the same time"
		error 198
	}

	/* Expanding the expression */
	if (`"`expression'"' != "") {
		mata: st_local("files",parallel_expand_expr(`"`expression'"'))
	}
	else if (regexm(`"`files'"', "[*]")) local files : dir . files "`files'"

	/* Checking cmd/dofile */
	cap confirm file `"`do'"'
	if (_rc) cap confirm file `"`do'.do"'
	if (!_rc) local do = `"`do'.do"'
	else {
		cap `do'
		if (_rc == 199) {
			di as error `"Error: No file or cmd nammed -`do'-"'
			exit 199
		}
	}

	/* Checking out the number of files */
	tokenize `"`files'"'
	local n = 0
	local i = 0
	local nerr = 0
	while ("``++n''" != "") {

		/* Checking whether it exists*/
		local fn = `"``n''"'
		if (!regexm(`"`fn'"',"\.dta$")) cap confirm file `"``n''.dta"'
		else cap confirm file `"``n''"'

		if (!_rc) {
			if (!regexm(`"`fn'"',"\.dta$")) local ext = ".dta"
			else local ext = ""
			local file`++i' = "``n''`ext'"
		}
		else {
			di "{result:Warning:}{text:The file -``n''.dta- couldn't be found.}"
		}
	}
	local n = `i'

	/* If no files had been found */
	if (!`n') {
		di as error "No files found"
		error 601
	}

	/* Showing the files that will be used */
	di "{result:The following files will be processed:}"
	forval i=1/`n' {
		di " " as result %03.0f `i' as text " `file`i''"
	}

	/* Checking the groups clusters */
	local size = `n'/$PLL_CLUSTERS

	local oldclusters = $PLL_CLUSTERS
	local olddir = $PLL_STATA_PATH
	if (`size' < 1) {
		qui parallel setclusters `n', statapath(`olddir') f
		local g = 1
		forval i=1/`n' {
			local group`g' `group`g'' `file`i''
		}
	}
	else {
		/* Grouping files */
		local g = 1
		forval i=1/`n' {
			local group`g' `group`g'' `file`i''
			if (!mod(`i',$PLL_CLUSTERS)) local ++g
		}
	}

	local ng = ceil(`size')

	di "{result:The files will be processed in the following order:}"
	forval i=1/`ng' {
		di " " as result %03.0f `i' as text " `group`i''"
	}

	/* Getting a common id for the files */
	mata: parallel_sandbox(5)
	local parallelid0 = "`parallelid'"
	local tmpid = "__pll`parallelid'_append"
	mkdir `tmpid'

	local nsave = 0
	forval i=1/`ng' {

		/* Writing the file */
		local f `tmpid'.do
		tempname fh
		qui file open `fh' using `f', w replace

		tokenize `group`i''
		local j = 0
		local k = 0
		file write `fh' `"cd "`c(pwd)'""' _newline
		while (`"``++j''"' != "") {
			file write `fh' `"if (\`pll_instance' == `++k') {"' _newline
			file write `fh' `"    use ``j'' `if' `in'"' _newline
			file write `fh' `"    local filename = `"``j''"'"' _newline
			file write `fh' `"    local tmpn = string(`++nsave',"%04.0f")"'_newline _newline "}" _newline
		}

		cap findfile `do'
		if (_rc) {
			file write `fh' `"`do'"' _newline
		}

		file write `fh' `"gen dta_source = "\`filename'""' _newline
		file write `fh' "compress" _newline
		file write `fh' "save `tmpid'/`tmpid'\`tmpn', replace" _newline

		file close `fh'

		qui parallel setclusters `--j', s(`olddir') f

		mata: parallel_sandbox(5)
		local parallelid`i' = "`parallelid'"
		cap noi parallel do `f', `options' nodata setparallelid(`parallelid') `keep' `keeplast'

		/* Checking if an error has occurred */
		if (_rc) {

			mata: parallel_sandbox(2, "$LAST_PLL_ID")
			if ("`keep'"=="" & "`keeplast'"=="") qui parallel clean, e(${LAST_PLL_ID}) nologs
			forval j=0/`i' {
				mata: parallel_sandbox(2, "`parallelid`j''")
				if ("`keep'"=="" & "`keeplast'"=="") qui parallel clean, e(`parallelid`j'') nologs
			}
			qui parallel setclusters `oldclusters', s(`olddir') f
			di as error "An error -`=_rc'- has occured while running parallel"
			cap rm `f'
			exit `=_rc'
		}

		rm `f'
	}

	qui clear

	/* Appending all the results */
	forval i=1/`nsave' {
		local tmpn = "`tmpid'"+string(`i',"%04.0f")+".dta"
		cap {
			append using `tmpid'/`tmpn'
			rm `tmpid'/`tmpn'
		}

		if (!c(N)) cap use `tmpid'/`tmpn'
		if (_rc) local err `err' `file`i''
	}

	/* Labeling */
	quietly {
		if (c(N)) {
			encode dta_source, gen(dta_source2)
			drop dta_source
			ren dta_source2 dta_source
			lab var dta_source "Original dataset of the observation"
		}
	}

	/* Removing the tmp dir and free id */
	forval i = 0/`ng' {
		mata: parallel_sandbox(2, "`parallelid`i''")
		qui parallel clean, e(`parallelid`i'') nologs
	}

	if (`"`err'"'!="") di "{result:Warning:}{text:The following files could't be found}" _newline as text `"`=regexr(`"`err'"',"^[0]","")'"'

	qui parallel setclusters `oldclusters', s(`olddir') f

end


iematch : help file writing test

iematch : help file writing test

help file as it was prepared before the writing test

{smcl}
{* }{...}
{hline}
help for {hi:iematch}
{hline}

{title:Title}

{phang2}{cmdab:iematch} {hline 2} Matching base observations towards target observations using on a single continous variable.

{title:Syntax}

{phang2}
{cmdab:iematch}
, {cmdab:grp:dummy(}{it:varname}{cmd:)} {cmdab:match:var(}{it:varname}{cmd:)}
[{cmdab:id:var(}{it:varlist}{cmd:)} {cmdab:m1} {cmdab:maxdiff(}{it:numlist}{cmd:)} {cmdab:seed(}{it:numlist}{cmd:)}
{cmdab:matchid:name(}{it:string}{cmd:)} {cmdab:matchdi:ffname(}{it:string}{cmd:)}
{cmdab:matchre:sultname(}{it:string}{cmd:)} {cmdab:matchco:untname(}{it:string}{cmd:)}
]

{marker opts}{...}
{synoptset 22}{...}
{synopthdr:options}
{synoptline}
{synopt :{cmdab:grp:dummy(}{it:varname}{cmd:)}}The group dummy variable where 1 indicates base observations and 0 target observations{p_end}
{synopt :{cmdab:match:var(}{it:varname}{cmd:)}}The variable with a continous value to match on{p_end}
{synopt :{cmdab:id:var(}{it:varlist}{cmd:)}}The uniquely and fully identifying ID varaible. Used to indicate which target observation a base observation is match with. If omitted an ID variable will be created.{p_end}
{synopt :{cmdab:m1}}Allows many-to-one matches. The default is to allow only one-to-one matches{p_end}
{synopt :{cmdab:maxdiff(}{it:numlist}{cmd:)}}Set a maximum difference allowed in {cmdab:matchvar()}. If a base observation has no match within this difference then it will remain unmatched{p_end}
{synopt :{cmdab:seed(}{it:numlist}{cmd:)}}If seed is not set outside the command (recommended), then this option can be used to set the seed to make matches stable even when multiple observations have the same value in {cmdab:matchvar()}{p_end}
{synopt :{cmdab:matchid:name(}{it:string}{cmd:)}}Manually sets the name of the variable that indicates which target observation each base observation is matched with. The default is _matchID{p_end}
{synopt :{cmdab:matchdi:ffname(}{it:string}{cmd:)}}Manually sets the name of the variable that indicates the differnece in {cmdab:matchvar()} in each match pair/group. The default is _matchDiff{p_end}
{synopt :{cmdab:matchre:sultname(}{it:string}{cmd:)}}Manually sets the name of the variable that indicates if an observation was matched, or provide a reason why it was not. The default is _matchOrReason{p_end}
{synopt :{cmdab:matchco:untname(}{it:string}{cmd:)}}Manually sets the name of the variable that indicates how many observations a target obsersvation is matched with in a many-to-one matches. The default is _matchCount{p_end}
{synoptline}

{title:Description}

{pstd}{cmdab:iematch} matches base observations towards base observations in terms
	of nearest value in {cmd:matchvar()}. Base observations are observations with
	value 1 in {cmd:grpdummy()} and target observations are observations with value
	0. For example, in a typical p-score match, base observations are treatment and target is control, however, there are many other examples of matching where it could
	be different.

{pstd}{cmdab:iematch} base its matching on algorithm on the Stable Marriage algorithm. This algorithm was chosen because it always a solution and because it was possible to implement a stable matching even if multiple observations has identical match values (see {cmd:seed()} option for more details). One disadvantage is of this algorithm is that it only take local efficiency into account and not global. This means that it is possible that other matching algorithms might generate a more efficient match in terms of the sum of the difference of all matched pairs.

{pstd}{hi:{ul:One-to-one matching algorithm}}{p_end}
{pstd}The algorithm used in a one-to-one match starts by evaluating which
	target observation each base observation is closest to and vice versa for target observations. If a base and a target observation pair mutually prefer each other, then those two observations are a match. The algorithm then repeats these two steps but excludes observations after they are matched until all base observations are either matched or excluded due to the option {cmdab:maxdiff()}.

{pstd}In a one-to-one match there have to be at least as many target observations as there are base observation. A one-to-one match returns three variables. One variable indicates the result of the matching for each observation. For example, matched, not matched, no match within the max difference etc. The other two variables hold info on the matched pair. One variable is the ID of the target observation in the match pair, and the other variable is the difference in {cmd:matchvar()} between the two observation in the match pair. See the {help iematch##rtrnvrs:options} for more details.

{pstd}{hi:{ul:Many-to-one matching algorithm}}{p_end}
{pstd}The algorithm used here is the same as in a one-to-one match. But instead of matching only when there is mutual preference, all base observations are matched towards their preferred target observation in the first step, as long as the match is within the max-value if {cmdab:maxdiff()} is used.

{pstd}In a many-to-one match there is no restriction in terms of the number of target observations in relation to base observations. The many-to-one matching results in four variables. Three of the variables are the same as in a one-to-one match. The fourth variable indicates how many base observations that were matched towards each target observation. See the {help iematch##rtrnvrs:options} for more details.

{title:Options}

{phang}{cmdab:grp:dummy(}{it:varname}{cmd:)} indicates which variable is the group dummy. This variable, must be numeric and is only allowed to have the values 1, 0 or missing. 1 indicates a base observation, 0 indicates a target observation, and observations with a missing value will be excluded from the missing.

{phang}{cmdab:match:var(}{it:varname}{cmd:)} indicates the variable used in the matching. This must be a numeric variable, and it is typically a continuous variable. This variable must not be missing for any observations that has either the value 1 or 0 in the group dummy variable.

{phang}{cmdab:id:var(}{it:varlist}{cmd:)} indicates the variable that uniquely and fully identifies the data set. The values in this variable is the values that will be used in the variable that indicates which target observation each base observations matched against.

{phang}{cmdab:m1}

{phang}{cmdab:maxdiff(}{it:numlist}{cmd:)}

{phang}{cmdab:seed(}{it:numlist}{cmd:)}

{marker rtrnvrs}{...}
{phang}{cmdab:matchid:name(}{it:string}{cmd:)} The names {inp:_ID}, {inp:_matchDiff}, {inp:_matchOrReason} and {inp:_matchCount} are not allowed.

{phang}{cmdab:matchdi:ffname(}{it:string}{cmd:)} The names {inp:_ID}, {inp:_matchID}, {inp:_matchOrReason} and {inp:_matchCount} are not allowed.

{phang}{cmdab:matchre:sultname(}{it:string}{cmd:)} The names {inp:_ID}, {inp:_matchID}, {inp:_matchDiff} and {inp:_matchCount} are not allowed.

{phang}{cmdab:matchco:untname(}{it:string}{cmd:)} The names {inp:_ID}, {inp:_matchID}, {inp:_matchDiff} and {inp:_matchOrReason} are not allowed.

{title:Stored results}

{pstd}
{cmdab:iematch} stores the following results in {hi:r()}:

{title:Examples}

{pstd}
A series of examples on how to specify command, and how to evaluate output:

{pstd}{hi:Example 1.}

{pstd}{hi:Example 2.}

{title:Acknowledgements}

{phang}I would like to acknowledge the help in testing and proofreading I received in relation to this command and help file from (in alphabetic order):{p_end}

{title:Author}

{phang}Kristoffer Bj�rkefur, The World Bank, DECIE

{phang}Please send bug-reports, suggestions and requests for clarifications
		 writing "ietools iematch" in the subject line to:{break}
		 kbjarkefur@worldbank.org

{phang}You can also see the code, make comments to the code, see the version
		 history of the code, and submit additions or edits to the code through
		 the github repository of ietoolkit:{break}
		 {browse "https://github.com/worldbank/ietoolkit"}


iematch : input test - id var and match var

Last commit was not tested in Stata. This commit makes the code run
properly in Stata. Testing of ID var and match vars input is not done

	clear

	*command starts here

	cap program drop 	iematch
	    program define	iematch

		syntax    , ///
			GRPdummy(varname) 		///
			MATCHcont(varname) 		///
			[						///
			idvar(varname)			///
			matchidname(string)		///
			matchdiffname(string)	///
			]

		*****
		*1. [if] [in]
		*2 make match only on obs with group
		*3 m:1 match

		noi di "Syntax OK!"

		qui {

		********************************
		*
		*	Checking ID var
		*
		********************************

		*Checking set-up for the ID var used for _matchID
		if "`idvar'" == "" {

			*Copy this name
			local idvar _ID

			*Make sure that the default name _ID is not already used
			cap confirm variable `idvar'
			if _rc == 0 {

				di as error "A variable with name `idvar' is already defined. Either drop this variable or specify a variable using idvar() that fully and uniquely identifies the data set."
				error 110

			}

			*Generate the ID var from row number
			gen `idvar' = _n

		}
		else {

			*Make sure that user specified ID is uniquely and fully identifying the observations
			cap isid `idvar'
			if _rc != 0 {

				di as error "Variable `idvar' specified in idvar() does not fully and uniquely identify the observations in the data set. Meaning that `idvar' is not allowed to have any duplicates or missing values. Make `idvar' uniquly and fully idnetifying or exclude the varaibleas using if or in."
				error 450

			}
		}

		********************************
		*
		*	Checking and generating the MatchID and the MatchDiff var
		*
		********************************

		*See function below
		iematchMatchVarCheck `matchidname' _matchID
		local matchIDname "`r(validVarName)'"
		gen  `matchIDname' = .

		iematchMatchVarCheck `matchdiffname' _matchDiff
		local matchDiffName "`r(validVarName)'"
		gen `matchDiffName' = .

		********************************
		*
		*	Creating tempvar used
		*
		********************************

		tempvar hi_diff lo_diff pref match newMatch

		gen `hi_diff' = .
		gen `lo_diff' = .

		gen `pref' = .
		gen byte `match' = 0
		gen byte `newMatch' = 0

		********************************
		*
		*	Start matching
		*
		********************************

		count if `grpdummy' == 1 & `match' == 0
		local left2Match = `r(N)'

		while (`left2Match' > 0) {

			sort `match' `matchcont'

			replace `lo_diff' 	= .
			replace `hi_diff' 	= .
			replace `pref'		= . if `match' == 0
			replace `newMatch' 	= 0

			replace `lo_diff' = abs(`matchcont' - `matchcont'[_n-1]) if `match' == 0 & `grpdummy' != `grpdummy'[_n-1]
			replace `hi_diff' = abs(`matchcont' - `matchcont'[_n+1]) if `match' == 0 & `grpdummy' != `grpdummy'[_n+1]

			replace `pref' = `idvar'[_n-1] if `match' == 0 & `lo_diff' <  `hi_diff'
			replace `pref' = `idvar'[_n+1] if `match' == 0 & `lo_diff' >= `hi_diff'

			replace `newMatch' = 1 if `match' == 0 & `pref' == `idvar'[_n-1] & `pref'[_n-1] == `idvar'
			replace `newMatch' = 1 if `match' == 0 & `pref' == `idvar'[_n+1] & `pref'[_n+1] == `idvar'

			replace `match' = 1 							if `newMatch' == 1
			replace `matchDiffName' = min(`lo_diff', `hi_diff') 	if `newMatch' == 1
			replace `matchIDname' = `idvar' 						if `newMatch' == 1 & `grpdummy' == 1
			replace `matchIDname' = `pref' 						if `newMatch' == 1 & `grpdummy' == 0

			*order _match* , last

		noi	count if `grpdummy' == 1 & `match' == 0
			local left2Match = `r(N)'

			*pause

		}
	}

	end

	*Cheack matchVar input
	cap program drop 	iematchMatchVarCheck
		program define 	iematchMatchVarCheck  , rclass

		args defaultName optionName

		*Testing that the option name is not the same as the other variable's default name.
		if "`defaultName'" == "_matchID" & "`optionName'" == "_matchDiff" {

			noi di as error "The new name specified in matchidname() is not allowed to be _matchDiff"
			error 198
		}
		if "`defaultName'" == "_matchDiff" & "`optionName'" == "_matchID" {

			noi di as error "The new name specified in matchdiffname() is not allowed to be _matchID"
			error 198
		}

		*Checking set-up for the ID var used for _matchID
		if "``optionName''" == "" {

			*CSet the default name
			local validMatchVarname `defaultName'

			*Make sure that the default name _ID is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				noi di as error "A variable with name `validMatchVarname' is already defined. Either drop this variable or specify a new variable name using `optionName'()."
				error 110

			}

		}

		else {

			*CSet the default name
			local validMatchVarname `optionName'

			*Make sure that the default name _ID is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				noi di as error "The variable name `validMatchVarname' specified in `optionName'() is already defined in the data set. Either drop this variable or specify a another variable name using `optionName'()."
				error 110

			}

		}

		*Testing that the new name is valid by creating a variable that is dropped immedeatly afterwards
		cap gen `validMatchVarname' = .

		if _rc != 0 {

			noi di as error "The variable name specified is not a valid variable name."
			error _rc

		}
		drop `validMatchVarname'

		return local validVarName 	"`validMatchVarname'"

	end

	set seed 1235324

	set obs 10000

	gen id = _n

	gen rand1 = uniform()

	gen tmt = (rand1 < .40)

	drop rand1
	tab tmt

	gen p_hat = uniform()

	iematch if id < 5532  , grp(tmt) match(p_hat) idvar(id)

	clear

	*command starts here

	cap program drop 	iematch
	    program define	iematch

		syntax  [if] [in] , ///
			GRPdummy(varname) 		///
			MATCHcont(varname) 		///
			[						///
			idvar(varname)			///
			matchidname(string)	///
			matchdiffname(string)	///

			]

		qui {

		********************************
		*
		*	Checking ID var
		*
		********************************

		*Checking set-up for the ID var used for _matchID
		if "`idvar'" == "" {

			*Copy this name
			local idvar _ID

			*Make sure that the default name _ID is not already used
			cap confirm variable `idvar'
			if _rc == 0 {

				di as error "A variable with name `idvar' is already defined. Either drop this variable or specify a variable using idvar() that fully and uniquely identifies the data set."
				error 110

			}

			*Generate the ID var from row number
			gen `idvar' = _n

		} else {

			isid `idvar'
			di as error "Variable `idvar' specified in idvar() does not fully and uniquely identify the observations in the data set. Meaning that `idvar' is not allowed to have any duplicates or missing values. Make `idvar' uniquly and fully idnetifying or exclude the varaibleas using if or in."
			error 450

		}

		********************************
		*
		*	Checking and generating the MatchID and the MatchDiff var
		*
		********************************

		*See function below
		iematchMatchVarCheck `matchidname' _matchID
		local matchIDname "`r(validVarName)'"
		gen  `matchIDname' = .

		iematchMatchVarCheck `matchdiffname' _matchDiff
		local matchDiffName "`r(validVarName)'"
		gen matchDiffName = .

		********************************
		*
		*	Creating tempvar used
		*
		********************************

		tempvar hi_diff lo_diff pref match newMatch

		gen `hi_diff' = .
		gen `lo_diff' = .

		gen `pref' = .
		gen byte `match' = 0
		gen byte `newMatch' = 0

		********************************
		*
		*	Start matching
		*
		********************************

		count if `grpdummy' == 1 & `match' == 0
		local left2Match = `r(N)'

		while (`left2Match' > 0) {

			sort `match' `matchcont'

			replace `lo_diff' 	= .
			replace `hi_diff' 	= .
			replace `pref'		= . if `match' == 0
			replace `newMatch' 	= 0

			replace `lo_diff' = abs(`matchcont' - `matchcont'[_n-1]) if `match' == 0 & `grpdummy' != `grpdummy'[_n-1]
			replace `hi_diff' = abs(`matchcont' - `matchcont'[_n+1]) if `match' == 0 & `grpdummy' != `grpdummy'[_n+1]

			replace `pref' = `idvar'[_n-1] if `match' == 0 & `lo_diff' <  `hi_diff'
			replace `pref' = `idvar'[_n+1] if `match' == 0 & `lo_diff' >= `hi_diff'

			replace `newMatch' = 1 if `match' == 0 & `pref' == `idvar'[_n-1] & `pref'[_n-1] == `idvar'
			replace `newMatch' = 1 if `match' == 0 & `pref' == `idvar'[_n+1] & `pref'[_n+1] == `idvar'

			replace `match' = 1 							if `newMatch' == 1
			replace matchDiffName = min(`lo_diff', `hi_diff') 	if `newMatch' == 1
			replace `matchIDname' = `idvar' 						if `newMatch' == 1 & `grpdummy' == 1
			replace `matchIDname' = `pref' 						if `newMatch' == 1 & `grpdummy' == 0

			*order _match* , last

		noi	count if `grpdummy' == 1 & `match' == 0
			local left2Match = `r(N)'

			*pause

		}
	}

	end

	iematchMatchVarCheck optionName defaultName

	cap program drop 	iematchMatchVarCheck
		program define 	iematchMatchVarCheck

		args optionName defaultName

		*Testing that the option name is not the same as the other variable's default name.
		if "`defaultName'" == "_matchID" & "`optionName'" == "_matchDiff" {

			di as error "The new name specified in matchidname() is not allowed to be _matchDiff"
			error 198
		}
		if "`defaultName'" == "_matchDiff" & "`optionName'" == "_matchID" {

			di as error "The new name specified in matchdiffname() is not allowed to be _matchID"
			error 198
		}

		*Checking set-up for the ID var used for _matchID
		if "``optionName''" == "" {

			*CSet the default name
			local validMatchVarname `defaultName'

			*Make sure that the default name _ID is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				di as error "A variable with name `validMatchVarname' is already defined. Either drop this variable or specify a new variable name using `optionName'()."
				error 110

			}

		} else {

			*CSet the default name
			local validMatchVarname `optionName'

			*Make sure that the default name _ID is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				di as error "The variable name `validMatchVarname' specified in `optionName'() is already defined in the data set. Either drop this variable or specify a another variable name using `optionName'()."
				error 110

			}

		}

		*Testing that the new name is valid by creating a variable that is dropped immedeatly afterwards
		cap gen `validMatchVarname' = .
		if _rc != 0 {

			di as error "The variable name specified is not a valid variable name."
			error _rc

		}
		drop `validMatchVarname'

		return local validVarName 	"`validMatchVarname'"

	end

	set seed 1235324

	set obs 10000

	gen id = _n

	gen rand1 = uniform()

	gen tmt = (rand1 < .40)

	drop rand1
	tab tmt

	gen p_hat = uniform()

	iematch   , grp(tmt) match(p_hat) idvar(id)


update Norris source URL

update Norris source URL

* Norris Democracy Timeseries Data Release 3.0, January 2009
* http://www.hks.harvard.edu/fs/pnorris/Data/Data.htm

local URL "https://www.hks.harvard.edu/fs/pnorris/Data/Democracy%20TimeSeries%20Data/Democracy_TimeSeries_Data_January2009_StataSE.dta"
local PREFIX "norris"
local YEARVAR "Year"
local COUNTRYVAR "Refno"
local MERGEWITH "Norris"
local MERGEUSING "Numeric"

tempfile merge
prep_mergefile `MERGEWITH' `MERGEUSING' `merge'

* If the file source file doesn't already exist then download and create it.
use_or_import_source `PREFIX' `URL'

/*
Norris 2009 Notes:
This merge uses Refno accounting for the inconsistencies below. Refno may be not work if there is an update to the dataset.

* duplication with Congo Braz CCODE 484 (2005,2006) and Angola CCODE 540 (1980)
* Refno 7, Year: 1971 - Likely Argentina
* Refno 65, Year: 1971 - Likely Ghana
* Refno 87, Year: 1971 - Likely Kenya
* Refno 46,84,85 have missing Natcode, but not inconsistent
*/

prefix `PREFIX'
merge_ids `PREFIX' `YEARVAR' `COUNTRYVAR' `MERGEUSING' `merge'

duplicates drop UCINumeric year, force

* Merge with the Dataset
merge_with_master `FILENAME'

* Norris Democracy Timeseries Data Release 3.0, January 2009
* http://www.hks.harvard.edu/fs/pnorris/Data/Data.htm

local URL "http://www.hks.harvard.edu/fs/pnorris/Data/Democracy%20TimeSeries%20Data/Democracy_TimeSeries_Data_January2009_StataSE.dta"
local PREFIX "norris"
local YEARVAR "Year"
local COUNTRYVAR "Refno"
local MERGEWITH "Norris"
local MERGEUSING "Numeric"

tempfile merge
prep_mergefile `MERGEWITH' `MERGEUSING' `merge'

* If the file source file doesn't already exist then download and create it.
use_or_import_source `PREFIX' `URL'

/*
Norris 2009 Notes:
This merge uses Refno accounting for the inconsistencies below. Refno may be not work if there is an update to the dataset.

* duplication with Congo Braz CCODE 484 (2005,2006) and Angola CCODE 540 (1980)
* Refno 7, Year: 1971 - Likely Argentina
* Refno 65, Year: 1971 - Likely Ghana
* Refno 87, Year: 1971 - Likely Kenya
* Refno 46,84,85 have missing Natcode, but not inconsistent
*/

prefix `PREFIX'
merge_ids `PREFIX' `YEARVAR' `COUNTRYVAR' `MERGEUSING' `merge'

duplicates drop UCINumeric year, force

* Merge with the Dataset
merge_with_master `FILENAME'


iegraph : expanded the description section

{smcl}
{* 26 Dec 2016}{...}
{hline}
help for {hi:iegraph}
{hline}

{title:Title}

{phang2}{cmdab:iegraph} {hline 2} Generates graphs based on regressions with treatment dummies common in impact evaluations.

{title:Syntax}

{phang2}
{cmdab:iegraph} {varlist}
, [{cmd:noconfbars} {cmdab:ti:tle(}{it:string}{cmd:)}
{cmdab:save(}{it:string}{cmd:)} {cmdab:confbarsnone(}{it:varlist}{cmd:)}
{it:any towway graph options}
]

{marker opts}{...}
{synoptset 22}{...}
{synopthdr:options}
{synoptline}
{synopt :{cmd:noconfbars}} Removes the confidence interval bars from graphs for all treatments.{p_end}
{synopt :{cmdab:ti:tle(}{it:string}{cmd:)}} Manually sets the title of the graph.{p_end}
{synopt :{cmdab:save(}{it:string}{cmd:)}} Sets the filename and the directory to which the graph will be set.{p_end}
{synopt :{cmdab:confbarsnone(}{it:varlist}{cmd:)}} Removes confidence interval bars from only the {it:varlist} listed.{p_end}
{synopt :{cmdab:yzero}} Forces y-axis on the graph to start at 0.{p_end}
{synopt :{cmdab: Graph Options}} Options that can be used with normal graph commands can also be used. See example 2 for details.{p_end}
{synoptline}

{marker desc}
{title:Description}

{pstd}{cmdab:iegraph} This command creates bar graphs on the based on the coefficients
	of treatment dummies in regression results. This command is developed for reading
	stored results of from two types of impact evaluation regression models, but there
	are countless of other examples where the command also can be used. {cmd:iegraph} must
	be used immedeatly after running the regression or as long as teh regression result is
	still stored in or restored to Stata's {help ereturn} results.

{pstd}{bf:Model 1: OLS with Treatment Dummies}{break}The most typical impact evaluation regression is
	to have the outcome variable as the dependent varaible and one dummy for each
	treatment arm where control is the omitted category. This regressions can also include
	covariates, fixed effects etc., but as long as the treatment status is defined by
	mutually exclusive dummy variables. See especially example 1 and 2 below. This command
	works with any number of treatment arms but works best from two arms (treatment
	and control) to five treatment arms (4 different treatments and control). More
	arms than that may result in a still correct but perhaps cluttered  graph.

{pstd}{bf:Model 2: Difference-in-Differneces}{break}Another typical regression model in impact
	evaluations are difference-in-difference (DD) models with two treatment arms (treatment
	and control) and two time periods. If the DD regression is specified as having the
	outcome variable as the dependent variable and three dummy variables (time, treatment
	and timeXtreatment) as the independent varaibles, then this command will produce a nice
	graph. Controls, treatment effects etc. may be added to the regression model. See especially example 3.

{pstd}{bf:Graph Output}{break}The graph generated by this command is created using the following values. The
	control bar is the mean of the outcome variable for the control group. It is not
	the constant from the regression as those are not identical if, for example, fixed effects
	and covariates were used. For each treatment group the bar is the sum of the value
	of the control bar and the beta coefficient in the regression of the corresponding
	treatment dummy. The confidence intervals are calculated from the variance in the
	beta coefficients in the regression.{p_end}

{pstd}The graph also includes the N for each treatment arm in the regression and uses
	that value as labels on the x-axis. Stars are added to this value if the corresponding
	coefficient is statistically different from zero in the regression{p_end}

{marker optslong}
{title:Options}

{phang}{cmd:noconfbars} Removes the confidence interval bars from graphs for all
	treatments. The default value for the confidence interval bars ar 95%. {p_end}

{phang}{cmdab:ti:tle(}{it:string}{cmd:)} Manually sets the title of the graph.{p_end}

{phang}{cmdab:save(}{it:string}{cmd:)} Sets the filename and the directory to which
	the graph will be set.{p_end}

{phang}{cmdab:confbarsnone(}{it:varlist}{cmd:)} Removes confidence interval bars
	from only the {it:varlist} listed. The remaining variables in the graphs which
	have not been specified in {cmdab:confbarsnone} will still have the confidence
	interval bars. {p_end}

{phang}{cmdab:yzero} Manually sets the y-axis of the graph to start at zero instead of the Stata default.{p_end}

{marker optslong}
{title:Examples}

{pstd} {hi:Example 1.}

{pmore} {inp:regress outcomevar treatment_dummy1 treatment_dummy2}{break}
		{inp:iegraph treatment_dummy1 treatment_dummy2 , title({it:string})}

{pmore}In the example above, graphs comparing the effects of {it:treatmentVar1} and
		{it:treatmentVar2} to the mean of the control will be generated. The option
		title({it:string}) gives the graph the title that the user puts in the string.

{pstd} {hi:Example 2.}

{pmore}{inp:iegraph treatment, noconfbars title("Treatment effect on price") save("$Output/Graph1.gph") xlabel(,angle(45)) ylabel(minmax)}

{pmore}In the example above, the observations with value 1 in {it:tmt} will be matched
	towards the nearest, in terms of {it:p_hat}, observations with value 0 in {it:tmt} as
	long as the difference in {it:p_hat} is less than .001. Only observations that has the
	value 1 in variable {it:baseline} will be included in the match.

{title:Acknowledgements}

{phang}I would like to acknowledge the help in testing and proofreading I received in relation to this command and help file from (in alphabetic order):{p_end}
{pmore}{break}

{title:Author}

{phang}Kristoffer Bjarkefur & Mrijan Rimal, The World Bank, DECIE

{phang}Please send bug-reports, suggestions and requests for clarifications
		 writing "ietools iegraph" in the subject line to:{break}
		 kbjarkefur@worldbank.org

{phang}You can also see the code, make comments to the code, see the version
		 history of the code, and submit additions or edits to the code through
		 the github repository of ietoolkit:{break}
		 {browse "https://github.com/worldbank/ietoolkit"}

{smcl}
{* 26 Dec 2016}{...}
{hline}
help for {hi:iegraph}
{hline}

{title:Title}

{phang2}{cmdab:iegraph} {hline 2} Generates graphs based on regressions done during typical impact evaluation.

{title:Syntax}

{phang2}
{cmdab:iegraph} {varlist}
, [{cmd:noconfbars} {cmdab:TI:tle(}{it:string}{cmd:)}
{cmdab:save(}{it:string}{cmd:)} {cmdab:confbarsnone(}{it:varlist}{cmd:)}
{it: Graph Options}
]

{marker opts}{...}
{synoptset 22}{...}
{synopthdr:options}
{synoptline}
{synopt :{cmd:noconfbars}} Removes the confidence interval bars from graphs for all treatments.{p_end}
{synopt :{cmdab:TI:tle(}{it:string}{cmd:)}} Manually sets the title of the graph.{p_end}
{synopt :{cmdab:save(}{it:string}{cmd:)}} Sets the filename and the directory to which the graph will be set.{p_end}
{synopt :{cmdab:confbarsnone(}{it:varlist}{cmd:)}} Removes confidence interval bars from only the {it:varlist} listed.{p_end}
{synopt :{cmdab:yzero}} Forces y-axis on the graph to start at 0.{p_end}
{synopt :{cmdab: Graph Options}} Options that can be used with normal graph commands can also be used. See example 2 for details.{p_end}
{synoptline}

{marker desc}
{title:Description}

{pstd}{cmdab:iegraph} This command creates bar graphs on the average treatment effects
	for multiple treatments in a impact evaluation project. The command creates bar
	graphs for the mean of the control group and for the various treatment groups. {p_end}

{marker optslong}
{title:Options}

{phang}{cmd:noconfbars} Removes the confidence interval bars from graphs for all
	treatments.{p_end}

{phang}{cmdab:TI:tle(}{it:string}{cmd:)} Manually sets the title of the graph.{p_end}

{phang}{cmdab:save(}{it:string}{cmd:)} Sets the filename and the directory to which
	the graph will be set.{p_end}

{phang}{cmdab:confbarsnone(}{it:varlist}{cmd:)} Removes confidence interval bars
	from only the {it:varlist} listed. The remaining variables in the graphs which
	have not been specified in {cmdab:confbarsnone} will still have the confidence
	interval bars. {p_end}

{phang}{cmdab:yzero} Manually sets the y-axis of the graph to start at zero instead of the Stata default.{p_end}

{title:Examples}

{pstd} {hi:Example 1.}

{pmore}{inp:iegraph treatmentVar1 treatmentVar2 , title({it:string})}

{pmore}In the example above, graphs comparing the effects of {it:treatmentVar1} and
		{it:treatmentVar2} to the mean of the control will be generated. The option
		title({it:string}) gives the graph the title that the user puts in the string.

{pstd} {hi:Example 2.}

{pmore}{inp:iegraph treatment, noconfbars title("Treatment effect on price") save("$Output/Graph1.gph") xlabel(,angle(45)) ylabel(minmax)}

{pmore}In the example above, the observations with value 1 in {it:tmt} will be matched
	towards the nearest, in terms of {it:p_hat}, observations with value 0 in {it:tmt} as
	long as the difference in {it:p_hat} is less than .001. Only observations that has the
	value 1 in variable {it:baseline} will be included in the match.

{title:Acknowledgements}

{phang}I would like to acknowledge the help in testing and proofreading I received in relation to this command and help file from (in alphabetic order):{p_end}
{pmore}{break}

{title:Author}

{phang}Kristoffer Bjarkefur & Mrijan Rimal, The World Bank, DECIE

{phang}Please send bug-reports, suggestions and requests for clarifications
		 writing "ietools iegraph" in the subject line to:{break}
		 kbjarkefur@worldbank.org

{phang}You can also see the code, make comments to the code, see the version
		 history of the code, and submit additions or edits to the code through
		 the github repository of ietoolkit:{break}
		 {browse "https://github.com/worldbank/ietoolkit"}