ieimpgraph :unique beta macname

ieimpgraph :unique beta macname

cap	program drop	ieimpgraph
	program define 	ieimpgraph

	syntax anything

	mat list e(b)

    local counter =0

	foreach var of local varlist {

		local counter = `counter' + 1

        local beta_`counter' = _b[`var']

		di "`beta'"

	}

end

cap	program drop	tobeused
	program define 	tobeused

	capture file close `textfile'_`date'
	file open  `textfile'_`date' using `textfile'_`date'.txt, text write replace
	file write `textfile'_`date' ///
		"Variable" 	_tab "Tmt-Status"	_tab "mean" _tab "coeff" _tab "conf_int_min" _tab "conf_int_max" _tab "obs" _tab "star" _n
	file close `textfile'_`date'

	local REG_RESULTS ""
	foreach depvar of local depvars {

		rename lag2_`depvar' lagvar

		local regname `depvar'
			eststo `regname' : areg `depvar' tmt_reg tmt_sdp tmt_inc lagvar NoCropBoroBL NoCropBL incorrect_no_bl NoPlotDataBL NoHarvValueDataBL if t == 2, a(d_code) cl(v_code)
		local  REG_RESULTS "`REG_RESULTS' `regname'"

		foreach tmt_arm in reg sdp inc {
			scalar coeff_`tmt_arm' 		= _b[tmt_`tmt_arm']
			scalar coeff_se_`tmt_arm' 	= _se[tmt_`tmt_arm']

			sum `depvar' if e(sample) & tmt_`tmt_arm' == 1
			scalar obs_`tmt_arm'	= r(N)
		}

		sum `depvar' if e(sample) & tmt_reg == 0 & tmt_sdp == 0 & tmt_inc == 0
		scalar ctl_N		= r(N)
		scalar ctl_mean	  	= r(mean)
		scalar ctl_mean_sd 	= r(sd)

		estadd scalar ctl_mean_table	=r(mean)
		estadd scalar ctl_N_table		=r(N)
		estadd scalar ctl_mean_sd_table	=r(sd)

		sum lagvar if e(sample) & !(NoCropBoroBL == 1 | NoCropBL == 1 | incorrect_no_bl == 1 | NoPlotDataBL == 1 | NoHarvValueDataBL == 1)
		scalar BL_mean 		= r(mean)
		scalar BL_N 	= r(N)

		estadd scalar BL_mean_table	=r(mean)
		estadd scalar BL_N_table	=r(N)

		rename lagvar lag2_`depvar'

		foreach tmt_arm in reg sdp inc {
			scalar  conf_int_min_`tmt_arm'   =coeff_`tmt_arm'-(1.96*coeff_se_`tmt_arm') + ctl_mean
			scalar  conf_int_max_`tmt_arm'   =coeff_`tmt_arm'+(1.96*coeff_se_`tmt_arm') + ctl_mean

			test    tmt_`tmt_arm'

			scalar  pvalue =r(p)
			local star_`tmt_arm'
			if pvalue < 0.10 {
				local star_`tmt_arm' "*"
			}
			if pvalue < 0.05 {
				local star_`tmt_arm' "**"
			}
			if pvalue < 0.01 {
				local star_`tmt_arm' "***"
			}

			scalar tmt_mean_`tmt_arm' = ctl_mean + coeff_`tmt_arm'
		}

		file open  `textfile'_`date' using `textfile'_`date'.txt, text write append
		file write `textfile'_`date' ///
			"`regname'" _tab "Control"  _tab %9.3f (ctl_mean)      _tab  		 			_tab 		 			  	  _tab 			 				_tab %9.3f (ctl_N) 	   _tab			  	  _n ///
			"`regname'" _tab "reg"		_tab %9.3f (tmt_mean_reg)  _tab  %9.3f (coeff_reg)  _tab %9.3f (conf_int_min_reg) _tab %9.3f (conf_int_max_reg) _tab %9.3f (obs_reg)   _tab "`star_reg'"  _n ///
			"`regname'" _tab "sdp"		_tab %9.3f (tmt_mean_sdp)  _tab  %9.3f (coeff_sdp)  _tab %9.3f (conf_int_min_sdp) _tab %9.3f (conf_int_max_sdp) _tab %9.3f (obs_sdp)   _tab "`star_sdp'"  _n ///
			"`regname'" _tab "inc"		_tab %9.3f (tmt_mean_inc)  _tab  %9.3f (coeff_inc)  _tab %9.3f (conf_int_min_inc) _tab %9.3f (conf_int_max_inc) _tab %9.3f (obs_inc)   _tab "`star_inc'"  _n
		file close `textfile'_`date'

	}
	esttab `REG_RESULTS' using "$csv/outcomeVar_AllTreat_`date'.csv",      ///
		title(`tabeleFileName') stats(BL_mean_table BL_N_table ctl_mean_table ctl_N_table ctl_mean_sd_table N) mtitles nonumbers plain br  star(* 0.10 ** 0.05 *** 0.01) ///
		b(a3)  se(%9.2f) alignment(l) replace

	/***************************************

		Produce the graphs

	***************************************/

	insheet using `textfile'_`date'.txt, clear tab

	pause on

	cd "$graphs/gRaw"

	gen tmt = .
	replace tmt = 1 if tmtstatus == "Control"
	replace tmt = 2 if tmtstatus == "reg"
	replace tmt = 3 if tmtstatus == "sdp"
	replace tmt = 4 if tmtstatus == "inc"

	gen placement = tmt

	*gen 	placement = tmt if demo_code == 1
	*replace placement = tmt + 5 if demo_code == 2

	foreach percentVar in mean coeff conf_int_min conf_int_max {

		replace `percentVar' =  `percentVar' * 100 if variable == "c1_commerce"
	}

	gen 	labelString 		= ""
	gen 	labelCategory		= .
	local 	continVars_Large  	c1_harv_value c1_net_yield c1_gross_yield c1_total_earnings c2_inp_total_spending c1_IAAP_harv_value c1_net_yield_nonLabor c1_net_yield_allLabor
	local 	continVars_Small  	c1_total_plotsize
	local 	percentage_Vars   	c1_commerce

	foreach L_var of local continVars_Large {

		replace labelCategory = 1 if variable == "`L_var'"
	}
	foreach S_var of local continVars_Small {

		replace labelCategory = 2 if variable == "`S_var'"
	}
	foreach P_var of local percentage_Vars {

		replace labelCategory = 3 if variable == "`P_var'"
	}

	tempvar labelString_temp decimal_postion

	tostring mean,	gen(`labelString_temp') force
	gen 	`decimal_postion' = strpos(`labelString_temp',".")

	*For continious values without decimal point (means include all in practice)
	replace `decimal_postion' = 10 if  `decimal_postion' == 0

	replace labelString = substr(`labelString_temp',1,(`decimal_postion'-1)) 		if labelCategory == 1
	replace labelString = substr(`labelString_temp',1,4) 							if labelCategory == 2
	replace labelString = substr(`labelString_temp',1,`decimal_postion'+1) + "%"	if labelCategory == 3

	tempfile restart_loop
	save 	`restart_loop', replace

	levelsof variable
	foreach level in `r(levels)' {
		use `restart_loop', clear
		di "`level'"
		keep if variable == "`level'"

		foreach tmt in reg sdp inc {
			preserve
				keep if tmtstatus == "`tmt'"
				local obs_`tmt'  = obs[1]
				local star_`tmt' = star[1]
			restore
		}

		preserve
			keep if tmtstatus == "Control"
			local obs_ctrl  = obs[1]
		restore

		local title "ERROR"

		if "`level'" == "c1_IAAP_harv_value"	local title "Harvest value IAPP crops, Taka"
		if "`level'" == "c1_commerce"			local title "Commercialization, earnings/production %"
		if "`level'" == "c1_gross_yield"		local title "Gross yield, Taka/Ha"
		if "`level'" == "c1_gross_yield_comp"	local title "Gross yield, plots with completed harvest, Taka/Ha"
		if "`level'" == "c1_gross_yield_harv" 	local title "Gross yield, actaully harvested all plots, Taka/Ha"
		if "`level'" == "c1_harv_value"			local title "Total value all harvest, Taka"
		if "`level'" == "c1_harv_value_int"		local title "Total value harvest intercropped, Taka"
		if "`level'" == "c1_harv_value_mono" 	local title "Total value harvest monocropped, Taka"
		if "`level'" == "c1_net_yield"			local title "Net yield, Taka/Ha"
		if "`level'" == "c1_total_earnings"		local title "Total earnings all crop sales, Taka"
		if "`level'" == "c1_total_plotsize"		local title "Total plotsize, Ha"
		if "`level'" == "c2_inp_total_spending"	local title "Total input spending, Taka"

		local noyscale ylabel(#3) ytitle("")
		*if "`title'" != "Boro" & "`title'" != "Lentil" & "`title'" != "Mustard" local noyscale ysc(off)

		if "`level'" == "c2_inp_total_spending"	local noyscale ylabel(#2) ytitle("")
		if "`level'" == "c1_gross_yield"		local noyscale ylabel(#2) ytitle("")

		local graphname	gph_`textfile'_`level'
		sort placement
		graph twoway (bar mean placement if tmt == 1, color("17 33 44")	 ) ///
			(bar mean placement if tmt == 2 , color("228 202 194") 		) ///
			(bar mean placement if tmt == 3 , color("215 107 96" ) 		 ) ///
			(bar mean placement if tmt == 4 , color("156 36  24" ) 		) ///
			(rcap conf_int_max conf_int_min placement, lc(gs) 						) ///
			(scatter  mean placement, msym(none) mlab(labelString) mlabs(medium) mlabpos(10) mlabcolor(black))	, ///
			legend(order(1 2 3 4) lab(1 "Control") lab(2 "Regular Treatment") lab(3 "Shared Demo Treatment") lab(4 "Incentives Treatment")) ///
			 plotregion(margin(b+3 t+3)) ///
			xlabel(	1 `" (N = `obs_ctrl') "' ///
					2 `" "(N = `obs_reg')" " `star_reg' " "' ///
					3 `" "(N = `obs_sdp')" " `star_sdp' " "' ///
					4 `" "(N = `obs_inc')" " `star_inc' " "', noticks labsize(medsmall)) ///
			graphregion( lcolor("182 222 255") fcolor("182 222 255")) ///
			plotregion(fcolor("247 247 247") ) ///
		   	xtitle("") ``level'_lab' `noyscale'  ///
			title("`title'") saving(`graphname'.gph, replace)
			graph export `graphname'.png, replace
		*pause
	}

	*Main outcomes
	cd "$graphs/gRaw"
	grc1leg gph_ag_outcome_log_c1_harv_value.gph 	///
			gph_ag_outcome_log_c1_net_yield.gph 	///
			gph_ag_outcome_log_c1_gross_yield.gph 	///
			gph_ag_outcome_log_c1_total_earnings.gph 	///
		,saving(gph_ag_outcome_log_MAIN.png, replace) /*ycom*/ ///
		cols(2) imargin(small) graphregion( lcolor("182 222 255") fcolor("182 222 255"))
	graph use gph_ag_outcome_log_MAIN.png
	graph display, xsize(8.5)

	cd "$graphs/"
	graph export gph_ag_outcome_log_MAIN.eps, preview(on) replace

	pause

	*Other ag outcomes
	cd "$graphs/gRaw"
	grc1leg gph_ag_outcome_log_c2_inp_total_spending.gph 	///
			gph_ag_outcome_log_c1_total_plotsize.gph 	///
			gph_ag_outcome_log_c1_IAAP_harv_value.gph 	///
			gph_ag_outcome_log_c1_commerce.gph 	///
		,saving(gph_ag_outcome_log_other.png, replace) /*ycom*/ ///
		cols(2) imargin(small) graphregion( lcolor("182 222 255") fcolor("182 222 255"))
	graph use gph_ag_outcome_log_other.png
	graph display, xsize(8.5)

		cd "$graphs/"
	graph export gph_ag_outcome_log_other.eps, preview(on) replace

	cap log close `logname'
end

cap	program drop	ieimpgraph
	program define 	ieimpgraph

	syntax anything

	mat list e(b)

	foreach var of local varlist {

		local beta = _b[`var']

		di "`beta'"

	}

end

cap	program drop	tobeused
	program define 	tobeused

	capture file close `textfile'_`date'
	file open  `textfile'_`date' using `textfile'_`date'.txt, text write replace
	file write `textfile'_`date' ///
		"Variable" 	_tab "Tmt-Status"	_tab "mean" _tab "coeff" _tab "conf_int_min" _tab "conf_int_max" _tab "obs" _tab "star" _n
	file close `textfile'_`date'

	local REG_RESULTS ""
	foreach depvar of local depvars {

		rename lag2_`depvar' lagvar

		local regname `depvar'
			eststo `regname' : areg `depvar' tmt_reg tmt_sdp tmt_inc lagvar NoCropBoroBL NoCropBL incorrect_no_bl NoPlotDataBL NoHarvValueDataBL if t == 2, a(d_code) cl(v_code)
		local  REG_RESULTS "`REG_RESULTS' `regname'"

		foreach tmt_arm in reg sdp inc {
			scalar coeff_`tmt_arm' 		= _b[tmt_`tmt_arm']
			scalar coeff_se_`tmt_arm' 	= _se[tmt_`tmt_arm']

			sum `depvar' if e(sample) & tmt_`tmt_arm' == 1
			scalar obs_`tmt_arm'	= r(N)
		}

		sum `depvar' if e(sample) & tmt_reg == 0 & tmt_sdp == 0 & tmt_inc == 0
		scalar ctl_N		= r(N)
		scalar ctl_mean	  	= r(mean)
		scalar ctl_mean_sd 	= r(sd)

		estadd scalar ctl_mean_table	=r(mean)
		estadd scalar ctl_N_table		=r(N)
		estadd scalar ctl_mean_sd_table	=r(sd)

		sum lagvar if e(sample) & !(NoCropBoroBL == 1 | NoCropBL == 1 | incorrect_no_bl == 1 | NoPlotDataBL == 1 | NoHarvValueDataBL == 1)
		scalar BL_mean 		= r(mean)
		scalar BL_N 	= r(N)

		estadd scalar BL_mean_table	=r(mean)
		estadd scalar BL_N_table	=r(N)

		rename lagvar lag2_`depvar'

		foreach tmt_arm in reg sdp inc {
			scalar  conf_int_min_`tmt_arm'   =coeff_`tmt_arm'-(1.96*coeff_se_`tmt_arm') + ctl_mean
			scalar  conf_int_max_`tmt_arm'   =coeff_`tmt_arm'+(1.96*coeff_se_`tmt_arm') + ctl_mean

			test    tmt_`tmt_arm'

			scalar  pvalue =r(p)
			local star_`tmt_arm'
			if pvalue < 0.10 {
				local star_`tmt_arm' "*"
			}
			if pvalue < 0.05 {
				local star_`tmt_arm' "**"
			}
			if pvalue < 0.01 {
				local star_`tmt_arm' "***"
			}

			scalar tmt_mean_`tmt_arm' = ctl_mean + coeff_`tmt_arm'
		}

		file open  `textfile'_`date' using `textfile'_`date'.txt, text write append
		file write `textfile'_`date' ///
			"`regname'" _tab "Control"  _tab %9.3f (ctl_mean)      _tab  		 			_tab 		 			  	  _tab 			 				_tab %9.3f (ctl_N) 	   _tab			  	  _n ///
			"`regname'" _tab "reg"		_tab %9.3f (tmt_mean_reg)  _tab  %9.3f (coeff_reg)  _tab %9.3f (conf_int_min_reg) _tab %9.3f (conf_int_max_reg) _tab %9.3f (obs_reg)   _tab "`star_reg'"  _n ///
			"`regname'" _tab "sdp"		_tab %9.3f (tmt_mean_sdp)  _tab  %9.3f (coeff_sdp)  _tab %9.3f (conf_int_min_sdp) _tab %9.3f (conf_int_max_sdp) _tab %9.3f (obs_sdp)   _tab "`star_sdp'"  _n ///
			"`regname'" _tab "inc"		_tab %9.3f (tmt_mean_inc)  _tab  %9.3f (coeff_inc)  _tab %9.3f (conf_int_min_inc) _tab %9.3f (conf_int_max_inc) _tab %9.3f (obs_inc)   _tab "`star_inc'"  _n
		file close `textfile'_`date'

	}
	esttab `REG_RESULTS' using "$csv/outcomeVar_AllTreat_`date'.csv",      ///
		title(`tabeleFileName') stats(BL_mean_table BL_N_table ctl_mean_table ctl_N_table ctl_mean_sd_table N) mtitles nonumbers plain br  star(* 0.10 ** 0.05 *** 0.01) ///
		b(a3)  se(%9.2f) alignment(l) replace

	/***************************************

		Produce the graphs

	***************************************/

	insheet using `textfile'_`date'.txt, clear tab

	pause on

	cd "$graphs/gRaw"

	gen tmt = .
	replace tmt = 1 if tmtstatus == "Control"
	replace tmt = 2 if tmtstatus == "reg"
	replace tmt = 3 if tmtstatus == "sdp"
	replace tmt = 4 if tmtstatus == "inc"

	gen placement = tmt

	*gen 	placement = tmt if demo_code == 1
	*replace placement = tmt + 5 if demo_code == 2

	foreach percentVar in mean coeff conf_int_min conf_int_max {

		replace `percentVar' =  `percentVar' * 100 if variable == "c1_commerce"
	}

	gen 	labelString 		= ""
	gen 	labelCategory		= .
	local 	continVars_Large  	c1_harv_value c1_net_yield c1_gross_yield c1_total_earnings c2_inp_total_spending c1_IAAP_harv_value c1_net_yield_nonLabor c1_net_yield_allLabor
	local 	continVars_Small  	c1_total_plotsize
	local 	percentage_Vars   	c1_commerce

	foreach L_var of local continVars_Large {

		replace labelCategory = 1 if variable == "`L_var'"
	}
	foreach S_var of local continVars_Small {

		replace labelCategory = 2 if variable == "`S_var'"
	}
	foreach P_var of local percentage_Vars {

		replace labelCategory = 3 if variable == "`P_var'"
	}

	tempvar labelString_temp decimal_postion

	tostring mean,	gen(`labelString_temp') force
	gen 	`decimal_postion' = strpos(`labelString_temp',".")

	*For continious values without decimal point (means include all in practice)
	replace `decimal_postion' = 10 if  `decimal_postion' == 0

	replace labelString = substr(`labelString_temp',1,(`decimal_postion'-1)) 		if labelCategory == 1
	replace labelString = substr(`labelString_temp',1,4) 							if labelCategory == 2
	replace labelString = substr(`labelString_temp',1,`decimal_postion'+1) + "%"	if labelCategory == 3

	tempfile restart_loop
	save 	`restart_loop', replace

	levelsof variable
	foreach level in `r(levels)' {
		use `restart_loop', clear
		di "`level'"
		keep if variable == "`level'"

		foreach tmt in reg sdp inc {
			preserve
				keep if tmtstatus == "`tmt'"
				local obs_`tmt'  = obs[1]
				local star_`tmt' = star[1]
			restore
		}

		preserve
			keep if tmtstatus == "Control"
			local obs_ctrl  = obs[1]
		restore

		local title "ERROR"

		if "`level'" == "c1_IAAP_harv_value"	local title "Harvest value IAPP crops, Taka"
		if "`level'" == "c1_commerce"			local title "Commercialization, earnings/production %"
		if "`level'" == "c1_gross_yield"		local title "Gross yield, Taka/Ha"
		if "`level'" == "c1_gross_yield_comp"	local title "Gross yield, plots with completed harvest, Taka/Ha"
		if "`level'" == "c1_gross_yield_harv" 	local title "Gross yield, actaully harvested all plots, Taka/Ha"
		if "`level'" == "c1_harv_value"			local title "Total value all harvest, Taka"
		if "`level'" == "c1_harv_value_int"		local title "Total value harvest intercropped, Taka"
		if "`level'" == "c1_harv_value_mono" 	local title "Total value harvest monocropped, Taka"
		if "`level'" == "c1_net_yield"			local title "Net yield, Taka/Ha"
		if "`level'" == "c1_total_earnings"		local title "Total earnings all crop sales, Taka"
		if "`level'" == "c1_total_plotsize"		local title "Total plotsize, Ha"
		if "`level'" == "c2_inp_total_spending"	local title "Total input spending, Taka"

		local noyscale ylabel(#3) ytitle("")
		*if "`title'" != "Boro" & "`title'" != "Lentil" & "`title'" != "Mustard" local noyscale ysc(off)

		if "`level'" == "c2_inp_total_spending"	local noyscale ylabel(#2) ytitle("")
		if "`level'" == "c1_gross_yield"		local noyscale ylabel(#2) ytitle("")

		local graphname	gph_`textfile'_`level'
		sort placement
		graph twoway (bar mean placement if tmt == 1, color("17 33 44")	 ) ///
			(bar mean placement if tmt == 2 , color("228 202 194") 		) ///
			(bar mean placement if tmt == 3 , color("215 107 96" ) 		 ) ///
			(bar mean placement if tmt == 4 , color("156 36  24" ) 		) ///
			(rcap conf_int_max conf_int_min placement, lc(gs) 						) ///
			(scatter  mean placement, msym(none) mlab(labelString) mlabs(medium) mlabpos(10) mlabcolor(black))	, ///
			legend(order(1 2 3 4) lab(1 "Control") lab(2 "Regular Treatment") lab(3 "Shared Demo Treatment") lab(4 "Incentives Treatment")) ///
			 plotregion(margin(b+3 t+3)) ///
			xlabel(	1 `" (N = `obs_ctrl') "' ///
					2 `" "(N = `obs_reg')" " `star_reg' " "' ///
					3 `" "(N = `obs_sdp')" " `star_sdp' " "' ///
					4 `" "(N = `obs_inc')" " `star_inc' " "', noticks labsize(medsmall)) ///
			graphregion( lcolor("182 222 255") fcolor("182 222 255")) ///
			plotregion(fcolor("247 247 247") ) ///
		   	xtitle("") ``level'_lab' `noyscale'  ///
			title("`title'") saving(`graphname'.gph, replace)
			graph export `graphname'.png, replace
		*pause
	}

	*Main outcomes
	cd "$graphs/gRaw"
	grc1leg gph_ag_outcome_log_c1_harv_value.gph 	///
			gph_ag_outcome_log_c1_net_yield.gph 	///
			gph_ag_outcome_log_c1_gross_yield.gph 	///
			gph_ag_outcome_log_c1_total_earnings.gph 	///
		,saving(gph_ag_outcome_log_MAIN.png, replace) /*ycom*/ ///
		cols(2) imargin(small) graphregion( lcolor("182 222 255") fcolor("182 222 255"))
	graph use gph_ag_outcome_log_MAIN.png
	graph display, xsize(8.5)

	cd "$graphs/"
	graph export gph_ag_outcome_log_MAIN.eps, preview(on) replace

	pause

	*Other ag outcomes
	cd "$graphs/gRaw"
	grc1leg gph_ag_outcome_log_c2_inp_total_spending.gph 	///
			gph_ag_outcome_log_c1_total_plotsize.gph 	///
			gph_ag_outcome_log_c1_IAAP_harv_value.gph 	///
			gph_ag_outcome_log_c1_commerce.gph 	///
		,saving(gph_ag_outcome_log_other.png, replace) /*ycom*/ ///
		cols(2) imargin(small) graphregion( lcolor("182 222 255") fcolor("182 222 255"))
	graph use gph_ag_outcome_log_other.png
	graph display, xsize(8.5)

		cd "$graphs/"
	graph export gph_ag_outcome_log_other.eps, preview(on) replace

	cap log close `logname'
end


fix minor bug where error not thrown correctly

*! version 1.16.9000 17may2016
*! PARALLEL: Stata module for parallel computing
*! by George G. Vega [cre,aut], Brian Quistorff [ctb]
*!
*! Project website:
*!   https://github.com/gvegayon/parallel
*! Bug reports:
*!   https://github.com/gvegayon/parallel/issues
/*
Copyright (c) 2014  <George G. Vega>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

// Syntax parser
program def parallel
    version 11.0

	// Checks wether if is parallel prefix or not
	if  (regexm(`"`0'"', "^(do|clean|setclusters|break|version|append|printlog|viewlog)")) {
	/* If not prefix */
		parallel_`0'
	}
	else if (regexm(`"`0'"', "^(bs|sim)[,]?[\s ]?")) {
	/* Prefix bootstrap or simulate */
		local cmd = regexs(1)
		local 0   = regexr(`"`0'"', "^(bs|sim)", "")
		gettoken x 0 : 0, parse(":") bind
		local 0 = regexr(`"`0'"', "^[:]", "")
		gettoken x options : x, parse(",") bind

		gettoken 0 argopt : 0, parse(",") bind
		parallel_`cmd' `0', argopt(`argopt') `options'
	}
	else if (regexm(`"`0'"',"^([,]?.*[:])")) {
	/* if prefix */
		gettoken x 0 : 0, parse(":") bind
		local 0 = regexr(`"`0'"', "^[:]", "")
		// Gets the options (if these exists) of parallel
		gettoken x options : x, parse(",") bind

		gettoken 0 argopt : 0, parse(",") bind
		parallel_do `0', `options' prefix argopt(`argopt')
	}
	else {
		local val = regexm(`"`0'"', "^([a-zA-Z0-9_]*).*")
		local subcommand = regexs(1)
		di as result `"-`subcommand'-"' as error " invalid parallel subcommand" as text " ({stata h parallel:{it:help parallel}})"
		exit 198
	}
end

/* Returns the version of parallel */
program def parallel_version, rclass
	version 11.0
	di as result "parallel" as text " Stata module for parallel computing"
	di as result "vers" as text " 1.16.9000 17may2016"
	di as result "auth" as text " George G. Vega [cre,aut], Brian Quistorff [ctb]"

	return local pll_vers = "1.16.9000"
end

/* Take a look to logfiles */
program def parallel_printlog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)]
	parallel_checklog `pll_instance', e(`event') action(print)
end

program def parallel_viewlog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)]
	parallel_checklog `pll_instance', e(`event') action(view)
end

program def parallel_checklog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)] action(string)

	if ("`event'"=="") local event = "$LAST_PLL_ID"

	if ("`event'"!=".") {
		/* By default uses 1 */
		if ("`pll_instance'" == "") local pll_instance 1

		local pll_instance : di %04.0f `pll_instance'

		/* Does de file exists */
		local logname = "__pll`event'_do`pll_instance'.log"

		if (c(os) == "Windows") local logname = "`c(tmpdir)'`logname'"
		else local logname = "`c(tmpdir)'/`logname'"

		/* If the logfile does not exists, do nothing*/
		cap confirm file "`logname'"
		if (_rc) {
			di as result "No logfile for instance -`pll_instance'- of parallel process -`event'- found"
			exit
		}

		/* Showing the log in screen */
		if ("`action'"=="print") {
			di as result "{hline 80}"
			di as result %~80s "beginning of file -`logname'-"
			di as result "{hline 80}"
			type `"`logname'"'
			di as result "{hline 80}"
			di as result %~80s "end of file -`logname'-"
			di as result "{hline 80}"
		}
		else view `"`logname'"'
	}
	else {
		di as error "It seems that you haven't use -parallel- yet."
		exit 601
	}

end

////////////////////////////////////////////////////////////////////////////////
// Splits the dataset into clusters
program def parallel_spliter
	version 11.0
	syntax [namelist(name=xtstructure)] [,parallelid(string) sorting(integer 0) force(integer 0) keepusing(varlist) orig_cl_local(string)]
	//args xtstructure parallelid Sorting Force

	if length("$PLL_CLUSTERS") == 0 {
		di as err "Number of clusters not fixed." as text " Set the number of clusters with " _newline as err "-{cmd:parallel setclusters #}-"
		exit 198
	}

	//quietly {
		if (length("`xtstructure'")) {
			_assert `sorting' | `force', msg("Data not sorted") rc(5)

			/* Checking the data types */
			foreach var of varlist `xtstructure' {
				capture confirm string var `var'
				if (`=_rc') local numvars `numvars' `var'
				else local strvars `strvars' `var'
			}

			/* calculate max possible clusters */
			egen _`parallelid'grp = group(`xtstructure'), missing
			summ _`parallelid'grp, meanonly
			local max_n_cl = `r(max)'
			drop _`parallelid'grp
		}
		else {
			local max_n_cl = _N
		}

		/* Do we have too many clusters? */
		if (`max_n_cl'<$PLL_CLUSTERS){
			c_local `orig_cl_local' ${PLL_CLUSTERS}
			global PLL_CLUSTERS = `max_n_cl'
			di "Small workload/num groups. Temporarily setting number of clusters to ${PLL_CLUSTERS}"
		}

		gen _`parallelid'cut = .

		/* Checking types of data */
		if (length("`numvars'")) local numvars st_data(.,"`numvars'")
		else local numvars J(0,0,.)

		if (length("`strvars'")) local strvars st_sdata(.,"`strvars'")
		else local strvars J(0,0,"")

		/* Processing in MATA */
		mata: st_store(., "_`parallelid'cut", parallel_divide_index(`numvars', `strvars'))

		if (length("`keepusing'")) {
			keep _`parallelid'cut `keepusing'
			gen __pllnobs`parallelid' = _n
		}

		qui save __pll`parallelid'_dataset, replace
		drop _all
	//}

end

////////////////////////////////////////////////////////////////////////////////
// MAIN PROGRAM
// There are things we need to possibly restore on exit
// - Data/S_FN (this is taken care of by -preserve-)
// - PWD (non-indented capture block)
// - Temporaliy changed number of clusters (non-indented capture block)
// - Turn off the timer (do within catch for capture blocks)
program def parallel_do, rclass
	version 11.0

	#delimit ;
	syntax anything(name=dofile equalok everything)
		[, by(string)
		Keep
		KEEPLast
		prefix
		Force
		PROGrams(string)
		Mata
		NOGlobals
		KEEPTiming
		Seeds(string)
		NOData
		Randtype(string)
		Timeout(integer 60)
		PROCessors(integer 0)
		argopt(string)
		KEEPUsing(string)
		SETparallelid(string)
		];
	#delimit cr

	if length("$PLL_CLUSTERS") == 0 {
		di "{error:You haven't set the number of clusters}" _n "{error:Please set it with: {cmd:parallel setclusters} {it:#}}"
		exit 198
	}

	if `c(noisily)'==0 & "`programs'"!="" {
		di "{error:If you want to pass programs in memory (using the option programs)}" _n "{error:then the program must be run noisily.}"
		exit 198
	}

	// Initial checks
	foreach opt in macrolist keep keeplast prefix force mata noglobals keeptiming nodata {
		local `opt' = length("``opt''") > 0
	}

	/* Randtype */
	if ("`randtype'" == "") local randtype = "datetime"

	/* If no data parsing has to be done (because of no data!) */
	if (!(c(N)*c(k))) local nodata 1

	if (!`keeptiming') {
		timer clear 97
		timer clear 98
		timer clear 99
	}

	timer on 98

	// Deletes last parallel instance ran
	if (`keeplast' & length("`r(pll_id)'")) cap parallel_clean, e(`r(pll_id)') nologs

	if length("`by'") != 0 {
		local sortlist: sortedby
		local sorting = regexm("`sortlist'","^`by'")
	}
	else local sorting = 0

	// Gets the directory where to work at
	if (!`prefix') {

		/* First checks if the file exists */
		mata: parallel_normalizepath(`"`dofile'"',1)
		local pll_dir = "`filedir'"
		local dofile = "`filename'"
	}
	else local pll_dir = c(pwd)+"/"

	local initialdir = c(pwd)
	qui cd "`pll_dir'"
	capture noisily { //Start capture block for PWD

	/* Creates a unique ID for the process and secures it */
	if ("`setparallelid'"=="") mata: parallel_sandbox(5)
	else local parallelid = "`setparallelid'"

	/* Generates database clusters */
	capture noisily { //Start capture block for PLL_CLUSTERS
	if (!`nodata'){
		local sfn = "$S_FN" // be able to "fake" restore this
		preserve

		parallel_spliter `by' , parallelid(`parallelid') sorting(`sorting') force(`force') keepusing(`keepusing') orig_cl_local(orig_PLL_CLUSTERS)
	}

	global LAST_PLL_ID = "`parallelid'"
	global LAST_PLL_DIR = "`pll_dir'"
	global LAST_PLL_N = $PLL_CLUSTERS

	/* Starts building the files */
	local work_around_no_cwd = 0
	quietly {

		/* Saves mata objects */
		if (`mata') {
			mata: mata mlib create l__pll`parallelid'_mlib, replace
			cap mata: mata mlib add l__pll`parallelid'_mlib *()
			cap which l__pll`parallelid'_mlib.mlib
			if _rc!=0{
				local work_around_no_cwd = 1
				noi di "Note: In order to pass mata objects to the clusters, they are saved to a temporary mlib file in the current directly."
				noi di "      Your ado-path doesn't contain the current directory."
				noi di "      We have added the current directory to the end of the ado-path for the clusters.."
			}
			cap mata: mata matsave __pll`parallelid'_mata.mmat *, replace
			if (`=_rc') local matasave = 0
			else local matasave = 1
		}
		else local matasave = 0
	}

	/* Writing the dofile */
	mata: st_local("errornum", strofreal(parallel_write_do(strtrim(`"`dofile' `argopt'"'), "`parallelid'", $PLL_CLUSTERS, `prefix', `matasave', !`noglobals', "`seeds'", "`randtype'", `nodata', "`pll_dir'", "`programs'", `processors',`work_around_no_cwd')))
	if (`errornum') {
		/* Removes the sandbox file (unprotect the files) */
		mata: parallel_sandbox(2, "`parallelid'")

		error `errornum'
	}

	timer off 98
	cap timer list
	if (r(t98) == .) local pll_t_setu = 0
	else local pll_t_setu = r(t98)

	/* Running the dofiles */
	timer on 99
	mata: st_local("nerrors", strofreal(parallel_run("`parallelid'",$PLL_CLUSTERS,`"$PLL_STATA_PATH"',`=`timeout'*1000')))
	timer off 99

	cap timer list
	if (r(t99) == .) local pll_t_calc = 0
	else local pll_t_calc = r(t99)
	local pll_t_reps = r(nt99)

	timer on 97
	// Paste the databases
	if (!`nodata' & !`nerrors') {
		parallel_fusion `parallelid', clusters($PLL_CLUSTERS) keepusing(`keepusing')
		cap drop _`parallelid'cut

		global S_FN = "`sfn'" // Restores original S_FN (file name) value
		restore, not
	}

	return scalar pll_n = $PLL_CLUSTERS
	} //End capture block for $PLL_CLUSTERS
	if _rc {
		local orig_rc = _rc
		if "`orig_PLL_CLUSTERS'"!="" global PLL_CLUSTERS=`orig_PLL_CLUSTERS'
		cap timer off 97
		cap timer off 98
		cap timer off 99
		exit `orig_rc'
	}
	if "`orig_PLL_CLUSTERS'"!="" global PLL_CLUSTERS=`orig_PLL_CLUSTERS'

	/* Removes the sandbox file (unprotect the files) */
	if ("`setparallelid'" == "") {
		mata: parallel_sandbox(2, "`parallelid'")
		if (!`keep' & !`keeplast') parallel_clean, e("`parallelid'") nologs
	}

	} //End capture block for PWD
	if _rc {
		local orig_rc = _rc
		qui cd "`initialdir'"
		cap timer off 97
		cap timer off 98
		cap timer off 99
		exit `orig_rc'
	}
	qui cd "`initialdir'"

	timer off 97
	cap timer list
	if (r(t97) == .) local pll_t_fini = 0
	else local pll_t_fini = r(t97)

	qui timer list
	return local  pll_seeds="`pllseeds'"
	return scalar pll_errs = `nerrors'
	return local  pll_dir "`pll_dir'"
	return scalar pll_t_reps = `pll_t_reps'
	return scalar pll_t_setu = `pll_t_setu'
	return scalar pll_t_calc = `pll_t_calc'
	return scalar pll_t_fini = `pll_t_fini'
	return local pll_id = "`parallelid'"

	if `nerrors' {
		di as err "`nerrors' child processes encountered errors. Throwing last error."
		exit `pll_last_error'
	}
end

////////////////////////////////////////////////////////////////////////////////
// Cleans all files generated by parallel
// If you want to keep the logs, then specify -, nologs-
program def parallel_clean
	version 11.0
	syntax [, Event(string) All Force nologs]

	if (length("`event'") != 0 & length("`all'") != 0) {
		di as error `"invalid syntax: Using -pll_id- and -all- jointly is not allowed."'
		exit 198
	}
	local do_logs = ("`logs'"!="nologs")
	mata: parallel_clean("`event'", `=length("`all'") > 0', `=length("`force'") > 0', `do_logs')
end

////////////////////////////////////////////////////////////////////////////////
// Sets the number of clusters as a global macro
program parallel_setclusters
	version 11.0
	syntax anything(name=nclusters)  [, Force Statapath(string asis) Gateway(string) Includefile(string) procexec(int 2)]

	local nclusters = int(real(`"`nclusters'"'))
	_assert `nclusters'!=., msg(`"Not allowed: "#" Should be a number"') rc(109)
	_assert `nclusters'>0,  msg(`"Not allowed: "#" Should be positive"') rc(109)
	_assert inlist(`procexec',0,1,2), msg("procexec() must be 0, 1, or 2") rc(198)

	global USE_PROCEXEC = `procexec'

	local force = (length("`force'")>0)
	mata: parallel_setclusters(`nclusters', `force')
	mata: st_local("error", strofreal(parallel_setstatapath(`"`statapath'"', `force')))
	_assert (!`error'), msg("Can not set Stata directory, try using -statapath()- option") rc(`error')

	if "`c(mode)'"=="batch" & "`c(os)'"=="Windows" & "$USE_PROCEXEC"=="0" {
		if `"`gateway'"'=="" local gateway "pll_gateway.sh"

		cap confirm file `"`gateway'"'
		_assert `=!_rc', msg(`"On Windows batch-mode with command-line process execution, requires a gateway file is required but not found (path: `c(pwd)')"') rc(`=_rc')
		global PLL_GATEWAY_FNAME `"`gateway'"'
	}

	if `"`includefile'"'!=""{
		cap confirm file `"`includefile'"'
		_assert `=!_rc', msg("Can not find the include file (`includefile').") msg(`=_rc')
		global PLL_INCLUDE_FILE `"`includefile'"'
	}
end

////////////////////////////////////////////////////////////////////////////////
// Exports

// Exports a copy of programs
program def program_export
	version 11.0
	syntax using/ [,Programlist(string) Inname(string)]

	mata: program_export("`using'", "`programlist'", "`inname'")
end

////////////////////////////////////////////////////////////////////////////////
// Appends the clusterized dataset
program def parallel_fusion
	version 11.0
	syntax anything(name=parallelid) , clusters(integer) [keepusing(string)]

	cap use "__pll`parallelid'_dta0001.dta", clear
	if (_rc){
		di "{error:No dataset for instance 0001.}"
		exit _rc
	}
	local sortlist: sortedby

	forval i = 2/`clusters' {
		cap append using `"__pll`parallelid'_dta`=string(`i',"%04.0f")'.dta"'
		if (_rc){
			di "{error:No dataset for instance `=string(`i',"%04.0f")'.}"
			exit _rc
		}
	}

	/* If it just used a set of variables */
	if (length("`keepusing'")) {
		qui merge 1:1 __pllnobs`parallelid' using __pll`parallelid'_dataset, keep(3) nogen
		drop __pllnobs`parallelid'
	}

	//restore the sort
	if "`sortlist'"!="" sort `sortlist'

end

////////////////////////////////////////////////////////////////////////////////
// Checks whether the user pressed break inside a loop
program def parallel_break
	version 11.0
	mata: parallel_break()
end

*! version 1.16.9000 17may2016
*! PARALLEL: Stata module for parallel computing
*! by George G. Vega [cre,aut], Brian Quistorff [ctb]
*!
*! Project website:
*!   https://github.com/gvegayon/parallel
*! Bug reports:
*!   https://github.com/gvegayon/parallel/issues
/*
Copyright (c) 2014  <George G. Vega>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

// Syntax parser
program def parallel
    version 11.0

	// Checks wether if is parallel prefix or not
	if  (regexm(`"`0'"', "^(do|clean|setclusters|break|version|append|printlog|viewlog)")) {
	/* If not prefix */
		parallel_`0'
	}
	else if (regexm(`"`0'"', "^(bs|sim)[,]?[\s ]?")) {
	/* Prefix bootstrap or simulate */
		local cmd = regexs(1)
		local 0   = regexr(`"`0'"', "^(bs|sim)", "")
		gettoken x 0 : 0, parse(":") bind
		local 0 = regexr(`"`0'"', "^[:]", "")
		gettoken x options : x, parse(",") bind

		gettoken 0 argopt : 0, parse(",") bind
		parallel_`cmd' `0', argopt(`argopt') `options'
	}
	else if (regexm(`"`0'"',"^([,]?.*[:])")) {
	/* if prefix */
		gettoken x 0 : 0, parse(":") bind
		local 0 = regexr(`"`0'"', "^[:]", "")
		// Gets the options (if these exists) of parallel
		gettoken x options : x, parse(",") bind

		gettoken 0 argopt : 0, parse(",") bind
		parallel_do `0', `options' prefix argopt(`argopt')
	}
	else {
		local val = regexm(`"`0'"', "^([a-zA-Z0-9_]*).*")
		local subcommand = regexs(1)
		di as result `"-`subcommand'-"' as error " invalid parallel subcommand" as text " ({stata h parallel:{it:help parallel}})"
		exit 198
	}
end

/* Returns the version of parallel */
program def parallel_version, rclass
	version 11.0
	di as result "parallel" as text " Stata module for parallel computing"
	di as result "vers" as text " 1.16.9000 17may2016"
	di as result "auth" as text " George G. Vega [cre,aut], Brian Quistorff [ctb]"

	return local pll_vers = "1.16.9000"
end

/* Take a look to logfiles */
program def parallel_printlog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)]
	parallel_checklog `pll_instance', e(`event') action(print)
end

program def parallel_viewlog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)]
	parallel_checklog `pll_instance', e(`event') action(view)
end

program def parallel_checklog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)] action(string)

	if ("`event'"=="") local event = "$LAST_PLL_ID"

	if ("`event'"!=".") {
		/* By default uses 1 */
		if ("`pll_instance'" == "") local pll_instance 1

		local pll_instance : di %04.0f `pll_instance'

		/* Does de file exists */
		local logname = "__pll`event'_do`pll_instance'.log"

		if (c(os) == "Windows") local logname = "`c(tmpdir)'`logname'"
		else local logname = "`c(tmpdir)'/`logname'"

		/* If the logfile does not exists, do nothing*/
		cap confirm file "`logname'"
		if (_rc) {
			di as result "No logfile for instance -`pll_instance'- of parallel process -`event'- found"
			exit
		}

		/* Showing the log in screen */
		if ("`action'"=="print") {
			di as result "{hline 80}"
			di as result %~80s "beginning of file -`logname'-"
			di as result "{hline 80}"
			type `"`logname'"'
			di as result "{hline 80}"
			di as result %~80s "end of file -`logname'-"
			di as result "{hline 80}"
		}
		else view `"`logname'"'
	}
	else {
		di as error "It seems that you haven't use -parallel- yet."
		exit 601
	}

end

////////////////////////////////////////////////////////////////////////////////
// Splits the dataset into clusters
program def parallel_spliter
	version 11.0
	syntax [namelist(name=xtstructure)] [,parallelid(string) sorting(integer 0) force(integer 0) keepusing(varlist) orig_cl_local(string)]
	//args xtstructure parallelid Sorting Force

	if length("$PLL_CLUSTERS") == 0 {
		di as err "Number of clusters not fixed." as text " Set the number of clusters with " _newline as err "-{cmd:parallel setclusters #}-"
		exit 198
	}

	//quietly {
		if (length("`xtstructure'")) {
			_assert `sorting' | `force', msg("Data not sorted") rc(5)

			/* Checking the data types */
			foreach var of varlist `xtstructure' {
				capture confirm string var `var'
				if (`=_rc') local numvars `numvars' `var'
				else local strvars `strvars' `var'
			}

			/* calculate max possible clusters */
			egen _`parallelid'grp = group(`xtstructure'), missing
			summ _`parallelid'grp, meanonly
			local max_n_cl = `r(max)'
			drop _`parallelid'grp
		}
		else {
			local max_n_cl = _N
		}

		/* Do we have too many clusters? */
		if (`max_n_cl'<$PLL_CLUSTERS){
			c_local `orig_cl_local' ${PLL_CLUSTERS}
			global PLL_CLUSTERS = `max_n_cl'
			di "Small workload/num groups. Temporarily setting number of clusters to ${PLL_CLUSTERS}"
		}

		gen _`parallelid'cut = .

		/* Checking types of data */
		if (length("`numvars'")) local numvars st_data(.,"`numvars'")
		else local numvars J(0,0,.)

		if (length("`strvars'")) local strvars st_sdata(.,"`strvars'")
		else local strvars J(0,0,"")

		/* Processing in MATA */
		mata: st_store(., "_`parallelid'cut", parallel_divide_index(`numvars', `strvars'))

		if (length("`keepusing'")) {
			keep _`parallelid'cut `keepusing'
			gen __pllnobs`parallelid' = _n
		}

		qui save __pll`parallelid'_dataset, replace
		drop _all
	//}

end

////////////////////////////////////////////////////////////////////////////////
// MAIN PROGRAM
// There are things we need to possibly restore on exit
// - Data/S_FN (this is taken care of by -preserve-)
// - PWD (non-indented capture block)
// - Temporaliy changed number of clusters (non-indented capture block)
// - Turn off the timer (do within catch for capture blocks)
program def parallel_do, rclass
	version 11.0

	#delimit ;
	syntax anything(name=dofile equalok everything)
		[, by(string)
		Keep
		KEEPLast
		prefix
		Force
		PROGrams(string)
		Mata
		NOGlobals
		KEEPTiming
		Seeds(string)
		NOData
		Randtype(string)
		Timeout(integer 60)
		PROCessors(integer 0)
		argopt(string)
		KEEPUsing(string)
		SETparallelid(string)
		];
	#delimit cr

	if length("$PLL_CLUSTERS") == 0 {
		di "{error:You haven't set the number of clusters}" _n "{error:Please set it with: {cmd:parallel setclusters} {it:#}}"
		exit 198
	}

	if `c(noisily)'==0 & "`programs'"!="" {
		di "{error:If you want to pass programs in memory (using the option programs)}" _n "{error:then the program must be run noisily.}"
		exit 198
	}

	// Initial checks
	foreach opt in macrolist keep keeplast prefix force mata noglobals keeptiming nodata {
		local `opt' = length("``opt''") > 0
	}

	/* Randtype */
	if ("`randtype'" == "") local randtype = "datetime"

	/* If no data parsing has to be done (because of no data!) */
	if (!(c(N)*c(k))) local nodata 1

	if (!`keeptiming') {
		timer clear 97
		timer clear 98
		timer clear 99
	}

	timer on 98

	// Deletes last parallel instance ran
	if (`keeplast' & length("`r(pll_id)'")) cap parallel_clean, e(`r(pll_id)') nologs

	if length("`by'") != 0 {
		local sortlist: sortedby
		local sorting = regexm("`sortlist'","^`by'")
	}
	else local sorting = 0

	// Gets the directory where to work at
	if (!`prefix') {

		/* First checks if the file exists */
		mata: parallel_normalizepath(`"`dofile'"',1)
		local pll_dir = "`filedir'"
		local dofile = "`filename'"
	}
	else local pll_dir = c(pwd)+"/"

	local initialdir = c(pwd)
	qui cd "`pll_dir'"
	capture noisily { //Start capture block for PWD

	/* Creates a unique ID for the process and secures it */
	if ("`setparallelid'"=="") mata: parallel_sandbox(5)
	else local parallelid = "`setparallelid'"

	/* Generates database clusters */
	capture noisily { //Start capture block for PLL_CLUSTERS
	if (!`nodata'){
		local sfn = "$S_FN" // be able to "fake" restore this
		preserve

		parallel_spliter `by' , parallelid(`parallelid') sorting(`sorting') force(`force') keepusing(`keepusing') orig_cl_local(orig_PLL_CLUSTERS)
	}

	global LAST_PLL_ID = "`parallelid'"
	global LAST_PLL_DIR = "`pll_dir'"
	global LAST_PLL_N = $PLL_CLUSTERS

	/* Starts building the files */
	local work_around_no_cwd = 0
	quietly {

		/* Saves mata objects */
		if (`mata') {
			mata: mata mlib create l__pll`parallelid'_mlib, replace
			cap mata: mata mlib add l__pll`parallelid'_mlib *()
			cap which l__pll`parallelid'_mlib.mlib
			if _rc!=0{
				local work_around_no_cwd = 1
				noi di "Note: In order to pass mata objects to the clusters, they are saved to a temporary mlib file in the current directly."
				noi di "      Your ado-path doesn't contain the current directory."
				noi di "      We have added the current directory to the end of the ado-path for the clusters.."
			}
			cap mata: mata matsave __pll`parallelid'_mata.mmat *, replace
			if (`=_rc') local matasave = 0
			else local matasave = 1
		}
		else local matasave = 0
	}

	/* Writing the dofile */
	mata: st_local("errornum", strofreal(parallel_write_do(strtrim(`"`dofile' `argopt'"'), "`parallelid'", $PLL_CLUSTERS, `prefix', `matasave', !`noglobals', "`seeds'", "`randtype'", `nodata', "`pll_dir'", "`programs'", `processors',`work_around_no_cwd')))
	if (`errornum') {
		/* Removes the sandbox file (unprotect the files) */
		mata: parallel_sandbox(2, "`parallelid'")

		error `errornum'
	}

	timer off 98
	cap timer list
	if (r(t98) == .) local pll_t_setu = 0
	else local pll_t_setu = r(t98)

	/* Running the dofiles */
	timer on 99
	mata: st_local("nerrors", strofreal(parallel_run("`parallelid'",$PLL_CLUSTERS,`"$PLL_STATA_PATH"',`=`timeout'*1000')))
	timer off 99

	cap timer list
	if (r(t99) == .) local pll_t_calc = 0
	else local pll_t_calc = r(t99)
	local pll_t_reps = r(nt99)

	timer on 97
	// Paste the databases
	if (!`nodata' & !`nerrors') {
		parallel_fusion `parallelid', clusters($PLL_CLUSTERS) keepusing(`keepusing')
		cap drop _`parallelid'cut

		global S_FN = "`sfn'" // Restores original S_FN (file name) value
		restore, not
	}

	return scalar pll_n = $PLL_CLUSTERS
	} //End capture block for $PLL_CLUSTERS
	if _rc {
		if "`orig_PLL_CLUSTERS'"!="" global PLL_CLUSTERS=`orig_PLL_CLUSTERS'
		cap timer off 97
		cap timer off 98
		cap timer off 99
		exit _rc
	}
	if "`orig_PLL_CLUSTERS'"!="" global PLL_CLUSTERS=`orig_PLL_CLUSTERS'

	/* Removes the sandbox file (unprotect the files) */
	if ("`setparallelid'" == "") {
		mata: parallel_sandbox(2, "`parallelid'")
		if (!`keep' & !`keeplast') parallel_clean, e("`parallelid'") nologs
	}

	} //End capture block for PWD
	if _rc {
		qui cd "`initialdir'"
		cap timer off 97
		cap timer off 98
		cap timer off 99
		exit _rc
	}
	qui cd "`initialdir'"

	timer off 97
	cap timer list
	if (r(t97) == .) local pll_t_fini = 0
	else local pll_t_fini = r(t97)

	qui timer list
	return local  pll_seeds="`pllseeds'"
	return scalar pll_errs = `nerrors'
	return local  pll_dir "`pll_dir'"
	return scalar pll_t_reps = `pll_t_reps'
	return scalar pll_t_setu = `pll_t_setu'
	return scalar pll_t_calc = `pll_t_calc'
	return scalar pll_t_fini = `pll_t_fini'
	return local pll_id = "`parallelid'"

	if `nerrors' {
		di as err "`nerrors' child processes encountered errors. Throwing last error."
		exit `pll_last_error'
	}
end

////////////////////////////////////////////////////////////////////////////////
// Cleans all files generated by parallel
// If you want to keep the logs, then specify -, nologs-
program def parallel_clean
	version 11.0
	syntax [, Event(string) All Force nologs]

	if (length("`event'") != 0 & length("`all'") != 0) {
		di as error `"invalid syntax: Using -pll_id- and -all- jointly is not allowed."'
		exit 198
	}
	local do_logs = ("`logs'"!="nologs")
	mata: parallel_clean("`event'", `=length("`all'") > 0', `=length("`force'") > 0', `do_logs')
end

////////////////////////////////////////////////////////////////////////////////
// Sets the number of clusters as a global macro
program parallel_setclusters
	version 11.0
	syntax anything(name=nclusters)  [, Force Statapath(string asis) Gateway(string) Includefile(string) procexec(int 2)]

	local nclusters = int(real(`"`nclusters'"'))
	_assert `nclusters'!=., msg(`"Not allowed: "#" Should be a number"') rc(109)
	_assert `nclusters'>0,  msg(`"Not allowed: "#" Should be positive"') rc(109)
	_assert inlist(`procexec',0,1,2), msg("procexec() must be 0, 1, or 2") rc(198)

	global USE_PROCEXEC = `procexec'

	local force = (length("`force'")>0)
	mata: parallel_setclusters(`nclusters', `force')
	mata: st_local("error", strofreal(parallel_setstatapath(`"`statapath'"', `force')))
	_assert (!`error'), msg("Can not set Stata directory, try using -statapath()- option") rc(`error')

	if "`c(mode)'"=="batch" & "`c(os)'"=="Windows" & "$USE_PROCEXEC"=="0" {
		if `"`gateway'"'=="" local gateway "pll_gateway.sh"

		cap confirm file `"`gateway'"'
		_assert `=!_rc', msg(`"On Windows batch-mode with command-line process execution, requires a gateway file is required but not found (path: `c(pwd)')"') rc(`=_rc')
		global PLL_GATEWAY_FNAME `"`gateway'"'
	}

	if `"`includefile'"'!=""{
		cap confirm file `"`includefile'"'
		_assert `=!_rc', msg("Can not find the include file (`includefile').") msg(`=_rc')
		global PLL_INCLUDE_FILE `"`includefile'"'
	}
end

////////////////////////////////////////////////////////////////////////////////
// Exports

// Exports a copy of programs
program def program_export
	version 11.0
	syntax using/ [,Programlist(string) Inname(string)]

	mata: program_export("`using'", "`programlist'", "`inname'")
end

////////////////////////////////////////////////////////////////////////////////
// Appends the clusterized dataset
program def parallel_fusion
	version 11.0
	syntax anything(name=parallelid) , clusters(integer) [keepusing(string)]

	cap use "__pll`parallelid'_dta0001.dta", clear
	if (_rc){
		di "{error:No dataset for instance 0001.}"
		exit _rc
	}
	local sortlist: sortedby

	forval i = 2/`clusters' {
		cap append using `"__pll`parallelid'_dta`=string(`i',"%04.0f")'.dta"'
		if (_rc){
			di "{error:No dataset for instance `=string(`i',"%04.0f")'.}"
			exit _rc
		}
	}

	/* If it just used a set of variables */
	if (length("`keepusing'")) {
		qui merge 1:1 __pllnobs`parallelid' using __pll`parallelid'_dataset, keep(3) nogen
		drop __pllnobs`parallelid'
	}

	//restore the sort
	if "`sortlist'"!="" sort `sortlist'

end

////////////////////////////////////////////////////////////////////////////////
// Checks whether the user pressed break inside a loop
program def parallel_break
	version 11.0
	mata: parallel_break()
end


iegroph : error code and small edits

iegroph : error code and small edits

error message and error code and small edits to the file format addition

cap	program drop	iegraph
	program define 	iegraph
	preserve
	syntax varlist, [noconfbars TItle(string) save(string) confbarsnone(varlist) GREYscale yzero *]
	mat beta_ = e(b)

	local counter = 0
	//local textfile = "Mrijan"
	qui{
		if "`confbars'" 			!= "" {
		local CONFINT_BAR 	= 0
		}
		else if "`confbars'" 			== "" {
		local CONFINT_BAR 	= 1
		}

		local varTest : list confbarsnone in varlist

		if `varTest' == 0 {
					noi display as error "{phang} Variables defined in confbarsnone cannot be found in the graph variable list. {p_end}"
					noi display ""
					error 111
					 }

		foreach var of local varlist{
			cap assert inlist(`var',0,1) | missing(`var')
				if _rc {
					noi display as error "{phang} The variable `var' is not a dummy. Treatment variable needs to be a dummy(0 or 1) variable. {p_end}"
					noi display ""
					error 149
					}
				else {
					}

		local counter = `counter' + 1
		di `counter'

		scalar coeff_`var' = _b[`var']
		//noi display coeff_`var'

		scalar coeff_se_`var' = _se[`var']
		//noi display coeff_se_`var'

		count if e(sample) == 1 & `var' == 1 //Count one tmt group at the time
		scalar obs_`var' = r(N)
		//noi display obs_`var'

			}
	if "`save'" != "" {

        *Find index for where the file type suffix start
        local dot_index     = strpos("`save'",".") + 1

        *Extract the file index
        local file_suffix   = substr("`save'", `dot_index', .)
		//noi di "`file_suffix'"

		*
        local nonGPH_formats png tiff gph ps eps pdf wmf emf

        *If no file format suffix is specified, use the default .xlsx

        if `dot_index' == 1 | "`file_suffix'" == "gph" {
			local save_export = 0
		}

        *If a file format suffix is specified make sure that it is one of the two allowed.

        else if `:list file_suffix in nonGPH_formats' != 0  {
            local save_export = 1

			if ("`file_suffix'" == "wmf" | "`file_suffix'" == "emf") & "`c(os)'" != "Windows" {
				di as error "The file formats .wmf and .emf are only allowed when using Stata on a Windows computer."
                error 198
			}
		}
        else {

            di as error "You are not using a allowed file format in save(`save'). Only the following formats are allowed: gph `nonGPH_formats'"
            error 198
        }
    }

	//noi di `"iegraph `varlist', `noconfbars' title(`title') save(`save') `confbarsnone' `yzero' `anything'"'

	local count: word count `varlist'
	local graphCount = `count' + 1
	//tokenize "`varlist'"
	//noi di `count'

	//Make all vars tempvars (maybe do later)
	//Make sure that missing is properly handled
	tempvar anyTMT
	egen `anyTMT' = rowmax(`varlist') if e(sample) == 1
	tempvar control
	gen `control' = (`anyTMT' == 0) if !missing(`anyTMT')

	sum `e(depvar)' if e(sample) == 1 & `control' == 1
	scalar ctl_N		= r(N)
	scalar ctl_mean	  	= r(mean)
	scalar ctl_mean_sd 	= r(sd)

		foreach var of local varlist {
			scalar  conf_int_min_`var'   =	coeff_`var'-(1.96*coeff_se_`var') + ctl_mean
			scalar  conf_int_max_`var'   =	coeff_`var'+(1.96*coeff_se_`var') + ctl_mean

			test `var'
			local star_`var' " "

			scalar  pvalue =r(p)
			if pvalue < 0.10 {
				local star_`var' "*"
			}
			if pvalue < 0.05 {
				local star_`var' "**"
			}
			if pvalue < 0.01 {
				local star_`var' "***"
			}
			*noi display "`star_`var''"
			scalar tmt_mean_`var' = ctl_mean + coeff_`var'
		}

	tempname 	newHandle
	tempfile	newTextFile
	cap file close 	`newHandle'
	file open  	`newHandle' using "`newTextFile'", text write append

	*Write headers and control value
	file write `newHandle' ///
		"order" 	_tab "xLabeled"	_tab "mean" _tab "coeff" _tab "conf_int_min" _tab "conf_int_max" _tab "obs" _tab "star" _n 	///
		%9.3f (1) _tab "Control"  _tab %9.3f (ctl_mean)      _tab  		 			_tab 		 			  	  _tab 			 				_tab %9.3f (ctl_N) 	   _tab			  	  _n

	tempvar newCounter
	gen `newCounter' = 2  //First tmt group starts at 2 (1 is control)

	foreach var in `varlist' {
	     if `: list var in confbarsnone' {
		file write `newHandle' ///
			%9.3f (`newCounter') _tab "`var'" _tab %9.3f (tmt_mean_`var') _tab  %9.3f (coeff_`var')  _tab %9.3f  _tab _tab %9.3f (obs_`var')   _tab "`star_`var''"  _n

		replace `newCounter' = `newCounter' + 1
			}
		else {
		file write `newHandle' ///
			%9.3f (`newCounter') _tab "`var'" _tab %9.3f (tmt_mean_`var') _tab  %9.3f (coeff_`var')  _tab %9.3f (conf_int_min_`var') _tab %9.3f (conf_int_max_`var') _tab %9.3f (obs_`var')   _tab "`star_`var''"  _n

		replace `newCounter' = `newCounter' + 1
		}
	}

	file close `newHandle'

	copy "`newTextFile'"  "mainMasterDofile.txt" , replace

	/*************************************

		Create the graph

	*************************************/

	insheet using mainMasterDofile.txt, clear

	local tmtGroupBars 	""
	local xAxisLabels 	`"xlabel( "'
	local legendLabels	`"lab(1 "`obsLabelControl'")"'
	local legendNumbers	""

	forval tmtGroupCount = 1/`graphCount' {

		************
		*Create the bar for this group

		if "`greyscale'" == "" {
			colorPicker `tmtGroupCount' `graphCount'
		}
		else {
			greyPicker `tmtGroupCount' `graphCount'
		}

		local tmtGroupBars `"`tmtGroupBars' (bar mean order if order == `tmtGroupCount', color("`r(color)'") lcolor(black) ) "'

		************
		*Create labels etc. for this group

		local obs 			= obs[`tmtGroupCount']
		local stars 		= star[`tmtGroupCount']
		local legendLabel 	= xlabeled[`tmtGroupCount']

		local xAxisLabels 	`"`xAxisLabels' `tmtGroupCount' "(N = `obs') `stars'" "'
		local legendLabels 	`"`legendLabels' lab(`tmtGroupCount' "`legendLabel'") "'
		local legendNumbers	`"`legendNumbers' `tmtGroupCount'"'
	}

	*Close or comple some strings
	local xAxisLabels `"`xAxisLabels' ,noticks labsize(medsmall)) "'
	local legendOption `"legend(order(`legendNumbers') `legendLabels')"'

	*Create the confidence interval bars
	if `CONFINT_BAR' == 0 {
		//local confIntGraph == `"(scatter mean order,  msym(none)  mlabs(medium) mlabpos(10) mlabcolor(black)), xtitle("") ytitle("`e(depvar)'") "'
		local confIntGraph = ""
	}
		else if `CONFINT_BAR' == 1 {
		local confIntGraph = `"(rcap conf_int_max conf_int_min order, lc(gs)) (scatter mean order,  msym(none)  mlabs(medium) mlabpos(10) mlabcolor(black))"'
	}

	local titleOption `" , xtitle("") ytitle("`e(depvar)'") "'

	if "`save'" != "" {
		local saveOption saving("`save'", replace)
	}

	if "`yzero'" != "" {

		gen maxvalue = max(mean , conf_int_max)

		sum maxvalue
		local max = `r(max)'

		local logmax = log10(`max')

		local logmax = round(`logmax') - 1

		local tenpower = 10 ^ (`logmax')

		local up = `tenpower' * ceil(`max' / `tenpower')

		local quarter = (`up') / 4

		noi di "up: `up'"
		noi di "quarter: `quarter'"

		local yzero_option ylabel(0(`quarter')`up')
	}

		//noi di `" graph twoway `tmtGroupBars' `confIntGraph' `titleOption' `options' `legendOption' `xAxisLabels' `saveOption' title("`title'") `yzero_option'  "'
	if `save_export' == 0 {
		graph twoway `tmtGroupBars' `confIntGraph' `titleOption'  `legendOption' `xAxisLabels' `saveOption' title("`title'") `yzero_option' `options'
		noi di `" graph twoway `tmtGroupBars' `confIntGraph' `titleOption' `options' `legendOption' `xAxisLabels' `saveOption' title("`title'") `yzero_option'  "'

		}
	else if `save_export' == 1 {
		graph twoway `tmtGroupBars' `confIntGraph' `titleOption'  `legendOption' `xAxisLabels' title("`title'") `yzero_option' `options'
		graph export "`save'", replace
		}

	restore
}

end

cap	program drop	colorPicker
	program define 	colorPicker , rclass

	args groupCount totalNumGroups

	if `totalNumGroups' == 2 {

		if `groupCount' == 1 return local color "215 25 28"
		if `groupCount' == 2 return local color "43 123 182"
	}
	else if `totalNumGroups' == 3 {

		if `groupCount' == 1 return local color "215 25 28"
		if `groupCount' == 2 return local color "255 255 191"
		if `groupCount' == 3 return local color "43 123 182"
	}
	else if `totalNumGroups' == 4 {

		if `groupCount' == 1 return local color "215 25 28"
		if `groupCount' == 2 return local color "255 255 191"
		if `groupCount' == 3 return local color "171 217 233"
		if `groupCount' == 4 return local color "43 123 182"
	}
	else {

		*For five or more colors we repeat the same pattern

		local colourNum = mod(`groupCount', 5)
		if `colourNum' == 1 return local color "215 25 28"
		if `colourNum' == 2 return local color "253 174 93"
		if `colourNum' == 3 return local color "255 255 191"
		if `colourNum' == 4 return local color "171 217 233"
		if `colourNum' == 0 return local color "43 123 182"

	}

end

cap	program drop	greyPicker
	program define 	greyPicker , rclass

	args groupCount totalNumGroups

	if `groupCount' == 1 {

		return local color "black"
	}
	else if `groupCount' == 2 & `totalNumGroups' <= 3 {

		return local color "gs14"
	}
	else {

		local grayscale =  round( (`groupCount'-1) * (100 / (`totalNumGroups'-1) ))

		return local color "`grayscale' `grayscale' `grayscale' `grayscale'"
	}

end

cap	program drop	iegraph
	program define 	iegraph
	preserve
	syntax varlist, [noconfbars TItle(string) save(string) confbarsnone(varlist) GREYscale yzero *]
	mat beta_ = e(b)

	local counter = 0
	//local textfile = "Mrijan"
	qui{
		if "`confbars'" 			!= "" {
		local CONFINT_BAR 	= 0
		}
		else if "`confbars'" 			== "" {
		local CONFINT_BAR 	= 1
		}

		local varTest : list confbarsnone in varlist

		if `varTest' == 0 {
					noi display as error "{phang} Variables defined in confbarsnone cannot be found in the graph variable list. {p_end}"
					noi display ""
					error 111
					 }

		foreach var of local varlist{
			cap assert inlist(`var',0,1) | missing(`var')
				if _rc {
					noi display as error "{phang} The variable `var' is not a dummy. Treatment variable needs to be a dummy(0 or 1) variable. {p_end}"
					noi display ""
					error 149
					}
				else {
					}

		local counter = `counter' + 1
		di `counter'

		scalar coeff_`var' = _b[`var']
		//noi display coeff_`var'

		scalar coeff_se_`var' = _se[`var']
		//noi display coeff_se_`var'

		count if e(sample) == 1 & `var' == 1 //Count one tmt group at the time
		scalar obs_`var' = r(N)
		//noi display obs_`var'

			}
	if "`save'" != "" {

        *Find index for where the file type suffix start
        local dot_index     = strpos("`save'",".") + 1

        *Extract the file index

        local file_suffix   = substr("`save'", `dot_index', .)
		noi di "`file_suffix'"

        local nonGPH_formats png tiff gph ps eps pdf wmf emf

        *If no file format suffix is specified, use the default .xlsx

        if `dot_index' == 1 | "`file_suffix'" == "gph" {
                   local save_export = 0
				}

        *If a file format suffix is specified make sure that it is one of the two allowed.

        else if `:list file_suffix in nonGPH_formats' != 0  {
            local save_export = 1

			if ("`file_suffix'" == "wmf" | "`file_suffix'" == "emf") & "`c(os)'" != "Windows" {
				di as error ""
                error
                   }
			}

        else {

            di as error "You are not using a allowed file format in save(`save'). Only the following formats are allowed: gph `nonGPH_formats'"
            error
        }
    }

	//noi di `"iegraph `varlist', `noconfbars' title(`title') save(`save') `confbarsnone' `yzero' `anything'"'

	local count: word count `varlist'
	local graphCount = `count' + 1
	//tokenize "`varlist'"
	//noi di `count'

	//Make all vars tempvars (maybe do later)
	//Make sure that missing is properly handled
	tempvar anyTMT
	egen `anyTMT' = rowmax(`varlist') if e(sample) == 1
	tempvar control
	gen `control' = (`anyTMT' == 0) if !missing(`anyTMT')

	sum `e(depvar)' if e(sample) == 1 & `control' == 1
	scalar ctl_N		= r(N)
	scalar ctl_mean	  	= r(mean)
	scalar ctl_mean_sd 	= r(sd)

		foreach var of local varlist {
			scalar  conf_int_min_`var'   =	coeff_`var'-(1.96*coeff_se_`var') + ctl_mean
			scalar  conf_int_max_`var'   =	coeff_`var'+(1.96*coeff_se_`var') + ctl_mean

			test `var'
			local star_`var' " "

			scalar  pvalue =r(p)
			if pvalue < 0.10 {
				local star_`var' "*"
			}
			if pvalue < 0.05 {
				local star_`var' "**"
			}
			if pvalue < 0.01 {
				local star_`var' "***"
			}
			*noi display "`star_`var''"
			scalar tmt_mean_`var' = ctl_mean + coeff_`var'
		}

	tempname 	newHandle
	tempfile	newTextFile
	cap file close 	`newHandle'
	file open  	`newHandle' using "`newTextFile'", text write append

	*Write headers and control value
	file write `newHandle' ///
		"order" 	_tab "xLabeled"	_tab "mean" _tab "coeff" _tab "conf_int_min" _tab "conf_int_max" _tab "obs" _tab "star" _n 	///
		%9.3f (1) _tab "Control"  _tab %9.3f (ctl_mean)      _tab  		 			_tab 		 			  	  _tab 			 				_tab %9.3f (ctl_N) 	   _tab			  	  _n

	tempvar newCounter
	gen `newCounter' = 2  //First tmt group starts at 2 (1 is control)

	foreach var in `varlist' {
	     if `: list var in confbarsnone' {
		file write `newHandle' ///
			%9.3f (`newCounter') _tab "`var'" _tab %9.3f (tmt_mean_`var') _tab  %9.3f (coeff_`var')  _tab %9.3f  _tab _tab %9.3f (obs_`var')   _tab "`star_`var''"  _n

		replace `newCounter' = `newCounter' + 1
			}
		else {
		file write `newHandle' ///
			%9.3f (`newCounter') _tab "`var'" _tab %9.3f (tmt_mean_`var') _tab  %9.3f (coeff_`var')  _tab %9.3f (conf_int_min_`var') _tab %9.3f (conf_int_max_`var') _tab %9.3f (obs_`var')   _tab "`star_`var''"  _n

		replace `newCounter' = `newCounter' + 1
		}
	}

	file close `newHandle'

	copy "`newTextFile'"  "mainMasterDofile.txt" , replace

	/*************************************

		Create the graph

	*************************************/

	insheet using mainMasterDofile.txt, clear

	local tmtGroupBars 	""
	local xAxisLabels 	`"xlabel( "'
	local legendLabels	`"lab(1 "`obsLabelControl'")"'
	local legendNumbers	""

	forval tmtGroupCount = 1/`graphCount' {

		************
		*Create the bar for this group

		if "`greyscale'" == "" {
			colorPicker `tmtGroupCount' `graphCount'
		}
		else {
			greyPicker `tmtGroupCount' `graphCount'
		}

		local tmtGroupBars `"`tmtGroupBars' (bar mean order if order == `tmtGroupCount', color("`r(color)'") lcolor(black) ) "'

		************
		*Create labels etc. for this group

		local obs 			= obs[`tmtGroupCount']
		local stars 		= star[`tmtGroupCount']
		local legendLabel 	= xlabeled[`tmtGroupCount']

		local xAxisLabels 	`"`xAxisLabels' `tmtGroupCount' "(N = `obs') `stars'" "'
		local legendLabels 	`"`legendLabels' lab(`tmtGroupCount' "`legendLabel'") "'
		local legendNumbers	`"`legendNumbers' `tmtGroupCount'"'
	}

	*Close or comple some strings
	local xAxisLabels `"`xAxisLabels' ,noticks labsize(medsmall)) "'
	local legendOption `"legend(order(`legendNumbers') `legendLabels')"'

	*Create the confidence interval bars
	if `CONFINT_BAR' == 0 {
		//local confIntGraph == `"(scatter mean order,  msym(none)  mlabs(medium) mlabpos(10) mlabcolor(black)), xtitle("") ytitle("`e(depvar)'") "'
		local confIntGraph = ""
	}
		else if `CONFINT_BAR' == 1 {
		local confIntGraph = `"(rcap conf_int_max conf_int_min order, lc(gs)) (scatter mean order,  msym(none)  mlabs(medium) mlabpos(10) mlabcolor(black))"'
	}

	local titleOption `" , xtitle("") ytitle("`e(depvar)'") "'

	if "`save'" != "" {
		local saveOption saving("`save'", replace)
	}

	if "`yzero'" != "" {

		gen maxvalue = max(mean , conf_int_max)

		sum maxvalue
		local max = `r(max)'

		local logmax = log10(`max')

		local logmax = round(`logmax') - 1

		local tenpower = 10 ^ (`logmax')

		local up = `tenpower' * ceil(`max' / `tenpower')

		local quarter = (`up') / 4

		noi di "up: `up'"
		noi di "quarter: `quarter'"

		local yzero_option ylabel(0(`quarter')`up')
	}

		//noi di `" graph twoway `tmtGroupBars' `confIntGraph' `titleOption' `options' `legendOption' `xAxisLabels' `saveOption' title("`title'") `yzero_option'  "'
	if `save_export' == 0 {
		graph twoway `tmtGroupBars' `confIntGraph' `titleOption'  `legendOption' `xAxisLabels' `saveOption' title("`title'") `yzero_option' `options'
		noi di `" graph twoway `tmtGroupBars' `confIntGraph' `titleOption' `options' `legendOption' `xAxisLabels' `saveOption' title("`title'") `yzero_option'  "'

		}
	else if `save_export' == 1 {
		graph twoway `tmtGroupBars' `confIntGraph' `titleOption'  `legendOption' `xAxisLabels' title("`title'") `yzero_option' `options'
		graph export "`save'", replace
		}

	restore
}

end

cap	program drop	colorPicker
	program define 	colorPicker , rclass

	args groupCount totalNumGroups

	if `totalNumGroups' == 2 {

		if `groupCount' == 1 return local color "215 25 28"
		if `groupCount' == 2 return local color "43 123 182"
	}
	else if `totalNumGroups' == 3 {

		if `groupCount' == 1 return local color "215 25 28"
		if `groupCount' == 2 return local color "255 255 191"
		if `groupCount' == 3 return local color "43 123 182"
	}
	else if `totalNumGroups' == 4 {

		if `groupCount' == 1 return local color "215 25 28"
		if `groupCount' == 2 return local color "255 255 191"
		if `groupCount' == 3 return local color "171 217 233"
		if `groupCount' == 4 return local color "43 123 182"
	}
	else {

		*For five or more colors we repeat the same pattern

		local colourNum = mod(`groupCount', 5)
		if `colourNum' == 1 return local color "215 25 28"
		if `colourNum' == 2 return local color "253 174 93"
		if `colourNum' == 3 return local color "255 255 191"
		if `colourNum' == 4 return local color "171 217 233"
		if `colourNum' == 0 return local color "43 123 182"

	}

end

cap	program drop	greyPicker
	program define 	greyPicker , rclass

	args groupCount totalNumGroups

	if `groupCount' == 1 {

		return local color "black"
	}
	else if `groupCount' == 2 & `totalNumGroups' <= 3 {

		return local color "gs14"
	}
	else {

		local grayscale =  round( (`groupCount'-1) * (100 / (`totalNumGroups'-1) ))

		return local color "`grayscale' `grayscale' `grayscale' `grayscale'"
	}

end


iematch : if/in, maintain sort, merge to full data set

This commit implements if/in. Obs kept (match sample)will be matched and
then merged back to the full data set. Obs with missing in groudummy
will also be dropped from the match sample.

Extended missin values are added to the _match* vars to indicate why not
matched. For example, excluded by if/in or missing in groupdummy = .m,
control not matched = .c

Original sort is maintained and preserve/restore is implemented

	clear

	*command starts here

	cap program drop 	iematch
	    program define	iematch

		syntax  [if] [in]  , ///
			GRPdummy(varname) 		///
			MATCHcont(varname) 		///
			[						///
			idvar(varname)			///
			matchidname(string)		///
			matchdiffname(string)	///
			]

		*****
		*1. if in implemantation
		*2 make match only on obs with value in group dummy
		*3 m:1 match

		noi di "Syntax OK!"

		pause on

		qui {

		********************************
		*
		*	Gen temp sort and merge var
		*
		********************************

		tempvar originalSort

		gen `originalSort' = _n

		preserve

			if "`if'`in'" != "" keep `if'`in'

			********************************
			*
			*	Checking group dummy
			*
			********************************

			*Test that the group dummy is 1, 0 or missing
			cap assert inlist(`grpdummy', 1, 0, .) | `grpdummy' > .
			if _rc != 0 {

				di as error "The variable in grpdummy(`grpdummy') is not a dummy variable. The variable is only allowed to have the values 1, 0 or missing. Observations with missing varaibles in the grpdummy are ignored by this command."
				error _rc
			}

			*Exclude obs with missing value in groupdummy
			drop if `grpdummy' > .

			********************************
			*
			*	Checking match continous var
			*
			********************************

			*Test that the continous is numeric
			cap confirm numeric variable `matchcont'
			if _rc != 0 {

				di as error "The variable to match on specified in matchcont(`matchcont') is not a numeric variable."
				error 109

			}

			********************************
			*
			*	Checking ID var
			*
			********************************

			*Checking set-up for the ID var used for _matchID
			if "`idvar'" == "" {

				*Copy this name
				local idvar _ID

				*Make sure that the default name _ID is not already used
				cap confirm variable `idvar'
				if _rc == 0 {

					di as error "A variable with name `idvar' is already defined. Either drop this variable or specify a variable using idvar() that fully and uniquely identifies the data set."
					error 110

				}

				*Generate the ID var from row number
				gen `idvar' = _n

			}
			else {

				*Make sure that user specified ID is uniquely and fully identifying the observations
				cap isid `idvar'
				if _rc != 0 {

					di as error "Variable `idvar' specified in idvar() does not fully and uniquely identify the observations in the data set. Meaning that `idvar' is not allowed to have any duplicates or missing values. Make `idvar' uniquly and fully idnetifying or exclude the varaibleas using if or in."
					error 450

				}
			}

			********************************
			*
			*	Checking and generating the MatchID and the MatchDiff var
			*
			********************************

			*See function below
			iematchMatchVarCheck `matchidname' _matchID
			local matchIDname "`r(validVarName)'"
			gen  `matchIDname' = .

			iematchMatchVarCheck `matchdiffname' _matchDiff
			local matchDiffName "`r(validVarName)'"
			gen `matchDiffName' = .

			********************************
			*
			*	Creating tempvar used
			*
			********************************

			*Initiate the temporary variables used by this command
			tempvar hi_diff lo_diff pref match newMatch rand

			gen `hi_diff' = .
			gen `lo_diff' = .

			gen `pref' = .
			gen byte `match' = 0
			gen byte `newMatch' = 0

			** Generate random variable that seperate two obs
			*  with the same value in the continuos variable
			gen `rand' = uniform()

			********************************
			*
			*	Start matching
			*
			********************************

			count if `grpdummy' == 1 & `match' == 0
			local left2Match = `r(N)'

			while (`left2Match' > 0) {

				sort `match' `matchcont' `rand'

				replace `lo_diff' 	= .
				replace `hi_diff' 	= .
				replace `pref'		= . if `match' == 0
				replace `newMatch' 	= 0

				replace `lo_diff' = abs(`matchcont' - `matchcont'[_n-1]) if `match' == 0 & `grpdummy' != `grpdummy'[_n-1]
				replace `hi_diff' = abs(`matchcont' - `matchcont'[_n+1]) if `match' == 0 & `grpdummy' != `grpdummy'[_n+1]

				replace `pref' = `idvar'[_n-1] if `match' == 0 & `lo_diff' <  `hi_diff'
				replace `pref' = `idvar'[_n+1] if `match' == 0 & `lo_diff' >= `hi_diff'

				replace `newMatch' = 1 if `match' == 0 & `pref' == `idvar'[_n-1] & `pref'[_n-1] == `idvar'
				replace `newMatch' = 1 if `match' == 0 & `pref' == `idvar'[_n+1] & `pref'[_n+1] == `idvar'

				replace `match' = 1 							if `newMatch' == 1
				replace `matchDiffName' = min(`lo_diff', `hi_diff') 	if `newMatch' == 1
				replace `matchIDname' = `idvar' 						if `newMatch' == 1 & `grpdummy' == 1
				replace `matchIDname' = `pref' 						if `newMatch' == 1 & `grpdummy' == 0

				*order _match* , last

			noi	count if `grpdummy' == 1 & `match' == 0
				local left2Match = `r(N)'

			}

			replace `matchIDname' 	= .c if `match' == 0 & `grpdummy' == 0
			replace `matchDiffName' = .c if `match' == 0 & `grpdummy' == 0

			keep `matchIDname' `matchDiffName' `idvar' `originalSort'

			tempfile mergefile

			save `mergefile'

			*pause

		restore

		tempvar mergevar

		merge 1:1  `originalSort' using `mergefile', gen(`mergevar')

		replace `matchIDname' 	= .m if `mergevar' == 1
		replace `matchDiffName' = .m if `mergevar' == 1

		sort `originalSort'

	}

	end

	*Cheack matchVar input
	cap program drop 	iematchMatchVarCheck
		program define 	iematchMatchVarCheck  , rclass

		args defaultName optionName

		*Testing that the option name is not the same as the other variable's default name.
		if "`defaultName'" == "_matchID" & "`optionName'" == "_matchDiff" {

			noi di as error "The new name specified in matchidname() is not allowed to be _matchDiff"
			error 198
		}
		if "`defaultName'" == "_matchDiff" & "`optionName'" == "_matchID" {

			noi di as error "The new name specified in matchdiffname() is not allowed to be _matchID"
			error 198
		}

		*Checking set-up for the ID var used for _matchID
		if "``optionName''" == "" {

			*CSet the default name
			local validMatchVarname `defaultName'

			*Make sure that the default name _ID is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				noi di as error "A variable with name `validMatchVarname' is already defined. Either drop this variable or specify a new variable name using `optionName'()."
				error 110

			}

		}

		else {

			*CSet the default name
			local validMatchVarname `optionName'

			*Make sure that the default name _ID is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				noi di as error "The variable name `validMatchVarname' specified in `optionName'() is already defined in the data set. Either drop this variable or specify a another variable name using `optionName'()."
				error 110

			}

		}

		*Testing that the new name is valid by creating a variable that is dropped immedeatly afterwards
		cap gen `validMatchVarname' = .

		if _rc != 0 {

			noi di as error "The variable name specified is not a valid variable name."
			error _rc

		}
		drop `validMatchVarname'

		return local validVarName 	"`validMatchVarname'"

	end

	clear

*	set seed 1235324

	set obs 50000

	gen id = _n

	gen rand1 = uniform()

	gen tmt = (rand1 < .40)

	drop rand1
	tab tmt

	gen p_hat = uniform()

	iematch  , grp(tmt) match(p_hat) idvar(id)

	clear

	*command starts here

	cap program drop 	iematch
	    program define	iematch

		syntax  [if] [in]  , ///
			GRPdummy(varname) 		///
			MATCHcont(varname) 		///
			[						///
			idvar(varname)			///
			matchidname(string)		///
			matchdiffname(string)	///
			]

		*****
		*1. if in implemantation
		*2 make match only on obs with value in group dummy
		*3 m:1 match

		noi di "Syntax OK!"

		qui {

		********************************
		*
		*	Checking group dummy
		*
		********************************

		*Test that the group dummy is 1, 0 or missing
		cap assert inlist(`grpdummy', 1, 0, .) | `grpdummy' > .

		if _rc != 0 {

			di as error "The variable in grpdummy(`grpdummy') is not a dummy variable. The variable is only allowed to have the values 1, 0 or missing. Observations with missing varaibles in the grpdummy are ignored by this command."
			error _rc
		}

		********************************
		*
		*	Checking match continous var
		*
		********************************

		*Test that the continous is numeric
		cap confirm numeric variable `matchcont'
		if _rc != 0 {

			di as error "The variable to match on specified in matchcont(`matchcont') is not a numeric variable."
			error 109

		}

		********************************
		*
		*	Checking ID var
		*
		********************************

		*Checking set-up for the ID var used for _matchID
		if "`idvar'" == "" {

			*Copy this name
			local idvar _ID

			*Make sure that the default name _ID is not already used
			cap confirm variable `idvar'
			if _rc == 0 {

				di as error "A variable with name `idvar' is already defined. Either drop this variable or specify a variable using idvar() that fully and uniquely identifies the data set."
				error 110

			}

			*Generate the ID var from row number
			gen `idvar' = _n

		}
		else {

			*Make sure that user specified ID is uniquely and fully identifying the observations
			cap isid `idvar'
			if _rc != 0 {

				di as error "Variable `idvar' specified in idvar() does not fully and uniquely identify the observations in the data set. Meaning that `idvar' is not allowed to have any duplicates or missing values. Make `idvar' uniquly and fully idnetifying or exclude the varaibleas using if or in."
				error 450

			}
		}

		********************************
		*
		*	Checking and generating the MatchID and the MatchDiff var
		*
		********************************

		*See function below
		iematchMatchVarCheck `matchidname' _matchID
		local matchIDname "`r(validVarName)'"
		gen  `matchIDname' = .

		iematchMatchVarCheck `matchdiffname' _matchDiff
		local matchDiffName "`r(validVarName)'"
		gen `matchDiffName' = .

		********************************
		*
		*	Creating tempvar used
		*
		********************************

		*Initiate the temporary variables used by this command
		tempvar hi_diff lo_diff pref match newMatch rand

		gen `hi_diff' = .
		gen `lo_diff' = .

		gen `pref' = .
		gen byte `match' = 0
		gen byte `newMatch' = 0

		** Generate random variable that seperate two obs
		*  with the same value in the continuos variable
		gen `rand' = uniform()

		********************************
		*
		*	Start matching
		*
		********************************

		count if `grpdummy' == 1 & `match' == 0
		local left2Match = `r(N)'

		while (`left2Match' > 0) {

			sort `match' `matchcont' `rand'

			replace `lo_diff' 	= .
			replace `hi_diff' 	= .
			replace `pref'		= . if `match' == 0
			replace `newMatch' 	= 0

			replace `lo_diff' = abs(`matchcont' - `matchcont'[_n-1]) if `match' == 0 & `grpdummy' != `grpdummy'[_n-1]
			replace `hi_diff' = abs(`matchcont' - `matchcont'[_n+1]) if `match' == 0 & `grpdummy' != `grpdummy'[_n+1]

			replace `pref' = `idvar'[_n-1] if `match' == 0 & `lo_diff' <  `hi_diff'
			replace `pref' = `idvar'[_n+1] if `match' == 0 & `lo_diff' >= `hi_diff'

			replace `newMatch' = 1 if `match' == 0 & `pref' == `idvar'[_n-1] & `pref'[_n-1] == `idvar'
			replace `newMatch' = 1 if `match' == 0 & `pref' == `idvar'[_n+1] & `pref'[_n+1] == `idvar'

			replace `match' = 1 							if `newMatch' == 1
			replace `matchDiffName' = min(`lo_diff', `hi_diff') 	if `newMatch' == 1
			replace `matchIDname' = `idvar' 						if `newMatch' == 1 & `grpdummy' == 1
			replace `matchIDname' = `pref' 						if `newMatch' == 1 & `grpdummy' == 0

			*order _match* , last

		noi	count if `grpdummy' == 1 & `match' == 0
			local left2Match = `r(N)'

			*pause

		}
	}

	end

	*Cheack matchVar input
	cap program drop 	iematchMatchVarCheck
		program define 	iematchMatchVarCheck  , rclass

		args defaultName optionName

		*Testing that the option name is not the same as the other variable's default name.
		if "`defaultName'" == "_matchID" & "`optionName'" == "_matchDiff" {

			noi di as error "The new name specified in matchidname() is not allowed to be _matchDiff"
			error 198
		}
		if "`defaultName'" == "_matchDiff" & "`optionName'" == "_matchID" {

			noi di as error "The new name specified in matchdiffname() is not allowed to be _matchID"
			error 198
		}

		*Checking set-up for the ID var used for _matchID
		if "``optionName''" == "" {

			*CSet the default name
			local validMatchVarname `defaultName'

			*Make sure that the default name _ID is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				noi di as error "A variable with name `validMatchVarname' is already defined. Either drop this variable or specify a new variable name using `optionName'()."
				error 110

			}

		}

		else {

			*CSet the default name
			local validMatchVarname `optionName'

			*Make sure that the default name _ID is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				noi di as error "The variable name `validMatchVarname' specified in `optionName'() is already defined in the data set. Either drop this variable or specify a another variable name using `optionName'()."
				error 110

			}

		}

		*Testing that the new name is valid by creating a variable that is dropped immedeatly afterwards
		cap gen `validMatchVarname' = .

		if _rc != 0 {

			noi di as error "The variable name specified is not a valid variable name."
			error _rc

		}
		drop `validMatchVarname'

		return local validVarName 	"`validMatchVarname'"

	end

	clear

*	set seed 1235324

	set obs 100000

	gen id = _n

	gen rand1 = uniform()

	gen tmt = (rand1 < .40)

	drop rand1
	tab tmt

	gen p_hat = uniform()

	iematch if id < 5532  , grp(tmt) match(p_hat) idvar(id)


signal names corrected

signal names corrected

onerror {resume}
quietly WaveActivateNextPane {} 0
add wave -noupdate /ppwa_rtl_tb/sl_clk
add wave -noupdate /ppwa_rtl_tb/sl_reset_n
add wave -noupdate /ppwa_rtl_tb/sl_measure_signal
add wave -noupdate -radix unsigned /ppwa_rtl_tb/usig_period_count
add wave -noupdate -radix unsigned /ppwa_rtl_tb/usig_hightime_count
add wave -noupdate -expand -subitemconfig {/ppwa_rtl_tb/my_unit_under_test/ri.usig_counter {-radix unsigned}} /ppwa_rtl_tb/my_unit_under_test/ri
TreeUpdate [SetDefaultTree]
WaveRestoreCursors {{Cursor 1} {400000 ps} 0}
configure wave -namecolwidth 150
configure wave -valuecolwidth 100
configure wave -justifyvalue left
configure wave -signalnamewidth 1
configure wave -snapdistance 10
configure wave -datasetprefix 0
configure wave -rowmargin 4
configure wave -childrowmargin 2
configure wave -gridoffset 0
configure wave -gridperiod 1
configure wave -griddelta 40
configure wave -timeline 0
configure wave -timelineunits ns
update
WaveRestoreZoom {0 ps} {8416800 ps}

onerror {resume}
quietly WaveActivateNextPane {} 0
add wave -noupdate /ppwe_rtl_tb/sl_clk
add wave -noupdate /ppwe_rtl_tb/sl_reset_n
add wave -noupdate /ppwe_rtl_tb/sl_measure_signal
add wave -noupdate -radix unsigned /ppwe_rtl_tb/usig_period_count
add wave -noupdate -radix unsigned /ppwe_rtl_tb/usig_hightime_count
add wave -noupdate -expand -subitemconfig {/ppwe_rtl_tb/my_unit_under_test/ri.usig_counter {-radix unsigned}} /ppwe_rtl_tb/my_unit_under_test/ri
TreeUpdate [SetDefaultTree]
WaveRestoreCursors {{Cursor 1} {400000 ps} 0}
configure wave -namecolwidth 150
configure wave -valuecolwidth 100
configure wave -justifyvalue left
configure wave -signalnamewidth 1
configure wave -snapdistance 10
configure wave -datasetprefix 0
configure wave -rowmargin 4
configure wave -childrowmargin 2
configure wave -gridoffset 0
configure wave -gridperiod 1
configure wave -griddelta 40
configure wave -timeline 0
configure wave -timelineunits ns
update
WaveRestoreZoom {0 ps} {8416800 ps}


added global changeset\!

added global changeset\!

/// 0. program lib for prepare basic data
// THIS PROGRAM MAKES THE CHANGESET FILE
// AND THE CLEAN COUNTY FILE

program preparebasic

// 0.1 clean changeset data
insheet using ${osmchange}changesets-geocoded.csv, clear
insheet using ${osmchange}world-geocode.csv, clear
renamevar
cleanvar
droplargeuser
save ${stash}cleanchangeset1, replace

// 0.2 clean Urban Area (UA) data
import excel ${rawmaps}ua_list_all.xls, clear firstrow
qui cleanua
save ${stash}cleanua, replace

// 0.3.1 clean my county color data
// TODO: doublecheck the color data
insheet using ${rawmaps}county_lookup.csv, clear
// TODO: calculate area
// TODO: match to census data from
// http://www.census.gov/support/USACdataDownloads.html#PEN
// American Fact Finder - "download center"

keep fips color treat
gen str5 fips2 = string(fips, "%05.0f")
drop fips
rename fips2 fips
save ${stash}countylookup, replace

// 0.3.2.1 clean related county data == census (race, pop)
census_pop

// 0.3.2.2 make annual population file (seer data)
county_pop

// 0.3.3 clean related county data == acs (educ, income etc)
acs_pop

// 0.3.4 area (adds land area for county)
savearea

// 0.3.5 clean county data and merge

insheet using ${rawmaps}CO-EST2012-Alldata.csv, clear
cleancnty

merge 1:1 fips using ${stash}countylookup, keep(match) nogen

merge 1:1 fips using ${rawmaps}census_pop, keep(match) nogen

merge 1:1 fips using ${rawmaps}acs, keep(match) nogen

merge 1:1 fips using ${stash}area_tmp, keep(match) nogen

save ${stash}cleancnty, replace

end

program savearea

insheet using ${rawmaps}area.txt, clear names

gen str5 fips = string(geoid, "%05.0f")
keep fips aland_sqmi

save ${stash}area_tmp, replace

end

program cleancnty

// TODO: add income, race, poverty information
// layout: http://www.census.gov/popest/data/counties/totals/2012/files/CO-EST2012-alldata.pdf

// this drops all the states
drop if sumlev == 40

gen str5 fips = string(state, "%02.0f") + string(county, "%03.0f")
rename census2010 cntypop
rename ctyname cntyname

keep fips region division state county stname cntyname cntypop

end

program cleanua
// see here for descriptions:
// http://www.census.gov/geo/reference/ua/ualists_layout.html

// UACE and geoid10 are the same thing
rename UACE geoid10
rename NAME uaname
rename POP uapop
rename HU uahu
rename AREALANDSQMI uaarealand
rename POPDEN uapopden
gen uaclustertype = "area" if LSADC == "75"
replace uaclustertype = "cluster" if LSADC == "76"
drop AREA* LSADC
end

program droplargeuser
// drop large users
// TODO: make this process more systematic?
drop if user == "DaveHansenTiger"
drop if user == "woodpeck_fixbot"
drop if user == "woodpeck_repair"
drop if user == "nmixter"
drop if user == "jumbanho"
drop if user == "-"
drop if user == "balrog-kun"
drop if user == "jremillard-massgis"
drop if user == "pnorman_mechanical"
drop if user == "CanvecImports"
drop if user == "TIGERcnl"
drop if user == "canvec_fsteggink"
drop if user == "OSMF Redaction Account"
drop if user == "bot-mode"
drop if num_changes > 40000
end

program renamevar
rename v1 changesetid
rename v2 uid
rename v3 created_at
rename v4 min_lat
rename v5 max_lat
rename v6 min_lon
rename v7 max_lon
rename v8 closed_at
rename v9 open
rename v10 num_changes
rename v11 user
rename v12 lat
rename v13 lon
rename v14 fips
rename v15 geoid10
end

program cleanvar

gen tstamp = clock(created_at, "YMD#hms#")
format tstamp %tc

gen tstamp_date = dofc(tstamp)
format tstamp_date %td

gen month = mofd(tstamp_date)
format month %tm

drop if abs(min_lat - max_lat) > 1
drop if abs(min_lon - max_lon) > 1

find_nonus

// drop non-US changesets
drop if fips == "NA" & geoid10 == "NA"
drop if fips == "I"

drop open closed_at created_at min_* max_*

end

program find_nonus

sort uid tstamp
gen tmp = (fips=="NA" & geoid10=="NA")
replace tmp = 1 if fips == "I"

bysort uid: egen nonus = total(tmp)
bysort uid: egen nonus10 = total(tmp & _n < 11)
bysort uid: egen nonus20 = total(tmp & _n < 21)
bysort uid: egen nonus100 = total(tmp & _n < 101)
drop tmp

end

/// 0. program lib for prepare basic data
// THIS PROGRAM MAKES THE CHANGESET FILE
// AND THE CLEAN COUNTY FILE

program preparebasic

// 0.1 clean changeset data
insheet using ${osmchange}changesets-geocoded.csv, clear
renamevar
cleanvar
droplargeuser
save ${stash}cleanchangeset1, replace

// 0.2 clean Urban Area (UA) data
import excel ${rawmaps}ua_list_all.xls, clear firstrow
qui cleanua
save ${stash}cleanua, replace

// 0.3.1 clean my county color data
// TODO: doublecheck the color data
insheet using ${rawmaps}county_lookup.csv, clear
// TODO: calculate area
// TODO: match to census data from
// http://www.census.gov/support/USACdataDownloads.html#PEN
// American Fact Finder - "download center"

keep fips color treat
gen str5 fips2 = string(fips, "%05.0f")
drop fips
rename fips2 fips
save ${stash}countylookup, replace

// 0.3.2.1 clean related county data == census (race, pop)
census_pop

// 0.3.2.2 make annual population file (seer data)
county_pop

// 0.3.3 clean related county data == acs (educ, income etc)
acs_pop

// 0.3.4 area (adds land area for county)
savearea

// 0.3.5 clean county data and merge

insheet using ${rawmaps}CO-EST2012-Alldata.csv, clear
cleancnty

merge 1:1 fips using ${stash}countylookup, keep(match) nogen

merge 1:1 fips using ${rawmaps}census_pop, keep(match) nogen

merge 1:1 fips using ${rawmaps}acs, keep(match) nogen

merge 1:1 fips using ${stash}area_tmp, keep(match) nogen

save ${stash}cleancnty, replace

end

program savearea

insheet using ${rawmaps}area.txt, clear names

gen str5 fips = string(geoid, "%05.0f")
keep fips aland_sqmi

save ${stash}area_tmp, replace

end

program cleancnty

// TODO: add income, race, poverty information
// layout: http://www.census.gov/popest/data/counties/totals/2012/files/CO-EST2012-alldata.pdf

// this drops all the states
drop if sumlev == 40

gen str5 fips = string(state, "%02.0f") + string(county, "%03.0f")
rename census2010 cntypop
rename ctyname cntyname

keep fips region division state county stname cntyname cntypop

end

program cleanua
// see here for descriptions:
// http://www.census.gov/geo/reference/ua/ualists_layout.html

// UACE and geoid10 are the same thing
rename UACE geoid10
rename NAME uaname
rename POP uapop
rename HU uahu
rename AREALANDSQMI uaarealand
rename POPDEN uapopden
gen uaclustertype = "area" if LSADC == "75"
replace uaclustertype = "cluster" if LSADC == "76"
drop AREA* LSADC
end

program droplargeuser
// drop large users
// TODO: make this process more systematic?
drop if user == "DaveHansenTiger"
drop if user == "woodpeck_fixbot"
drop if user == "woodpeck_repair"
drop if user == "nmixter"
drop if user == "jumbanho"
drop if user == "-"
drop if user == "balrog-kun"
drop if user == "jremillard-massgis"
drop if user == "pnorman_mechanical"
drop if user == "CanvecImports"
drop if user == "TIGERcnl"
drop if user == "canvec_fsteggink"
drop if user == "OSMF Redaction Account"
drop if user == "bot-mode"
drop if num_changes > 40000
end

program renamevar
rename v1 changesetid
rename v2 uid
rename v3 created_at
rename v4 min_lat
rename v5 max_lat
rename v6 min_lon
rename v7 max_lon
rename v8 closed_at
rename v9 open
rename v10 num_changes
rename v11 user
rename v12 lat
rename v13 lon
rename v14 fips
rename v15 geoid10
end

program cleanvar

gen tstamp = clock(created_at, "YMD#hms#")
format tstamp %tc

gen tstamp_date = dofc(tstamp)
format tstamp_date %td

gen month = mofd(tstamp_date)
format month %tm

drop if abs(min_lat - max_lat) > 1
drop if abs(min_lon - max_lon) > 1

find_nonus

// drop non-US changesets
drop if fips == "NA" & geoid10 == "NA"

drop open closed_at created_at min_* max_*

end

program find_nonus

sort uid tstamp
bysort uid: egen nonus = total(fips=="NA" & geoid10=="NA")
bysort uid: egen nonus10 = total((fips=="NA" & geoid10=="NA") & _n < 11)

end


* always update feed

* always update feed

for f in `ls -1 *.input.* | sed 's/\(.*\)\.input.*/\1/'`; do
redo-ifchange $f.html
ALBUM=`vorbiscomment -l $f.oga|grep --only-matching --perl-regexp '(?<=^album=).*$'`
done

redo-always feed.atom

cat << EOF >> $3
<!DOCTYPE html>
<meta charset=utf-8>
<title>$ALBUM</title>
<link rel=alternate type=application/atom+xml href=feed.atom>
<link rel=stylesheet href=style.css>

$ALBUM

Folgen

EOF

for f in `ls --reverse -1 *.html | sed 's/\(.*\)\..*/\1/'`; do
test "$f" = "index" || TRACKNUMBER=`vorbiscomment -l $f.oga|grep --only-matching --perl-regexp '(?<=^tracknumber=).*$'`
test "$f" = "index" || DATE=`vorbiscomment -l $f.oga|grep --only-matching --perl-regexp '(?<=^date=).*$'`
test "$f" = "index" || DESCRIPTION=`vorbiscomment -l $f.oga|grep --only-matching --perl-regexp '(?<=^description=).*$'`
test "$f" = "index" || cat << EOF >> $3

Folge $TRACKNUMBER vom `date +%-d.%-m.%Y -d$DATE` — $DESCRIPTION

EOF
done

cat << EOF >> $3

EOF

for f in `ls -1 *.input.* | sed 's/\(.*\)\.input.*/\1/'`; do
redo-ifchange $f.html
ALBUM=`vorbiscomment -l $f.oga|grep --only-matching --perl-regexp '(?<=^album=).*$'`
done

redo-ifchange feed.atom

cat << EOF >> $3
<!DOCTYPE html>
<meta charset=utf-8>
<title>$ALBUM</title>
<link rel=alternate type=application/atom+xml href=feed.atom>
<link rel=stylesheet href=style.css>

$ALBUM

Folgen

EOF

for f in `ls --reverse -1 *.html | sed 's/\(.*\)\..*/\1/'`; do
test "$f" = "index" || TRACKNUMBER=`vorbiscomment -l $f.oga|grep --only-matching --perl-regexp '(?<=^tracknumber=).*$'`
test "$f" = "index" || DATE=`vorbiscomment -l $f.oga|grep --only-matching --perl-regexp '(?<=^date=).*$'`
test "$f" = "index" || DESCRIPTION=`vorbiscomment -l $f.oga|grep --only-matching --perl-regexp '(?<=^description=).*$'`
test "$f" = "index" || cat << EOF >> $3

Folge $TRACKNUMBER vom `date +%-d.%-m.%Y -d$DATE` — $DESCRIPTION

EOF
done

cat << EOF >> $3

EOF