UpdatE

UpdatE

*! version 5.1 31MAY2017  Kristoffer Bjarkefur kbjarkefur@worldbank.org

cap	program drop	iegraph2
	program define 	iegraph2 , rclass

	preserve

	syntax varlist, [noconfbars BASICTItle(string) save(string) confbarsnone(varlist) VARLabels BAROPTions(string) GREYscale yzero *]

	qui{

	version 11

	*Load regression results
	mat beta_ = e(b)

	local counter = 0

	*Checking to see if the noconfbars option has been used and assigning 1 and 0 based
	*on that to the CONFINT_BAR variable.
	if "`confbars'" 		!= "" {
		local CONFINT_BAR 	= 0
	}
	else if "`confbars'" 	== "" {
		local CONFINT_BAR 	= 1
	}

	*Testing to see if the variables used in confbarsnone are actually in the list of
	*variables used for the regression/graph.
	local varTest : list confbarsnone in varlist

	if `varTest' == 0 {
		*Error displayed if variables defined in confbarsnone aren't in the list of variables for the graph.
		noi display as error "{phang} Variables defined in confbarsnone(`confbarsnone') cannot be found in the graph variable list. {p_end}"
		noi display ""
		error 111
	}

	*Testing to see if the variables used in the regressions are actual dummy variables as treatment vars need to be dummy variables.
	foreach var of local varlist{
		cap assert inlist(`var',0,1) | missing(`var')
		if _rc {
			noi display as error "{phang} The variable `var' is not a dummy. Treatment variable needs to be a dummy (0 or 1) variable. {p_end}"
			noi display ""
			error 149
		}

		local counter = `counter' + 1

		*Assigning variable coefficient/standard errors/no of obs. to scalars with the name Coeff_(`variable name')
		*coeff_se_(`variable name'), obs_(`variable name').

		scalar coeff_`var' = _b[`var']

		scalar coeff_se_`var' = _se[`var']

		count if e(sample) == 1 & `var' == 1 //Count one tmt group at the time
		scalar obs_`var' = r(N)

	}

	*Checking to see if the save option is used what is the extension related to it.
	if "`save'" != "" {

        *Find index for where the file type suffix start
        local dot_index     = strpos("`save'",".") + 1

        *Extract the file index
        local file_suffix   = substr("`save'", `dot_index', .)

		*List of formats to which the file can be exported
        local nonGPH_formats png tiff gph ps eps pdf wmf emf

        *If no file format suffix is specified, use the default .gph
        if `dot_index' == 1 | "`file_suffix'" == "gph" {
			local save_export = 0
		}

        *If a file format suffix is specified make sure that it is one of the eight formats allowed.
        else if `:list file_suffix in nonGPH_formats' != 0  {
            local save_export = 1

			if ("`file_suffix'" == "wmf" | "`file_suffix'" == "emf") & "`c(os)'" != "Windows" {
				di as error "{pstd}The file formats .wmf and .emf are only allowed when using Stata on a Windows computer.{p_end}"
                error 198
			}
		}
		*If a different extension was used then displaying an error.
        else {

            di as error "{pstd}You are not using a allowed file format in save(`save'). Only the following formats are allowed: gph `nonGPH_formats'{p_end}"
            error 198
        }
    }
	else {

		*Save option is not used, therefore save export will not be used
		local save_export = 0
	}

	local count: word count `varlist' // Counting the number of total vars used as treatment.
	local graphCount = `count' + 1 // Number of vars needed for the graph is total treatment vars plus one(control).

	//Make all vars tempvars (maybe do later)
	//Make sure that missing is properly handled

	tempvar anyTMT control
	egen `anyTMT' = rowmax(`varlist') 	if e(sample) == 1
	gen `control' = (`anyTMT' == 0) 	if !missing(`anyTMT')

	sum `e(depvar)' if e(sample) == 1 & `control' == 1
	scalar ctl_N		= r(N)
	scalar ctl_mean	  	= r(mean)
	scalar ctl_mean_sd 	= r(sd)

	foreach var of local varlist {

		*Caculating confidnece interval
		scalar  conf_int_min_`var'   =	coeff_`var'-(1.96*coeff_se_`var') + ctl_mean
		scalar  conf_int_max_`var'   =	coeff_`var'+(1.96*coeff_se_`var') + ctl_mean

		**Assigning stars to the treatment vars.

		*Perform the test to get p-values
		test `var'
		local star_`var' " "

		scalar  pvalue =r(p)
		if pvalue < 0.10 {
			local star_`var' "*"
		}
		if pvalue < 0.05 {
			local star_`var' "**"
		}
		if pvalue < 0.01 {
			local star_`var' "***"
		}

		scalar tmt_mean_`var' = ctl_mean + coeff_`var'
	}

	/*************************************

		Set up temp file where results are written

	*************************************/

	tempfile		 newTextFile
	tempname 		 newHandle
	cap file close 	`newHandle'
	file open  		`newHandle' using "`newTextFile'", text write append

	*Write headers and control value
	file write `newHandle' ///
		"order" 	_tab "xLabeled"	_tab "mean" _tab "coeff" _tab "conf_int_min" _tab "conf_int_max" _tab "obs" _tab "star" _n 	///
		%9.3f (1) _tab "Control"  _tab %9.3f (ctl_mean)      _tab  	_tab   _tab _tab %9.3f (ctl_N) 	 _tab  _n

	tempvar newCounter
	gen `newCounter' = 2  //First tmt group starts at 2 (1 is control)

	foreach var in `varlist' {

		if "`varlabels'" == "" {

			*Default is to use the varname in legend
			local var_legend "`var'"

		}
		else {

			*Option to use the variable label in the legen instead
			local var_legend : variable label `var'

		}

		*Writing the necessary tables for the graph to list to file.
	    if `: list var in confbarsnone' {
			file write `newHandle' 	%9.3f (`newCounter') _tab `"`var_legend'"'  ///
			_tab %9.3f (tmt_mean_`var') _tab  %9.3f (coeff_`var')  _tab _tab ///
			_tab %9.3f (obs_`var') _tab "`star_`var''"  _n

			replace `newCounter' = `newCounter' + 1
		}
		else {
			file write `newHandle' %9.3f (`newCounter') _tab `"`var_legend'"' ///
			_tab %9.3f (tmt_mean_`var') _tab  %9.3f (coeff_`var')  			///
			_tab %9.3f (conf_int_min_`var') _tab %9.3f (conf_int_max_`var')	///
			_tab %9.3f (obs_`var')   _tab "`star_`var''"  _n

			replace `newCounter' = `newCounter' + 1
		}
	}

	file close `newHandle'

	/*************************************

		Create the graph

	*************************************/

	*Rread file with results
	insheet using `newTextFile', clear

	*Defining various options to go on the graph option.

	local tmtGroupBars 	""
	local xAxisLabels 	`"xlabel( "'
	local legendLabels	""
	local legendNumbers	""

	forval tmtGroupCount = 1/`graphCount' {

		************
		*Create the bar for this group

		if "`greyscale'" == "" {
			colorPicker `tmtGroupCount' `graphCount'
		}
		else {
			greyPicker `tmtGroupCount' `graphCount'
		}

		local tmtGroupBars `"`tmtGroupBars' (bar mean order if order == `tmtGroupCount', `baroptions' color("`r(color)'") lcolor(black) ) "'

		************
		*Create labels etc. for this group

		local obs 			= obs[`tmtGroupCount']
		local stars 		= star[`tmtGroupCount']
		local legendLabel 	= xlabeled[`tmtGroupCount']

		local xAxisLabels 	`"`xAxisLabels' `tmtGroupCount' "(N = `obs') `stars'" "'
		local legendLabels 	`"`legendLabels' lab(`tmtGroupCount' "`legendLabel'") "'
		local legendNumbers	`"`legendNumbers' `tmtGroupCount'"'
	}

	*Close or comple some strings
	local xAxisLabels `"`xAxisLabels' ,noticks labsize(medsmall)) "'
	local legendOption `"legend(order(`legendNumbers') `legendLabels')"'

	*Create the confidence interval bars

	if `CONFINT_BAR' == 0 {

		local confIntGraph = ""
	}
		else if `CONFINT_BAR' == 1 {
		local confIntGraph = `"(rcap conf_int_max conf_int_min order, lc(gs)) (scatter mean order,  msym(none)  mlabs(medium) mlabpos(10) mlabcolor(black))"'
	}

	local titleOption `" , xtitle("") ytitle("`e(depvar)'") "'

	if "`save'" != "" {
		local saveOption saving("`save'", replace)
	}

	*******************************************************************************
	*** Generating the graph axis labels for the y-zero option used..
	*******************************************************************************

	if "`yzero'" != "" {

		*Finding the max value that is needed in the Y-axis.
		gen maxvalue = max(mean , conf_int_max)
		sum maxvalue

		*From the max of mean values and conf_int_max values.
		local max = `r(max)'

		*Log10 of the Max value to find the order necessary.
		local logmax = log10(`max')
		local logmax = round(`logmax') - 1

		*Generating tenpower which is 1 order smaller than the max value,
		*so we can log to that.
		local tenpower = 10 ^ (`logmax')

		*Rounding up for max value.
		local up = `tenpower' * ceil(`max' / `tenpower')

		*Generating quarter value for y-axis markers.
		local quarter = (`up') / 4

		*Specifying the option itself.
		local yzero_option ylabel(0(`quarter')`up')
	}

	*******************************************************************************
	***Graph generation based on if the option save has a export or a save feature.
	*******************************************************************************

	*Store all the options in one local
	local commandline 		`" `tmtGroupBars' `confIntGraph' `titleOption'  `legendOption' `xAxisLabels' title("`basictitle'") `yzero_option' `options'  "'

	*Error message used in both save-option cases below.
	local graphErrorMessage `" Something went wrong while trying to generate the graph. Click {stata di r(cmd) :display graph options } to see what graph options iegraph used. This can help in locating the source of the error in the command. "'

	if `save_export' == 0 {

		*Generate a return local with the code that will be used to generate the graph
		return local cmd `"graph twoway `tmtGroupBars' `confIntGraph' `titleOption'  `legendOption' `xAxisLabels' `saveOption' title("`basictitle'") `yzero_option' `options'"'

		*Generate the graph
		cap graph twoway `commandline' `saveOption'

		*If error, provide error message and then run the code again allowing the program to crash
		if _rc {

			di as error "{pstd}`graphErrorMessage'{p_end}"
            graph twoway `commandline' `saveOption'
		}
	}
	else if `save_export' == 1 {

		*Generate a return local with the code that will be used to generate the graph
		return local cmd `"graph twoway `commandline'"'

		*Generate the graph
		cap graph twoway `commandline'

		*If error, provide error message and then run the code again allowing the program to crash
		if _rc {

			di as error "{pstd}`graphErrorMessage'{p_end}"
            graph twoway `commandline'
		}

		*Export graph to preferred option
		graph export "`save'", replace

	}

	restore
}

end
	*******************************************
	*******************************************
		******* To pick colours based *******
		******* on the number of vars *******
	*******************************************
	*******************************************
	cap	program drop	colorPicker
		program define 	colorPicker , rclass

		args groupCount totalNumGroups

		if `totalNumGroups' == 2 {

			if `groupCount' == 1 return local color "215 25 28"
			if `groupCount' == 2 return local color "43 123 182"
		}
		else if `totalNumGroups' == 3 {

			if `groupCount' == 1 return local color "215 25 28"
			if `groupCount' == 2 return local color "255 255 191"
			if `groupCount' == 3 return local color "43 123 182"
		}
		else if `totalNumGroups' == 4 {

			if `groupCount' == 1 return local color "215 25 28"
			if `groupCount' == 2 return local color "255 255 191"
			if `groupCount' == 3 return local color "171 217 233"
			if `groupCount' == 4 return local color "43 123 182"
		}
		else {

			*For five or more colors we repeat the same pattern

			local colourNum = mod(`groupCount', 5)
			if `colourNum' == 1 return local color "215 25 28"
			if `colourNum' == 2 return local color "253 174 93"
			if `colourNum' == 3 return local color "255 255 191"
			if `colourNum' == 4 return local color "171 217 233"
			if `colourNum' == 0 return local color "43 123 182"

		}

	end
	*******************************************
	*******************************************
		******* Greyscale Option *******
		******* Colour Picker    *******
	*******************************************
	*******************************************

	cap	program drop	greyPicker
		program define 	greyPicker , rclass

		args groupCount totalNumGroups

		if `groupCount' == 1 {

			return local color "black"
		}
		else if `groupCount' == 2 & `totalNumGroups' <= 3 {

			return local color "gs14"
		}
		else {

			local grayscale =  round( (`groupCount'-1) * (100 / (`totalNumGroups'-1) ))

			return local color "`grayscale' `grayscale' `grayscale' `grayscale'"
		}

	end

*! version 5.1 31MAY2017  Kristoffer Bjarkefur kbjarkefur@worldbank.org

cap	program drop	iegraph2
	program define 	iegraph2 , rclass

	preserve

	syntax varlist, [noconfbars BASICTItle(string) save(string) confbarsnone(varlist) VARLabels BAROPTions(string) GREYscale yzero *]

	qui{

	version 11

	*Load regression results
	mat beta_ = e(b)

	local counter = 0

	*Checking to see if the noconfbars option has been used and assigning 1 and 0 based
	*on that to the CONFINT_BAR variable.
	if "`confbars'" 		!= "" {
		local CONFINT_BAR 	= 0
	}
	else if "`confbars'" 	== "" {
		local CONFINT_BAR 	= 1
	}

	*Testing to see if the variables used in confbarsnone are actually in the list of
	*variables used for the regression/graph.
	local varTest : list confbarsnone in varlist

	if `varTest' == 0 {
		*Error displayed if variables defined in confbarsnone aren't in the list of variables for the graph.
		noi display as error "{phang} Variables defined in confbarsnone(`confbarsnone') cannot be found in the graph variable list. {p_end}"
		noi display ""
		error 111
	}

	*Testing to see if the variables used in the regressions are actual dummy variables as treatment vars need to be dummy variables.
	foreach var of local varlist{
		cap assert inlist(`var',0,1) | missing(`var')
		if _rc {
			noi display as error "{phang} The variable `var' is not a dummy. Treatment variable needs to be a dummy (0 or 1) variable. {p_end}"
			noi display ""
			error 149
		}

		local counter = `counter' + 1

		*Assigning variable coefficient/standard errors/no of obs. to scalars with the name Coeff_(`variable name')
		*coeff_se_(`variable name'), obs_(`variable name').

		scalar coeff_`var' = _b[`var']

		scalar coeff_se_`var' = _se[`var']

		count if e(sample) == 1 & `var' == 1 //Count one tmt group at the time
		scalar obs_`var' = r(N)

	}

	*Checking to see if the save option is used what is the extension related to it.
	if "`save'" != "" {

        *Find index for where the file type suffix start
        local dot_index     = strpos("`save'",".") + 1

        *Extract the file index
        local file_suffix   = substr("`save'", `dot_index', .)

		*List of formats to which the file can be exported
        local nonGPH_formats png tiff gph ps eps pdf wmf emf

        *If no file format suffix is specified, use the default .gph
        if `dot_index' == 1 | "`file_suffix'" == "gph" {
			local save_export = 0
		}

        *If a file format suffix is specified make sure that it is one of the eight formats allowed.
        else if `:list file_suffix in nonGPH_formats' != 0  {
            local save_export = 1

			if ("`file_suffix'" == "wmf" | "`file_suffix'" == "emf") & "`c(os)'" != "Windows" {
				di as error "{pstd}The file formats .wmf and .emf are only allowed when using Stata on a Windows computer.{p_end}"
                error 198
			}
		}
		*If a different extension was used then displaying an error.
        else {

            di as error "{pstd}You are not using a allowed file format in save(`save'). Only the following formats are allowed: gph `nonGPH_formats'{p_end}"
            error 198
        }
    }
	else {

		*Save option is not used, therefore save export will not be used
		local save_export = 0
	}

	local count: word count `varlist' // Counting the number of total vars used as treatment.
	local graphCount = `count' + 1 // Number of vars needed for the graph is total treatment vars plus one(control).

	//Make all vars tempvars (maybe do later)
	//Make sure that missing is properly handled

	tempvar anyTMT control
	egen `anyTMT' = rowmax(`varlist') 	if e(sample) == 1
	gen `control' = (`anyTMT' == 0) 	if !missing(`anyTMT')

	sum `e(depvar)' if e(sample) == 1 & `control' == 1
	scalar ctl_N		= r(N)
	scalar ctl_mean	  	= r(mean)
	scalar ctl_mean_sd 	= r(sd)

	foreach var of local varlist {

		*Caculating confidnece interval
		scalar  conf_int_min_`var'   =	coeff_`var'-(1.96*coeff_se_`var') + ctl_mean
		scalar  conf_int_max_`var'   =	coeff_`var'+(1.96*coeff_se_`var') + ctl_mean

		**Assigning stars to the treatment vars.

		*Perform the test to get p-values
		test `var'
		local star_`var' " "

		scalar  pvalue =r(p)
		if pvalue < 0.10 {
			local star_`var' "*"
		}
		if pvalue < 0.05 {
			local star_`var' "**"
		}
		if pvalue < 0.01 {
			local star_`var' "***"
		}

		scalar tmt_mean_`var' = ctl_mean + coeff_`var'
	}

	/*************************************

		Set up temp file where results are written

	*************************************/

	tempfile		 newTextFile
	tempname 		 newHandle
	cap file close 	`newHandle'
	file open  		`newHandle' using "`newTextFile'", text write append

	*Write headers and control value
	file write `newHandle' ///
		"order" 	_tab "xLabeled"	_tab "mean" _tab "coeff" _tab "conf_int_min" _tab "conf_int_max" _tab "obs" _tab "star" _n 	///
		%9.3f (1) _tab "Control"  _tab %9.3f (ctl_mean)      _tab  	_tab   _tab _tab %9.3f (ctl_N) 	 _tab  _n

	tempvar newCounter
	gen `newCounter' = 2  //First tmt group starts at 2 (1 is control)

	foreach var in `varlist' {

		if "`varlabels'" == "" {

			*Default is to use the varname in legend
			local var_legend "`var'"

		}
		else {

			*Option to use the variable label in the legen instead
			local var_legend : variable label `var'

		}

		*Writing the necessary tables for the graph to list to file.
	    if `: list var in confbarsnone' {
			file write `newHandle' 	%9.3f (`newCounter') _tab `"`var_legend'"'  ///
			_tab %9.3f (tmt_mean_`var') _tab  %9.3f (coeff_`var')  _tab _tab ///
			_tab %9.3f (obs_`var') _tab "`star_`var''"  _n

			replace `newCounter' = `newCounter' + 1
		}
		else {
			file write `newHandle' %9.3f (`newCounter') _tab `"`var_legend'"' ///
			_tab %9.3f (tmt_mean_`var') _tab  %9.3f (coeff_`var')  			///
			_tab %9.3f (conf_int_min_`var') _tab %9.3f (conf_int_max_`var')	///
			_tab %9.3f (obs_`var')   _tab "`star_`var''"  _n

			replace `newCounter' = `newCounter' + 1
		}
	}

	file close `newHandle'

	/*************************************

		Create the graph

	*************************************/

	*Rread file with results
	insheet using `newTextFile', clear

	*Defining various options to go on the graph option.

	local tmtGroupBars 	""
	local xAxisLabels 	`"xlabel( "'
	local legendLabels	""
	local legendNumbers	""

	forval tmtGroupCount = 1/`graphCount' {

		************
		*Create the bar for this group

		if "`greyscale'" == "" {
			colorPicker `tmtGroupCount' `graphCount'
		}
		else {
			greyPicker `tmtGroupCount' `graphCount'
		}

		local tmtGroupBars `"`tmtGroupBars' (bar mean order if order == `tmtGroupCount', `baroptions' color("`r(color)'") lcolor(black) ) "'

		************
		*Create labels etc. for this group

		local obs 			= obs[`tmtGroupCount']
		local stars 		= star[`tmtGroupCount']
		local legendLabel 	= xlabeled[`tmtGroupCount']

		local xAxisLabels 	`"`xAxisLabels' `tmtGroupCount' "(N = `obs') `stars'" "'
		local legendLabels 	`"`legendLabels' lab(`tmtGroupCount' "`legendLabel'") "'
		local legendNumbers	`"`legendNumbers' `tmtGroupCount'"'
	}

	*Close or comple some strings
	local xAxisLabels `"`xAxisLabels' ,noticks labsize(medsmall)) "'
	local legendOption `"legend(order(`legendNumbers') `legendLabels')"'

	*Create the confidence interval bars

	if `CONFINT_BAR' == 0 {

		local confIntGraph = ""
	}
		else if `CONFINT_BAR' == 1 {
		local confIntGraph = `"(rcap conf_int_max conf_int_min order, lc(gs)) (scatter mean order,  msym(none)  mlabs(medium) mlabpos(10) mlabcolor(black))"'
	}

	local titleOption `" , xtitle("") ytitle("`e(depvar)'") "'

	if "`save'" != "" {
		local saveOption saving("`save'", replace)
	}

	*******************************************************************************
	*** Generating the graph axis labels for the y-zero option used..
	*******************************************************************************

	if "`yzero'" != "" {

		*Finding the max value that is needed in the Y-axis.
		gen maxvalue = max(mean , conf_int_max)
		sum maxvalue

		*From the max of mean values and conf_int_max values.
		local max = `r(max)'

		*Log10 of the Max value to find the order necessary.
		local logmax = log10(`max')
		local logmax = round(`logmax') - 1

		*Generating tenpower which is 1 order smaller than the max value,
		*so we can log to that.
		local tenpower = 10 ^ (`logmax')

		*Rounding up for max value.
		local up = `tenpower' * ceil(`max' / `tenpower')

		*Generating quarter value for y-axis markers.
		local quarter = (`up') / 4

		*Specifying the option itself.
		local yzero_option ylabel(0(`quarter')`up')
	}

	*******************************************************************************
	***Graph generation based on if the option save has a export or a save feature.
	*******************************************************************************

	*Store all the options in one local
	local commandline 		`" `tmtGroupBars' `confIntGraph' `titleOption'  `legendOption' `xAxisLabels' title("`basictitle'") `yzero_option' `options'  "'

	*Error message used in both save-option cases below.
	local graphErrorMessage `" Something went wrong while trying to generate the graph. Click {stata di r(cmd) :display graph options } to see what graph options iegraph used. This can help in de-bugging your code. "'

	if `save_export' == 0 {

		*Generate a return local with the code that will be used to generate the graph
		return local cmd `"graph twoway `tmtGroupBars' `confIntGraph' `titleOption'  `legendOption' `xAxisLabels' `saveOption' title("`basictitle'") `yzero_option' `options'"'

		*Generate the graph
		cap graph twoway `commandline' `saveOption'

		*If error, provide error message and then run the code again allowing the program to crash
		if _rc {

			di as error "{pstd}`graphErrorMessage'{p_end}"
            graph twoway `commandline' `saveOption'
		}
	}
	else if `save_export' == 1 {

		*Generate a return local with the code that will be used to generate the graph
		return local cmd `"graph twoway `commandline'"'

		*Generate the graph
		cap graph twoway `commandline'

		*If error, provide error message and then run the code again allowing the program to crash
		if _rc {

			di as error "{pstd}`graphErrorMessage'{p_end}"
            graph twoway `commandline'
		}

		*Export graph to preferred option
		graph export "`save'", replace

	}

	restore
}

end
	*******************************************
	*******************************************
		******* To pick colours based *******
		******* on the number of vars *******
	*******************************************
	*******************************************
	cap	program drop	colorPicker
		program define 	colorPicker , rclass

		args groupCount totalNumGroups

		if `totalNumGroups' == 2 {

			if `groupCount' == 1 return local color "215 25 28"
			if `groupCount' == 2 return local color "43 123 182"
		}
		else if `totalNumGroups' == 3 {

			if `groupCount' == 1 return local color "215 25 28"
			if `groupCount' == 2 return local color "255 255 191"
			if `groupCount' == 3 return local color "43 123 182"
		}
		else if `totalNumGroups' == 4 {

			if `groupCount' == 1 return local color "215 25 28"
			if `groupCount' == 2 return local color "255 255 191"
			if `groupCount' == 3 return local color "171 217 233"
			if `groupCount' == 4 return local color "43 123 182"
		}
		else {

			*For five or more colors we repeat the same pattern

			local colourNum = mod(`groupCount', 5)
			if `colourNum' == 1 return local color "215 25 28"
			if `colourNum' == 2 return local color "253 174 93"
			if `colourNum' == 3 return local color "255 255 191"
			if `colourNum' == 4 return local color "171 217 233"
			if `colourNum' == 0 return local color "43 123 182"

		}

	end
	*******************************************
	*******************************************
		******* Greyscale Option *******
		******* Colour Picker    *******
	*******************************************
	*******************************************

	cap	program drop	greyPicker
		program define 	greyPicker , rclass

		args groupCount totalNumGroups

		if `groupCount' == 1 {

			return local color "black"
		}
		else if `groupCount' == 2 & `totalNumGroups' <= 3 {

			return local color "gs14"
		}
		else {

			local grayscale =  round( (`groupCount'-1) * (100 / (`totalNumGroups'-1) ))

			return local color "`grayscale' `grayscale' `grayscale' `grayscale'"
		}

	end


Small fixes on parallel append and bs

Small fixes on parallel append and bs

*! vers 0.14.5 9may2014
*! auth George G. Vega

program def parallel_append

	vers 11.0

	syntax [anything(name=files)] , Do(string asis)[ in(string asis) if (string asis) Expression(string) *]

	if ("`in'" != "") local in in `in'
	if ("`if'" != "") local if if `if'

	/* Checking arguments */
	if (`"`anything'"' == "" & `"`expression'"' == "") {
		di as error "One of -files- or -expr()- must be specified."
		error 1
	}
	else if (`"`anything'"' != "" & `"`expression'"' != "") {
		di as error "-files- and -expr()- cannot be specified at the same time"
		error 1
	}

	/* Expanding the expression */
	if (`"`expression'"' != "") {
		mata: st_local("files",parallel_expand_expr(`"`expression'"'))
	}

	/* Checking out the number of files */
	tokenize `files'
	local n = 0
	local i = 0
	local nerr = 0
	while ("``++n''" != "") {

		/* Checking whether it exists*/
		local fn = `"``n''"'
		if (!regexm(`"`fn'"',"\.dta$")) cap confirm file `"``n''.dta"'
		else cap confirm file `"``n''"'

		if (!_rc) {
			if (!regexm(`"`fn'"',"\.dta$")) local ext = ".dta"
			else local ext = ""
			local file`++i' = "``n''`ext'"
		}
		else {
			di as result "{it:Warning:}{text:The file -``n''.dta- couldn't be found.}"
		}
	}
	local n = `i'

	/* If no files had been found */
	if (!`n') {
		di as error "No files found"
		error 1
	}

	/* Showing the files that will be used */
	di "{result:The following files will be processed:}"
	forval i=1/`n' {
		di as text %03.0f "`i'" as result " `file`i''"
	}

	/* Checking the groups clusters */
	local size = `n'/$PLL_CLUSTERS

	local oldclusters = $PLL_CLUSTERS
	local olddir = $PLL_DIR
	if (`size' < 1) {
		qui parallel setclusters `n', statadir(`olddir') f
		local g = 1
		forval i=1/`n' {
			local group`g' `group`g'' `file`i''
		}
	}
	else {
		/* Grouping files */
		local g = 1
		forval i=1/`n' {
			local group`g' `group`g'' `file`i''
			if (!mod(`i',$PLL_CLUSTERS)) local ++g
		}
	}

	local ng = ceil(`size')

	forval i=1/`ng' {
		di "{result:The files will be processed in the following order:}"
		di as text " Group " %02.0f "`i':" as result "`group`i''"
	}

	/* Getting a common id for the files */
	local err = 1
	while (`err') {
		mata: (void) parallel_randomid(10,"",1,1,1)
		local tmpid = "__pll`r(id1)'_append"
		cap mkdir `tmpid'
		local err = _rc
	}

	local nsave = 0
	forval i=1/`ng' {

		/* Writing the file */
		local f `tmpid'.do
		qui file open fh using `f', w replace

		tokenize `group`i''
		local j = 0
		local k = 0
		file write fh "cd `c(pwd)'" _newline
		while (`"``++j''"' != "") {
			file write fh `"if (\`pll_instance' == `++k') {"' _newline
			file write fh `"    use ``j'' `if' `in'"' _newline
			file write fh `"    local tmpn = string(`++nsave',"%04.0f")"'_newline _newline "}" _newline
		}

		cap findfile `do'
		if (_rc) {
			file write fh `"`do'"' _newline
		}

		file write fh `"gen filename = "``j''""'
		file write fh "compress" _newline
		file write fh "save `tmpid'/`tmpid'\`tmpn', replace" _newline

		file close fh

		qui parallel setclusters `--j', s(`olddir') f
		parallel do `f', `options'

		/*!less __pll`=r(pll_id)'_do1.do
		!less __pll`=r(pll_id)'_do1.log*/
		rm `f'
	}

	qui clear

	/* Appending all the results */
	forval i=1/`nsave' {
		local tmpn = "`tmpid'"+string(`i',"%04.0f")+".dta"
		cap {
			append using `tmpid'/`tmpn'
			rm `tmpid'/`tmpn'
		}

		if (!c(N)) cap use `tmpid'/`tmpn'

		if (_rc) local err `err' `file`i''
	}

	/* Removing the tmp dir */
	cap rmdir `tmpid'

	if (length("`err'")) di as result "{it:Warning:The following files could't be found}" _newline as text "`err'"

	qui parallel setclusters `oldclusters', s(`olddir') f

end

*! vers 0.14.5 9may2014
*! auth George G. Vega

program def parallel_append

	vers 11.0

	syntax [anything(name=files)] , Do(string asis)[ in(string asis) if (string asis) Expression(string) *]

	if ("`in'" != "") local in in `in'
	if ("`if'" != "") local if if `if'

	/* Checking arguments */
	if (`"`anything'"' == "" & `"`expression'"' == "") {
		di as error "One of -files- or -expr()- must be specified."
		error 1
	}
	else if (`"`anything'"' != "" & `"`expression'"' != "") {
		di as error "-files- and -expr()- cannot be specified at the same time"
		error 1
	}

	/* Expanding the expression */
	if (`"`expression'"' != "") {
		mata: st_local("files",parallel_expand_expr(`"`expression'"'))
	}

	/* Checking out the number of files */
	tokenize `files'
	local n = 0
	local i = 0
	local nerr = 0
	while ("``++n''" != "") {

		/* Checking whether it exists*/
		local fn = `"``n''"'
		if (!regexm(`"`fn'"',"\.dta$")) cap confirm file `"``n''.dta"'
		else cap confirm file `"``n''"'

		if (!_rc) {
			if (!regexm(`"`fn'"',"\.dta$")) local ext = ".dta"
			else local ext = ""
			local file`++i' = "``n''`ext'"
		}
		else {
			di as result "{it:Warning:}{text:The file -``n''.dta- couldn't be found.}"
		}
	}
	local n = `i'

	/* If no files had been found */
	if (!`n') {
		di as error "No files found"
		error 1
	}

	/* Showing the files that will be used */
	di "{result:The following files will be processed:}"
	forval i=1/`n' {
		di as text %03.0f "`i'" as result " `file`i''"
	}

	/* Checking the groups clusters */
	local size = `n'/$PLL_CLUSTERS

	local oldclusters = $PLL_CLUSTERS
	local olddir = $PLL_DIR
	if (`size' < 1) {
		qui parallel setclusters `n', statadir(`olddir') f
		local g = 1
		forval i=1/`n' {
			local group`g' `group`g'' `file`i''
		}
	}
	else {
		/* Grouping files */
		local g = 1
		forval i=1/`n' {
			local group`g' `group`g'' `file`i''
			if (!mod(`i',$PLL_CLUSTERS)) local ++g
		}
	}

	local ng = ceil(`size')

	forval i=1/`ng' {
		di "{result:The files will be processed in the following order:}"
		di as text " Group " %02.0f "`i':" as result "`group`i''"
	}

	/* Getting a common id for the files */
	local err = 1
	while (`err') {
		mata: (void) parallel_randomid(10,"",1,1,1)
		local tmpid = "__pllappend`r(id1)'"
		cap mkdir `tmpid'
		local err = _rc
	}

	local nsave = 0
	forval i=1/`ng' {

		/* Writing the file */
		local f `tmpid'.do
		qui file open fh using `f', w replace

		tokenize `group`i''
		local j = 0
		local k = 0
		file write fh "cd `c(pwd)'" _newline
		while (`"``++j''"' != "") {
			file write fh "if (\`pll_instance' == `++k') {" _newline
			file write fh "    use ``j'' `if' `in'" _newline
			file write fh `"    local tmpn = string(`++nsave',"%04.0f")"' _newline "}" _newline
		}

		cap findfile `do'
		if (_rc) {
			file write fh `"`do'"' _newline
		}

		file write fh "compress" _newline
		file write fh "save `tmpid'/`tmpid'\`tmpn', replace" _newline

		file close fh

		qui parallel setclusters `--j', s(`olddir') f
		parallel do `f', `options'

		/*!less __pll`=r(pll_id)'_do1.do
		!less __pll`=r(pll_id)'_do1.log*/
		rm `f'
	}

	qui clear

	/* Appending all the results */
	forval i=1/`nsave' {
		local tmpn = "`tmpid'"+string(`i',"%04.0f")+".dta"
		cap {
			append using `tmpid'/`tmpn'
			rm `tmpid'/`tmpn'
		}

		if (!c(N)) cap use `tmpid'/`tmpn'

		if (_rc) local err `err' `file`i''
	}

	/* Removing the tmp dir */
	cap rmdir `tmpid'

	if (length("`err'")) di as result "{it:Warning:The following files could't be found}" _newline as text "`err'"

	qui parallel setclusters `oldclusters', s(`olddir') f

end


[iegitaddmd ?] add skipfolders() to helpfile

{smcl}
{* 5 Nov 2019}{...}
{hline}
help for {hi:iegitaddmd}
{hline}

{title:Title}

{cmd:iegitaddmd} {hline 2} Creates a placeholder file in subfolders of a GitHub repository folder, which allows committing folder structures with empty folders.

{phang2}For a more descriptive discussion on the intended usage and work flow of this
command please see the {browse "https://dimewiki.worldbank.org/wiki/Iegitaddmd":DIME Wiki}.

{title:Syntax}

{phang} {cmdab:iegitaddmd} , {opt folder(full_file_path)} [{opt comparefolder(full_file_path)} {cmd:customfile(}{it:{help filename}}{cmd:)} {opt all} {opt skip} {opt replace} {opt skipfolders(folder_name)}  {opt auto:matic} {opt dry:run}]

{marker opts}{...}
{synoptset 28}{...}
{synopthdr:options}
{synoptline}
{synopt :{opt folder(full_file_path)}}Specifies the folder path to the project folder clone where placeholder files will be created{p_end}
{synopt :{opt comparefolder(full_file_path)}}Specifies the top folder of a folder tree to be compared with {opt folder()}{p_end}
{synopt :{cmd:customfile(}{it:{help filename}}{cmd:)}}Specifies a file saved on disk that is used instead of the default file as placeholder{p_end}
{synopt :{cmd:all}}Creates the placeholder file in every subfolder of {opt folder()}, whether empty or not{p_end}
{synopt :{opt skip}}If option {opt all} is used and a folder has a file with same name as placeholder file, then nothing is done{p_end}
{synopt :{opt replace}}If option {opt all} is used and a folder has a file with same name as placeholder file, then the file is overwritten{p_end}
{synopt :{opt skipfolders(folder_name)}}List of folders to be skipped. The folder {it:.git} is always added to this list, even when option is not used{p_end}
{synopt :{opt auto:matic}}Makes the command create placeholder files without prompting the user for each file{p_end}
{synopt :{opt dry:run}}Makes the command list all the files that would have been created without this option.{p_end}
{synoptline}

{marker desc}
{title:Description}

{pstd}GitHub does not sync empty folders, or folder that only contain ignored files. However, it is common in research projects that these types of folders needs to be shared anyways. {cmd:iegitaddmd} provides a solution to this by creating placeholder files in in these folders. This placeholder file can then be shared to GitHub and that way the folder is also shared. {cmd:iegitaddmd} have two ways to identify which those folders are that are developed with the two use cases described below in mind.{p_end}

{pstd}{bf:Use case 1.} It is common in the beginning of a research project that a standardized folder structure is set up (for example with {help iefolder}) where some folders are created that are not yet needed. Examples of folders like that could be the output folders. If these folders are not shared at the time they are created, there is a risk that different team members name them differently which will be confusing and could lead to errors. {cmd:iegitaddmd} can solve this by scanning a folder and all its sub-folders for completely empty folders, and create a placeholder file in that folder.{p_end}

{pstd}The solution to use case 1 in {cmd:iegitaddmd} is a Stata adaptation of {it:Solution B} in {browse "http://bytefreaks.net/gnulinux/bash/how-to-add-automatically-all-empty-folders-in-git-repository" :this post}.{p_end}

{pstd}{bf:Use case 2.} A common method when transferring an already ongoing research project to GitHub is to copy all of the data work folder from wherever the files and folders where shared before (for example DropBox) to a clone of the repository. Then only the files that are meant to be shared on GitHub are uploaded to the cloud using a {it:.gitignore file}. Empty folders or folders where all files are ignored is then not shared through GitHub. {cmd:iegitaddmd} can solve this by scanning the DropBox folder (or wherever the project folder was shared before) for sub-folders that exist in that folder but not in the clone and crate that folder in the clone and put a placeholder in it.{p_end}

{pstd}The way you differentiate between these two use cases when using {cmd:iegitaddmd} is whether the option {opt comparefolder()} is used or not. If that option is used, then {cmd:iegitaddmd} will identify all folders that are in {opt comparefolder()} that is not in {opt folder()}, and create a placeholder file there. If {opt comparefolder()} is not used, then {cmd:iegitaddmd} will identify all completely empty folders (no files or subfolders) and create a placeholder file there.{p_end}

{pstd}The default placeholder file used if no file is specified using option {opt customfile()} is called README.md, which is a name and format recognized by GitHub.com so that the content of the file is displayed on GitHub.com when navigating to that folder using a browser. The placeholder file may be removed as	soon as files that are not ignored have been added to the folder, or even better is to keep the file but edit the content of it so it documents the content of that folder for other and future team members.{p_end}

{marker optslong}
{title:Options}

{phang}{opt folder(file_path)} is the folder path to the project folder clone where the placeholder files will be created. Unless option {opt comparefolder()} or {opt all} is used, then placeholder files will be created in all empty folders. See option {opt manual} for a way to manually confirm each folder before a placeholder file is created in it.{p_end}

{phang}{opt comparefolder(file_path)} is used to compare a file tree in a different location and create the folders with placeholder files in all folders that exist in {opt comparefolder()} but not in {opt folder()}. If a folder that does no exist in {opt folder()} has a subfolder, then both the folder and the subfolder will be created and the placeholder file is only crated in the sub-folder. This is enough to share both folder and the subfolder in GitHub. If the {opt comparefolder()} and {opt folder()} ends with folders with different names, for example {opt folder(${project}/datawork/baseline)} and {opt comparefolder(${project}/datawork)}, then the command will throw an error as it is likely that the paths do not point to corresponding folders in the two folder trees. See option {opt manual} for a way to manually confirm each folder before a placeholder file is created in it.{p_end}

{phang}{cmd:file(}{it:{help filename}}{cmd:)} allows the user to specify a file saved on
	the computer to be used as the placeholder file	instead of the default README.md file. This
	allows people and organizations to write their own template placeholder files according to their own
	preferences. We recommend that a file of type .md (markdown) and name README.md is used as
	GitHub.com recognizes this name and format and will display the content of the file when someone
	browse to that folder on GitHub.com. But this is not a technical requirement, any file type
	and name can be used as placeholder file.{p_end}

{phang}{opt all} creates the placeholder file in {opt folder()} and in every subfolder,
	regardless if they are empty or not. This allows the user to create a placeholder file in every
	folder that can be edited with documentation and instructions to the purpose and usage of
	each subfolder. This option can also be important when .{it:gitignore} is used, as {cmd:iegitaddmd} will not
	create files to subfolders that only has ignored file -- in which case the folder will not
	be synced by GitHub.{p_end}

{phang}{opt skip} and {opt replace} tells {cmd:iegitaddmd} what to do if the option {opt all} is used
	and if any of the folders contain a file with the same name as the placeholder file the command
	is trying to create. If a file with the same name as the placeholder file exist and neither of these options
	are used, then the command will throw an error. If {opt skip} is used, then nothing is done in the
	folder where the file with the same name exists and the command proceeds to the next folder.
	If {opt replace} is used then the file with the same name is overwritten with the new placeholder
	file before the command proceeds to the next folder.{p_end}

{phang}{opt skipfolders(folder_name)} can be used to tell {cmd:iegitaddmd} which folders
in which a placeholder file should never be created. The best example of this is the {it:.git}
folder in which placeholders never should be created. That name is always skipped regardless if
this option is used or not. In this option additional folders to be skipped can be listed. You
should not list the full folder path in this option, just the folder name. All folders with
that name will be skipped regardless of location in the project folder.

{phang}{opt auto:matic} can be used to speed up the creation of placeholders by telling the
command to not prompt the users for confirmation for each file before it is created. The
default is that the command is asking the user before creating each place holder file.
This option should only be used when you are confident you have specified the correct folder
paths. We recommend that you use the {opt dryrun} with this option to make sure that the
folder paths are correct.{p_end}

{phang}{opt dry:run} can be used to safely test that the folder paths are specified correctly
before any placeholder files are created. When this option it used the command simply lists
the file that would have been created if this option were not used. Once you are confident
that list is correct, you can remove this option and re-run the command and the files will
be created.{p_end}

{title:Example}

{pstd}{inp:global github_folder "C:\Users\JohnSmith\Documents\GitHub\ProjectA"}{break}{inp:iegitaddmd , folder({it:"$github_folder"})}{p_end}

{pstd}In the example above, there is a GitHub repository in the folder ProjectA. This
	repository has a folder structure where some folders are still empty but will later
	be populated with files. In order to have all folders, even the empty ones, synced on all
	collaborators' cloned local copies of the repository, the folders need to contain at least
	one file, which is being created by the command.{p_end}

{title:Acknowledgements}

{pstd}I would like to acknowledge the help in testing and proofreading I received in relation to this command and help file from (in alphabetic order):{p_end}
{pmore}Guadalupe Bedoya{break}Luiza Cardoso de Andrade{break}Mrijan Rimal{break}Benjamin Daniels{break}

{title:Author}

{phang}All commands in ietoolkit is developed by DIME Analytics at DECIE, The World Bank's unit for Development Impact Evaluations.

{phang}Main author: Kristoffer Bjarkefur, DIME Analytics, The World Bank Group

{phang}Please send bug-reports, suggestions and requests for clarifications
		 writing "ietoolkit iegitaddmd" in the subject line to:{break}
		 dimeanalytics@worldbank.org

{phang}You can also see the code, make comments to the code, see the version
		 history of the code, and submit additions or edits to the code through {browse "https://github.com/worldbank/ietoolkit":the GitHub repository of ietoolkit}.{p_end}

{smcl}
{* 5 Nov 2019}{...}
{hline}
help for {hi:iegitaddmd}
{hline}

{title:Title}

{cmd:iegitaddmd} {hline 2} Creates a placeholder file in subfolders of a GitHub repository folder, which allows committing folder structures with empty folders.

{phang2}For a more descriptive discussion on the intended usage and work flow of this
command please see the {browse "https://dimewiki.worldbank.org/wiki/Iegitaddmd":DIME Wiki}.

{title:Syntax}

{phang} {cmdab:iegitaddmd} , {opt folder(file_path)} [{opt comparefolder(file_path)} {cmd:customfile(}{it:{help filename}}{cmd:)} {opt all} {opt skip} {opt replace} {opt auto:matic} {opt dry:run}]

{marker opts}{...}
{synoptset 23}{...}
{synopthdr:options}
{synoptline}
{synopt :{opt folder(file_path)}}Specifies the folder path to the project folder clone where placeholder files will be created{p_end}
{synopt :{opt comparefolder(file_path)}}Specifies the top folder of a folder tree to be compared with {opt folder()}{p_end}
{synopt :{cmd:customfile(}{it:{help filename}}{cmd:)}}Specifies a file saved on disk that is used instead of the default file as placeholder{p_end}
{synopt :{cmd:all}}Creates the placeholder file in every subfolder of {opt folder()}, whether empty or not{p_end}
{synopt :{opt skip}}If option {opt all} is used and a folder has a file with same name as placeholder file, then nothing is done{p_end}
{synopt :{opt replace}}If option {opt all} is used and a folder has a file with same name as placeholder file, then the file is overwritten{p_end}
{synopt :{opt auto:matic}}Makes the command create placeholder files without prompting the user for each file{p_end}
{synopt :{opt dry:run}}Makes the command list all the files that would have been created without this option.{p_end}
{synoptline}

{marker desc}
{title:Description}

{pstd}GitHub does not sync empty folders, or folder that only contain ignored files. However, it is common in research projects that these types of folders needs to be shared anyways. {cmd:iegitaddmd} provides a solution to this by creating placeholder files in in these folders. This placeholder file can then be shared to GitHub and that way the folder is also shared. {cmd:iegitaddmd} have two ways to identify which those folders are that are developed with the two use cases described below in mind.{p_end}

{pstd}{bf:Use case 1.} It is common in the beginning of a research project that a standardized folder structure is set up (for example with {help iefolder}) where some folders are created that are not yet needed. Examples of folders like that could be the output folders. If these folders are not shared at the time they are created, there is a risk that different team members name them differently which will be confusing and could lead to errors. {cmd:iegitaddmd} can solve this by scanning a folder and all its sub-folders for completely empty folders, and create a placeholder file in that folder.{p_end}

{pstd}The solution to use case 1 in {cmd:iegitaddmd} is a Stata adaptation of {it:Solution B} in {browse "http://bytefreaks.net/gnulinux/bash/how-to-add-automatically-all-empty-folders-in-git-repository" :this post}.{p_end}

{pstd}{bf:Use case 2.} A common method when transferring an already ongoing research project to GitHub is to copy all of the data work folder from wherever the files and folders where shared before (for example DropBox) to a clone of the repository. Then only the files that are meant to be shared on GitHub are uploaded to the cloud using a {it:.gitignore file}. Empty folders or folders where all files are ignored is then not shared through GitHub. {cmd:iegitaddmd} can solve this by scanning the DropBox folder (or wherever the project folder was shared before) for sub-folders that exist in that folder but not in the clone and crate that folder in the clone and put a placeholder in it.{p_end}

{pstd}The way you differentiate between these two use cases when using {cmd:iegitaddmd} is whether the option {opt comparefolder()} is used or not. If that option is used, then {cmd:iegitaddmd} will identify all folders that are in {opt comparefolder()} that is not in {opt folder()}, and create a placeholder file there. If {opt comparefolder()} is not used, then {cmd:iegitaddmd} will identify all completely empty folders (no files or subfolders) and create a placeholder file there.{p_end}

{pstd}The default placeholder file used if no file is specified using option {opt customfile()} is called README.md, which is a name and format recognized by GitHub.com so that the content of the file is displayed on GitHub.com when navigating to that folder using a browser. The placeholder file may be removed as	soon as files that are not ignored have been added to the folder, or even better is to keep the file but edit the content of it so it documents the content of that folder for other and future team members.{p_end}

{marker optslong}
{title:Options}

{phang}{opt folder(file_path)} is the folder path to the project folder clone where the placeholder files will be created. Unless option {opt comparefolder()} or {opt all} is used, then placeholder files will be created in all empty folders. See option {opt manual} for a way to manually confirm each folder before a placeholder file is created in it.{p_end}

{phang}{opt comparefolder(file_path)} is used to compare a file tree in a different location and create the folders with placeholder files in all folders that exist in {opt comparefolder()} but not in {opt folder()}. If a folder that does no exist in {opt folder()} has a subfolder, then both the folder and the subfolder will be created and the placeholder file is only crated in the sub-folder. This is enough to share both folder and the subfolder in GitHub. If the {opt comparefolder()} and {opt folder()} ends with folders with different names, for example {opt folder(${project}/datawork/baseline)} and {opt comparefolder(${project}/datawork)}, then the command will throw an error as it is likely that the paths do not point to corresponding folders in the two folder trees. See option {opt manual} for a way to manually confirm each folder before a placeholder file is created in it.{p_end}

{phang}{cmd:file(}{it:{help filename}}{cmd:)} allows the user to specify a file saved on
	the computer to be used as the placeholder file	instead of the default README.md file. This
	allows people and organizations to write their own template placeholder files according to their own
	preferences. We recommend that a file of type .md (markdown) and name README.md is used as
	GitHub.com recognizes this name and format and will display the content of the file when someone
	browse to that folder on GitHub.com. But this is not a technical requirement, any file type
	and name can be used as placeholder file.{p_end}

{phang}{opt all} creates the placeholder file in {opt folder()} and in every subfolder,
	regardless if they are empty or not. This allows the user to create a placeholder file in every
	folder that can be edited with documentation and instructions to the purpose and usage of
	each subfolder. This option can also be important when .{it:gitignore} is used, as {cmd:iegitaddmd} will not
	create files to subfolders that only has ignored file -- in which case the folder will not
	be synced by GitHub.{p_end}

{phang}{opt skip} and {opt replace} tells {cmd:iegitaddmd} what to do if the option {opt all} is used
	and if any of the folders contain a file with the same name as the placeholder file the command
	is trying to create. If a file with the same name as the placeholder file exist and neither of these options
	are used, then the command will throw an error. If {opt skip} is used, then nothing is done in the
	folder where the file with the same name exists and the command proceeds to the next folder.
	If {opt replace} is used then the file with the same name is overwritten with the new placeholder
	file before the command proceeds to the next folder.{p_end}

{phang}{opt auto:matic} can be used to speed up the creation of placeholders by telling the command to not prompt the users for confirmation for each file before it is created. The default is that the command is asking the user before creating each place holder file. This option should only be used when you are confident you have specified the correct folder paths. We recommend that you use the {opt dryrun} with this option to make sure that the folder paths are correct.{p_end}

{phang}{opt dry:run} can be used to safely test that the folder paths are specified correctly before any placeholder files are created. When this option it used the command simply lists the file that would have been created if this option were not used. Once you are confident that list is correct, you can remove this option and re-run the command and the files will be created.{p_end}

{title:Example}

{pstd}{inp:global github_folder "C:\Users\JohnSmith\Documents\GitHub\ProjectA"}{break}{inp:iegitaddmd , folder({it:"$github_folder"})}{p_end}

{pstd}In the example above, there is a GitHub repository in the folder ProjectA. This
	repository has a folder structure where some folders are still empty but will later
	be populated with files. In order to have all folders, even the empty ones, synced on all
	collaborators' cloned local copies of the repository, the folders need to contain at least
	one file, which is being created by the command.{p_end}

{title:Acknowledgements}

{pstd}I would like to acknowledge the help in testing and proofreading I received in relation to this command and help file from (in alphabetic order):{p_end}
{pmore}Guadalupe Bedoya{break}Luiza Cardoso de Andrade{break}Mrijan Rimal{break}Benjamin Daniels{break}

{title:Author}

{phang}All commands in ietoolkit is developed by DIME Analytics at DECIE, The World Bank's unit for Development Impact Evaluations.

{phang}Main author: Kristoffer Bjarkefur, DIME Analytics, The World Bank Group

{phang}Please send bug-reports, suggestions and requests for clarifications
		 writing "ietoolkit iegitaddmd" in the subject line to:{break}
		 dimeanalytics@worldbank.org

{phang}You can also see the code, make comments to the code, see the version
		 history of the code, and submit additions or edits to the code through {browse "https://github.com/worldbank/ietoolkit":the GitHub repository of ietoolkit}.{p_end}


iematch : including Mrijan's edits

iematch : including Mrijan's edits

Used Mrijan's edits from his writing test

{smcl}
{* }{...}
{hline}
help for {hi:iematch}
{hline}

{title:Title}

{phang2}{cmdab:iematch} {hline 2} Matching base observations towards target observations using on a single continous variable.

{title:Syntax}

{phang2}
{cmdab:iematch}
, {cmdab:grp:dummy(}{it:varname}{cmd:)} {cmdab:match:var(}{it:varname}{cmd:)}
[{cmdab:id:var(}{it:varlist}{cmd:)} {cmdab:m1} {cmdab:maxdiff(}{it:numlist}{cmd:)} {cmd:seedok}
{cmdab:matchid:name(}{it:string}{cmd:)} {cmdab:matchdi:ffname(}{it:string}{cmd:)}
{cmdab:matchre:sultname(}{it:string}{cmd:)} {cmdab:matchco:untname(}{it:string}{cmd:)}
]

{marker opts}{...}
{synoptset 22}{...}
{synopthdr:options}
{synoptline}
{synopt :{cmdab:grp:dummy(}{it:varname}{cmd:)}}The group dummy variable where 1
	indicates base observations and 0 target observations{p_end}
{synopt :{cmdab:match:var(}{it:varname}{cmd:)}}The variable with a continous value
	to match on{p_end}
{synopt :{cmdab:id:var(}{it:varlist}{cmd:)}}The uniquely and fully identifying ID
	varaible. Used to indicate which target observation a base observation is match
	with. If omitted an ID variable will be created.{p_end}
{synopt :{cmdab:m1}}Allows many-to-one matches. The default is to allow only
	one-to-one matches. See the {help iematch##desc:description} section.{p_end}
{synopt :{cmdab:maxdiff(}{it:numlist}{cmd:)}}Set a maximum difference allowed in
	{cmdab:matchvar()}. If a base observation has no match within this difference
 then it will remain unmatched{p_end}
{synopt :{cmd:seedok}}Supresses the error maessage thrown when there are duplicates
	in {cmd:matchvar()}. When there are duplicates, the seed needs to be set in order
	to have a replicable match. The {help seed} should be set before this command.{p_end}
{synopt :{cmdab:matchre:sultname(}{it:string}{cmd:)}}Manually sets the name of
	the variable that indicates if an observation was matched, or provide a reason why
	it was not. The default is _matchOrReason{p_end}
{synopt :{cmdab:matchid:name(}{it:string}{cmd:)}}Manually sets the name of the
	variable that indicates which target observation each base observation is
	matched with. The default is _matchID{p_end}
{synopt :{cmdab:matchdi:ffname(}{it:string}{cmd:)}}Manually sets the name of the
	variable that indicates the differnece in {cmdab:matchvar()} in each match
	pair/group. The default is _matchDiff{p_end}
{synopt :{cmdab:matchco:untname(}{it:string}{cmd:)}}Manually sets the name of the
	variable that indicates how many observations a target obsersvation is matched
	with in a many-to-one matches. The default is _matchCount{p_end}
{synoptline}

{marker desc}
{title:Description}

{pstd}{cmdab:iematch} matches base observations towards target observations in terms
	of nearest value in {cmd:matchvar()}. Base observations are observations with
	value 1 in {cmd:grpdummy()} and target observations are observations with value
	0. For example, in a typical p-score match, base observations are treatment and
	target is control, however, there are many other examples of matching where it could
	be different.

{pstd}{cmdab:iematch} bases its matching algorithm on the Stable Marriage algorithm.
	This algorithm was chosen because it always provides a solution and allows stable
	matching even if multiple observations have identical match
	values (see {cmd:seed()} option for more details). One disadvantage of this
	algorithm is that it takes into account local efficiency only, and not global efficiency. This
	means that it is possible that other matching algorithms might generate a more
	efficient match in terms of the sum of the difference of all matched pairs/groups.

{pstd}{hi:{ul:One-to-one matching algorithm}}{break}
	The algorithm used in a one-to-one match starts by evaluating which
	target observation each base observation is closest to and vice versa for each
	target observations. If a base and target observation pair mutually prefer
	each other, then these two observations are matched. The algorithm then repeats
	the initial two evaluation steps, and excludes observations after they are matched,
	until all base observations are either matched or excluded due to the option {cmdab:maxdiff()}.
	Matched observations ends up in pairs of exactly one base observation and one
	target observation.

{pstd}In a one-to-one match, there has to be at least as many target observations
	as there are base observations. A one-to-one match returns three variables. One
	variable indicates the result of the matching for each observation, such as,
	matched, not matched, no match within the max difference, etc. The other two variables
	hold information on the matched pair. One variable is the ID of the target observation
	in the matched pair, and the other variable is the difference in {cmd:matchvar()}
	between the two observations in the match pair. See the below in this section
	for more details on the variables generated.

{pstd}{hi:{ul:Many-to-one matching algorithm}}{break}
	The algorithm used for many-to-one matching is the same as in a one-to-one match. But instead
	of matching only when there is a mutual preference, all base observations are
	matched towards their preferred target observation in the first step, as long
	as the match is within the max-value if {cmdab:maxdff()} is used. Matched observations
	end up in groups in which there is exactly one target observation but where there are
	either one or several base observations.

{pstd}In a many-to-one match, there is no restriction in terms of the number of
	base observations in relation to target observations. The many-to-one matching
	yields four variables. Three of the variables are the same as in one-to-one
	matching, and only the variable listing the difference  The fourth variable indicates how many base observations that were matched
	towards each target observation. See the below in this section for more details on
	the variables generated.

{pstd}{hi:{ul:Variables generated}}{break}
	This section explains the variables that are generated by this command. All variables
	will be referred to by their default names, but those names can be manually set to somthing
	else (see the options for this command).

{pstd}{hi:_matchOrReason}{break}
	This variable indicates the result of the match for all observations. Observations
	that ended up in a match pair/group has the value 1. Target observations not matched have the
	value 0. Base observations without a valid match due to {cmdab:maxdff()} are
	assigned the value .d. Observations excluded from the match using {inp:if}/{inp:in} are
	assigned the value .i. Observations excluded from the match due to missing value in {cmd:grpdummy()} are
	assigned the value .g. Observations excluded from the match due to missing value in {cmd:matchvar()} are
	assigned the value .m. All values have descreptive value labels.

{pstd}{hi:_matchID}{break}
	This variable indicates the ID of the target observations in each match pair/group. For matched
	target observations, the value in _matchID will be equal to the value in the ID
	variable. Since the values in the ID varaible are unique and there is exactly one target
	observation in each matched pair/group, this variable functions as a unique pair/group ID.
	In addition to indicating which observations are included in the same pair/group, this
	varaible can be used to include a pair/group fixed effect in a regression.

{pstd}{hi:_matchDiff}{break}
	This variable indicates the difference in {cmd:matchvar()} between matched base observations
	and target observations. In a one-to-one match this value is the identical for
	both the base observation and the target observation in each matched pair.
	In a many-to-one match, this value is only indicated for base observations. It is
	missing for target observations as there are potentially multiple matches, and
	subsequently multiple differences.

{pstd}{hi:_matchCount}{break}
	In a many-to-one match, this variable indicates how many base observations were
	matched towards each matched target observation. This variable can be used as regression
	weights when controlling for the fact that some observations are matched towards multiple
	observations.

{marker optslong}
{title:Options}

{phang}{cmdab:grp:dummy(}{it:varname}{cmd:)} is the dummy variable that indicates if an observation
	is a base or target observation. This variable, must be numeric and is only allowed to have
	the values 1, 0 or missing. 1 indicates a base observation, 0 indicates a target observation,
	and observations with a missing value will be excluded from the missing.

{phang}{cmdab:match:var(}{it:varname}{cmd:)} is the convariable used in the
	matching. This must be a numeric variable, and it is typically a continuous
	variable. This variable must not be missing for any observations that has
	either the value 1 or 0 in the group dummy variable.

{phang}{cmdab:id:var(}{it:varlist}{cmd:)} indicates the variable that uniquely
	and fully identifies the data set. The values in this variable is the values
	that will be used in the variable that indicates which target observation each
	base observations matched against.

{phang}{cmdab:m1} set the match to a many-to-one match. This allows multiple base
	observations to be matched towards a single target observation. The default is
	a one-to-one match when each matched target observation only is mathced towards
	a single base observation. This option allows the number of base observations
	to be larger then the number of target observations. See {help iematch##desc:description}
	section for more details.

{phang}{cmdab:maxdiff(}{it:numlist}{cmd:)} sets a maximum allowed difference between
	a base observation and a target observation for a match to be valid. Any base
	observation without a match within this difference will end up unmatched.

{phang}{cmd:seedok} supresses the error maessage thrown when there are duplicates among
	the base observations or the target observations in {cmd:matchvar()}. When there
	are duplicates a random variable is used to distinguish which variable to match.
	Setting the seed makes this randomization replicable meaning that the matching
	is replicable. The {help seed} should be set before this command by the user. When the
	seed is set, or if replicable matching is not important, then the option {cmd:seedok} can
	be used to supress the error message. Duplicates between base observations and
	target observations are allowed.

{phang}{cmdab:matchre:sultname(}{it:string}{cmd:)} allows the user to manually set the
	name of the variable that indicates the result of the match for each observation.
	If omitted, the default name is {inp:_matchOrReason}. The names {inp:_ID},
	{inp:_matchID}, {inp:_matchDiff} and {inp:_matchCount} are not allowed.

{phang}{cmdab:matchid:name(}{it:string}{cmd:)} allows the user to manually set the
	name of the variable that indicates the target observations in each match pair/group.
	If omitted, the default name is {inp:_matchID}. The names {inp:_ID}, {inp:_matchDiff}, {inp:_matchOrReason}
	and {inp:_matchCount} are not allowed.

{phang}{cmdab:matchdi:ffname(}{it:string}{cmd:)} allows the user to manually set
	the name of the variable that indicates the difference between the matched base observations
	and target observations. If omitted, the default name is {inp:_matchDiff}. The
	names {inp:_ID}, {inp:_matchID}, {inp:_matchOrReason} and {inp:_matchCount} are
	not allowed.

{phang}{cmdab:matchco:untname(}{it:string}{cmd:)} allows the user to manually set the
	name of the variable that indicates the number of base observations that has
	been matched towards each matched matcehd target observation. This option may
	only be used in combination with option {cmd:m1}. If omitted, the default
	name is {inp:_matchCount}. The names {inp:_ID}, {inp:_matchID}, {inp:_matchDiff}
	and {inp:_matchOrReason} are not allowed.

{title:Acknowledgements}

{phang}I would like to acknowledge the help in testing and proofreading I received in relation to this command and help file from (in alphabetic order):{p_end}
{pmore}Mrijan Rimal{break}

{title:Author}

{phang}Kristoffer Bjarkefur, The World Bank, DECIE

{phang}Please send bug-reports, suggestions and requests for clarifications
		 writing "ietools iematch" in the subject line to:{break}
		 kbjarkefur@worldbank.org

{phang}You can also see the code, make comments to the code, see the version
		 history of the code, and submit additions or edits to the code through
		 the github repository of ietoolkit:{break}
		 {browse "https://github.com/worldbank/ietoolkit"}

{smcl}
{* }{...}
{hline}
help for {hi:iematch}
{hline}

{title:Title}

{phang2}{cmdab:iematch} {hline 2} Matching base observations towards target observations using on a single continous variable.

{title:Syntax}

{phang2}
{cmdab:iematch}
, {cmdab:grp:dummy(}{it:varname}{cmd:)} {cmdab:match:var(}{it:varname}{cmd:)}
[{cmdab:id:var(}{it:varlist}{cmd:)} {cmdab:m1} {cmdab:maxdiff(}{it:numlist}{cmd:)} {cmd:seedok}
{cmdab:matchid:name(}{it:string}{cmd:)} {cmdab:matchdi:ffname(}{it:string}{cmd:)}
{cmdab:matchre:sultname(}{it:string}{cmd:)} {cmdab:matchco:untname(}{it:string}{cmd:)}
]

{marker opts}{...}
{synoptset 22}{...}
{synopthdr:options}
{synoptline}
{synopt :{cmdab:grp:dummy(}{it:varname}{cmd:)}}The group dummy variable where 1
	indicates base observations and 0 target observations{p_end}
{synopt :{cmdab:match:var(}{it:varname}{cmd:)}}The variable with a continous value
	to match on{p_end}
{synopt :{cmdab:id:var(}{it:varlist}{cmd:)}}The uniquely and fully identifying ID
	varaible. Used to indicate which target observation a base observation is match
	with. If omitted an ID variable will be created.{p_end}
{synopt :{cmdab:m1}}Allows many-to-one matches. The default is to allow only
	one-to-one matches. See the {help iematch##desc:description} section.{p_end}
{synopt :{cmdab:maxdiff(}{it:numlist}{cmd:)}}Set a maximum difference allowed in
	{cmdab:matchvar()}. If a base observation has no match within this difference
 then it will remain unmatched{p_end}
{synopt :{cmd:seedok}}Supresses the error maessage thrown when there are duplicates
	in {cmd:matchvar()}. When there are duplicates, the seed needs to be set in order
	to have a replicable match. The {help seed} should be set before this command.{p_end}
{synopt :{cmdab:matchid:name(}{it:string}{cmd:)}}Manually sets the name of the
	variable that indicates which target observation each base observation is
	matched with. The default is _matchID{p_end}
{synopt :{cmdab:matchdi:ffname(}{it:string}{cmd:)}}Manually sets the name of the
	variable that indicates the differnece in {cmdab:matchvar()} in each match
	pair/group. The default is _matchDiff{p_end}
{synopt :{cmdab:matchre:sultname(}{it:string}{cmd:)}}Manually sets the name of
	the variable that indicates if an observation was matched, or provide a reason why
	it was not. The default is _matchOrReason{p_end}
{synopt :{cmdab:matchco:untname(}{it:string}{cmd:)}}Manually sets the name of the
	variable that indicates how many observations a target obsersvation is matched
	with in a many-to-one matches. The default is _matchCount{p_end}
{synoptline}

{marker desc}
{title:Description}

{pstd}{cmdab:iematch} matches base observations towards target observations in terms
	of nearest value in {cmd:matchvar()}. Base observations are observations with
	value 1 in {cmd:grpdummy()} and target observations are observations with value
	0. For example, in a typical p-score match, base observations are treatment and
	target is control, however, there are many other examples of matching where it could
	be different.

{pstd}{cmdab:iematch} base its matching algorithm on the Stable Marriage algorithm.
	This algorithm was chosen because it always a solution and because it was possible
	to implement a stable matching even if multiple observations has identical match
	values (see {cmd:seed()} option for more details). One disadvantage is of this
	algorithm is that it only take local efficiency into account and not global. This
	means that it is possible that other matching algorithms might generate a more
	efficient match in terms of the sum of the difference of all matched pairs/groups.

{pstd}{hi:{ul:One-to-one matching algorithm}}{break}
	The algorithm used in a one-to-one match starts by evaluating which
	target observation each base observation is closest to and vice versa for
	target observations. If a base and a target observation pair mutually prefer
	each other, then those two observations are a match. The algorithm then repeats
	these two steps but excludes observations after they are matched until all base
	observations are either matched or excluded due to the option {cmdab:maxdiff()}.
	Matched observations ends up in pairs of exactly one base observation and one
	target observation.

{pstd}In a one-to-one match there have to be at least as many target observations
	as there are base observation. A one-to-one match returns three variables. One
	variable indicates the result of the matching for each observation. For example,
	matched, not matched, no match within the max difference etc. The other two variables
	hold info on the matched pair. One variable is the ID of the target observation
	in the match pair, and the other variable is the difference in {cmd:matchvar()}
	between the two observation in the match pair. See the below in this section
	for more details on the variables generated.

{pstd}{hi:{ul:Many-to-one matching algorithm}}{break}
	The algorithm used here is the same as in a one-to-one match. But instead
	of matching only when there is mutual preference, all base observations are
	matched towards their preferred target observation in the first step, as long
	as the match is within the max-value if {cmdab:maxdff()} is used. Matched observations
	end up in groups in which there is exactly one target observation but where there are
	either one or several base observations.

{pstd}In a many-to-one match there is no restriction in terms of the number of
	target observations in relation to base observations. The many-to-one matching
	results in four variables. Three of the variables are the same as in a one-to-one
	match. The fourth variable indicates how many base observations that were matched
	towards each target observation. See the below in this section for more details on
	the variables generated.

{pstd}{hi:{ul:Variables generated}}{break}
	This section explains the variables that are generated by this command. All variables
	will be referred to by their default names, but those names can be manually set to somthing
	else using the corresponding options.

{pstd}{hi:_matchID}{break}
	This variable indicates the ID if the target observations in each match pair/group. For matched
	target observations, the value in this variable will be the same value as in teh ID
	variable. Since the values in the ID varaible are unique and there is exactly one target
	observation in each matched pair/group, this variable function as a unique pair/group ID.
	In addition to indicating which observations are included in the same pair/group, this
	varaible can be used to include a pair/group fixed effect in a regression.

{pstd}{hi:_matchDiff}{break}
	This variable indicates the difference in {cmd:matchvar()} between matched base observations
	and target observations. In a one-to-one match this value is the identical for
	both the base observation and the target observation in each matched pair.
	In a many-to-one match, this value is only indicated for base observations. It is
	missing for target observations as there are potentially multiple matches,. and
	then there are not one single difference.

{pstd}{hi:_matchOrReason}{break}
	This variable indicates the result of the match for all observations. Observations
	that ended up in a match pair or group has the value 1. Target observations
	not matched has the value 0. Any base observation without a valid match due to {cmdab:maxdff()} are
	assigned the value .n with a descriptive label. Any observations excluded from the match completely,
	either by having a missing value in {cmd:grpdummy()} or by using {inp:if} or {inp:in} are
	assigned the value .m and a descriptive label.

{pstd}{hi:_matchCount}{break}
	In a many-to-one match, this variable indicates how many target observations were
	matched towards each matched base observation. This variable can be used as regression
	weights when controlling for the fact that some observations are matched towards multiple
	observations.

{marker optslong}
{title:Options}

{phang}{cmdab:grp:dummy(}{it:varname}{cmd:)} is the dummy variable that indicates if an observation
	is a base or target observation. This variable, must be numeric and is only allowed to have
	the values 1, 0 or missing. 1 indicates a base observation, 0 indicates a target observation,
	and observations with a missing value will be excluded from the missing.

{phang}{cmdab:match:var(}{it:varname}{cmd:)} is the convariable used in the
	matching. This must be a numeric variable, and it is typically a continuous
	variable. This variable must not be missing for any observations that has
	either the value 1 or 0 in the group dummy variable.

{phang}{cmdab:id:var(}{it:varlist}{cmd:)} indicates the variable that uniquely
	and fully identifies the data set. The values in this variable is the values
	that will be used in the variable that indicates which target observation each
	base observations matched against.

{phang}{cmdab:m1} set the match to a many-to-one match. This allows multiple base
	observations to be matched towards a single target observation. The default is
	a one-to-one match when each matched target observation only is mathced towards
	a single base observation. This option allows the number of base observations
	to be larger then the number of target observations. See {help iematch##desc:description}
	section for more details.

{phang}{cmdab:maxdiff(}{it:numlist}{cmd:)} sets a maximum allowed difference between
	a base observation and a target observation for a match to be valid. Any base
	observation without a match within this difference will end up unmatched.

{phang}{cmd:seedok} supresses the error maessage thrown when there are duplicates among
	the base observations or the target observations in {cmd:matchvar()}. When there
	are duplicates a random variable is used to distinguish which variable to match.
	Setting the seed makes this randomization replicable meaning that the matching
	is replicable. The {help seed} should be set before this command by the user. When the
	seed is set, or if replicable matching is not important, then the option {cmd:seedok} can
	be used to supress the error message. Duplicates between base observations and
	target observations are allowed.

{phang}{cmdab:matchid:name(}{it:string}{cmd:)} allows the user to manually set the
	name of the variable that indicates the target observations in each match pair/group.
	If omitted, the default name is {inp:_matchID}. The names {inp:_ID}, {inp:_matchDiff}, {inp:_matchOrReason}
	and {inp:_matchCount} are not allowed.

{phang}{cmdab:matchdi:ffname(}{it:string}{cmd:)} allows the user to manually set
	the name of the variable that indicates the difference between the matched base observations
	and target observations. If omitted, the default name is {inp:_matchDiff}. The
	names {inp:_ID}, {inp:_matchID}, {inp:_matchOrReason} and {inp:_matchCount} are
	not allowed.

{phang}{cmdab:matchre:sultname(}{it:string}{cmd:)} allows the user to manually set the
	name of the variable that indicates the result of the match for each observation.
	If omitted, the default name is {inp:_matchOrReason}. The names {inp:_ID},
	{inp:_matchID}, {inp:_matchDiff} and {inp:_matchCount} are not allowed.

{phang}{cmdab:matchco:untname(}{it:string}{cmd:)} allows the user to manually set the
	name of the variable that indicates the number of base observations that has
	been matched towards each matched matcehd target observation. This option may
	only be used in combination with option {cmd:m1}. If omitted, the default
	name is {inp:_matchCount}. The names {inp:_ID}, {inp:_matchID}, {inp:_matchDiff}
	and {inp:_matchOrReason} are not allowed.

{title:Author}

{phang}Kristoffer Bj�rkefur, The World Bank, DECIE

{phang}Please send bug-reports, suggestions and requests for clarifications
		 writing "ietools iematch" in the subject line to:{break}
		 kbjarkefur@worldbank.org

{phang}You can also see the code, make comments to the code, see the version
		 history of the code, and submit additions or edits to the code through
		 the github repository of ietoolkit:{break}
		 {browse "https://github.com/worldbank/ietoolkit"}


Update PolityIV to 2014

Update PolityIV to 2014

* 2014 Polity 4
* http://www.systemicpeace.org/polity/polity4.htm

local URL "http://www.systemicpeace.org/inscr/p4v2014.xls"
local PREFIX "polity"
local YEARVAR "year"
local COUNTRYVAR "ccode"
local MERGEWITH "Polity"
local MERGEUSING "Numeric"

tempfile merge
prep_mergefile `MERGEWITH' `MERGEUSING' `merge'

* If the file source file doesn't already exist then download and create it.
use_or_import_source_excel `PREFIX' `URL'

* Fixes:
drop if year == .
* Drop Sudan Transition
drop if cyear == 6252011
* Drop Ethiopian Transition
drop if cyear == 5301993
* Drop Vietnam Transition
drop if cyear == 8161976

prefix `PREFIX'

merge_ids `PREFIX' `YEARVAR' `COUNTRYVAR' `MERGEUSING' `merge'

* Merge with the dataset
merge_with_master

* 2013 Polity 4
* http://www.systemicpeace.org/polity/polity4.htm

local URL "http://www.systemicpeace.org/inscr/p4v2013.xls"
local PREFIX "polity"
local YEARVAR "year"
local COUNTRYVAR "ccode"
local MERGEWITH "Polity"
local MERGEUSING "Numeric"

tempfile merge
prep_mergefile `MERGEWITH' `MERGEUSING' `merge'

* If the file source file doesn't already exist then download and create it.
use_or_import_source_excel `PREFIX' `URL'

* Fixes:
drop if year == .
* Drop Sudan Transition
drop if cyear == 6252011
* Drop Ethiopian Transition
drop if cyear == 5301993
* Drop Vietnam Transition
drop if cyear == 8161976

prefix `PREFIX'

merge_ids `PREFIX' `YEARVAR' `COUNTRYVAR' `MERGEUSING' `merge'

* Merge with the dataset
merge_with_master


iegrpah : help expanded examples

iegrpah : help expanded examples

{smcl}
{* 26 Dec 2016}{...}
{hline}
help for {hi:iegraph}
{hline}

{title:Title}

{phang2}{cmdab:iegraph} {hline 2} Generates graphs based on regressions with treatment dummies common in impact evaluations.

{title:Syntax}

{phang2}
{cmdab:iegraph} {varlist}
, [{cmd:noconfbars} {cmdab:ti:tle(}{it:string}{cmd:)}
{cmdab:save(}{it:string}{cmd:)} {cmdab:confbarsnone(}{it:varlist}{cmd:)}
{it:any towway graph options}
]

{marker opts}{...}
{synoptset 22}{...}
{synopthdr:options}
{synoptline}
{synopt :{cmd:noconfbars}} Removes the confidence interval bars from graphs for all treatments.{p_end}
{synopt :{cmdab:ti:tle(}{it:string}{cmd:)}} Manually sets the title of the graph.{p_end}
{synopt :{cmdab:save(}{it:string}{cmd:)}} Sets the filename and the directory to which the graph will be set.{p_end}
{synopt :{cmdab:confbarsnone(}{it:varlist}{cmd:)}} Removes confidence interval bars from only the {it:varlist} listed.{p_end}
{synopt :{cmdab:yzero}} Forces y-axis on the graph to start at 0.{p_end}
{synopt :{cmdab: Graph Options}} Options that can be used with normal graph commands can also be used. See example 2 for details.{p_end}
{synoptline}

{marker desc}
{title:Description}

{pstd}{cmdab:iegraph} This command creates bar graphs on the based on the coefficients
	of treatment dummies in regression results. This command is developed for reading
	stored results of from two types of impact evaluation regression models, but there
	are countless of other examples where the command also can be used. {cmd:iegraph} must
	be used immedeatly after running the regression or as long as teh regression result is
	still stored in or restored to Stata's {help ereturn} results.

{pstd}{bf:Model 1: OLS with Treatment Dummies}{break}The most typical impact evaluation regression is
	to have the outcome variable as the dependent varaible and one dummy for each
	treatment arm where control is the omitted category. This regressions can also include
	covariates, fixed effects etc., but as long as the treatment status is defined by
	mutually exclusive dummy variables. See especially example 1 and 2 below. This command
	works with any number of treatment arms but works best from two arms (treatment
	and control) to five treatment arms (4 different treatments and control). More
	arms than that may result in a still correct but perhaps cluttered  graph.

{pstd}{bf:Model 2: Difference-in-Differneces}{break}Another typical regression model in impact
	evaluations are difference-in-difference (Diff-in-Diff) models with two treatment arms (treatment
	and control) and two time periods. If the Diff-in-Diff regression is specified as having the
	outcome variable as the dependent variable and three dummy variables (time, treatment
	and timeXtreatment) as the independent varaibles, then this command will produce a nice
	graph. Controls, treatment effects etc. may be added to the regression model. See especially example 3.

{pstd}{bf:Graph Output}{break}The graph generated by this command is created using the following values. The
	control bar is the mean of the outcome variable for the control group. It is not
	the constant from the regression as those are not identical if, for example, fixed effects
	and covariates were used. For each treatment group the bar is the sum of the value
	of the control bar and the beta coefficient in the regression of the corresponding
	treatment dummy. The confidence intervals are calculated from the variance in the
	beta coefficients in the regression.{p_end}

{pstd}The graph also includes the N for each treatment arm in the regression and uses
	that value as labels on the x-axis. Stars are added to this value if the corresponding
	coefficient is statistically different from zero in the regression{p_end}

{marker optslong}
{title:Options}

{phang}{cmd:noconfbars} Removes the confidence interval bars from graphs for all
	treatments. The default value for the confidence interval bars ar 95%. {p_end}

{phang}{cmdab:ti:tle(}{it:string}{cmd:)} Manually sets the title of the graph.{p_end}

{phang}{cmdab:save(}{it:string}{cmd:)} Sets the filename and the directory to which
	the graph will be set.{p_end}

{phang}{cmdab:confbarsnone(}{it:varlist}{cmd:)} Removes confidence interval bars
	from only the {it:varlist} listed. The remaining variables in the graphs which
	have not been specified in {cmdab:confbarsnone} will still have the confidence
	interval bars. {p_end}

{phang}{cmdab:yzero} Manually sets the y-axis of the graph to start at zero instead of the Stata default.{p_end}

{marker optslong}
{title:Examples}

{pstd} {hi:Example 1.}

{pmore} {inp:regress} {it:outcomevar treatment_dummy}{break}
		{inp:iegraph} {it:treatment_dummy} , {inp:title({it:"Treatment Effect on Outcome"})}

{pmore}In the example above, there is only two treatment arms (treatment and
		control). {it:treatment_dummy} has a 1 for all treatment observations and
		a 0 for all control observations. The graph will have one bar for control and
		it shows the mean for {it:outcomevar} for all observations in control. The
		second bar in the graph will be the sum of that mean and the coefficient
		for {it:treatment_dummy} in the regression. The graph will also have the
		title: Treatment Effect on Outcome.

{pstd} {hi:Example 2.}

{pmore} {inp:regress} {it:income tmt_1 tmt_2 age education}{inp:, cluster(}{it:district}{inp:)}{break}
		{inp:iegraph} {it:tmt_1 tmt_2}{inp:, noconfbars yzero title({it:"Treatment effect on income"}) }

{pmore}In the example above, the treatment effect on income in reserached. There
		are three treatment arms; control, treatment 1 ({it:tmt_1}) and treatment
		2 ({it:tmt_2}). It is important that no observation has the value 1 in
		both {it:tmt_1} and {it:tmt_2} (i.e. no observation is in more than one
		treatment) and some observations must have the value 0 in both {it:tmt_1}
		and {it:tmt_2} (i.e. control observations). The variables {it:age} and
		{it:education} are covariates (control variables) and is not included
		in {cmd:iegraph}. {inp:noconfbars} omitts the confidence intervall bars
		, and {inp:yzero} sets the y-axis to start at 0.

{pstd} {hi:Example 3.}

{pmore} {inp:regress} {it:chld_wght time treat timeXtreat}{break}
		{inp:iegraph} {it:time treat timeXtreat} {inp:, title({it:"Treatment effect on Child Weight (Diff-in-Diff)"})}

{pmore}In the example above, the data set is a panel data set with two time
		periods and the regression estimates the treatment effect on child weight
		using a Difference-in-Differences model.
		The dummy variable {it:time} indicates if it is time period 0 or 1.
		The dummy variable {it:treat} indicates if the observation is treatment
		or control. {it:timeXtreat} is the interaction term of {it:time}
		and {it:treat}. This the standard way to set up a Difference-in-Differences
		regression model.

{pstd} {hi:Example 4.}

{pmore} {inp:regress} {it:harvest T1 T2 T3 } {break}
		{inp:iegraph} {it:T1 T2 T3} {inp:, title({it:"Treatment effect on harvest"})
		xlabel(,angle(45)) yzero ylabel(minmax) save({it:"$Output/Graph1.gph"})}

{pmore}The example above shows how to save a graph to disk. It also shows that
	most two-way graph options can be used. In this example the {cmd:iegraph}
	option {cdm:yzero} conflicts with the two-way option {cmd:ylabel(minmax)}.
	In such a case the user specified option takes presidence over {cmd:iegraph}
	options like {cdm:yzero}.

{title:Acknowledgements}

{phang}We would like to acknowledge the help in testing and proofreading we received in relation to this command and help file from (in alphabetic order):{p_end}
{pmore}{break}

{title:Author}

{phang}Kristoffer Bjarkefur & Mrijan Rimal, The World Bank, DECIE

{phang}Please send bug-reports, suggestions and requests for clarifications
		 writing "ietools iegraph" in the subject line to:{break}
		 kbjarkefur@worldbank.org

{phang}You can also see the code, make comments to the code, see the version
		 history of the code, and submit additions or edits to the code through
		 the github repository of ietoolkit:{break}
		 {browse "https://github.com/worldbank/ietoolkit"}

{smcl}
{* 26 Dec 2016}{...}
{hline}
help for {hi:iegraph}
{hline}

{title:Title}

{phang2}{cmdab:iegraph} {hline 2} Generates graphs based on regressions with treatment dummies common in impact evaluations.

{title:Syntax}

{phang2}
{cmdab:iegraph} {varlist}
, [{cmd:noconfbars} {cmdab:ti:tle(}{it:string}{cmd:)}
{cmdab:save(}{it:string}{cmd:)} {cmdab:confbarsnone(}{it:varlist}{cmd:)}
{it:any towway graph options}
]

{marker opts}{...}
{synoptset 22}{...}
{synopthdr:options}
{synoptline}
{synopt :{cmd:noconfbars}} Removes the confidence interval bars from graphs for all treatments.{p_end}
{synopt :{cmdab:ti:tle(}{it:string}{cmd:)}} Manually sets the title of the graph.{p_end}
{synopt :{cmdab:save(}{it:string}{cmd:)}} Sets the filename and the directory to which the graph will be set.{p_end}
{synopt :{cmdab:confbarsnone(}{it:varlist}{cmd:)}} Removes confidence interval bars from only the {it:varlist} listed.{p_end}
{synopt :{cmdab:yzero}} Forces y-axis on the graph to start at 0.{p_end}
{synopt :{cmdab: Graph Options}} Options that can be used with normal graph commands can also be used. See example 2 for details.{p_end}
{synoptline}

{marker desc}
{title:Description}

{pstd}{cmdab:iegraph} This command creates bar graphs on the based on the coefficients
	of treatment dummies in regression results. This command is developed for reading
	stored results of from two types of impact evaluation regression models, but there
	are countless of other examples where the command also can be used. {cmd:iegraph} must
	be used immedeatly after running the regression or as long as teh regression result is
	still stored in or restored to Stata's {help ereturn} results.

{pstd}{bf:Model 1: OLS with Treatment Dummies}{break}The most typical impact evaluation regression is
	to have the outcome variable as the dependent varaible and one dummy for each
	treatment arm where control is the omitted category. This regressions can also include
	covariates, fixed effects etc., but as long as the treatment status is defined by
	mutually exclusive dummy variables. See especially example 1 and 2 below. This command
	works with any number of treatment arms but works best from two arms (treatment
	and control) to five treatment arms (4 different treatments and control). More
	arms than that may result in a still correct but perhaps cluttered  graph.

{pstd}{bf:Model 2: Difference-in-Differneces}{break}Another typical regression model in impact
	evaluations are difference-in-difference (DD) models with two treatment arms (treatment
	and control) and two time periods. If the DD regression is specified as having the
	outcome variable as the dependent variable and three dummy variables (time, treatment
	and timeXtreatment) as the independent varaibles, then this command will produce a nice
	graph. Controls, treatment effects etc. may be added to the regression model. See especially example 3.

{pstd}{bf:Graph Output}{break}The graph generated by this command is created using the following values. The
	control bar is the mean of the outcome variable for the control group. It is not
	the constant from the regression as those are not identical if, for example, fixed effects
	and covariates were used. For each treatment group the bar is the sum of the value
	of the control bar and the beta coefficient in the regression of the corresponding
	treatment dummy. The confidence intervals are calculated from the variance in the
	beta coefficients in the regression.{p_end}

{pstd}The graph also includes the N for each treatment arm in the regression and uses
	that value as labels on the x-axis. Stars are added to this value if the corresponding
	coefficient is statistically different from zero in the regression{p_end}

{marker optslong}
{title:Options}

{phang}{cmd:noconfbars} Removes the confidence interval bars from graphs for all
	treatments. The default value for the confidence interval bars ar 95%. {p_end}

{phang}{cmdab:ti:tle(}{it:string}{cmd:)} Manually sets the title of the graph.{p_end}

{phang}{cmdab:save(}{it:string}{cmd:)} Sets the filename and the directory to which
	the graph will be set.{p_end}

{phang}{cmdab:confbarsnone(}{it:varlist}{cmd:)} Removes confidence interval bars
	from only the {it:varlist} listed. The remaining variables in the graphs which
	have not been specified in {cmdab:confbarsnone} will still have the confidence
	interval bars. {p_end}

{phang}{cmdab:yzero} Manually sets the y-axis of the graph to start at zero instead of the Stata default.{p_end}

{marker optslong}
{title:Examples}

{pstd} {hi:Example 1.}

{pmore} {inp:regress outcomevar treatment_dummy1 treatment_dummy2}{break}
		{inp:iegraph treatment_dummy1 treatment_dummy2 , title({it:string})}

{pmore}In the example above, graphs comparing the effects of {it:treatmentVar1} and
		{it:treatmentVar2} to the mean of the control will be generated. The option
		title({it:string}) gives the graph the title that the user puts in the string.

{pstd} {hi:Example 2.}

{pmore}{inp:iegraph treatment, noconfbars title("Treatment effect on price") save("$Output/Graph1.gph") xlabel(,angle(45)) ylabel(minmax)}

{pmore}In the example above, the observations with value 1 in {it:tmt} will be matched
	towards the nearest, in terms of {it:p_hat}, observations with value 0 in {it:tmt} as
	long as the difference in {it:p_hat} is less than .001. Only observations that has the
	value 1 in variable {it:baseline} will be included in the match.

{title:Acknowledgements}

{phang}I would like to acknowledge the help in testing and proofreading I received in relation to this command and help file from (in alphabetic order):{p_end}
{pmore}{break}

{title:Author}

{phang}Kristoffer Bjarkefur & Mrijan Rimal, The World Bank, DECIE

{phang}Please send bug-reports, suggestions and requests for clarifications
		 writing "ietools iegraph" in the subject line to:{break}
		 kbjarkefur@worldbank.org

{phang}You can also see the code, make comments to the code, see the version
		 history of the code, and submit additions or edits to the code through
		 the github repository of ietoolkit:{break}
		 {browse "https://github.com/worldbank/ietoolkit"}


t/passfailtest.do: just return exit codes, don't print messages.

The exit code numbers are useful enough, and the messages are the sort of
thing that might turn into lies eventually.

rm -f pleasefail
redo passfail
[ -e passfail ] || exit 42
PF1=$(cat passfail)
touch pleasefail
redo passfail 2>/dev/null && exit 43
[ -e passfail ] || exit 44
PF2=$(cat passfail)
[ "$PF1" = "$PF2" ] || exit 45
rm -f pleasefail
redo passfail || exit 46
PF3=$(cat passfail)
[ "$PF1" != "$PF3" ] || exit 47

rm -f pleasefail
redo passfail
if [ ! -e passfail ]; then
	echo "passfail should exist" >&2
	exit 42
fi
PF1=$(cat passfail)
touch pleasefail
if redo passfail 2>/dev/null; then
	echo "redo passfail should have failed" >&2
	exit 42
fi
if [ ! -e passfail ]; then
	echo "passfail should STILL exist" >&2
	exit 42
fi
PF2=$(cat passfail)
if [ "$PF1" != "$PF2" ]; then
	echo "passfail changed even though it failed" >&2
	exit 42
fi
rm -f pleasefail
redo passfail || exit 43
PF3=$(cat passfail)
if [ "$PF1" = "$PF3" ]; then
	echo "passfail did not change even though it passed" >&2
	exit 42
fi