run file for iegitaddmd testin new .git skip functions

	* Set this local to a folder where folders can be creaetd for this experiment
	global ietoolkit_clone "C:\Users\wb462869\Documents\GitHub\ietoolkit"
	global test_folder "${ietoolkit_clone}\run\output\iegitaddmd"

	qui do "${ietoolkit_clone}/run/ie_recurse_rmdir.do"

	* Create a folder clone and a folder dropbox (see use case 2 in help file for why both these folders are needed)
	local highlevel_folders clone dropbox gitfilter1 gitfilter2
	foreach folder of local highlevel_folders {
		ie_recurse_rmdir, folder("${test_folder}/`folder'") okifnotexist
		mkdir "${test_folder}/`folder'"
	}

	* USe iefolder to create identical project folders
	iefolder new project , projectfolder("${test_folder}/clone")
	iefolder new project , projectfolder("${test_folder}/dropbox")
	iefolder new round Baseline , projectfolder("${test_folder}/clone")
	iefolder new round Baseline , projectfolder("${test_folder}/dropbox")

	iefolder new unitofobs students, projectfolder("${test_folder}/clone")
	iefolder new unitofobs teachers, projectfolder("${test_folder}/clone")
	iefolder new unitofobs schools, projectfolder("${test_folder}/clone")

	* Remove some folders from the clone to simulate use case 2 (see helpfile for description of use cases)
	rmdir "${test_folder}/clone/DataWork/Baseline/DataSets/Deidentified"
	rmdir "${test_folder}/clone/DataWork/Baseline/DataSets/Final"
	rmdir "${test_folder}/clone/DataWork/Baseline/DataSets/Intermediate"
	rmdir "${test_folder}/clone/DataWork/Baseline/DataSets"
	rmdir "${test_folder}/clone/DataWork/Baseline/Questionnaire/Questionnaire Documentation"

	* Create the git folder
	local gitfolders gitfilter1 gitfilter2
	foreach gitfolder of local gitfolders {
		mkdir "${test_folder}/`gitfolder'/.git"
		mkdir "${test_folder}/`gitfolder'/includeme"
		mkdir "${test_folder}/`gitfolder'/skipmein2"
		mkdir "${test_folder}/`gitfolder'/skipalsomein2"
		mkdir "${test_folder}/`gitfolder'/sub"
		mkdir "${test_folder}/`gitfolder'/sub/.git"
		mkdir "${test_folder}/`gitfolder'/sub/includealsome"
	}

	*Set global to ietoolkit clone
	qui do "${ietoolkit_clone}/src/ado_files/iegitaddmd.ado"

	*Use case 1 - see helpfile for description of use cases
	iegitaddmd , folder("${test_folder}/clone/DataWork") auto dryrun
	iegitaddmd , folder("${test_folder}/clone/DataWork") auto

	*Use case 2 - see helpfile for description of use cases
	iegitaddmd , folder("${test_folder}/clone\DataWork") comparefolder("${test_folder}/dropbox\DataWork") auto dryrun
	iegitaddmd , folder("${test_folder}/clone\DataWork") comparefolder("${test_folder}/dropbox\DataWork") auto

	*Test prompt
	mkdir "${test_folder}/clone/DataWork/Baseline/DataSets/Final/Publishable"
	*iegitaddmd , folder("${test_folder}/clone/DataWork")

	*Test skip git folders
	iegitaddmd , folder("${test_folder}/gitfilter1") auto

	*Test skip custom
	iegitaddmd , folder("${test_folder}/gitfilter2") auto skipfolders("skipmein2" "skipalsomein2" "folderthatnotexist")

	* Set this local to a folder where folders can be creaetd for this experiment
	global test_folder ""

	* Create a folder clone and a folder dropbox (see use case 2 in help file for why both these folders are needed)
	mkdir "${test_folder}/clone"
	mkdir "${test_folder}/dropbox"

	* USe iefolder to create identical project folders
	iefolder new project , projectfolder("${test_folder}/clone")
	iefolder new project , projectfolder("${test_folder}/dropbox")
	iefolder new round Baseline , projectfolder("${test_folder}/clone")
	iefolder new round Baseline , projectfolder("${test_folder}/dropbox")

	iefolder new unitofobs students, projectfolder("${test_folder}/clone")
	iefolder new unitofobs teachers, projectfolder("${test_folder}/clone")
	iefolder new unitofobs schools, projectfolder("${test_folder}/clone")

	* Remove some folders from the clone to simulate use case 2 (see helpfile for description of use cases)
	rmdir "${test_folder}/clone/DataWork/Baseline/DataSets/Deidentified"
	rmdir "${test_folder}/clone/DataWork/Baseline/DataSets/Final"
	rmdir "${test_folder}/clone/DataWork/Baseline/DataSets/Intermediate"
	rmdir "${test_folder}/clone/DataWork/Baseline/DataSets"
	rmdir "${test_folder}/clone/DataWork/Baseline/Questionnaire/Questionnaire Documentation"

	*Set global to ietoolkit clone
	global ietoolkit_clone ""
	qui do "${ietoolkit_clone}/src/ado_files/iegitaddmd.ado"

	*Use case 1 - see helpfile for description of use cases
	iegitaddmd , folder("${test_folder}/clone/DataWork") auto dryrun
	iegitaddmd , folder("${test_folder}/clone/DataWork") auto

	*Use case 2 - see helpfile for description of use cases
	iegitaddmd , folder("${test_folder}/clone\DataWork") comparefolder("${test_folder}/dropbox\DataWork") auto dryrun
	iegitaddmd , folder("${test_folder}/clone\DataWork") comparefolder("${test_folder}/dropbox\DataWork") auto

	*Test prompt
	mkdir "${test_folder}/clone/DataWork/Baseline/DataSets/Final/Publishable"
	iegitaddmd , folder("${test_folder}/clone/DataWork")


allow synonyms for bs and sim

allow synonyms for bs and sim

*! version 1.18.3 09jul2017
*! PARALLEL: Stata module for parallel computing
*! by George G. Vega [cre,aut], Brian Quistorff [ctb]
*!
*! Project website:
*!   https://github.com/gvegayon/parallel
*! Bug reports:
*!   https://github.com/gvegayon/parallel/issues
/*
Copyright (c) 2014  <George G. Vega>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

// Syntax parser
program def parallel
    version 11.0

	// Checks wether if is parallel prefix or not
	if  (regexm(`"`0'"', "^(do|clean|setclusters|break|version|append|printlog|viewlog|numprocessors)")) {
	/* If not prefix */
		parallel_`0'
	}
	else if (regexm(`"`0'"', "^(bootstrap|bstrap|bs|simulate|sim)[,]?[\s ]?")) {
	/* Prefix bootstrap or simulate */
		local cmd = regexs(1)
    if inlist("`cmd'","bootstrap","bstrap") local cmd = "bs"
    if "`cmd'"=="simulate" local cmd = "sim"
		mata: st_local("0", regexr(st_local("0"), "^(bootstrap|bstrap|bs|simulate|sim)", ""))
		gettoken x 0 : 0, parse(":") bind
		local 0 = regexr(`"`0'"', "^[:]", "")
		gettoken x options : x, parse(",") bind

		gettoken 0 argopt : 0, parse(",") bind
		parallel_`cmd' `0', argopt(`argopt') `options'
	}
	else if (regexm(`"`0'"',"^([,]?.*[:])")) {
	/* if prefix */
		gettoken x 0 : 0, parse(":") bind
		mata: st_local("0", regexr(st_local("0"), "^[:]", ""))
		// Gets the options (if these exists) of parallel
		gettoken x options : x, parse(",") bind

		gettoken 0 argopt : 0, parse(",") bind
		parallel_do `0', `options' prefix argopt(`argopt')
	}
	else {
		local val = regexm(`"`0'"', "^([a-zA-Z0-9_]*).*")
		local subcommand = regexs(1)
		di as result `"-`subcommand'-"' as error " invalid parallel subcommand" as text " ({stata h parallel:{it:help parallel}})"
		exit 198
	}
end

/*
 For Windows we use an environment variable that could be redefined by a parent
   (both a plus and minus). Alternative is to sum output from command:
    WMIC CPU Get DeviceID,NumberOfCores,NumberOfLogicalProcessors

 For *nix see http://stackoverflow.com/questions/6481005/ for alternatives.
   Note: 'nproc --all' does not return the right result on some cloud providers where systems are shared.
*/
program parallel_numprocessors, rclass
	if "`c(os)'"=="Windows" {
		local nproc : env NUMBER_OF_PROCESSORS
	}
	else {
		tempfile nproc_out
		! getconf _NPROCESSORS_ONLN > "`nproc_out'"
		tempname nproc_out_fhandle
		file open `nproc_out_fhandle' using "`nproc_out'", read text
		file read `nproc_out_fhandle' nproc
		local r_eof = `r(eof)'
		file close `nproc_out_fhandle'
		_assert `r_eof'==0, msg("Wasn't able to read output from system")
	}

	local nproc = int(real("`nproc'"))
	_assert "`nproc'"!=".", msg("Wasn't able to interpret output from system")

	di "Number of logical processors: `nproc'"
	return scalar numprocessors = `nproc'
end

/* Returns the version of parallel */
program def parallel_version, rclass
	version 11.0
	di as result "parallel" as text " Stata module for parallel computing"
	di as result "vers" as text " 1.18.3 09jul2017"
	di as result "auth" as text " George G. Vega [cre,aut], Brian Quistorff [ctb]"

	return local pll_vers = "1.18.3"
end

/* Take a look to logfiles */
program def parallel_printlog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)]
	parallel_checklog `pll_instance', e(`event') action(print)
end

program def parallel_viewlog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)]
	parallel_checklog `pll_instance', e(`event') action(view)
end

program def parallel_checklog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)] action(string)

	if ("`event'"=="") local event = "$LAST_PLL_ID"

	if ("`event'"!=".") {
		/* By default uses 1 */
		if ("`pll_instance'" == "") local pll_instance 1

		local pll_instance : di %04.0f `pll_instance'

		/* Does de file exists */
		local logname = "__pll`event'_do`pll_instance'.log"

		if (c(os) == "Windows") local logname = "`c(tmpdir)'`logname'"
		else local logname = "`c(tmpdir)'/`logname'"

		/* If the logfile does not exists, do nothing*/
		cap confirm file "`logname'"
		if (_rc) {
			di as result "No logfile for instance -`pll_instance'- of parallel process -`event'- found"
			exit
		}

		/* Showing the log in screen */
		if ("`action'"=="print") {
			di as result "{hline 80}"
			di as result %~80s "beginning of file -`logname'-"
			di as result "{hline 80}"
			type `"`logname'"'
			di as result "{hline 80}"
			di as result %~80s "end of file -`logname'-"
			di as result "{hline 80}"
		}
		else view `"`logname'"'
	}
	else {
		di as error "It seems that you haven't use -parallel- yet."
		exit 601
	}

end

////////////////////////////////////////////////////////////////////////////////
// Splits the dataset into clusters
program def parallel_spliter
	version 11.0
	syntax [namelist(name=xtstructure)] [,parallelid(string) sorting(integer 0) force(integer 0) keepusing(varlist) orig_cl_local(string)]
	//args xtstructure parallelid Sorting Force

	if length("$PLL_CLUSTERS") == 0 {
		di as err "Number of clusters not fixed." as text " Set the number of clusters with " _newline as err "-{cmd:parallel setclusters #}-"
		exit 198
	}

	//quietly {
		if (length("`xtstructure'")) {
			_assert `sorting' | `force', msg("Data not sorted") rc(5)

			/* Checking the data types */
			foreach var of varlist `xtstructure' {
				capture confirm string var `var'
				if (`=_rc') local numvars `numvars' `var'
				else local strvars `strvars' `var'
			}

			/* calculate max possible clusters */
			egen _`parallelid'grp = group(`xtstructure'), missing
			summ _`parallelid'grp, meanonly
			local max_n_cl = `r(max)'
		}
		else {
			local max_n_cl = _N
		}

		/* Do we have too many clusters? */
		if (`max_n_cl'<$PLL_CLUSTERS){
			c_local `orig_cl_local' ${PLL_CLUSTERS}
			global PLL_CLUSTERS = `max_n_cl'
			di "Small workload/num groups. Temporarily setting number of clusters to ${PLL_CLUSTERS}"
		}

		if (length("`xtstructure'")) {
			preserve
			keep _`parallelid'grp
			contract _`parallelid'grp
			sort _freq
			//Figuring out the mapping from groupID to cut that equalizes sizes is the "Partition Problem"
			//which is hard to solve exactly (it's NP-complete). A rough solution suffices here though.
			gen _`parallelid'cut = mod(_n, ${PLL_CLUSTERS}) + 1
			tempfile grp_to_cut_map
			qui save `"`grp_to_cut_map'"'
			restore

			qui merge m:1 _`parallelid'grp using `"`grp_to_cut_map'"', keepusing(_`parallelid'cut) nogenerate
			drop _`parallelid'grp
			sort `xtstructure'
		}
		else {
			gen _`parallelid'cut = ceil(_n*${PLL_CLUSTERS}/_N) //each of size _N/$PLL_CLUSTERS
		}

		if (length("`keepusing'")) {
			keep _`parallelid'cut `keepusing'
			gen __pllnobs`parallelid' = _n
		}

		qui save __pll`parallelid'_dataset, replace
		drop _all
	//}

end

//in a subprogram because some of the outputopts may have the same name as -parallel- options
program pll_remove_replace
	syntax, pll_argopt(string) pll_outputopts(string)

	//parse the original options
	foreach pll_outputopt of local pll_outputopts{
		local pll_outputopts_syn "`pll_outputopts_syn' `pll_outputopt'(string)"
	}
	local 0 : copy local pll_argopt
	syntax [anything(equalok everything)][, `pll_outputopts_syn' *]

	//remove the ones were are subbing out
	c_local argopt `"`anything', `options'"'

	//some may be null so only sub out real ones
	foreach pll_outputopt of local pll_outputopts{
		if "`pll_outputopt'"!="" local outputopts "`outputopts' `pll_outputopt'"
	}
	c_local outputopts `outputopts'
	/* old method (doesn't account for syntax eliding blank ones
	//make version of argopt that doesn't have the options that will be swapped out
	local argopt_orig : copy local argopt
	foreach outputopt of local outputopts{
		local argopt = regexr(`"`argopt'"',"`outputopt'\([^\)]*\)","")
	}*/
end

//allow some or all files to not have been created
//If at least one file was made, even if it was empty, make a file
// (same save dynamics as user program)
program pll_collect
	syntax, folder(string) parallelid(string) outputopts(string) argopt_orig(string)
	preserve

	foreach outputopt of local outputopts{
		mata: st_local("val", strofreal(regexm(st_local("argopt_orig"),st_local("outputopt")+"\(([^)]+)\)")))
		if `val'{
			mata: st_local("outfile", regexs(1))
			//get rid of quotes
			local outfile `outfile'
			local emptyok ""
			drop _all
			forval i=1/$PLL_CLUSTERS{
				cap append using `"`folder'__pll`parallelid'_out_`outputopt'`=strofreal(`i',"%04.0f")'"'
				if (_rc==0) local emptyok "emptyok"
			}

			qui save `"`outfile'"', replace `emptyok'
		}
	}
end

////////////////////////////////////////////////////////////////////////////////
// MAIN PROGRAM
// There are things we need to possibly restore on exit
// - Data/S_FN (this is taken care of by -preserve-)
// - PWD (non-indented capture block)
// - Temporaliy changed number of clusters (non-indented capture block)
// - Turn off the timer (do within catch for capture blocks)
program def parallel_do, rclass
	version 11.0

	#delimit ;
	syntax anything(name=dofile equalok everything)
		[, by(string)
		Keep
		KEEPLast
		prefix
		Force
		PROGrams(string)
		Mata
		NOGlobals
		KEEPTiming
		Seeds(string)
		NOData
		Randtype(string)
		Timeout(integer 60)
		PROCessors(integer 0)
		argopt(string)
		KEEPUsing(string)
		SETparallelid(string)
		OUTputopts(string)
		DETerministicoutput
		];
	#delimit cr

	if length("$PLL_CLUSTERS") == 0 {
		di as err "You haven't set the number of clusters" _n "Please set it with: {cmd:parallel setclusters} {it:#}"
		exit 198
	}

	if `c(noisily)'==0 & "`programs'"!="" {
		di as err "If you want to pass programs in memory (using the option programs)" _n "then the program must be run noisily."
		exit 198
	}

	// Initial checks
	foreach opt in macrolist keep keeplast prefix force mata noglobals keeptiming nodata deterministicoutput {
		local `opt' = length("``opt''") > 0
	}

	/* Randtype */
	if ("`randtype'" == "") local randtype = "datetime"

	/* If no data parsing has to be done (because of no data!) */
	if (!(c(N)*c(k))) local nodata 1

	if (!`keeptiming') {
		timer clear 97
		timer clear 98
		timer clear 99
	}

	timer on 98

	// Deletes last parallel instance ran
	local last_id = cond("`r(pll_id)'"!="","`r(pll_id)'", "$LAST_PLL_ID")
	if (`keeplast' & length("`last_id'")) cap parallel_clean, e(`last_id') nologs

	if length("`by'") != 0 {
		local sortlist: sortedby
		local sorting = regexm("`sortlist'","^`by'")
	}
	else local sorting = 0

	// Gets the directory where to work at
	if (!`prefix') {

		/* First checks if the file exists */
		mata: parallel_normalizepath(`"`dofile'"',1)
		local pll_dir = "`filedir'"
		local dofile = "`filename'"
	}
	else local pll_dir = c(pwd)+"/"

	local initialdir = c(pwd)
	qui cd "`pll_dir'"
	capture noisily { //Start capture block for PWD & $PLL_CLUSTERS

	/* Creates a unique ID for the process and secures it */
	if ("`setparallelid'"=="") mata: parallel_sandbox(5)
	else local parallelid = "`setparallelid'"

	/* Generates database clusters */
	if (!`nodata'){
		local sfn = "$S_FN" // be able to "fake" restore this
		preserve

		parallel_spliter `by' , parallelid(`parallelid') sorting(`sorting') force(`force') keepusing(`keepusing') orig_cl_local(orig_PLL_CLUSTERS)
	}

	global LAST_PLL_ID = "`parallelid'"
	global LAST_PLL_DIR = "`pll_dir'"
	global LAST_PLL_N = $PLL_CLUSTERS

	/* Starts building the files */
	local work_around_no_cwd = 0
	quietly {

		/* Saves mata objects */
		if (`mata') {
			mata: mata mlib create l__pll`parallelid'_mlib, replace
			cap mata: mata mlib add l__pll`parallelid'_mlib *()
			cap which l__pll`parallelid'_mlib.mlib
			if _rc!=0{
				local work_around_no_cwd = 1
				noi di "Note: In order to pass mata objects to the clusters, they are saved to a temporary mlib file in the current directly."
				noi di "      Your ado-path doesn't contain the current directory."
				noi di "      We have added the current directory to the end of the ado-path for the clusters.."
			}
			cap mata: mata matsave __pll`parallelid'_mata.mmat *, replace
			if (`=_rc') local matasave = 0
			else local matasave = 1
		}
		else local matasave = 0
	}

	//see if we handle extra output streams
	if "`outputopts'"!=""{
		local argopt_orig : copy local argopt
		pll_remove_replace, pll_argopt(`"`argopt'"') pll_outputopts(`outputopts')
	}

	/* Writing the dofile */
	mata: st_local("errornum", strofreal(parallel_write_do(strtrim(`"`dofile' `argopt'"'), "`parallelid'", $PLL_CLUSTERS, `prefix', `matasave', !`noglobals', "`seeds'", "`randtype'", `nodata', "`pll_dir'", "`programs'", `processors',`work_around_no_cwd',"`outputopts'")))
	if (`errornum') {
		/* Removes the sandbox file (unprotect the files) */
		mata: parallel_sandbox(2, "`parallelid'")

		error `errornum'
	}

	timer off 98
	cap timer list
	if (r(t98) == .) local pll_t_setu = 0
	else local pll_t_setu = r(t98)

	/* Running the dofiles */
	timer on 99
	mata: st_local("nerrors", strofreal(parallel_run("`parallelid'",$PLL_CLUSTERS,`"$PLL_STATA_PATH"',`=`timeout'*1000', `deterministicoutput')))
	timer off 99

	cap timer list
	if (r(t99) == .) local pll_t_calc = 0
	else local pll_t_calc = r(t99)
	local pll_t_reps = r(nt99)

	timer on 97

	if ("`outputopts'"!=""  & !`nerrors'){
		pll_collect, folder("`pll_dir'") parallelid("`parallelid'") outputopts(`outputopts') argopt_orig(`"`argopt_orig'"')
	}

	// Paste the databases
	if (!`nodata' & !`nerrors') {
		parallel_fusion `parallelid', clusters($PLL_CLUSTERS) keepusing(`keepusing')
		cap drop _`parallelid'cut

		global S_FN = "`sfn'" // Restores original S_FN (file name) value
		restore, not
	}

	return scalar pll_n = $PLL_CLUSTERS

	/* Removes the sandbox file (unprotect the files) */
	if ("`setparallelid'" == "") {
		mata: parallel_sandbox(2, "`parallelid'")
		if (!`keep' & !`keeplast') parallel_clean, e("`parallelid'") nologs
	}

	} //End capture block
	if _rc {
		local orig_rc = _rc
		qui cd "`initialdir'"
		if "`orig_PLL_CLUSTERS'"!="" global PLL_CLUSTERS=`orig_PLL_CLUSTERS'
		cap timer off 97
		cap timer off 98
		cap timer off 99
		exit `orig_rc'
	}
	qui cd "`initialdir'"
	if "`orig_PLL_CLUSTERS'"!="" global PLL_CLUSTERS=`orig_PLL_CLUSTERS'

	timer off 97
	cap timer list
	if (r(t97) == .) local pll_t_fini = 0
	else local pll_t_fini = r(t97)

	qui timer list
	return local  pll_seeds="`pllseeds'"
	return scalar pll_errs = `nerrors'
	return local  pll_dir "`pll_dir'"
	return scalar pll_t_reps = `pll_t_reps'
	return scalar pll_t_setu = `pll_t_setu'
	return scalar pll_t_calc = `pll_t_calc'
	return scalar pll_t_fini = `pll_t_fini'
	return local pll_id = "`parallelid'"

	if `nerrors' {
		di as err "`nerrors' child processes encountered errors. Throwing last error."
		exit `pll_last_error'
	}
end

////////////////////////////////////////////////////////////////////////////////
// Cleans all files generated by parallel
// If you want to keep the logs, then specify -, nologs-
program def parallel_clean
	version 11.0
	syntax [, Event(string) All Force nologs]

	if (length("`event'") != 0 & length("`all'") != 0) {
		di as error `"invalid syntax: Using -pll_id- and -all- jointly is not allowed."'
		exit 198
	}
	local do_logs = ("`logs'"!="nologs")
	mata: parallel_clean("`event'", `=length("`all'") > 0', `=length("`force'") > 0', `do_logs')
end

////////////////////////////////////////////////////////////////////////////////
// Sets the number of clusters as a global macro
program parallel_setclusters
	version 11.0
	syntax anything(name=nclusters)  [, Force Statapath(string asis) Gateway(string) Includefile(string) procexec(int 2)]

	_assert inlist(`procexec',0,1,2), msg("procexec() must be 0, 1, or 2") rc(198)
	cap parallel_numprocessors
	local nproc = int(real("`r(numprocessors)'"))
	if "`nclusters'"=="default"{
		_assert `nproc'!=., msg("Couldn't determine number of available processors for default configuration.")
		local nclusters = max(floor(`nproc'*3/4),1)
	}
	else{
		local nclusters = int(real(`"`nclusters'"'))
		_assert (`nclusters'>0 & `nclusters'!=.),  msg(`"Not allowed: "#" Should be a positive number"') rc(109)
	}
	global USE_PROCEXEC = `procexec'

	local force = (length("`force'")>0)
	mata: parallel_setclusters(`nclusters', `force', `nproc')
	mata: st_local("error", strofreal(parallel_setstatapath(`"`statapath'"', `force')))
	_assert (!`error'), msg("Can not set Stata directory, try using -statapath()- option") rc(`error')

	if "`c(mode)'"=="batch" & "`c(os)'"=="Windows" & "$USE_PROCEXEC"=="0" {
		if `"`gateway'"'=="" local gateway "pll_gateway.sh"

		cap confirm file `"`gateway'"'
		_assert `=!_rc', msg(`"On Windows batch-mode with command-line process execution, requires a gateway file is required but not found (path: `c(pwd)')"') rc(`=_rc')
		global PLL_GATEWAY_FNAME `"`gateway'"'
	}

	if `"`includefile'"'!=""{
		cap confirm file `"`includefile'"'
		_assert `=!_rc', msg("Can not find the include file (`includefile').") msg(`=_rc')
		global PLL_INCLUDE_FILE `"`includefile'"'
	}
end

////////////////////////////////////////////////////////////////////////////////
// Exports

// Exports a copy of programs
program def program_export
	version 11.0
	syntax using/ [,Programlist(string) Inname(string)]

	mata: program_export("`using'", "`programlist'", "`inname'")
end

////////////////////////////////////////////////////////////////////////////////
// Appends the clusterized dataset
program def parallel_fusion
	version 11.0
	syntax anything(name=parallelid) , clusters(integer) [keepusing(string)]

	cap use "__pll`parallelid'_dta0001.dta", clear
	if (_rc){
		di as err "No dataset for instance 0001."
		exit _rc
	}
	local sortlist: sortedby

	forval i = 2/`clusters' {
		cap append using `"__pll`parallelid'_dta`=string(`i',"%04.0f")'.dta"'
		if (_rc){
			di as err "No dataset for instance `=string(`i',"%04.0f")'."
			exit _rc
		}
	}

	/* If it just used a set of variables */
	if (length("`keepusing'")) {
		qui merge 1:1 __pllnobs`parallelid' using __pll`parallelid'_dataset, keep(3) nogen
		drop __pllnobs`parallelid'
	}

	//restore the sort
	if "`sortlist'"!="" sort `sortlist'

end

////////////////////////////////////////////////////////////////////////////////
// Checks whether the user pressed break inside a loop
program def parallel_break
	version 11.0
	mata: parallel_break()
end

*! version 1.18.3 09jul2017
*! PARALLEL: Stata module for parallel computing
*! by George G. Vega [cre,aut], Brian Quistorff [ctb]
*!
*! Project website:
*!   https://github.com/gvegayon/parallel
*! Bug reports:
*!   https://github.com/gvegayon/parallel/issues
/*
Copyright (c) 2014  <George G. Vega>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

// Syntax parser
program def parallel
    version 11.0

	// Checks wether if is parallel prefix or not
	if  (regexm(`"`0'"', "^(do|clean|setclusters|break|version|append|printlog|viewlog|numprocessors)")) {
	/* If not prefix */
		parallel_`0'
	}
	else if (regexm(`"`0'"', "^(bs|sim)[,]?[\s ]?")) {
	/* Prefix bootstrap or simulate */
		local cmd = regexs(1)
		mata: st_local("0", regexr(st_local("0"), "^(bs|sim)", ""))
		gettoken x 0 : 0, parse(":") bind
		local 0 = regexr(`"`0'"', "^[:]", "")
		gettoken x options : x, parse(",") bind

		gettoken 0 argopt : 0, parse(",") bind
		parallel_`cmd' `0', argopt(`argopt') `options'
	}
	else if (regexm(`"`0'"',"^([,]?.*[:])")) {
	/* if prefix */
		gettoken x 0 : 0, parse(":") bind
		mata: st_local("0", regexr(st_local("0"), "^[:]", ""))
		// Gets the options (if these exists) of parallel
		gettoken x options : x, parse(",") bind

		gettoken 0 argopt : 0, parse(",") bind
		parallel_do `0', `options' prefix argopt(`argopt')
	}
	else {
		local val = regexm(`"`0'"', "^([a-zA-Z0-9_]*).*")
		local subcommand = regexs(1)
		di as result `"-`subcommand'-"' as error " invalid parallel subcommand" as text " ({stata h parallel:{it:help parallel}})"
		exit 198
	}
end

/*
 For Windows we use an environment variable that could be redefined by a parent
   (both a plus and minus). Alternative is to sum output from command:
    WMIC CPU Get DeviceID,NumberOfCores,NumberOfLogicalProcessors

 For *nix see http://stackoverflow.com/questions/6481005/ for alternatives.
   Note: 'nproc --all' does not return the right result on some cloud providers where systems are shared.
*/
program parallel_numprocessors, rclass
	if "`c(os)'"=="Windows" {
		local nproc : env NUMBER_OF_PROCESSORS
	}
	else {
		tempfile nproc_out
		! getconf _NPROCESSORS_ONLN > "`nproc_out'"
		tempname nproc_out_fhandle
		file open `nproc_out_fhandle' using "`nproc_out'", read text
		file read `nproc_out_fhandle' nproc
		local r_eof = `r(eof)'
		file close `nproc_out_fhandle'
		_assert `r_eof'==0, msg("Wasn't able to read output from system")
	}

	local nproc = int(real("`nproc'"))
	_assert "`nproc'"!=".", msg("Wasn't able to interpret output from system")

	di "Number of logical processors: `nproc'"
	return scalar numprocessors = `nproc'
end

/* Returns the version of parallel */
program def parallel_version, rclass
	version 11.0
	di as result "parallel" as text " Stata module for parallel computing"
	di as result "vers" as text " 1.18.3 09jul2017"
	di as result "auth" as text " George G. Vega [cre,aut], Brian Quistorff [ctb]"

	return local pll_vers = "1.18.3"
end

/* Take a look to logfiles */
program def parallel_printlog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)]
	parallel_checklog `pll_instance', e(`event') action(print)
end

program def parallel_viewlog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)]
	parallel_checklog `pll_instance', e(`event') action(view)
end

program def parallel_checklog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)] action(string)

	if ("`event'"=="") local event = "$LAST_PLL_ID"

	if ("`event'"!=".") {
		/* By default uses 1 */
		if ("`pll_instance'" == "") local pll_instance 1

		local pll_instance : di %04.0f `pll_instance'

		/* Does de file exists */
		local logname = "__pll`event'_do`pll_instance'.log"

		if (c(os) == "Windows") local logname = "`c(tmpdir)'`logname'"
		else local logname = "`c(tmpdir)'/`logname'"

		/* If the logfile does not exists, do nothing*/
		cap confirm file "`logname'"
		if (_rc) {
			di as result "No logfile for instance -`pll_instance'- of parallel process -`event'- found"
			exit
		}

		/* Showing the log in screen */
		if ("`action'"=="print") {
			di as result "{hline 80}"
			di as result %~80s "beginning of file -`logname'-"
			di as result "{hline 80}"
			type `"`logname'"'
			di as result "{hline 80}"
			di as result %~80s "end of file -`logname'-"
			di as result "{hline 80}"
		}
		else view `"`logname'"'
	}
	else {
		di as error "It seems that you haven't use -parallel- yet."
		exit 601
	}

end

////////////////////////////////////////////////////////////////////////////////
// Splits the dataset into clusters
program def parallel_spliter
	version 11.0
	syntax [namelist(name=xtstructure)] [,parallelid(string) sorting(integer 0) force(integer 0) keepusing(varlist) orig_cl_local(string)]
	//args xtstructure parallelid Sorting Force

	if length("$PLL_CLUSTERS") == 0 {
		di as err "Number of clusters not fixed." as text " Set the number of clusters with " _newline as err "-{cmd:parallel setclusters #}-"
		exit 198
	}

	//quietly {
		if (length("`xtstructure'")) {
			_assert `sorting' | `force', msg("Data not sorted") rc(5)

			/* Checking the data types */
			foreach var of varlist `xtstructure' {
				capture confirm string var `var'
				if (`=_rc') local numvars `numvars' `var'
				else local strvars `strvars' `var'
			}

			/* calculate max possible clusters */
			egen _`parallelid'grp = group(`xtstructure'), missing
			summ _`parallelid'grp, meanonly
			local max_n_cl = `r(max)'
		}
		else {
			local max_n_cl = _N
		}

		/* Do we have too many clusters? */
		if (`max_n_cl'<$PLL_CLUSTERS){
			c_local `orig_cl_local' ${PLL_CLUSTERS}
			global PLL_CLUSTERS = `max_n_cl'
			di "Small workload/num groups. Temporarily setting number of clusters to ${PLL_CLUSTERS}"
		}

		if (length("`xtstructure'")) {
			preserve
			keep _`parallelid'grp
			contract _`parallelid'grp
			sort _freq
			//Figuring out the mapping from groupID to cut that equalizes sizes is the "Partition Problem"
			//which is hard to solve exactly (it's NP-complete). A rough solution suffices here though.
			gen _`parallelid'cut = mod(_n, ${PLL_CLUSTERS}) + 1
			tempfile grp_to_cut_map
			qui save `"`grp_to_cut_map'"'
			restore

			qui merge m:1 _`parallelid'grp using `"`grp_to_cut_map'"', keepusing(_`parallelid'cut) nogenerate
			drop _`parallelid'grp
			sort `xtstructure'
		}
		else {
			gen _`parallelid'cut = ceil(_n*${PLL_CLUSTERS}/_N) //each of size _N/$PLL_CLUSTERS
		}

		if (length("`keepusing'")) {
			keep _`parallelid'cut `keepusing'
			gen __pllnobs`parallelid' = _n
		}

		qui save __pll`parallelid'_dataset, replace
		drop _all
	//}

end

//in a subprogram because some of the outputopts may have the same name as -parallel- options
program pll_remove_replace
	syntax, pll_argopt(string) pll_outputopts(string)

	//parse the original options
	foreach pll_outputopt of local pll_outputopts{
		local pll_outputopts_syn "`pll_outputopts_syn' `pll_outputopt'(string)"
	}
	local 0 : copy local pll_argopt
	syntax [anything(equalok everything)][, `pll_outputopts_syn' *]

	//remove the ones were are subbing out
	c_local argopt `"`anything', `options'"'

	//some may be null so only sub out real ones
	foreach pll_outputopt of local pll_outputopts{
		if "`pll_outputopt'"!="" local outputopts "`outputopts' `pll_outputopt'"
	}
	c_local outputopts `outputopts'
	/* old method (doesn't account for syntax eliding blank ones
	//make version of argopt that doesn't have the options that will be swapped out
	local argopt_orig : copy local argopt
	foreach outputopt of local outputopts{
		local argopt = regexr(`"`argopt'"',"`outputopt'\([^\)]*\)","")
	}*/
end

//allow some or all files to not have been created
//If at least one file was made, even if it was empty, make a file
// (same save dynamics as user program)
program pll_collect
	syntax, folder(string) parallelid(string) outputopts(string) argopt_orig(string)
	preserve

	foreach outputopt of local outputopts{
		mata: st_local("val", strofreal(regexm(st_local("argopt_orig"),st_local("outputopt")+"\(([^)]+)\)")))
		if `val'{
			mata: st_local("outfile", regexs(1))
			//get rid of quotes
			local outfile `outfile'
			local emptyok ""
			drop _all
			forval i=1/$PLL_CLUSTERS{
				cap append using `"`folder'__pll`parallelid'_out_`outputopt'`=strofreal(`i',"%04.0f")'"'
				if (_rc==0) local emptyok "emptyok"
			}

			qui save `"`outfile'"', replace `emptyok'
		}
	}
end

////////////////////////////////////////////////////////////////////////////////
// MAIN PROGRAM
// There are things we need to possibly restore on exit
// - Data/S_FN (this is taken care of by -preserve-)
// - PWD (non-indented capture block)
// - Temporaliy changed number of clusters (non-indented capture block)
// - Turn off the timer (do within catch for capture blocks)
program def parallel_do, rclass
	version 11.0

	#delimit ;
	syntax anything(name=dofile equalok everything)
		[, by(string)
		Keep
		KEEPLast
		prefix
		Force
		PROGrams(string)
		Mata
		NOGlobals
		KEEPTiming
		Seeds(string)
		NOData
		Randtype(string)
		Timeout(integer 60)
		PROCessors(integer 0)
		argopt(string)
		KEEPUsing(string)
		SETparallelid(string)
		OUTputopts(string)
		DETerministicoutput
		];
	#delimit cr

	if length("$PLL_CLUSTERS") == 0 {
		di as err "You haven't set the number of clusters" _n "Please set it with: {cmd:parallel setclusters} {it:#}"
		exit 198
	}

	if `c(noisily)'==0 & "`programs'"!="" {
		di as err "If you want to pass programs in memory (using the option programs)" _n "then the program must be run noisily."
		exit 198
	}

	// Initial checks
	foreach opt in macrolist keep keeplast prefix force mata noglobals keeptiming nodata deterministicoutput {
		local `opt' = length("``opt''") > 0
	}

	/* Randtype */
	if ("`randtype'" == "") local randtype = "datetime"

	/* If no data parsing has to be done (because of no data!) */
	if (!(c(N)*c(k))) local nodata 1

	if (!`keeptiming') {
		timer clear 97
		timer clear 98
		timer clear 99
	}

	timer on 98

	// Deletes last parallel instance ran
	local last_id = cond("`r(pll_id)'"!="","`r(pll_id)'", "$LAST_PLL_ID")
	if (`keeplast' & length("`last_id'")) cap parallel_clean, e(`last_id') nologs

	if length("`by'") != 0 {
		local sortlist: sortedby
		local sorting = regexm("`sortlist'","^`by'")
	}
	else local sorting = 0

	// Gets the directory where to work at
	if (!`prefix') {

		/* First checks if the file exists */
		mata: parallel_normalizepath(`"`dofile'"',1)
		local pll_dir = "`filedir'"
		local dofile = "`filename'"
	}
	else local pll_dir = c(pwd)+"/"

	local initialdir = c(pwd)
	qui cd "`pll_dir'"
	capture noisily { //Start capture block for PWD & $PLL_CLUSTERS

	/* Creates a unique ID for the process and secures it */
	if ("`setparallelid'"=="") mata: parallel_sandbox(5)
	else local parallelid = "`setparallelid'"

	/* Generates database clusters */
	if (!`nodata'){
		local sfn = "$S_FN" // be able to "fake" restore this
		preserve

		parallel_spliter `by' , parallelid(`parallelid') sorting(`sorting') force(`force') keepusing(`keepusing') orig_cl_local(orig_PLL_CLUSTERS)
	}

	global LAST_PLL_ID = "`parallelid'"
	global LAST_PLL_DIR = "`pll_dir'"
	global LAST_PLL_N = $PLL_CLUSTERS

	/* Starts building the files */
	local work_around_no_cwd = 0
	quietly {

		/* Saves mata objects */
		if (`mata') {
			mata: mata mlib create l__pll`parallelid'_mlib, replace
			cap mata: mata mlib add l__pll`parallelid'_mlib *()
			cap which l__pll`parallelid'_mlib.mlib
			if _rc!=0{
				local work_around_no_cwd = 1
				noi di "Note: In order to pass mata objects to the clusters, they are saved to a temporary mlib file in the current directly."
				noi di "      Your ado-path doesn't contain the current directory."
				noi di "      We have added the current directory to the end of the ado-path for the clusters.."
			}
			cap mata: mata matsave __pll`parallelid'_mata.mmat *, replace
			if (`=_rc') local matasave = 0
			else local matasave = 1
		}
		else local matasave = 0
	}

	//see if we handle extra output streams
	if "`outputopts'"!=""{
		local argopt_orig : copy local argopt
		pll_remove_replace, pll_argopt(`"`argopt'"') pll_outputopts(`outputopts')
	}

	/* Writing the dofile */
	mata: st_local("errornum", strofreal(parallel_write_do(strtrim(`"`dofile' `argopt'"'), "`parallelid'", $PLL_CLUSTERS, `prefix', `matasave', !`noglobals', "`seeds'", "`randtype'", `nodata', "`pll_dir'", "`programs'", `processors',`work_around_no_cwd',"`outputopts'")))
	if (`errornum') {
		/* Removes the sandbox file (unprotect the files) */
		mata: parallel_sandbox(2, "`parallelid'")

		error `errornum'
	}

	timer off 98
	cap timer list
	if (r(t98) == .) local pll_t_setu = 0
	else local pll_t_setu = r(t98)

	/* Running the dofiles */
	timer on 99
	mata: st_local("nerrors", strofreal(parallel_run("`parallelid'",$PLL_CLUSTERS,`"$PLL_STATA_PATH"',`=`timeout'*1000', `deterministicoutput')))
	timer off 99

	cap timer list
	if (r(t99) == .) local pll_t_calc = 0
	else local pll_t_calc = r(t99)
	local pll_t_reps = r(nt99)

	timer on 97

	if ("`outputopts'"!=""  & !`nerrors'){
		pll_collect, folder("`pll_dir'") parallelid("`parallelid'") outputopts(`outputopts') argopt_orig(`"`argopt_orig'"')
	}

	// Paste the databases
	if (!`nodata' & !`nerrors') {
		parallel_fusion `parallelid', clusters($PLL_CLUSTERS) keepusing(`keepusing')
		cap drop _`parallelid'cut

		global S_FN = "`sfn'" // Restores original S_FN (file name) value
		restore, not
	}

	return scalar pll_n = $PLL_CLUSTERS

	/* Removes the sandbox file (unprotect the files) */
	if ("`setparallelid'" == "") {
		mata: parallel_sandbox(2, "`parallelid'")
		if (!`keep' & !`keeplast') parallel_clean, e("`parallelid'") nologs
	}

	} //End capture block
	if _rc {
		local orig_rc = _rc
		qui cd "`initialdir'"
		if "`orig_PLL_CLUSTERS'"!="" global PLL_CLUSTERS=`orig_PLL_CLUSTERS'
		cap timer off 97
		cap timer off 98
		cap timer off 99
		exit `orig_rc'
	}
	qui cd "`initialdir'"
	if "`orig_PLL_CLUSTERS'"!="" global PLL_CLUSTERS=`orig_PLL_CLUSTERS'

	timer off 97
	cap timer list
	if (r(t97) == .) local pll_t_fini = 0
	else local pll_t_fini = r(t97)

	qui timer list
	return local  pll_seeds="`pllseeds'"
	return scalar pll_errs = `nerrors'
	return local  pll_dir "`pll_dir'"
	return scalar pll_t_reps = `pll_t_reps'
	return scalar pll_t_setu = `pll_t_setu'
	return scalar pll_t_calc = `pll_t_calc'
	return scalar pll_t_fini = `pll_t_fini'
	return local pll_id = "`parallelid'"

	if `nerrors' {
		di as err "`nerrors' child processes encountered errors. Throwing last error."
		exit `pll_last_error'
	}
end

////////////////////////////////////////////////////////////////////////////////
// Cleans all files generated by parallel
// If you want to keep the logs, then specify -, nologs-
program def parallel_clean
	version 11.0
	syntax [, Event(string) All Force nologs]

	if (length("`event'") != 0 & length("`all'") != 0) {
		di as error `"invalid syntax: Using -pll_id- and -all- jointly is not allowed."'
		exit 198
	}
	local do_logs = ("`logs'"!="nologs")
	mata: parallel_clean("`event'", `=length("`all'") > 0', `=length("`force'") > 0', `do_logs')
end

////////////////////////////////////////////////////////////////////////////////
// Sets the number of clusters as a global macro
program parallel_setclusters
	version 11.0
	syntax anything(name=nclusters)  [, Force Statapath(string asis) Gateway(string) Includefile(string) procexec(int 2)]

	_assert inlist(`procexec',0,1,2), msg("procexec() must be 0, 1, or 2") rc(198)
	cap parallel_numprocessors
	local nproc = int(real("`r(numprocessors)'"))
	if "`nclusters'"=="default"{
		_assert `nproc'!=., msg("Couldn't determine number of available processors for default configuration.")
		local nclusters = max(floor(`nproc'*3/4),1)
	}
	else{
		local nclusters = int(real(`"`nclusters'"'))
		_assert (`nclusters'>0 & `nclusters'!=.),  msg(`"Not allowed: "#" Should be a positive number"') rc(109)
	}
	global USE_PROCEXEC = `procexec'

	local force = (length("`force'")>0)
	mata: parallel_setclusters(`nclusters', `force', `nproc')
	mata: st_local("error", strofreal(parallel_setstatapath(`"`statapath'"', `force')))
	_assert (!`error'), msg("Can not set Stata directory, try using -statapath()- option") rc(`error')

	if "`c(mode)'"=="batch" & "`c(os)'"=="Windows" & "$USE_PROCEXEC"=="0" {
		if `"`gateway'"'=="" local gateway "pll_gateway.sh"

		cap confirm file `"`gateway'"'
		_assert `=!_rc', msg(`"On Windows batch-mode with command-line process execution, requires a gateway file is required but not found (path: `c(pwd)')"') rc(`=_rc')
		global PLL_GATEWAY_FNAME `"`gateway'"'
	}

	if `"`includefile'"'!=""{
		cap confirm file `"`includefile'"'
		_assert `=!_rc', msg("Can not find the include file (`includefile').") msg(`=_rc')
		global PLL_INCLUDE_FILE `"`includefile'"'
	}
end

////////////////////////////////////////////////////////////////////////////////
// Exports

// Exports a copy of programs
program def program_export
	version 11.0
	syntax using/ [,Programlist(string) Inname(string)]

	mata: program_export("`using'", "`programlist'", "`inname'")
end

////////////////////////////////////////////////////////////////////////////////
// Appends the clusterized dataset
program def parallel_fusion
	version 11.0
	syntax anything(name=parallelid) , clusters(integer) [keepusing(string)]

	cap use "__pll`parallelid'_dta0001.dta", clear
	if (_rc){
		di as err "No dataset for instance 0001."
		exit _rc
	}
	local sortlist: sortedby

	forval i = 2/`clusters' {
		cap append using `"__pll`parallelid'_dta`=string(`i',"%04.0f")'.dta"'
		if (_rc){
			di as err "No dataset for instance `=string(`i',"%04.0f")'."
			exit _rc
		}
	}

	/* If it just used a set of variables */
	if (length("`keepusing'")) {
		qui merge 1:1 __pllnobs`parallelid' using __pll`parallelid'_dataset, keep(3) nogen
		drop __pllnobs`parallelid'
	}

	//restore the sort
	if "`sortlist'"!="" sort `sortlist'

end

////////////////////////////////////////////////////////////////////////////////
// Checks whether the user pressed break inside a loop
program def parallel_break
	version 11.0
	mata: parallel_break()
end


added age var

added age var

program make_merge

// balanced dataset of wikihandles
insheet using ${stash}tmp_word.csv, clear

// rename variables
renamevar

merge m:1 wikihandle using ${stash}bbk_master, nogen

merge 1:1 wikihandle year using ${stash}traf, nogen

//xtset the data
egen id = group(wikihandle)
xtset id year

// generate key DD vars
gen post = (year>2008)
gen treat = (debutyear < 1964)

// general log-vals
local outcomes "img text bd traf"
foreach x in `outcomes'{
    gen ln`x' = ln(`x'+1)
}

// keep relevant sample
drop if debut < 1944 | debut > 1984

// generate quality quartiles
genqual

// generate timevarying controls
gen decade = round(debut, 10)
egen dy = group(decade year)
egen qy = group(quality year)

drop if year == 2013
replace text = text/1000

// generate page age
bysort playerid: gen tmp = (text>0)*year
replace tmp = 2012 if tmp==0
bysort playerid: egen page_create_yr = min(tmp)
drop tmp

gen age = year - page_create_yr

keep wikihandle text img bd playerid ln* post treat quality qy debut traf year isbaseball id finalyear final firstall numa size playername everinducted age

save ${stash}master, replace

end

// helper programs

program renamevar

rename v1 wikihandle
rename v2 year
rename v3 user
rename v4 revid
rename v5 size
rename v6 text
rename v7 img
rename v8 bd

end

program genqual

gen quality = .
_pctile numallstar, p(25, 50, 75)

replace qual = 1 if numallstar <= `r(r1)' & isb == 1
replace qual = 2 if numallstar > `r(r1)'  & numall <=`r(r2)' & isb == 1
replace qual = 3 if numallstar > `r(r2)'  & numall <=`r(r3)' & isb == 1
replace qual = 4 if numallstar > `r(r3)' & isb == 1

_pctile minutesrank, p(33.333, 66.667 90)

replace qual = 1 if minutesrank <= `r(r1)' & isb == 0
replace qual = 2 if minutesrank > `r(r1)' & minutesrank <=`r(r2)' & isb == 0
replace qual = 3 if minutesrank > `r(r2)' & minutesrank <=`r(r3)' & isb == 0
replace qual = 4 if minutesrank > `r(r3)' & isb == 0

end

program make_merge

// balanced dataset of wikihandles
insheet using ${stash}tmp_word.csv, clear

// rename variables
renamevar

merge m:1 wikihandle using ${stash}bbk_master, nogen

merge 1:1 wikihandle year using ${stash}traf, nogen

//xtset the data
egen id = group(wikihandle)
xtset id year

// generate key DD vars
gen post = (year>2008)
gen treat = (debutyear < 1964)

// general log-vals
local outcomes "img text bd traf"
foreach x in `outcomes'{
    gen ln`x' = ln(`x'+1)
}

// keep relevant sample
drop if debut < 1944 | debut > 1984

// generate quality quartiles
genqual

// generate timevarying controls
gen decade = round(debut, 10)
egen dy = group(decade year)
egen qy = group(quality year)

drop if year == 2013
replace text = text/1000

// generate page age
bysort playerid: gen tmp = (text>0)*year
replace tmp = 2012 if tmp==0
bysort playerid: egen page_create_yr = min(tmp)
drop tmp

gen age = year - page_create_yr

keep wikihandle text img bd playerid ln* post treat quality qy debut traf year isbaseball id finalyear final firstall numa size playername everinducted

save ${stash}master, replace

end

// helper programs

program renamevar

rename v1 wikihandle
rename v2 year
rename v3 user
rename v4 revid
rename v5 size
rename v6 text
rename v7 img
rename v8 bd

end

program genqual

gen quality = .
_pctile numallstar, p(25, 50, 75)

replace qual = 1 if numallstar <= `r(r1)' & isb == 1
replace qual = 2 if numallstar > `r(r1)'  & numall <=`r(r2)' & isb == 1
replace qual = 3 if numallstar > `r(r2)'  & numall <=`r(r3)' & isb == 1
replace qual = 4 if numallstar > `r(r3)' & isb == 1

_pctile minutesrank, p(33.333, 66.667 90)

replace qual = 1 if minutesrank <= `r(r1)' & isb == 0
replace qual = 2 if minutesrank > `r(r1)' & minutesrank <=`r(r2)' & isb == 0
replace qual = 3 if minutesrank > `r(r2)' & minutesrank <=`r(r3)' & isb == 0
replace qual = 4 if minutesrank > `r(r3)' & isb == 0

end


all.do

all.do

exec 1>&2
agda-pkg -i. -pstdlib -pcrypto-agda/agda-nplib -pcrypto-agda/explore crypto-agda.agda


[INFRA] Updated wave.do file with newly added top signals

onerror {resume}
quietly WaveActivateNextPane {} 0
add wave -noupdate -expand -group {clock and reset} -radix hexadecimal /top_tb/T1/clk
add wave -noupdate -expand -group {clock and reset} -radix hexadecimal /top_tb/T1/reset
add wave -noupdate -expand -group pc -radix hexadecimal /top_tb/T1/curr_pc_top
add wave -noupdate -expand -group pc -radix hexadecimal /top_tb/T1/next_pc_top
add wave -noupdate -expand -group pc -radix hexadecimal /top_tb/T1/next_seq_pc_top
add wave -noupdate -expand -group pc -radix hexadecimal /top_tb/T1/next_jmp_pc_top
add wave -noupdate -expand -group pc -radix hexadecimal /top_tb/T1/next_beq_pc_top
add wave -noupdate -expand -group pc -radix hexadecimal /top_tb/T1/next_brn_eq_pc_top
add wave -noupdate -expand -group instr -radix hexadecimal /top_tb/T1/instr_top
add wave -noupdate -expand -group instr -radix hexadecimal /top_tb/T1/funct_top
add wave -noupdate -expand -group instr -radix hexadecimal /top_tb/T1/op_top
add wave -noupdate -expand -group instr /top_tb/T1/sign_ext_top
add wave -noupdate -expand -group instr /top_tb/T1/is_r_type_top
add wave -noupdate -expand -group instr /top_tb/T1/is_i_type_top
add wave -noupdate -expand -group instr /top_tb/T1/is_j_type_top
add wave -noupdate -expand -group control /top_tb/T1/reg_src_top
add wave -noupdate -expand -group control -radix hexadecimal /top_tb/T1/reg_dst_top
add wave -noupdate -expand -group control -radix hexadecimal /top_tb/T1/jump_top
add wave -noupdate -expand -group control -radix hexadecimal /top_tb/T1/branch_top
add wave -noupdate -expand -group control -radix hexadecimal /top_tb/T1/mem_read_top
add wave -noupdate -expand -group control -radix hexadecimal /top_tb/T1/mem_to_reg_top
add wave -noupdate -expand -group control -radix hexadecimal /top_tb/T1/alu_op_top
add wave -noupdate -expand -group control -radix hexadecimal /top_tb/T1/mem_wr_top
add wave -noupdate -expand -group control -radix hexadecimal /top_tb/T1/reg_wr_top
add wave -noupdate -expand -group control -radix hexadecimal /top_tb/T1/alu_src_top
add wave -noupdate -expand -group alu -radix hexadecimal /top_tb/T1/r_data_p1_top
add wave -noupdate -expand -group alu -radix hexadecimal /top_tb/T1/r_data_p2_top
add wave -noupdate -expand -group alu /top_tb/T1/n_top
add wave -noupdate -expand -group alu -radix hexadecimal /top_tb/T1/res_alu_top
add wave -noupdate -expand -group rf -radix hexadecimal /top_tb/T1/rd_top
add wave -noupdate -expand -group rf -radix hexadecimal /top_tb/T1/rs_top
add wave -noupdate -expand -group rf -radix hexadecimal /top_tb/T1/rt_top
add wave -noupdate -expand -group rf -radix hexadecimal /top_tb/T1/r_data_p1_top
add wave -noupdate -expand -group rf -radix hexadecimal /top_tb/T1/r_data_p2_rf_top
add wave -noupdate -expand -group rf -radix hexadecimal /top_tb/T1/wr_data_rf_top
add wave -noupdate -expand -group rf /top_tb/T1/rs_dec_top
add wave -noupdate -expand -group rf -radix hexadecimal /top_tb/T1/rd_dec_top
add wave -noupdate -radix hexadecimal /top_tb/T1/sign_imm_top
add wave -noupdate -radix hexadecimal /top_tb/T1/target_top
add wave -noupdate -group {alu block} -radix hexadecimal /top_tb/T1/A1/opr_a_alu_i
add wave -noupdate -group {alu block} -radix hexadecimal /top_tb/T1/A1/opr_b_alu_i
add wave -noupdate -group {alu block} -radix hexadecimal /top_tb/T1/A1/op_alu_i
add wave -noupdate -group {alu block} -radix hexadecimal /top_tb/T1/A1/cin_alu
add wave -noupdate -group {alu block} -radix hexadecimal /top_tb/T1/A1/opr_b_negated_alu
add wave -noupdate -group {alu block} -radix hexadecimal /top_tb/T1/A1/adder_out_alu
add wave -noupdate -group {alu block} -radix hexadecimal /top_tb/T1/A1/carry_out_alu
add wave -noupdate -group {alu block} -radix hexadecimal /top_tb/T1/A1/shifter_out_alu
add wave -noupdate -group {alu block} -radix hexadecimal /top_tb/T1/A1/logical_out_alu
add wave -noupdate -group {alu block} -radix hexadecimal /top_tb/T1/A1/res_alu_o
add wave -noupdate -group {alu block} /top_tb/T1/A1/n_alu_o
add wave -noupdate -group {alu block} -radix hexadecimal /top_tb/T1/A1/z_alu_o
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/alu_op_ctl_o
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/alu_src_ctl_o
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/branch_ctl_o
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/controls
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/instr_funct_ctl_i
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/instr_op_ctl_i
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/jump_ctl_o
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/mem_read_ctl_o
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/mem_to_reg_ctl_o
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/mem_wr_ctl_o
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/reg_dst_ctl_o
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/reg_wr_ctl_o
add wave -noupdate -group {control block} /top_tb/T1/C1/sign_ext_ctl_o
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/clk
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/r_data_p1
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/r_data_p1_rf_o
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/r_data_p2
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/r_data_p2_rf_o
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/r_reg_p1_rf_i
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/r_reg_p2_rf_i
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/reg_file
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/reset
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/w_data_rf_i
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/w_en_rf_i
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/w_reg_rf_i
TreeUpdate [SetDefaultTree]
WaveRestoreCursors {{Cursor 1} {50 ps} 0}
quietly wave cursor active 1
configure wave -namecolwidth 282
configure wave -valuecolwidth 187
configure wave -justifyvalue left
configure wave -signalnamewidth 0
configure wave -snapdistance 10
configure wave -datasetprefix 0
configure wave -rowmargin 4
configure wave -childrowmargin 2
configure wave -gridoffset 0
configure wave -gridperiod 1
configure wave -griddelta 40
configure wave -timeline 0
configure wave -timelineunits ps
update
WaveRestoreZoom {96 ps} {206 ps}

onerror {resume}
quietly WaveActivateNextPane {} 0
add wave -noupdate -expand -group {clock and reset} -radix hexadecimal /top_tb/T1/clk
add wave -noupdate -expand -group {clock and reset} -radix hexadecimal /top_tb/T1/reset
add wave -noupdate -expand -group pc -radix hexadecimal /top_tb/T1/curr_pc_top
add wave -noupdate -expand -group pc -radix hexadecimal /top_tb/T1/next_pc_top
add wave -noupdate -expand -group pc -radix hexadecimal /top_tb/T1/next_seq_pc_top
add wave -noupdate -expand -group pc -radix hexadecimal /top_tb/T1/next_jmp_pc_top
add wave -noupdate -expand -group pc -radix hexadecimal /top_tb/T1/next_beq_pc_top
add wave -noupdate -expand -group pc -radix hexadecimal /top_tb/T1/next_brn_eq_pc_top
add wave -noupdate -expand -group instr -radix hexadecimal /top_tb/T1/instr_top
add wave -noupdate -expand -group instr -radix hexadecimal /top_tb/T1/funct_top
add wave -noupdate -expand -group instr -radix hexadecimal /top_tb/T1/op_top
add wave -noupdate -expand -group instr /top_tb/T1/sign_ext_top
add wave -noupdate -expand -group instr /top_tb/T1/is_r_type_top
add wave -noupdate -expand -group instr /top_tb/T1/is_i_type_top
add wave -noupdate -expand -group instr /top_tb/T1/is_j_type_top
add wave -noupdate -expand -group control -radix hexadecimal /top_tb/T1/reg_dst_top
add wave -noupdate -expand -group control -radix hexadecimal /top_tb/T1/jump_top
add wave -noupdate -expand -group control -radix hexadecimal /top_tb/T1/branch_top
add wave -noupdate -expand -group control -radix hexadecimal /top_tb/T1/mem_read_top
add wave -noupdate -expand -group control -radix hexadecimal /top_tb/T1/mem_to_reg_top
add wave -noupdate -expand -group control -radix hexadecimal /top_tb/T1/alu_op_top
add wave -noupdate -expand -group control -radix hexadecimal /top_tb/T1/mem_wr_top
add wave -noupdate -expand -group control -radix hexadecimal /top_tb/T1/reg_wr_top
add wave -noupdate -expand -group control -radix hexadecimal /top_tb/T1/alu_src_top
add wave -noupdate -expand -group alu -radix hexadecimal /top_tb/T1/r_data_p1_top
add wave -noupdate -expand -group alu -radix hexadecimal /top_tb/T1/r_data_p2_top
add wave -noupdate -expand -group alu /top_tb/T1/n_top
add wave -noupdate -expand -group alu -radix hexadecimal /top_tb/T1/res_alu_top
add wave -noupdate -expand -group rf -radix hexadecimal /top_tb/T1/rd_top
add wave -noupdate -expand -group rf -radix hexadecimal /top_tb/T1/rs_top
add wave -noupdate -expand -group rf -radix hexadecimal /top_tb/T1/rt_top
add wave -noupdate -expand -group rf -radix hexadecimal /top_tb/T1/r_data_p1_top
add wave -noupdate -expand -group rf -radix hexadecimal /top_tb/T1/r_data_p2_rf_top
add wave -noupdate -expand -group rf -radix hexadecimal /top_tb/T1/wr_data_rf_top
add wave -noupdate -expand -group rf -radix hexadecimal /top_tb/T1/rd_dec_top
add wave -noupdate -radix hexadecimal /top_tb/T1/sign_imm_top
add wave -noupdate -radix hexadecimal /top_tb/T1/target_top
add wave -noupdate -group {alu block} -radix hexadecimal /top_tb/T1/A1/opr_a_alu_i
add wave -noupdate -group {alu block} -radix hexadecimal /top_tb/T1/A1/opr_b_alu_i
add wave -noupdate -group {alu block} -radix hexadecimal /top_tb/T1/A1/op_alu_i
add wave -noupdate -group {alu block} -radix hexadecimal /top_tb/T1/A1/cin_alu
add wave -noupdate -group {alu block} -radix hexadecimal /top_tb/T1/A1/opr_b_negated_alu
add wave -noupdate -group {alu block} -radix hexadecimal /top_tb/T1/A1/adder_out_alu
add wave -noupdate -group {alu block} -radix hexadecimal /top_tb/T1/A1/carry_out_alu
add wave -noupdate -group {alu block} -radix hexadecimal /top_tb/T1/A1/shifter_out_alu
add wave -noupdate -group {alu block} -radix hexadecimal /top_tb/T1/A1/logical_out_alu
add wave -noupdate -group {alu block} -radix hexadecimal /top_tb/T1/A1/res_alu_o
add wave -noupdate -group {alu block} /top_tb/T1/A1/n_alu_o
add wave -noupdate -group {alu block} -radix hexadecimal /top_tb/T1/A1/z_alu_o
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/alu_op_ctl_o
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/alu_src_ctl_o
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/branch_ctl_o
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/controls
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/instr_funct_ctl_i
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/instr_op_ctl_i
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/jump_ctl_o
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/mem_read_ctl_o
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/mem_to_reg_ctl_o
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/mem_wr_ctl_o
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/reg_dst_ctl_o
add wave -noupdate -group {control block} -radix hexadecimal /top_tb/T1/C1/reg_wr_ctl_o
add wave -noupdate -group {control block} /top_tb/T1/C1/sign_ext_ctl_o
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/clk
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/r_data_p1
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/r_data_p1_rf_o
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/r_data_p2
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/r_data_p2_rf_o
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/r_reg_p1_rf_i
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/r_reg_p2_rf_i
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/reg_file
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/reset
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/w_data_rf_i
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/w_en_rf_i
add wave -noupdate -group {rf block} -radix hexadecimal /top_tb/T1/R1/w_reg_rf_i
TreeUpdate [SetDefaultTree]
WaveRestoreCursors {{Cursor 1} {50 ps} 0}
quietly wave cursor active 1
configure wave -namecolwidth 282
configure wave -valuecolwidth 187
configure wave -justifyvalue left
configure wave -signalnamewidth 0
configure wave -snapdistance 10
configure wave -datasetprefix 0
configure wave -rowmargin 4
configure wave -childrowmargin 2
configure wave -gridoffset 0
configure wave -gridperiod 1
configure wave -griddelta 40
configure wave -timeline 0
configure wave -timelineunits ps
update
WaveRestoreZoom {96 ps} {206 ps}


iegraph : help edits to the help file

iegraph : help edits to the help file

{smcl}
{* 26 Dec 2016}{...}
{hline}
help for {hi:iegraph}
{hline}

{title:Title}

{phang2}{cmdab:iegraph} {hline 2} Generates graphs based on regressions with treatment dummies common in impact evaluations.

{title:Syntax}

{phang2}
{cmdab:iegraph} {varlist}
, [ {cmdab:ti:tle(}{it:string}{cmd:)} {cmdab:varl:abels}
{cmdab:save(}{it:string}{cmd:)} {cmdab:grey:scale} {cmdab:yzero}
{cmd:noconfbars} {cmdab:confbarsnone(}{it:varlist}{cmd:)}
{it:any towway graph options}
]

{marker opts}{...}
{synoptset 22}{...}
{synopthdr:options}
{synoptline}
{synopt :{cmdab:ti:tle(}{it:string}{cmd:)}} Manually sets the title of the graph.{p_end}
{synopt :{cmdab:varl:abels}} Uses variable labels for legends instead of variable names.{p_end}
{synopt :{cmdab:save(}{it:string}{cmd:)}} Sets the filename and the directory to which the graph will be set/exported.{p_end}
{synopt :{cmdab:grey:scale}} Uses greyscales for the bars instead of colors.{p_end}
{synopt :{cmdab:yzero}} Forces y-axis on the graph to start at 0.{p_end}
{synopt :{cmd:noconfbars}} Removes the confidence interval bars from graphs for all treatments.{p_end}
{synopt :{cmdab:confbarsnone(}{it:varlist}{cmd:)}} Removes confidence interval bars from only the {it:varlist} listed.{p_end}
{synoptline}

{pstd}Any towway graph options that can be used with normal twoway graph commands can
	also be used. If any of these commands conflict with any of the built in options,
	then the user specified settings have precedence. See example 2 for details.{p_end}

{marker desc}
{title:Description}

{pstd}{cmdab:iegraph} This command creates bar graphs on the based on the coefficients
	of treatment dummies in regression results. This command is developed for reading
	stored results of from two types of impact evaluation regression models, but there
	are countless of other examples where the command also can be used. {cmd:iegraph} must
	be used immediately after running the regression or as long as the regression result is
	still stored in or restored to Stata's {help ereturn} results.

{pstd}{bf:Model 1: OLS with Treatment Dummies}{break}The most typical impact evaluation regression is
	to have the outcome variable as the dependent variable and one dummy for each
	treatment arm where control is the omitted category. These regressions can also include
	covariates, fixed effects etc., but as long as the treatment status is defined by
	mutually exclusive dummy variables. See especially example 1 and 2 below. This command
	works with any number of treatment arms but works best from two arms (treatment
	and control) to five treatment arms (4 different treatments and control). More
	arms than that may result in a still correct but perhaps cluttered graph.

{pstd}{bf:Model 2: Difference-in-Differences}{break}Another typical regression model in impact
	evaluations are difference-in-difference (Diff-in-Diff) models with two treatment arms (treatment
	and control) and two time periods. If the Diff-in-Diff regression is specified as having the
	outcome variable as the dependent variable and three dummy variables (time, treatment
	and timeXtreatment) as the independent variables, then this command will produce a nice
	graph. Controls, treatment effects etc. may be added to the regression model. See especially example 3.

{pstd}{bf:Graph Output}{break}The graph generated by this command is created using the following values. The
	control bar is the mean of the outcome variable for the control group. It is not
	the constant from the regression as those are not identical if, for example, fixed effects
	and covariates were used. For each treatment group the bar is the sum of the value
	of the control bar and the beta coefficient in the regression of the corresponding
	treatment dummy. The confidence intervals are calculated from the variance in the
	beta coefficients in the regression.{p_end}

{pstd}The graph also includes the N for each treatment arm in the regression and uses
	that value as labels on the x-axis. Stars are added to this value if the corresponding
	coefficient is statistically different from zero in the regression{p_end}

{marker optslong}
{title:Options}

{phang}{cmdab:ti:tle(}{it:string}{cmd:)} Manually sets the title of the graph.{p_end}

{phang}{cmdab:varl:abels} Sets the legends to the variable labels for the
	variables instead of the variable names.{p_end}

{phang}{cmdab:save(}{it:string}{cmd:)} Sets the filename and the directory to which
	the graph will be set. If the filename ends with no extension or '.gph', it will be
	saved in the graph format. Extensions(png,pdf,tif,wmf,emf,ps,and eps) will be exported.
	Wmf and Emf are only available for the Windows version of Stata. Png and Tif for all
	versions of Stata except Console.{p_end}

{phang}{cmdab:grey:scale} Uses greyscales for the bars instead of colors. The color
	of the control bar will be black and the treatment bar will run in equal shade
	differnces from light grey to dark grey.{p_end}

{phang}{cmd:noconfbars} Removes the confidence interval bars from graphs for all
	treatments. The default value for the confidence interval bars ar 95%. {p_end}

{phang}{cmdab:confbarsnone(}{it:varlist}{cmd:)} Removes confidence interval bars
	from only the {it:varlist} listed. The remaining variables in the graphs which
	have not been specified in {cmdab:confbarsnone} will still have the confidence
	interval bars. {p_end}

{phang}{cmdab:yzero} Manually sets the y-axis of the graph to start at zero
	instead of the Stata default.{p_end}

{marker optslong}
{title:Examples}

{pstd} {hi:Example 1.}

{pmore} {inp:regress} {it:outcomevar treatment_dummy}{break}
		{inp:iegraph} {it:treatment_dummy} , {inp:title({it:"Treatment Effect on Outcome"})}

{pmore}In the example above, there is only two treatment arms (treatment and
		control). {it:treatment_dummy} has a 1 for all treatment observations and
		a 0 for all control observations. The graph will have one bar for control and
		it shows the mean for {it:outcomevar} for all observations in control. The
		second bar in the graph will be the sum of that mean and the coefficient
		for {it:treatment_dummy} in the regression. The graph will also have the
		title: Treatment Effect on Outcome.

{pstd} {hi:Example 2.}

{pmore} {inp:regress} {it:income tmt_1 tmt_2 age education}{inp:, cluster(}{it:district}{inp:)}{break}
		{inp:iegraph} {it:tmt_1 tmt_2}{inp:, noconfbars yzero title({it:"Treatment effect on income"}) }

{pmore}In the example above, the treatment effect on income in researched. There
		are three treatment arms; control, treatment 1 ({it:tmt_1}) and treatment
		2 ({it:tmt_2}). It is important that no observation has the value 1 in
		both {it:tmt_1} and {it:tmt_2} (i.e. no observation is in more than one
		treatment) and some observations must have the value 0 in both {it:tmt_1}
		and {it:tmt_2} (i.e. control observations). The variables {it:age} and
		{it:education} are covariates (control variables) and are not included
		in {cmd:iegraph}. {inp:noconfbars} omits the confidence interval bars
		, and {inp:yzero} sets the y-axis to start at 0.

{pstd} {hi:Example 3.}

{pmore} {inp:regress} {it:chld_wght time treat timeXtreat}{break}
		{inp:iegraph} {it:time treat timeXtreat} {inp:, title({it:"Treatment effect on Child Weight (Diff-in-Diff)"})}

{pmore}In the example above, the data set is a panel data set with two time
		periods and the regression estimates the treatment effect on child weight
		using a Difference-in-Differences model.
		The dummy variable {it:time} indicates if it is time period 0 or 1.
		The dummy variable {it:treat} indicates if the observation is treatment
		or control. {it:timeXtreat} is the interaction term of {it:time}
		and {it:treat}. This the standard way to set up a Difference-in-Differences
		regression model.

{pstd} {hi:Example 4.}

{pmore} {inp:regress} {it:harvest T1 T2 T3 } {break}
		{inp:iegraph} {it:T1 T2 T3} {inp:, title({it:"Treatment effect on harvest"})
		xlabel(,angle(45)) yzero ylabel(minmax) save({it:"$Output/Graph1.gph"})}

{pmore}The example above shows how to save a graph to disk. It also shows that
	most two-way graph options can be used. In this example the {cmd:iegraph}
	option {cdm:yzero} conflicts with the two-way option {cmd:ylabel(minmax)}.
	In such a case the user specified option takes precedence over {cmd:iegraph}
	options like {cdm:yzero}.

{title:Acknowledgements}

{phang}We would like to acknowledge the help in testing and proofreading we received in relation to this command and help file from (in alphabetic order):{p_end}
{pmore}Luiza Cardoso De Andrade{break}Michael Orevba{break}Ahmad Zia Wahdat{break}

{title:Authors}

{phang}Kristoffer Bjarkefur & Mrijan Rimal, The World Bank, DECIE

{phang}Please send bug-reports, suggestions and requests for clarifications
		 writing "ietoolkit iegraph" in the subject line to:{break}
		 kbjarkefur@worldbank.org

{phang}You can also see the code, make comments to the code, see the version
		 history of the code, and submit additions or edits to the code through
		 the github repository of ietoolkit:{break}
		 {browse "https://github.com/worldbank/ietoolkit"}

{smcl}
{* 26 Dec 2016}{...}
{hline}
help for {hi:iegraph}
{hline}

{title:Title}

{phang2}{cmdab:iegraph} {hline 2} Generates graphs based on regressions with treatment dummies common in impact evaluations.

{title:Syntax}

{phang2}
{cmdab:iegraph} {varlist}
, [{cmd:noconfbars} {cmdab:ti:tle(}{it:string}{cmd:)}
{cmdab:save(}{it:string}{cmd:)} {cmdab:confbarsnone(}{it:varlist}{cmd:)}
{it:any towway graph options}
]

{marker opts}{...}
{synoptset 22}{...}
{synopthdr:options}
{synoptline}
{synopt :{cmd:noconfbars}} Removes the confidence interval bars from graphs for all treatments.{p_end}
{synopt :{cmdab:ti:tle(}{it:string}{cmd:)}} Manually sets the title of the graph.{p_end}
{synopt :{cmdab:save(}{it:string}{cmd:)}} Sets the filename and the directory to which the graph will be set/exported.{p_end}
{synopt :{cmdab:confbarsnone(}{it:varlist}{cmd:)}} Removes confidence interval bars from only the {it:varlist} listed.{p_end}
{synopt :{cmdab:yzero}} Forces y-axis on the graph to start at 0.{p_end}
{synopt :{cmdab:varlabels}} Uses variable labels for legends instead of variable names.{p_end}
{synopt :{cmdab: Graph Options}} Options that can be used with normal graph commands can also be used. See example 2 for details.{p_end}
{synoptline}

{marker desc}
{title:Description}

{pstd}{cmdab:iegraph} This command creates bar graphs on the based on the coefficients
	of treatment dummies in regression results. This command is developed for reading
	stored results of from two types of impact evaluation regression models, but there
	are countless of other examples where the command also can be used. {cmd:iegraph} must
	be used immediately after running the regression or as long as the regression result is
	still stored in or restored to Stata's {help ereturn} results.

{pstd}{bf:Model 1: OLS with Treatment Dummies}{break}The most typical impact evaluation regression is
	to have the outcome variable as the dependent variable and one dummy for each
	treatment arm where control is the omitted category. These regressions can also include
	covariates, fixed effects etc., but as long as the treatment status is defined by
	mutually exclusive dummy variables. See especially example 1 and 2 below. This command
	works with any number of treatment arms but works best from two arms (treatment
	and control) to five treatment arms (4 different treatments and control). More
	arms than that may result in a still correct but perhaps cluttered graph.

{pstd}{bf:Model 2: Difference-in-Differences}{break}Another typical regression model in impact
	evaluations are difference-in-difference (Diff-in-Diff) models with two treatment arms (treatment
	and control) and two time periods. If the Diff-in-Diff regression is specified as having the
	outcome variable as the dependent variable and three dummy variables (time, treatment
	and timeXtreatment) as the independent variables, then this command will produce a nice
	graph. Controls, treatment effects etc. may be added to the regression model. See especially example 3.

{pstd}{bf:Graph Output}{break}The graph generated by this command is created using the following values. The
	control bar is the mean of the outcome variable for the control group. It is not
	the constant from the regression as those are not identical if, for example, fixed effects
	and covariates were used. For each treatment group the bar is the sum of the value
	of the control bar and the beta coefficient in the regression of the corresponding
	treatment dummy. The confidence intervals are calculated from the variance in the
	beta coefficients in the regression.{p_end}

{pstd}The graph also includes the N for each treatment arm in the regression and uses
	that value as labels on the x-axis. Stars are added to this value if the corresponding
	coefficient is statistically different from zero in the regression{p_end}

{marker optslong}
{title:Options}

{phang}{cmd:noconfbars} Removes the confidence interval bars from graphs for all
	treatments. The default value for the confidence interval bars ar 95%. {p_end}

{phang}{cmdab:ti:tle(}{it:string}{cmd:)} Manually sets the title of the graph.{p_end}

{phang}{cmdab:save(}{it:string}{cmd:)} Sets the filename and the directory to which
	the graph will be set. If the filename ends with no extension or '.gph', it will be
	saved in the graph format. Extensions(png,pdf,tif,wmf,emf,ps,and eps) will be exported.
	Wmf and Emf are only available for the Windows version of Stata. Png and Tif for all
	versions of Stata except Console.{p_end}

{phang}{cmdab:confbarsnone(}{it:varlist}{cmd:)} Removes confidence interval bars
	from only the {it:varlist} listed. The remaining variables in the graphs which
	have not been specified in {cmdab:confbarsnone} will still have the confidence
	interval bars. {p_end}

{phang}{cmdab:yzero} Manually sets the y-axis of the graph to start at zero instead of the Stata default.{p_end}

{phang}{cmdab:yzero} Sets the legends to the variable labels for the variables instead of the variable names.{p_end}

{marker optslong}
{title:Examples}

{pstd} {hi:Example 1.}

{pmore} {inp:regress} {it:outcomevar treatment_dummy}{break}
		{inp:iegraph} {it:treatment_dummy} , {inp:title({it:"Treatment Effect on Outcome"})}

{pmore}In the example above, there is only two treatment arms (treatment and
		control). {it:treatment_dummy} has a 1 for all treatment observations and
		a 0 for all control observations. The graph will have one bar for control and
		it shows the mean for {it:outcomevar} for all observations in control. The
		second bar in the graph will be the sum of that mean and the coefficient
		for {it:treatment_dummy} in the regression. The graph will also have the
		title: Treatment Effect on Outcome.

{pstd} {hi:Example 2.}

{pmore} {inp:regress} {it:income tmt_1 tmt_2 age education}{inp:, cluster(}{it:district}{inp:)}{break}
		{inp:iegraph} {it:tmt_1 tmt_2}{inp:, noconfbars yzero title({it:"Treatment effect on income"}) }

{pmore}In the example above, the treatment effect on income in researched. There
		are three treatment arms; control, treatment 1 ({it:tmt_1}) and treatment
		2 ({it:tmt_2}). It is important that no observation has the value 1 in
		both {it:tmt_1} and {it:tmt_2} (i.e. no observation is in more than one
		treatment) and some observations must have the value 0 in both {it:tmt_1}
		and {it:tmt_2} (i.e. control observations). The variables {it:age} and
		{it:education} are covariates (control variables) and are not included
		in {cmd:iegraph}. {inp:noconfbars} omits the confidence interval bars
		, and {inp:yzero} sets the y-axis to start at 0.

{pstd} {hi:Example 3.}

{pmore} {inp:regress} {it:chld_wght time treat timeXtreat}{break}
		{inp:iegraph} {it:time treat timeXtreat} {inp:, title({it:"Treatment effect on Child Weight (Diff-in-Diff)"})}

{pmore}In the example above, the data set is a panel data set with two time
		periods and the regression estimates the treatment effect on child weight
		using a Difference-in-Differences model.
		The dummy variable {it:time} indicates if it is time period 0 or 1.
		The dummy variable {it:treat} indicates if the observation is treatment
		or control. {it:timeXtreat} is the interaction term of {it:time}
		and {it:treat}. This the standard way to set up a Difference-in-Differences
		regression model.

{pstd} {hi:Example 4.}

{pmore} {inp:regress} {it:harvest T1 T2 T3 } {break}
		{inp:iegraph} {it:T1 T2 T3} {inp:, title({it:"Treatment effect on harvest"})
		xlabel(,angle(45)) yzero ylabel(minmax) save({it:"$Output/Graph1.gph"})}

{pmore}The example above shows how to save a graph to disk. It also shows that
	most two-way graph options can be used. In this example the {cmd:iegraph}
	option {cdm:yzero} conflicts with the two-way option {cmd:ylabel(minmax)}.
	In such a case the user specified option takes precedence over {cmd:iegraph}
	options like {cdm:yzero}.

{title:Acknowledgements}

{phang}We would like to acknowledge the help in testing and proofreading we received in relation to this command and help file from (in alphabetic order):{p_end}
{pmore}Luiza Cardoso De Andrade{break}Michael Orevba{break}Ahmad Zia Wahdat{break}

{title:Authors}

{phang}Kristoffer Bjarkefur & Mrijan Rimal, The World Bank, DECIE

{phang}Please send bug-reports, suggestions and requests for clarifications
		 writing "ietoolkit iegraph" in the subject line to:{break}
		 kbjarkefur@worldbank.org

{phang}You can also see the code, make comments to the code, see the version
		 history of the code, and submit additions or edits to the code through
		 the github repository of ietoolkit:{break}
		 {browse "https://github.com/worldbank/ietoolkit"}


do file

do file

*******************************************************************************
* AUTHORS: Leonel Fernandez
* DATE:January, 2015
* PURPOSE: Create clean database and indicators of official crime data in Mexico
* DATA IN:
*			-IncidenciaDelictiva_FueroComun_Estatal_1997-Diciembre2014.xls
*			(Raw monthly data of 66 crime types in the 32 federative entities
*			of Mxico, 1997-actual [Updates in the 20th of each month])
*
*			-state-population.csv (Cleaned estimated mid-year population data at
*			the state and county level from the CONAPO (2010) by Diego Valle
*			https://github.com/diegovalle/conapo-2010
*
* DATA OUT:
*			fuero-comun-estados.csv
*
*******************************************************************************
clear
set more off

* Defining globals. Change working directory HERE*
global files "/Users/leoxnv/desktop/Micrositio/incidenciaDelictiva/data"

*Downloading datasets*
cd "$files"

**IMPORTANT: Change updated URL after the 20th of each month*
copy "http://www.secretariadoejecutivo.gob.mx/docs/pdfs/incidencia%20delictiva%20del%20fuero%20comun/IncidenciaDelictiva_FueroComun_Estatal_1997-Diciembre2014.zip" incidencia.zip, replace
unzipfile "incidencia.zip", replace
copy "https://raw.githubusercontent.com/diegovalle/conapo-2010/master/clean-data/state-population.csv" statepop.csv
copy "$files/IncidenciaDelictiva_FueroComun_Estatal_1997-Diciembre2014.xlsx" IncidenciaDelictivaEstatal.xlsx, replace

*Cleaning population data set*
clear
import delimited "$files/statepop.csv"
drop if year > 2014 | year <1997
rename statecode state_code
save statepop.dta, replace

*Cleaning SESNSP's crime data set*

clear
import excel using "IncidenciaDelictivaEstatal.xlsx", firstrow
drop if AO == .

**Erase leading, trailing and intermediate blank spaces in all string variables
replace  ENTIDAD = trim(itrim(ENTIDAD))
replace  MODALIDAD = trim(itrim(MODALIDAD))
replace  TIPO = trim(itrim(TIPO))
replace  SUBTIPO = trim(itrim(SUBTIPO))

*Encoding state codes*
encode ENTIDAD, gen(state_code)
drop ENTIDAD TOTALAO

recode state_code (5=45) (6=46)
recode state_code (7=5) (8=6)
recode state_code (45=7) (46=8)

label define state_code 5 "COAHUILA", modify
label define state_code 6 "COLIMA", modify
label define state_code 7 "CHIAPAS", modify
label define state_code 8 "CHIHUAHUA", modify

*renaming variables*
rename (AO MODALIDAD TIPO SUBTIPO ENERO FEBRERO MARZO ABRIL MAYO JUNIO JULIO ///
		AGOSTO SEPTIEMBRE OCTUBRE NOVIEMBRE DICIEMBRE) 						 ///
       (year category type subtype count1 count2 count3 count4 count5 count6 ///
		count7 count8 count9 count10 count11 count12)

*Reshaping to long format*
reshape long count, i(year category type subtype state_code) j(month)
destring count, replace

*Creating crime variable wich groups all categories of "ROBO" in just one*
gen crime = category
replace crime = "ROBO" if word(category,1) == "ROBO"

*Merging with population data base*
merge m:1 year state_code using "$files/statepop.dta", keepusing(total)///
			nogenerate
rename total population

*Ordering variables and sorting cases*
order state_code year month crime category type subtype count population
sort state_code year crime category type subtype month

*Exporting to .cvs and saveing in .dta*
export delimited using "$files/fuero-comun-estados.csv", nolabel replace
save delitos-fuero-comun.dta, replace

*Stop here if you want to keep all files
*deleting files*
rm incidencia.zip
rm IncidenciaDelictiva_FueroComun_Estatal_1997-Diciembre2014.xlsx
rm statepop.dta
rm delitos-fuero-comun.dta

************************End of do-file*****************************************

*******************************************************************************
* AUTHORS: Leonel Fernandez
* DATE:January, 2015
* PURPOSE: Create clean database and indicators of official crime data in Mexico
* DATA IN:
*			-IncidenciaDelictiva_FueroComun_Estatal_1997-Diciembre2014.xls
*			(Raw monthly data of 66 crime types in the 32 federative entities
*			of Mxico, 1997-actual [Updates in the 20th of each month])
*
*			-state-population.csv (Cleaned estimated mid-year population data at
*			the state and county level from the CONAPO (2010) by Diego Valle
*			https://github.com/diegovalle/conapo-2010
*
* DATA OUT:
*			fuero-comun-estados.csv
*			fuero-comun-estados.dta
*******************************************************************************
clear
set more off

* Defining globals. Change working folder HERE*
global files "/Users/leoxnv/Micrositio/incidenciaDelictiva/data"

*Downloading datasets*
cd "$files"

**IMPORTANT: Change updated URL after the 20th of each month*
copy "http://www.secretariadoejecutivo.gob.mx/docs/pdfs/incidencia%20delictiva%20del%20fuero%20comun/IncidenciaDelictiva_FueroComun_Estatal_1997-Diciembre2014.zip" incidencia.zip, replace
unzipfile "incidencia.zip", replace
copy "https://raw.githubusercontent.com/diegovalle/conapo-2010/master/clean-data/state-population.csv" statepop.csv
copy "$files/IncidenciaDelictiva_FueroComun_Estatal_1997-Diciembre2014.xlsx" IncidenciaDelictivaEstatal.xlsx, replace

*Cleaning population data set*
clear
import delimited "$files/statepop.csv"
drop if year > 2014 | year <1997
rename statecode state_code
save statepop.dta, replace

*Cleaning SESNSP's crime data set*

clear
import excel using "IncidenciaDelictivaEstatal.xlsx", firstrow
drop if AO == .

**Erase leading, trailing and intermediate blank spaces in all string variables
replace  ENTIDAD = trim(itrim(ENTIDAD))
replace  MODALIDAD = trim(itrim(MODALIDAD))
replace  TIPO = trim(itrim(TIPO))
replace  SUBTIPO = trim(itrim(SUBTIPO))

*Encoding state codes*
encode ENTIDAD, gen(state_code)
drop ENTIDAD TOTALAO

recode state_code (5=45) (6=46)
recode state_code (7=5) (8=6)
recode state_code (45=7) (46=8)

label define state_code 5 "COAHUILA", modify
label define state_code 6 "COLIMA", modify
label define state_code 7 "CHIAPAS", modify
label define state_code 8 "CHIHUAHUA", modify

*renaming variables*
rename (AO MODALIDAD TIPO SUBTIPO ENERO FEBRERO MARZO ABRIL MAYO JUNIO JULIO ///
		AGOSTO SEPTIEMBRE OCTUBRE NOVIEMBRE DICIEMBRE) 						 ///
       (year category type subtype count1 count2 count3 count4 count5 count6 ///
		count7 count8 count9 count10 count11 count12)

*Reshaping to long format*
reshape long count, i(year category type subtype state_code) j(month)
destring count, replace

*Creating crime variable wich groups all categories of "ROBO" in just one*
gen crime = category
replace crime = "ROBO" if word(category,1) == "ROBO"

*Merging with population data base*
merge m:1 year state_code using "$files/statepop.dta", keepusing(total) nogenerate
rename total population

*Ordering variables and sorting cases*
order state_code year month crime category type subtype count population
sort state_code year crime category type subtype month

*Exporting to .cvs and saveing in .dta*
export delimited using "$files/fuero-comun-estados.csv", nolabel replace
save delitos-fuero-comun.dta, replace


reg tables, need checking

reg tables, need checking

clear all
set more off
set matsize 11000
program drop _all

qui adopath + "/mnt/nfs6/wikipedia.proj/wikibaseball/scripts/stata/ado"
declare_global

cd ${path}

// 1a. Produce Baseball / Basketball datasets
make_baseball

make_basketball

use ${lahman}bb_master, clear
append using ${basketball}bk_master
gen isbaseball = (minutesrank==.)
save ${stash}bbk_master, replace

outsheet wikihandle playerid using ${stash}wikilist.csv, replace nonames noquote

// 1b. Merge data
make_merge

/////////// ANALYSIS //////////////////

// 1. Summary Stats
summary

// 2. Meanline Charts
program drop _all
meanline img Images(Baseball) 1
meanline img Images(Basketball) 0

meanline text Text(Baseball) 1
meanline text Text(Basketball) 0

meanline bd Citations(Baseball) 1
meanline bd Citations(Basketball) 0

// 3. Baseline Digit & Copyright Regressions
reg_digit

reg_copy

// 4. DDD regressions

clear all
set more off
set matsize 11000
program drop _all

qui adopath + "/mnt/nfs6/wikipedia.proj/wikibaseball/scripts/stata/ado"
declare_global

cd ${path}

// 1a. Produce Baseball / Basketball datasets
make_baseball

make_basketball

use ${lahman}bb_master, clear
append using ${basketball}bk_master
gen isbaseball = (minutesrank==.)
save ${stash}bbk_master, replace

outsheet wikihandle playerid using ${stash}wikilist.csv, replace nonames noquote

// 1b. Merge data
make_merge

/////////// ANALYSIS //////////////////

// 1. Summary Stats
summary

// 2. Meanline Charts
program drop _all
meanline img Images(Baseball) 1
meanline img Images(Basketball) 0

meanline text Text(Baseball) 1
meanline text Text(Basketball) 0

meanline bd Citations(Baseball) 1
meanline bd Citations(Basketball) 0

// 3. Baseline Digit Regression
reg_digit


outofcopy on left

outofcopy on left

program t_test

use ${stash}master, clear

local flag `1'

keep if isb == `flag'
keep if year == 2013

labelvar

est clear
gen tmp = !treat
estpost ttest bd img text traf, by(tmp)

esttab, wide nonumber cells(`"mu_1(fmt(a3)) mu_2(fmt(a3)) b(fmt(a3) nostar) p(fmt(2) par("{ralign @modelwidth:{txt:(}" "{txt:)}}"))"') mlabels("") collabels("out-of-copy mean" "in-copy mean" "diff" "p-val") noobs replace

esttab using "${tables}ttest_`flag'.tex", wide nonumber cells("mu_1(fmt(a3)) mu_2(fmt(a3)) b(fmt(a3) nostar) p(fmt(2))") collabels("\textbf{out-of-copy mean}" "\textbf{in-copy mean}" "\textbf{diff}" "\textbf{p-val}") noobs replace booktabs label

end

program labelvar

label variable img "\emph{Number of Images}"
label variable text "\emph{Number of Words of Text}"
label variable bd "\emph{Number of Citations to Baseball Digest}"
label variable traf "\emph{Average Monthly Traffic}"

end

program t_test

use ${stash}master, clear

local flag `1'

keep if isb == `flag'
keep if year == 2013

labelvar

est clear
estpost ttest bd img text traf, by(treat)

esttab, wide nonumber cells(`"mu_1(fmt(a3)) mu_2(fmt(a3)) b(fmt(a3) nostar) p_l(fmt(2) par("{ralign @modelwidth:{txt:(}" "{txt:)}}"))"') mlabels("") collabels("in-copy mean" "out-of-copy mean" "diff" "p-val") noobs replace

esttab using "${tables}ttest_`flag'.tex", wide nonumber cells("mu_1(fmt(a3)) mu_2(fmt(a3)) b(fmt(a3) nostar) p_l(fmt(2))") collabels("\textbf{in-copy mean}" "\textbf{out-of-copy mean}" "\textbf{diff}" "\textbf{p-val}") noobs replace booktabs label

end

program labelvar

label variable img "\emph{Number of Images}"
label variable text "\emph{Number of Words of Text}"
label variable bd "\emph{Number of Citations to Baseball Digest}"
label variable traf "\emph{Average Monthly Traffic}"

end