Mejoras a ejemplos con BDA

Mejoras a ejemplos con BDA

clear all
set more off
set trace off
timer clear

cap log close _all
log using logbda.txt, replace text

global bda ~/../shared_bd/bases/bda/bases_stata
cd $bda

parallel clean , all // force
parallel setclusters 4, f

//////////////////////////////////////////////////////////////////////
// EJEMPLO PARALLEL BY
//////////////////////////////////////////////////////////////////////
use $bda/2013_06/pers

gen          nacimiento  = date(strofreal(fec_nac, "%12.0f"), "YMD")
parallel:gen nacimiento2 = date(strofreal(fec_nac, "%12.0f"), "YMD")
compare nacimiento nacimiento2

clear

//////////////////////////////////////////////////////////////////////
// EJEMPLO PARALLEL APPEND
//////////////////////////////////////////////////////////////////////
/* Ejemplo BDA 1 */
parallel append 2013_01/mcci 2013_02/mcci 2013_03/mcci, ///
	do(collapse (mean) monto_pe, by(perdev) fast) ///
	if(inlist(cod_mov, 11001, 11010))

/* Ejemplo BDA 2*/
cap program drop miprograma
program def miprograma
	/* Capturando la variable de sexo */
	merge m:m correl using $bda/2013_06/pers, keep(3) nogen ///
		keepusing(correl sexo)

	collapse (mean) mprom = monto_pesos (sd) msd = monto_pesos, ///
		by(perdev_rem sexo) fast
end

/* Aplicando programa */
parallel append, do(miprograma) programs(miprograma) ///
	e("%g_%02.0f/mcci.dta, 2010/2013, 1 6 12") if(inlist(cod_mov, 11001, 11010))

summ
d

/* Aplicando lo mismo pero a un set restringido */
parallel append, do(miprograma) programs(miprograma) ///
	e("%g_%02.0f/mcci.dta, 2010/2013, 1 6") ///
	if(inlist(cod_mov, 11001, 11010))

summ
d

log close

cd ~/parallel/ado
do compile

clear all
set more off
set trace off
timer clear

cap log close _all
log using logbda.txt, replace text

global bda ~/../shared_bd/bases/bda/bases_stata
cd $bda

parallel clean, all force
parallel setclusters 2, f

//////////////////////////////////////////////////////////////////////
// EJEMPLO PARALLEL BY
//////////////////////////////////////////////////////////////////////
use $bda/2013_06/pers

gen nacimiento = date(strofreal(fec_nac, "%12.0f"), "YMD")
parallel:gen nacimiento2 = date(strofreal(fec_nac, "%12.0f"), "YMD")
compare nacimiento nacimiento2

clear

//////////////////////////////////////////////////////////////////////
// EJEMPLO PARALLEL APPEND
//////////////////////////////////////////////////////////////////////
/* Ejemplo BDA 1 */
set trace on
parallel append 2013_01/mcci 2013_02/mcci 2013_03/mcci, ///
	do(collapse (mean) monto_pe, by(perdev) fast) ///
	if(inlist(cod_mov, 11001, 11010))

/* Ejemplo BDA 2*/
cap program drop miprograma
program def miprograma
	/* Capturando la variable de sexo */
	merge m:m correl using $bda/2013_06/pers, keep(3) nogen ///
		keepusing(correl sexo)

	collapse (mean) mprom = monto_pesos (sd) msd = monto_pesos, ///
		by(perdev_rem sexo) fast
end

/* Aplicando programa */
parallel append, do(miprograma) programs(miprograma) ///
	e("%g_%02.0f/mcci.dta, 2010/2013, 1 6 12") if(inlist(cod_mov, 11001, 11010))

summ
d

/* Aplicando lo mismo pero a un set restringido */
parallel append, do(miprograma) programs(miprograma) ///
	e("%g_%02.0f/mcci.dta, 2010/2013, 1 6") ///
	if(inlist(cod_mov, 11001, 11010))

summ
d

log close


redirect stderr for Stata child processes

parallel runs Stata child processes with the intent of monitoring them
in its own code. However, if stderr is not redirected, XStata and
noninteractive/batch Stata will wait for the child processes to complete
before returning from the shell command.

This change redirects stderr to the existing per-process log files.
On Linux, CLI Stata, batch Stata and XStata now exhibit the same
behavior and immediately return control to parallel.

*! version 0.14.7.22 22jul2014
*! author: George G. Vega Yon

/**
 * @brief Runs parallel child processes in batch mode.
 * @param parallelid Parallel id.
 * @param nchildren Number of child processes.
 * @param paralleldir Dir where the process should be running.
 * @param timeout Number of seconds to wait until stop the process for no conextion.
 * @param gateway_fname Name of file that a Cygwin process is listen to will execute from (Windows batch).
 * @returns Number of child processes that stopped with an error.
 */

mata:
real scalar parallel_run(
	string scalar parallelid,
	|real scalar nchildren,
	string scalar paralleldir,
	real scalar timeout,
	real scalar deterministicoutput,
	string matrix hostnames,
	string scalar ssh_str,
	string scalar gateway_fname
	) {

	real scalar fh, i, use_procexec
	string scalar tmpdir, tmpdir_i, line, line2, dofile_i, dofile_i_base, pidfile
	string scalar stata_quiet, folder, exec_cmd //, stata_batch
	string scalar hostname, env_tmp_assign, com_line_env, rmt_begin, rmt_end, fin_file
	string scalar finito_err_line, pid_err_line, log_err_cmd
	real colvector pids
	pids = J(0,1,.)

	// Setting default parameters
	if (nchildren == J(1,1,.)) nchildren = strtoreal(st_global("PLL_CHILDREN"))
	if (paralleldir == J(1,1,"")) paralleldir = st_global("PLL_STATA_PATH")

	// Message
	display(sprintf("{hline %g}",c("linesize") > 80?80:c("linesize")))
	display("{result:Parallel Computing with Stata}")
	if (!deterministicoutput) display("{text:Child processes:} {result:"+strofreal(nchildren)+"}")
	if (!deterministicoutput) display("{text:pll_id         :} {result:"+parallelid+"}")
	if (!deterministicoutput & length(hostnames)) display("{text:Hostnames :} {result:"+st_global("PLL_HOSTNAMES")+"}")
	if (!deterministicoutput) display("{text:Running at     :} {result:"+c("pwd")+"}")
	display("{text:Randtype       :} {result:"+st_local("randtype")+"}")

	tmpdir = c("tmpdir") + (regexm(c("tmpdir"),"(/|\\)$") ? "" : "/")
	//If there is a -cd- command in (sys)profile.do then we need to
	// specify the full path for the do file.  so grab the directory
	folder = st_global("LAST_PLL_DIR")

	if (c("os") != "Windows") { // MACOS/UNIX
		unlink("__pll"+parallelid+"_shell.sh")
		fh = fopen("__pll"+parallelid+"_shell.sh","w", 1)
		pidfile = "__pll"+parallelid+"_pids"
		unlink(pidfile)
		stata_quiet = " -q"
		//stata_batch = (c("os") == "Unix" ?" -b":" -e")
		// Writing file
		hostname = ""
		ssh_str = length(hostnames) ? (ssh_str == J(1,1,"")?"ssh ":ssh_str) : ""
		for(i=1;i<=nchildren;i++) {
			tmpdir_i = tmpdir+"__pll"+parallelid+"_tmpdir"+strofreal(i, "%04.0f")
			mkdir(tmpdir_i,1)
			dofile_i_base = "__pll"+parallelid+"_do"+strofreal(i,"%04.0f")
			env_tmp_assign = `"export STATATMP=""'+tmpdir_i+`"""'
			if(length(hostnames)>0) hostname = hostnames[1,mod(i-1,length(hostnames))+1]
			if(length(hostnames)>0 & hostname!="localhost"){
				com_line_env = `"cd ""'+folder+`"""'+ "; "+env_tmp_assign+"; "
				rmt_begin = ssh_str + hostname+" " + "'" + com_line_env + "nohup "
				rmt_end = "'"
				dofile_i = dofile_i_base+".do"
			}
			else{
				fput(fh, env_tmp_assign)
				dofile_i = folder+dofile_i_base+".do"
				rmt_begin = ""
				rmt_end = ""
			}
			//The standard batch-mode way of calling fumbles the automated name of the log file
			// if the folder has a space in it (it makes it the first word before the space,
			// rather than the base). So do the < > redirect way.
			//exec_cmd = ssh_str+hostname + paralleldir+stata_batch+stata_quiet+" "+`"do \""'+dofile_i + `"\""'
			exec_cmd = paralleldir+stata_quiet + `" < ""'+dofile_i + `"" > "' + dofile_i_base + ".log 2>&1"
			fput(fh, rmt_begin + exec_cmd + " & echo $!" + rmt_end + " >> "+pidfile)
			log_err_cmd = `"echo "Stata was not able to execute" > "'+dofile_i_base + ".log; "
			fin_file = "__pll"+parallelid+"_finito"+strofreal(i,"%04.0f")
			finito_err_line = `"echo -e "709\nCommand execution failed("'+hostname+`")" > "'+fin_file+"; "
			pid_err_line = "echo -1 >> "+pidfile+"; "
			fput(fh, "if [ $? -ne 0 ]; then "+log_err_cmd+finito_err_line+pid_err_line+" fi")
		}

		fclose(fh)

		stata("shell sh __pll"+parallelid+"_shell.sh") //wait for the pids to be full written

		fh = fopen(pidfile, "r")
		while ((line=fget(fh))!=J(0,0,"")) {
			pids = pids \ strtoreal(line)
		}
		fclose(fh)
		unlink(pidfile)
	}
	else { // WINDOWS
		use_procexec = strtoreal(st_global("USE_PROCEXEC"))
		if (!use_procexec){
			if (c("mode")=="batch"){ //Execute commands via Cygwin process
				if (gateway_fname == J(1,1,"")) gateway_fname = st_global("PLL_GATEWAY_FNAME")
				fh = fopen(gateway_fname,"a", 1)
				for(i=1;i<=nchildren;i++) {
					tmpdir_i = tmpdir+"__pll"+parallelid+"_tmpdir"+strofreal(i, "%04.0f")
					mkdir(tmpdir_i,1) // fput(fh, "mkdir "+c("tmpdir")+"/"+parallelid+strofreal(i,"%04.0f"))
					fput(fh, `"export STATATMP=""'+tmpdir_i+`"""')
					dofile_i = folder+"__pll"+parallelid+"_do"+strofreal(i,"%04.0f")+".do"
					fput(fh, paralleldir+`" -e -q do ""'+dofile_i+`"" &"')
				}
				fclose(fh)
			}
			else{
				unlink("__pll"+parallelid+"_shell.bat")
				fh = fopen("__pll"+parallelid+"_shell.bat","w", 1)

				fput(fh, "pushd "+pwd())

				// Writing file
				for(i=1;i<=nchildren;i++) {
					tmpdir_i = tmpdir+"__pll"+parallelid+"_tmpdir"+strofreal(i, "%04.0f")
					mkdir(tmpdir_i,1)
					fwrite(fh, "start /MIN /HIGH set STATATMP="+tmpdir_i+" ^& ")
					dofile_i = folder+"__pll"+parallelid+"_do"+strofreal(i,"%04.0f")+".do"
					fput(fh, paralleldir+`" /e /q do ""'+dofile_i+`""^&exit"')
				}

				fput(fh, "popd")
				fput(fh, "exit")

				fclose(fh)

				stata("winexec __pll"+parallelid+"_shell.bat")
			}
		}
		else{
			st_numscalar("PROCEXEC_HIDDEN",use_procexec)
			st_numscalar("PROCEXEC_ABOVE_NORMAL_PRIORITY",1)

			for(i=1;i<=nchildren;i++) {
				tmpdir_i = tmpdir+"__pll"+parallelid+"_tmpdir"+strofreal(i,"%04.0f")
				mkdir(tmpdir_i,1)
				dofile_i = folder+"__pll"+parallelid+"_do"+strofreal(i,"%04.0f")+".do"
				line2 = paralleldir+`" /e /q do ""'+dofile_i+`"""'
				stata("procenv set STATATMP="+tmpdir_i)
				stata("procexec "+line2)
				pids = pids\st_numscalar("r(pid)")
			}
			stata("procenv set STATATMP="+tmpdir)
		}
	}

	/* Waits until each process ends */
	return(parallel_finito(parallelid,nchildren,timeout,pids, deterministicoutput, hostnames, ssh_str))
}
end

*! version 0.14.7.22 22jul2014
*! author: George G. Vega Yon

/**
 * @brief Runs parallel child processes in batch mode.
 * @param parallelid Parallel id.
 * @param nchildren Number of child processes.
 * @param paralleldir Dir where the process should be running.
 * @param timeout Number of seconds to wait until stop the process for no conextion.
 * @param gateway_fname Name of file that a Cygwin process is listen to will execute from (Windows batch).
 * @returns Number of child processes that stopped with an error.
 */

mata:
real scalar parallel_run(
	string scalar parallelid,
	|real scalar nchildren,
	string scalar paralleldir,
	real scalar timeout,
	real scalar deterministicoutput,
	string matrix hostnames,
	string scalar ssh_str,
	string scalar gateway_fname
	) {

	real scalar fh, i, use_procexec
	string scalar tmpdir, tmpdir_i, line, line2, dofile_i, dofile_i_base, pidfile
	string scalar stata_quiet, folder, exec_cmd //, stata_batch
	string scalar hostname, env_tmp_assign, com_line_env, rmt_begin, rmt_end, fin_file
	string scalar finito_err_line, pid_err_line, log_err_cmd
	real colvector pids
	pids = J(0,1,.)

	// Setting default parameters
	if (nchildren == J(1,1,.)) nchildren = strtoreal(st_global("PLL_CHILDREN"))
	if (paralleldir == J(1,1,"")) paralleldir = st_global("PLL_STATA_PATH")

	// Message
	display(sprintf("{hline %g}",c("linesize") > 80?80:c("linesize")))
	display("{result:Parallel Computing with Stata}")
	if (!deterministicoutput) display("{text:Child processes:} {result:"+strofreal(nchildren)+"}")
	if (!deterministicoutput) display("{text:pll_id         :} {result:"+parallelid+"}")
	if (!deterministicoutput & length(hostnames)) display("{text:Hostnames :} {result:"+st_global("PLL_HOSTNAMES")+"}")
	if (!deterministicoutput) display("{text:Running at     :} {result:"+c("pwd")+"}")
	display("{text:Randtype       :} {result:"+st_local("randtype")+"}")

	tmpdir = c("tmpdir") + (regexm(c("tmpdir"),"(/|\\)$") ? "" : "/")
	//If there is a -cd- command in (sys)profile.do then we need to
	// specify the full path for the do file.  so grab the directory
	folder = st_global("LAST_PLL_DIR")

	if (c("os") != "Windows") { // MACOS/UNIX
		unlink("__pll"+parallelid+"_shell.sh")
		fh = fopen("__pll"+parallelid+"_shell.sh","w", 1)
		pidfile = "__pll"+parallelid+"_pids"
		unlink(pidfile)
		stata_quiet = " -q"
		//stata_batch = (c("os") == "Unix" ?" -b":" -e")
		// Writing file
		hostname = ""
		ssh_str = length(hostnames) ? (ssh_str == J(1,1,"")?"ssh ":ssh_str) : ""
		for(i=1;i<=nchildren;i++) {
			tmpdir_i = tmpdir+"__pll"+parallelid+"_tmpdir"+strofreal(i, "%04.0f")
			mkdir(tmpdir_i,1)
			dofile_i_base = "__pll"+parallelid+"_do"+strofreal(i,"%04.0f")
			env_tmp_assign = `"export STATATMP=""'+tmpdir_i+`"""'
			if(length(hostnames)>0) hostname = hostnames[1,mod(i-1,length(hostnames))+1]
			if(length(hostnames)>0 & hostname!="localhost"){
				com_line_env = `"cd ""'+folder+`"""'+ "; "+env_tmp_assign+"; "
				rmt_begin = ssh_str + hostname+" " + "'" + com_line_env + "nohup "
				rmt_end = "'"
				dofile_i = dofile_i_base+".do"
			}
			else{
				fput(fh, env_tmp_assign)
				dofile_i = folder+dofile_i_base+".do"
				rmt_begin = ""
				rmt_end = ""
			}
			//The standard batch-mode way of calling fumbles the automated name of the log file
			// if the folder has a space in it (it makes it the first word before the space,
			// rather than the base). So do the < > redirect way.
			//exec_cmd = ssh_str+hostname + paralleldir+stata_batch+stata_quiet+" "+`"do \""'+dofile_i + `"\""'
			exec_cmd = paralleldir+stata_quiet + `" < ""'+dofile_i + `"" > "' + dofile_i_base + ".log"
			fput(fh, rmt_begin + exec_cmd + " & echo $!" + rmt_end + " >> "+pidfile)
			log_err_cmd = `"echo "Stata was not able to execute" > "'+dofile_i_base + ".log; "
			fin_file = "__pll"+parallelid+"_finito"+strofreal(i,"%04.0f")
			finito_err_line = `"echo -e "709\nCommand execution failed("'+hostname+`")" > "'+fin_file+"; "
			pid_err_line = "echo -1 >> "+pidfile+"; "
			fput(fh, "if [ $? -ne 0 ]; then "+log_err_cmd+finito_err_line+pid_err_line+" fi")
		}

		fclose(fh)

		stata("shell sh __pll"+parallelid+"_shell.sh") //wait for the pids to be full written

		fh = fopen(pidfile, "r")
		while ((line=fget(fh))!=J(0,0,"")) {
			pids = pids \ strtoreal(line)
		}
		fclose(fh)
		unlink(pidfile)
	}
	else { // WINDOWS
		use_procexec = strtoreal(st_global("USE_PROCEXEC"))
		if (!use_procexec){
			if (c("mode")=="batch"){ //Execute commands via Cygwin process
				if (gateway_fname == J(1,1,"")) gateway_fname = st_global("PLL_GATEWAY_FNAME")
				fh = fopen(gateway_fname,"a", 1)
				for(i=1;i<=nchildren;i++) {
					tmpdir_i = tmpdir+"__pll"+parallelid+"_tmpdir"+strofreal(i, "%04.0f")
					mkdir(tmpdir_i,1) // fput(fh, "mkdir "+c("tmpdir")+"/"+parallelid+strofreal(i,"%04.0f"))
					fput(fh, `"export STATATMP=""'+tmpdir_i+`"""')
					dofile_i = folder+"__pll"+parallelid+"_do"+strofreal(i,"%04.0f")+".do"
					fput(fh, paralleldir+`" -e -q do ""'+dofile_i+`"" &"')
				}
				fclose(fh)
			}
			else{
				unlink("__pll"+parallelid+"_shell.bat")
				fh = fopen("__pll"+parallelid+"_shell.bat","w", 1)

				fput(fh, "pushd "+pwd())

				// Writing file
				for(i=1;i<=nchildren;i++) {
					tmpdir_i = tmpdir+"__pll"+parallelid+"_tmpdir"+strofreal(i, "%04.0f")
					mkdir(tmpdir_i,1)
					fwrite(fh, "start /MIN /HIGH set STATATMP="+tmpdir_i+" ^& ")
					dofile_i = folder+"__pll"+parallelid+"_do"+strofreal(i,"%04.0f")+".do"
					fput(fh, paralleldir+`" /e /q do ""'+dofile_i+`""^&exit"')
				}

				fput(fh, "popd")
				fput(fh, "exit")

				fclose(fh)

				stata("winexec __pll"+parallelid+"_shell.bat")
			}
		}
		else{
			st_numscalar("PROCEXEC_HIDDEN",use_procexec)
			st_numscalar("PROCEXEC_ABOVE_NORMAL_PRIORITY",1)

			for(i=1;i<=nchildren;i++) {
				tmpdir_i = tmpdir+"__pll"+parallelid+"_tmpdir"+strofreal(i,"%04.0f")
				mkdir(tmpdir_i,1)
				dofile_i = folder+"__pll"+parallelid+"_do"+strofreal(i,"%04.0f")+".do"
				line2 = paralleldir+`" /e /q do ""'+dofile_i+`"""'
				stata("procenv set STATATMP="+tmpdir_i)
				stata("procexec "+line2)
				pids = pids\st_numscalar("r(pid)")
			}
			stata("procenv set STATATMP="+tmpdir)
		}
	}

	/* Waits until each process ends */
	return(parallel_finito(parallelid,nchildren,timeout,pids, deterministicoutput, hostnames, ssh_str))
}
end


suest compatibility p2 (still not working)

*! version 1.1.0 10jul2014
* predict after reghdfe
* TODO: Not tested for -avge- variables!

program define reghdfe_p, // sortpreserve properties(default_xb)
	local version `clip(`c(version)', 11.2, 13.1)' // 11.2 minimum, 13+ preferred
	qui version `version'

	*if "`e(cmd)'" != "reghdfe" {
	*	error 301
	*}

	syntax anything [if] [in] , [XB XBD D Residuals SCores]
	if (`"`scores'"' != "") {
		_score_spec	`anything'
		local varlist `s(varlist)'
	}
	else {
		local 0 `anything'
		syntax newvarname  // [if] [in] , [XB XBD D Residuals SCores]
	}

	local option `xb' `xbd' `d' `residuals' `scores'
	if ("`option'"=="") local option xb // The default, as in -areg-
	local numoptions : word count `option'
	if (`numoptions'!=1) {
		di as error "(predict reghdfe) syntax error; specify one and only one option"
		exit 112
	}
	if ("`option'"=="scores") local option residuals

	local fixed_effects = e(absvars)

	* We need to have saved FEs and AvgEs for every option except -xb-
	if ("`option'"!="xb") {

		* Construct -d- if needed (sum of FEs)
		tempvar d
		qui gen double `d' = 0 `if' `in'

		forv g=1/`e(N_hdfe)' {
			local ok 0
			cap conf e `e(hdfe_target`g')'
			if (_rc==0) {
				cap conf numeric var `e(hdfe_target`g')'
			}
			if (_rc!=0) {
				di as error "In order to predict, all the FEs need to be saved with the absorb option (#`g' was not)" _n "For instance, instead of {it:absorb(i.year i.firm)}, set absorb(FE_YEAR=i.year FE_FIRM=i.firm)"
				exit 112
			}

			if missing("`e(hdfe_cvar`g')'") {
				qui replace `d' = `d' + `e(hdfe_target`g')'
			}
			else {
				qui replace `d' = `d' + `e(hdfe_target`g')' * `e(hdfe_cvar`g')'
			}
		}
		local K = cond( e(N_avge)==. , 0 , e(N_avge) )
		forv g=1/`K' {
			local ok 0
			cap conf e `e(avge_target`g')'
			if (_rc==0) {
				cap conf numeric var `e(avge_target`g')'
			}
			if (_rc!=0) {
				di as error "(predict reghdfe) you need to save all AvgEs in reghdfe, AvgE`g' not saved"
				exit 112
			}
			qui replace `d' = `d' + `e(avge_target`g')'
		}
	} // Finished creating `d' if needed

	* Construct -xb- if needed
	if ("`option'"!="d") {
		tempvar xb
		_predict double `xb' `if' `in', xb
	}

	* Adjusting for -noconstant- option
	local adj_cons = cond(e(_cons)<., e(_cons), 0)
	cap replace `xb' = `xb' + `adj_cons'
	cap replace `d' = `d' - `adj_cons'

	if ("`option'"=="xb") {
		rename `xb' `varlist'
	}
	else if ("`option'"=="d") {
		rename `d' `varlist'
	}
	else {
		qui replace `xb' = `xb' + `d' `if' `in'
		if ("`option'"=="xbd") {
			rename `xb' `varlist'
		}
		else if ("`option'"=="residuals") {
			qui replace `xb' = `e(depvar)' - `xb' `if' `in'
			rename `xb' `varlist'
		}
		else {
			error 112
		}
	}

	fvrevar `e(depvar)', list
	local format : format `r(varlist)'
	format `format' `varlist'
	* TODO: Allow [type] and recast to that, as -predict- usually does
end

*! version 1.1.0 10jul2014
* predict after reghdfe
* TODO: Not tested for -avge- variables!

program define reghdfe_p, // sortpreserve properties(default_xb)
	local version `clip(`c(version)', 11.2, 13.1)' // 11.2 minimum, 13+ preferred
	qui version `version'

	if "`e(cmd)'" != "reghdfe" {
		error 301
	}

	syntax newvarname [if] [in] , [XB XBD D Residuals SCores]

	local option `xb' `xbd' `d' `residuals' `scores'
	if ("`option'"=="") local option xb // The default, as in -areg-
	local numoptions : word count `option'
	if (`numoptions'!=1) {
		di as error "(predict reghdfe) syntax error; specify one and only one option"
		exit 112
	}
	if ("`option'"=="scores") local option residuals

	local fixed_effects = e(absvars)

	* We need to have saved FEs and AvgEs for every option except -xb-
	if ("`option'"!="xb") {

		* Construct -d- if needed (sum of FEs)
		tempvar d
		qui gen double `d' = 0 `if' `in'

		forv g=1/`e(N_hdfe)' {
			local ok 0
			cap conf e `e(hdfe_target`g')'
			if (_rc==0) {
				cap conf numeric var `e(hdfe_target`g')'
			}
			if (_rc!=0) {
				di as error "In order to predict, all the FEs need to be saved with the absorb option (#`g' was not)" _n "For instance, instead of {it:absorb(i.year i.firm)}, set absorb(FE_YEAR=i.year FE_FIRM=i.firm)"
				exit 112
			}

			if missing("`e(hdfe_cvar`g')'") {
				qui replace `d' = `d' + `e(hdfe_target`g')'
			}
			else {
				qui replace `d' = `d' + `e(hdfe_target`g')' * `e(hdfe_cvar`g')'
			}
		}
		local K = cond( e(N_avge)==. , 0 , e(N_avge) )
		forv g=1/`K' {
			local ok 0
			cap conf e `e(avge_target`g')'
			if (_rc==0) {
				cap conf numeric var `e(avge_target`g')'
			}
			if (_rc!=0) {
				di as error "(predict reghdfe) you need to save all AvgEs in reghdfe, AvgE`g' not saved"
				exit 112
			}
			qui replace `d' = `d' + `e(avge_target`g')'
		}
	} // Finished creating `d' if needed

	* Construct -xb- if needed
	if ("`option'"!="d") {
		tempvar xb
		_predict double `xb' `if' `in', xb
	}

	* Adjusting for -noconstant- option
	local adj_cons = cond(e(_cons)<., e(_cons), 0)
	cap replace `xb' = `xb' + `adj_cons'
	cap replace `d' = `d' - `adj_cons'

	if ("`option'"=="xb") {
		rename `xb' `varlist'
	}
	else if ("`option'"=="d") {
		rename `d' `varlist'
	}
	else {
		qui replace `xb' = `xb' + `d' `if' `in'
		if ("`option'"=="xbd") {
			rename `xb' `varlist'
		}
		else if ("`option'"=="residuals") {
			qui replace `xb' = `e(depvar)' - `xb' `if' `in'
			rename `xb' `varlist'
		}
		else {
			error 112
		}
	}

	fvrevar `e(depvar)', list
	local format : format `r(varlist)'
	format `format' `varlist'
	* TODO: Allow [type] and recast to that, as -predict- usually does
end


added year and post to merge

added year and post to merge

program mergebasic
use ${stash}cleanchangeset1, clear

drop if fips == "NA"

merge m:1 fips using ${stash}cleancnty, keep(master match) nogen

merge m:1 geoid10 using ${stash}cleanua, keep(master match) nogen

gen post =  month > mofd(date("10-1-2007","MDY"))
gen year = year(tstamp_date)

save ${stash}mergemaster1, replace
end

program mergebasic
use ${stash}cleanchangeset1, clear

drop if fips == "NA"

merge m:1 fips using ${stash}cleancnty, keep(master match) nogen

merge m:1 geoid10 using ${stash}cleanua, keep(master match) nogen

save ${stash}mergemaster1, replace
end


deals well with simple DD

deals well with simple DD

program diffindiff

local model `1'
local dv `2'
local unit `3'
local cutoff `4'
local mode `5'

di "OK, you asked me to generate DD charts"
di "for `model', unit: `unit', DVs: `dv', mode: `mode', cutoff: `cutoff'"

if "`mode'" == "run"{
    drop if year > `cutoff'
    runreg `model' "`dv'" `unit' `cutoff'
}

if "`mode'" == "write"{
    loadreg `model' "`dv'" `unit' `cutoff'
    writereg `model'_`unit'_`cutoff'
}

end

program runreg
local model `1'
local dv `2'
local unit `3'
local cutoff `4'

est clear
di "-----"
di "  Running `model' for `dv'"
di "-----"
foreach x in `dv'{
di "now processing `x'"
local command "`model' `x' 1.treat#1.post i.month, fe vce(robust)"
di "`command'"
`model' `x' 1.treat#1.post i.month, fe vce(robust)
esttab, keep(1.treat#1.post) p
estimates save ${myestimates}`model'_`x'_`cutoff'_`unit', replace
}
end

program loadreg

local model `1'
local dv `2'
local unit `3'
local cutoff `4'

di "storing estimates from `model' for `dv' (`unit')"

est clear
foreach x in `dv'{
estimates use ${myestimates}`model'_`x'_`cutoff'_`unit'
estadd local unitfe "Yes", replace
estadd local monthfe "Yes", replace
eststo est`x'
}
end

program writereg

local filename `1'

esttab using "${tables}`filename'.tex", keep(1.treat#1.post) se ar2 nonotes star(+ 0.15 * 0.10 ** 0.05 *** 0.01) coeflabels(1.treat#1.post "Post X TIGER" _cons "Constant") mtitles("Contrib" "Users" "New Users" "New Users(6+)" "New Super Users" "Super Users") replace booktabs  s(unitfe monthfe N, label("County FE" "Month FE"))

end

program diffindiff

local model `1'
local dv `2'
local unit `3'
local cutoff `4'
local mode `5'

di "OK, you asked me to generate DD charts"
di "for `model', unit: `unit', DVs: `dv', mode: `mode', cutoff: `cutoff'"

//testing
* local model xtpoisson
* local dv numuser
* local unit geoid10
//

if "`mode'" == "run"{
    drop if year > `cutoff'
    runreg `model' "`dv'" `unit' `cutoff'
}

if "`mode'" == "write"{
    loadreg `model' "`dv'" `unit' `cutoff'
    writereg `model'_`unit'_`cutoff'
}

end

program loadreg

local model `1'
local dv `2'
local unit `3'
local cutoff `4'

di "storing estimates from `model' for `dv' (`unit')"

est clear
foreach x in `dv'{
estimates use ${myestimates}`model'_`x'_`cutoff'_`unit'
estadd local unitfe "Yes", replace
estadd local monthfe "Yes", replace
eststo est`x'
}
end

program runreg
local model `1'
local dv `2'
local unit `3'
local cutoff `4'

est clear
di "-----"
di "  Running `model' for `dv'"
di "-----"
foreach x in `dv'{
di "now processing `x'"
local command "`model' `x' 1.treat#1.post i.month, fe vce(robust)"
di "`command'"
`model' `x' 1.treat#1.post i.month, fe vce(robust)
esttab, keep(1.treat#1.post) p
estimates save ${myestimates}`model'_`x'_`cutoff'_`unit', replace
}
end

program writereg

local filename `1'

esttab using "${tables}`filename'.tex", keep(1.treat#1.post) se ar2 nonotes star(+ 0.15 * 0.10 ** 0.05 *** 0.01) coeflabels(1.treat#1.post "Post X TIGER" _cons "Constant") mtitles("Contrib" "Users" "New Users" "New Users(6+)" "New Super Users" "Super Users") replace booktabs  s(unitfe monthfe N, label("County FE" "Month FE"))

end


Delete Do_File_Thesis59.do

Delete Do_File_Thesis59.do

deleted this due to new upload

/*
###############
Set Up:
###############

*/

clear
 cd "/Users/kevincarriere/Desktop/PoliceAssaults"
 insheet using "data/final/FinalData_nofed2.csv"
 *insheet using "data/final/FinalData.csv"
 set more off

 run "do/variables.do"

 /*
 ###############
 Stata Time, Set Your TimeSeries
 ###############
 */
 xtset  stateName ov_correcttime

 *Drop Florida because it only reports in December (looks like full assaults).
 drop if state=="Florida"
 *Canal Zone and Puerto Rico and Virgin Islands have no assaults.
 drop if state=="Canal Zone"
 drop if state=="Puerto Rico"
 drop if state=="Virgin Islands"
 *drop if state=="Guam"
 *Did not report their crime rates properly.
 drop if state=="Alabama"
 drop if state=="Rhode Island"
 *DC only has One Purchase
 drop if state=="District of Columbia"
 drop if unemployrate==0
 drop if year==2015
 drop if year==2016

/*
####################################################################
*/
 count if ov_quantitybought > 0
 count if assaults != .
 count if assaults == 0

 tab state, nofreq
 display r(r)

 count if ov_quantitybought > 0
 count if assaults != .
 count if assaults == 0

 sum assaults
 sum vicrimepop
 sum popmillion
 sum officerper1000
 sum unemployrate
 sum black_perc
 sum lincome
 sum ov_clothing_quantity
 sum ov_armor_quantity
 sum ov_weapon_quantity
 sum ov_record_quantity
pwcorr assaults vicrimepop popmillion officerper1000 unemployrate lincome black_perc l6_ov_clothing l6_ov_armor l6_ov_weapons l6_ov_photos, sig star(.05)

*
*XTNBREG
*

*8 Iterations
*This is non-stockpiled force on assaults without squared term.
xtnbreg assaults i.ov_correcttime c.l6_ov_force_hund c.l6_ov_photos_hund $controls, irr fe exposure(vicrimepop)
xtnbreg assaults i.ov_correcttime c.l6_ov_clothing_hund  c.l6_ov_armor_hund c.l6_ov_weapons_hund c.l6_ov_photos_hund $controls, irr fe exposure(vicrimepop)

*7 Iterations
*This is stockpiled force on assaults without squared term.
xtnbreg assaults i.ov_correcttime c.l6_ov_force_stockpile_hund c.l6_ov_photo_stockpile_hund $controls, irr fe exposure(vicrimepop)
xtnbreg assaults i.ov_correcttime c.l6_ov_clothing_stockpile_hund c.l6_ov_armor_stockpile_hund c.l6_ov_weapons_stockpile_hund c.l6_ov_photo_stockpile_hund  $controls, irr fe exposure(vicrimepop)

*7 Iterations
*This is stockpiled force on assaults, with squared term.
xtnbreg assaults i.ov_correcttime c.l6_ov_force_stockpile_hund##c.l6_ov_force_stockpile_hund c.l6_ov_photo_stockpile_hund##c.l6_ov_photo_stockpile_hund $controls, irr fe exposure(vicrimepop)
xtnbreg assaults i.ov_correcttime c.l6_ov_clothing_stockpile_hund##c.l6_ov_clothing_stockpile_hund c.l6_ov_armor_stockpile_hund##c.l6_ov_armor_stockpile_hund c.l6_ov_weapons_stockpile_hund##c.l6_ov_weapons_stockpile_hund c.l6_ov_photo_stockpile_hund##c.l6_ov_photo_stockpile_hund  $controls, irr fe exposure(vicrimepop)
*No Feds
 sum ov_armor_stockpile_hund
 margins, at(l6_ov_armor_stockpile_hund=(0(.5)21.5)) atmeans vsquish noatlegend

 tab state year if l6_ov_force_stockpile_hund > 330 & l6_ov_force_stockpile_hund!=.
 tab state year if l6_ov_photo_stockpile_hund > 75 & l6_ov_photo_stockpile_hund!=.

margins, at(l6_ov_force_stockpile_hund=(0(10)883)) atmeans vsquish noatlegend
marginsplot
margins, at(l6_ov_photo_stockpile_hund=(0(5)155)) atmeans vsquish noatlegend
marginsplot

 /*
 If you ask, "Well, what if we mean center?"

 tl;da: coefficients suck more, but in general same trend. photo drops down to 1% instead of 7%, armor 4% from 14%.
 */
xtnbreg assaults i.ov_correcttime c.l6_ov_force_c_hund c.l6_ov_photos_c_hund $controls, irr fe exposure(vicrimepop)
xtnbreg assaults i.ov_correcttime c.l6_ov_clothing_c_hund  c.l6_ov_armor_c_hund c.l6_ov_weapons_c_hund c.l6_ov_photos_c_hund $controls, irr fe exposure(vicrimepop)
*This is stockpiled force on assaults without squared term.
xtnbreg assaults i.ov_correcttime c.l6_ov_force_stockpile_c_hund c.l6_ov_photo_c_stockpile_hund $controls, irr fe exposure(vicrimepop)
xtnbreg assaults i.ov_correcttime c.l6_ov_clothing_stockpile_c_hund c.l6_ov_armor_stockpile_c_hund c.l6_ov_weapons_stockpile_c_hund c.l6_ov_photo_stockpile_c_hund  $controls, irr fe exposure(vicrimepop)
*This is stockpiled force on assaults, with squared term.
xtnbreg assaults i.ov_correcttime c.l6_ov_force_stockpile_c_hund##c.l6_ov_force_stockpile_c_hund c.l6_ov_photo_stockpile_c_hund##c.l6_ov_photo_stockpile_c_hund $controls, irr fe exposure(vicrimepop)
xtnbreg assaults i.ov_correcttime c.l6_ov_clothing_stockpile_c_hund##c.l6_ov_clothing_stockpile_c_hund c.l6_ov_armor_stockpile_c_hund##c.l6_ov_armor_stockpile_c_hund c.l6_ov_weapons_stockpile_c_hund##c.l6_ov_weapons_stockpile_c_hund c.l6_ov_photo_stockpile_c_hund##c.l6_ov_photo_stockpile_c_hund  $controls, irr fe exposure(vicrimepop)

/*
##### NPR #####
*/
xtnbreg assaults i.npr_correcttime c.l6_npr_clothing_hund  c.l6_npr_armor_hund c.l6_npr_weapons_hund c.l6_npr_photos_hund $controls, irr fe exposure(vicrimepop)
xtnbreg assaults i.npr_correcttime c.l6_npr_clothing_stockpile_hund c.l6_npr_armor_stockpile_hund c.l6_npr_weapons_stockpile_hund c.l6_npr_photo_stockpile_hund  $controls, irr fe exposure(vicrimepop)
xtnbreg assaults i.npr_correcttime c.l6_npr_clothing_stockpile_hund##c.l6_npr_clothing_stockpile_hund c.l6_npr_armor_stockpile_hund##c.l6_npr_armor_stockpile_hund c.l6_npr_weapons_stockpile_hund##c.l6_npr_weapons_stockpile_hund c.l6_npr_photo_stockpile_hund##c.l6_npr_photo_stockpile_hund  $controls, irr fe exposure(vicrimepop)

/*
##### NPR - OV #####
*/
xtnbreg assaults i.ov_correcttime c.l6_nprov_clothing_hund  c.l6_nprov_armor_hund c.l6_nprov_weapons_hund c.l6_nprov_photos_hund $controls, irr fe exposure(vicrimepop)
xtnbreg assaults i.ov_correcttime c.l6_nprov_clothing_stockpile_hund c.l6_nprov_armor_stockpile_hund c.l6_nprov_weapons_stockpile_hund c.l6_nprov_photo_stockpile_hund  $controls, irr fe exposure(vicrimepop)
xtnbreg assaults i.ov_correcttime c.l6_nprov_clothing_stockpile_hund##c.l6_nprov_clothing_stockpile_hund c.l6_nprov_armor_stockpile_hund##c.l6_nprov_armor_stockpile_hund c.l6_nprov_weapons_stockpile_hund##c.l6_nprov_weapons_stockpile_hund c.l6_nprov_photo_stockpile_hund##c.l6_nprov_photo_stockpile_hund  $controls, irr fe exposure(vicrimepop)

*******
*Logged Total Cost
*******

*8 Iterations
xtnbreg assaults i.ov_correcttime c.l6_ov_ln_clothing_totalcost c.l6_ov_ln_armor_totalcost c.l6_ov_ln_weapons_totalcost c.l6_ov_ln_photo_totalcost $controls, irr fe exposure(vicrimepop)

*Stockpile No Square
xtnbreg assaults i.ov_correcttime c.l6_ov_ln_clothing_cost_stockpile c.l6_ov_ln_armor_cost_stockpile c.l6_ov_ln_weapons_cost_stockpile c.l6_ov_ln_photo_cost_stockpile $controls, irr fe exposure(vicrimepop)

*Stockpile Squared
xtnbreg assaults i.ov_correcttime c.l6_ov_ln_clothing_cost_stockpile##c.l6_ov_ln_clothing_cost_stockpile c.l6_ov_ln_armor_cost_stockpile##c.l6_ov_ln_armor_cost_stockpile c.l6_ov_ln_weapons_cost_stockpile##c.l6_ov_ln_weapons_cost_stockpile c.l6_ov_ln_photo_cost_stockpile##c.l6_ov_ln_photo_cost_stockpile $controls, irr fe exposure(vicrimepop)