iemathc : add random var to matchcount()

A random variable is added to be able to sort between two observations
with the same value in the continuous variable to be used in the
matching

	clear

	*command starts here

	cap program drop 	iematch
	    program define	iematch

		syntax    , ///
			GRPdummy(varname) 		///
			MATCHcont(varname) 		///
			[						///
			idvar(varname)			///
			matchidname(string)		///
			matchdiffname(string)	///
			]

		*****
		*1. [if] [in]
		*2 make match only on obs with group
		*3 m:1 match

		noi di "Syntax OK!"

		qui {

		********************************
		*
		*	Checking group dummy
		*
		********************************

		*Test that the group dummy is 1, 0 or missing
		cap assert inlist(`grpdummy', 1, 0, .) | `grpdummy' > .

		if _rc != 0 {

			di as error "The variable in grpdummy(`grpdummy') is not a dummy variable. The variable is only allowed to have the values 1, 0 or missing. Observations with missing varaibles in the grpdummy are ignored by this command."
			error _rc
		}

		********************************
		*
		*	Checking ID var
		*
		********************************

		*Checking set-up for the ID var used for _matchID
		if "`idvar'" == "" {

			*Copy this name
			local idvar _ID

			*Make sure that the default name _ID is not already used
			cap confirm variable `idvar'
			if _rc == 0 {

				di as error "A variable with name `idvar' is already defined. Either drop this variable or specify a variable using idvar() that fully and uniquely identifies the data set."
				error 110

			}

			*Generate the ID var from row number
			gen `idvar' = _n

		}
		else {

			*Make sure that user specified ID is uniquely and fully identifying the observations
			cap isid `idvar'
			if _rc != 0 {

				di as error "Variable `idvar' specified in idvar() does not fully and uniquely identify the observations in the data set. Meaning that `idvar' is not allowed to have any duplicates or missing values. Make `idvar' uniquly and fully idnetifying or exclude the varaibleas using if or in."
				error 450

			}
		}

		********************************
		*
		*	Checking and generating the MatchID and the MatchDiff var
		*
		********************************

		*See function below
		iematchMatchVarCheck `matchidname' _matchID
		local matchIDname "`r(validVarName)'"
		gen  `matchIDname' = .

		iematchMatchVarCheck `matchdiffname' _matchDiff
		local matchDiffName "`r(validVarName)'"
		gen `matchDiffName' = .

		********************************
		*
		*	Creating tempvar used
		*
		********************************

		tempvar hi_diff lo_diff pref match newMatch rand

		gen `hi_diff' = .
		gen `lo_diff' = .

		gen `pref' = .
		gen byte `match' = 0
		gen byte `newMatch' = 0

		** Generate random variable that seperate two obs
		*  with the same value in the continuos variable
		gen `rand' = uniform()

		********************************
		*
		*	Start matching
		*
		********************************

		count if `grpdummy' == 1 & `match' == 0
		local left2Match = `r(N)'

		while (`left2Match' > 0) {

			sort `match' `matchcont' `rand'

			replace `lo_diff' 	= .
			replace `hi_diff' 	= .
			replace `pref'		= . if `match' == 0
			replace `newMatch' 	= 0

			replace `lo_diff' = abs(`matchcont' - `matchcont'[_n-1]) if `match' == 0 & `grpdummy' != `grpdummy'[_n-1]
			replace `hi_diff' = abs(`matchcont' - `matchcont'[_n+1]) if `match' == 0 & `grpdummy' != `grpdummy'[_n+1]

			replace `pref' = `idvar'[_n-1] if `match' == 0 & `lo_diff' <  `hi_diff'
			replace `pref' = `idvar'[_n+1] if `match' == 0 & `lo_diff' >= `hi_diff'

			replace `newMatch' = 1 if `match' == 0 & `pref' == `idvar'[_n-1] & `pref'[_n-1] == `idvar'
			replace `newMatch' = 1 if `match' == 0 & `pref' == `idvar'[_n+1] & `pref'[_n+1] == `idvar'

			replace `match' = 1 							if `newMatch' == 1
			replace `matchDiffName' = min(`lo_diff', `hi_diff') 	if `newMatch' == 1
			replace `matchIDname' = `idvar' 						if `newMatch' == 1 & `grpdummy' == 1
			replace `matchIDname' = `pref' 						if `newMatch' == 1 & `grpdummy' == 0

			*order _match* , last

		noi	count if `grpdummy' == 1 & `match' == 0
			local left2Match = `r(N)'

			*pause

		}
	}

	end

	*Cheack matchVar input
	cap program drop 	iematchMatchVarCheck
		program define 	iematchMatchVarCheck  , rclass

		args defaultName optionName

		*Testing that the option name is not the same as the other variable's default name.
		if "`defaultName'" == "_matchID" & "`optionName'" == "_matchDiff" {

			noi di as error "The new name specified in matchidname() is not allowed to be _matchDiff"
			error 198
		}
		if "`defaultName'" == "_matchDiff" & "`optionName'" == "_matchID" {

			noi di as error "The new name specified in matchdiffname() is not allowed to be _matchID"
			error 198
		}

		*Checking set-up for the ID var used for _matchID
		if "``optionName''" == "" {

			*CSet the default name
			local validMatchVarname `defaultName'

			*Make sure that the default name _ID is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				noi di as error "A variable with name `validMatchVarname' is already defined. Either drop this variable or specify a new variable name using `optionName'()."
				error 110

			}

		}

		else {

			*CSet the default name
			local validMatchVarname `optionName'

			*Make sure that the default name _ID is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				noi di as error "The variable name `validMatchVarname' specified in `optionName'() is already defined in the data set. Either drop this variable or specify a another variable name using `optionName'()."
				error 110

			}

		}

		*Testing that the new name is valid by creating a variable that is dropped immedeatly afterwards
		cap gen `validMatchVarname' = .

		if _rc != 0 {

			noi di as error "The variable name specified is not a valid variable name."
			error _rc

		}
		drop `validMatchVarname'

		return local validVarName 	"`validMatchVarname'"

	end

	clear

*	set seed 1235324

	set obs 10000

	gen id = _n

	gen rand1 = uniform()

	gen tmt = (rand1 < .40)
	replace tmt = . if (rand1 < .2)
	replace tmt = .a if (rand1 < .1)

	drop rand1
	tab tmt

	gen p_hat = uniform()

	iematch if id < 5532  , grp(tmt) match(p_hat) idvar(id)

	clear

	*command starts here

	cap program drop 	iematch
	    program define	iematch

		syntax    , ///
			GRPdummy(varname) 		///
			MATCHcont(varname) 		///
			[						///
			idvar(varname)			///
			matchidname(string)		///
			matchdiffname(string)	///
			]

		*****
		*1. [if] [in]
		*2 make match only on obs with group
		*3 m:1 match

		noi di "Syntax OK!"

		qui {

		********************************
		*
		*	Checking groupvar
		*
		********************************

		*Test that the group dummy is 1, 0 or missing
		cap assert inlist(`grpdummy', 1, 0, .) | `grpdummy' > .

		if _rc != 0 {

			di as error "The variable in grpdummy(`grpdummy') is not a dummy variable. The variable is only allowed to have the values 1, 0 or missing. Observations with missing varaibles in the grpdummy are ignored by this command."
			error _rc
		}

		********************************
		*
		*	Checking ID var
		*
		********************************

		*Checking set-up for the ID var used for _matchID
		if "`idvar'" == "" {

			*Copy this name
			local idvar _ID

			*Make sure that the default name _ID is not already used
			cap confirm variable `idvar'
			if _rc == 0 {

				di as error "A variable with name `idvar' is already defined. Either drop this variable or specify a variable using idvar() that fully and uniquely identifies the data set."
				error 110

			}

			*Generate the ID var from row number
			gen `idvar' = _n

		}
		else {

			*Make sure that user specified ID is uniquely and fully identifying the observations
			cap isid `idvar'
			if _rc != 0 {

				di as error "Variable `idvar' specified in idvar() does not fully and uniquely identify the observations in the data set. Meaning that `idvar' is not allowed to have any duplicates or missing values. Make `idvar' uniquly and fully idnetifying or exclude the varaibleas using if or in."
				error 450

			}
		}

		********************************
		*
		*	Checking and generating the MatchID and the MatchDiff var
		*
		********************************

		*See function below
		iematchMatchVarCheck `matchidname' _matchID
		local matchIDname "`r(validVarName)'"
		gen  `matchIDname' = .

		iematchMatchVarCheck `matchdiffname' _matchDiff
		local matchDiffName "`r(validVarName)'"
		gen `matchDiffName' = .

		********************************
		*
		*	Creating tempvar used
		*
		********************************

		tempvar hi_diff lo_diff pref match newMatch

		gen `hi_diff' = .
		gen `lo_diff' = .

		gen `pref' = .
		gen byte `match' = 0
		gen byte `newMatch' = 0

		********************************
		*
		*	Start matching
		*
		********************************

		count if `grpdummy' == 1 & `match' == 0
		local left2Match = `r(N)'

		while (`left2Match' > 0) {

			sort `match' `matchcont'

			replace `lo_diff' 	= .
			replace `hi_diff' 	= .
			replace `pref'		= . if `match' == 0
			replace `newMatch' 	= 0

			replace `lo_diff' = abs(`matchcont' - `matchcont'[_n-1]) if `match' == 0 & `grpdummy' != `grpdummy'[_n-1]
			replace `hi_diff' = abs(`matchcont' - `matchcont'[_n+1]) if `match' == 0 & `grpdummy' != `grpdummy'[_n+1]

			replace `pref' = `idvar'[_n-1] if `match' == 0 & `lo_diff' <  `hi_diff'
			replace `pref' = `idvar'[_n+1] if `match' == 0 & `lo_diff' >= `hi_diff'

			replace `newMatch' = 1 if `match' == 0 & `pref' == `idvar'[_n-1] & `pref'[_n-1] == `idvar'
			replace `newMatch' = 1 if `match' == 0 & `pref' == `idvar'[_n+1] & `pref'[_n+1] == `idvar'

			replace `match' = 1 							if `newMatch' == 1
			replace `matchDiffName' = min(`lo_diff', `hi_diff') 	if `newMatch' == 1
			replace `matchIDname' = `idvar' 						if `newMatch' == 1 & `grpdummy' == 1
			replace `matchIDname' = `pref' 						if `newMatch' == 1 & `grpdummy' == 0

			*order _match* , last

		noi	count if `grpdummy' == 1 & `match' == 0
			local left2Match = `r(N)'

			*pause

		}
	}

	end

	*Cheack matchVar input
	cap program drop 	iematchMatchVarCheck
		program define 	iematchMatchVarCheck  , rclass

		args defaultName optionName

		*Testing that the option name is not the same as the other variable's default name.
		if "`defaultName'" == "_matchID" & "`optionName'" == "_matchDiff" {

			noi di as error "The new name specified in matchidname() is not allowed to be _matchDiff"
			error 198
		}
		if "`defaultName'" == "_matchDiff" & "`optionName'" == "_matchID" {

			noi di as error "The new name specified in matchdiffname() is not allowed to be _matchID"
			error 198
		}

		*Checking set-up for the ID var used for _matchID
		if "``optionName''" == "" {

			*CSet the default name
			local validMatchVarname `defaultName'

			*Make sure that the default name _ID is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				noi di as error "A variable with name `validMatchVarname' is already defined. Either drop this variable or specify a new variable name using `optionName'()."
				error 110

			}

		}

		else {

			*CSet the default name
			local validMatchVarname `optionName'

			*Make sure that the default name _ID is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				noi di as error "The variable name `validMatchVarname' specified in `optionName'() is already defined in the data set. Either drop this variable or specify a another variable name using `optionName'()."
				error 110

			}

		}

		*Testing that the new name is valid by creating a variable that is dropped immedeatly afterwards
		cap gen `validMatchVarname' = .

		if _rc != 0 {

			noi di as error "The variable name specified is not a valid variable name."
			error _rc

		}
		drop `validMatchVarname'

		return local validVarName 	"`validMatchVarname'"

	end

	clear

*	set seed 1235324

	set obs 10000

	gen id = _n

	gen rand1 = uniform()

	gen tmt = (rand1 < .40)
	replace tmt = . if (rand1 < .2)
	replace tmt = .a if (rand1 < .1)

	drop rand1
	tab tmt

	gen p_hat = uniform()

	iematch if id < 5532  , grp(tmt) match(p_hat) idvar(id)


Add curse/all to t/all, now that it passes.

redo-ifchange hello yellow bellow c d example/all curse/all


Use 'confirm' instead of abusing 'syntax'.

Silly me.

* TODO: rename to 'clone'
/* generate a perfect copy of a variable, (type, labels, etc) of an variable

 syntax:
  clone newvar oldvar [if] [in]

 You can use 'if' and 'in' to control what values; values that don't match will be set to missing.
 If you want to clone a variable's metadata but not values use the idiom ". clone new old if 0".

 The reason the syntax is not "clone newvar = oldvar" is because but =/exp insists on numeric expressions,
 so string variables wouldn't be cloneable.
 */

program define clone
  // parse once to extract the basic pieces of syntax
  syntax namelist [if] [in]
  local _if = "`if'" //save these for later; the other syntax commands will smash them
  local _in = "`in'"

  gettoken target source : namelist

  // enforce types
  confirm new variable `target'
  confirm variable `source'

  // save attributes
  local T : type `source' //the data type
  local N : variable label `source' //the human readable description
  local V : value label `source' // the name of the label map in use, if there is one
                                 // Stata maintains a dictionary of dictionaries, each of which
                                 // maps integers to strings. Multiple variables can share a dictionary,
                                 // though it is rare except for e.g. "boolean"

  // make new variable
  generate `T' `target' = `source' `_if' `_in'

  // clone attributes if they exist
  // (except for type, which always exists and cannot be reassigned without
  //  another 'generate' doing a whole new malloc())
  if("`N'"!="") {
    label variable `target' "`N'"  //Yes, the setters and getters are...
  }
  if("`V'"!="") {
    label value `target' "`V'"     //...in fact reverses of each other
  }
end

* TODO: rename to 'clone'
/* generate a perfect copy of a variable, (type, labels, etc) of an variable

 syntax:
  clone newvar oldvar [if] [in]

 You can use 'if' and 'in' to control what values; values that don't match will be set to missing.
 If you want to clone a variable's metadata but not values use the idiom ". clone new old if 0".

 The reason the syntax is not "clone newvar = oldvar" is because but =/exp insists on numeric expressions,
 so string variables wouldn't be cloneable.
 */

program define clone
  // parse once to extract the basic pieces of syntax
  syntax namelist [if] [in]
  local _if = "`if'" //save these for later; the other syntax commands will smash them
  local _in = "`in'"

  gettoken target source : namelist

  // enforce types

  // newvar
  local 0 = "`target'"
  syntax newvarname

  // oldvar
  local 0 = "`source'"
  syntax varname

  // save attributes
  local T : type `source' //the data type
  local N : variable label `source' //the human readable description
  local V : value label `source' // the name of the label map in use, if there is one
                                 // Stata maintains a dictionary of dictionaries, each of which
                                 // maps integers to strings. Multiple variables can share a dictionary,
                                 // though it is rare except for e.g. "boolean"

  // make new variable
  generate `T' `target' = `source' `_if' `_in'

  // clone attributes if they exist
  // (except for type, which always exists and cannot be reassigned without
  //  another 'generate' doing a whole new malloc())
  if("`N'"!="") {
    label variable `target' "`N'"  //Yes, the setters and getters are...
  }
  if("`V'"!="") {
    label value `target' "`V'"     //...in fact reverses of each other
  }
end


debugged freq_table

debugged freq_table

substituted saved files with tempfile

// Apoorva Lal
// V 1.0
program define freq_table
	version 10
	/*
	SYNTAX:
	freq_table var1 var2 i.var3 i.var4#i.var4
	*/
	syntax anything // not quite, but easist to handle programmatically this way

	loc codedir "`c(pwd)'"
	qui cd "`codedir'"

	loc temp1 : list clean anything
	loc temp2 : list uniq temp1
	loc var: list clean temp2
	loc masterdata "`c(filename)'"
	qui count
	sca denom = `r(N)'

	tokenize `var'
	loc n: word count `var'

	preserve
	qui {
		clear
		set obs 1
		g raw = "ALL"
		g label = "ALL"
		g count = denom
		qui compress
		tempfile t0
		save "`t0'", replace
	}
	restore
	qui {
		forv i = 1/`n' {
			preserve
			loc prefix = substr("``i''",1,2)

			if ("`prefix'" == "i.") {			// stata's dummy notation
				loc interaction_split1 = strpos("``i''","*")
				loc interaction_split2 = strpos("``i''","#")
				loc interaction_split = max(`interaction_split1',`interaction_split2')
				if `interaction_split' != 0 {
					loc suffix1 = substr("``i''",3,`interaction_split'-3)
					cap conf var `suffix1'
					if _rc {
						noi di as error "`suffix1' does not exist."
						error
					}
					loc suffix2 = substr("``i''",`interaction_split'+3,.)
					cap conf var `suffix2'
					if _rc {
						noi di as error "`suffix2' does not exist"
						error
					}
					forv x = 1/2 {
						qui levelsof `suffix`x'', clean loc(`suffix`x''_levels) miss
						loc n_var`x': list sizeof `suffix`x''_levels
						loc lab`x' : var l `suffix`x''
						if "`lab`x''" == "" loc lab`x' `suffix`x''
						loc vlab`x' : val l `suffix`x''
						cap conf string var `suffix`x''
						if !_rc {
							loc counter_type`x' string
						}
						else {
							loc counter_type`x' numeric
						}
					}

					loc x = 1
					foreach a in ``suffix1'_levels' {
						foreach b in ``suffix2'_levels' {
							if "`vlab1'" != "" loc val_lab1: lab `vlab1' `a'
							else loc val_lab1 "`a'"
							if "`vlab2'" != "" loc val_lab2: lab `vlab2' `a'
							else loc val_lab2 "`b'"

							loc raw`x' = "`suffix1' == `a' X `suffix2' == `b' "
							loc labels`x' = "`lab1' == `val_lab1' X `lab2' == `val_lab2'"
							if "`counter_type1'" == "string" & ///
								 "`counter_type2'" == "string" {
								qui count if `suffix1' == "`a'" & `suffix2' == `b'
							}
							else if "`counter_type1'" == "numeric" & ///
								 "`counter_type2'" == "string" {
								 qui count if `suffix1' == `a' & `suffix2' == "`b'"
							}
							else if "`counter_type1'" == "string" & ///
								 "`counter_type2'" == "numeric" {
								 qui count if `suffix1' == "`a'" & `suffix2' == `b'
							}
							else if "`counter_type1'" == "numeric" & ///
								 "`counter_type2'" == "numeric" {
								 qui count if `suffix1' == `a' & `suffix2' == `b'
							}
							sca N_`x' = `r(N)'
							loc ++x
						}
					}
				}
				else {
					loc suffix1 = substr("``i''",3,.)
					cap conf var `suffix1'
					if _rc {
						noi di as error "`suffix1' does not exist."
						error
					}
					loc x 1
					qui levelsof `suffix`x'', clean loc(`suffix`x''_levels) miss
					loc n_var`x': list sizeof `suffix`x''_levels
					loc lab`x' : var l `suffix`x''
					if "`lab`x''" == "" loc lab`x' `suffix`x''
					loc vlab`x' : val l `suffix`x''
					cap conf string var `suffix`x''
					if !_rc {
						loc counter_type`x' string
					}
					else {
						loc counter_type`x' numeric
					}
					loc x = 1
					foreach a in ``suffix1'_levels' {
						if "`vlab1'" != "" loc val_lab1: lab `vlab1' `a'
						else loc val_lab1 "`a'"
						loc raw`x' = "`suffix1' == `a'"
						loc labels`x' = "`lab1' == `val_lab1'"
						if "`counter_type1'" == "string" {
							qui count if `suffix1' == "`a'"
						}
						else if "`counter_type1'" == "numeric" {
							qui count if `suffix1' == `a'
						}
						sca N_`x' = `r(N)'
						loc ++x
					}
				}

			}
			else {
				cap conf var ``i''
				if !_rc {
					loc lab : var l ``i''
					if "`lab'" == "" loc lab ``i''
					loc raw1 = "``i''"
					loc labels1 = "`lab' == 1"
					qui count if ``i'' & !mi(``i'')
					sca N_1 `r(N)'
					loc x = 2
				}
				else {
					di as error "``i'' does not exist"
					error
				}
			}
			clear
			set obs `=`x'-1'
			g raw = ""
			g label = ""
			g count = 0
			forv r = 1/`=`x'-1' {
				replace raw = "`raw`r''" if _n == `r'
				replace label = "`labels`r''" if _n == `r'
				replace count = N_`r' if _n == `r'
			}
			g pct = count/denom
			tempfile t`i'
			save "`t`i''", replace
			restore
			loc nrows `i'
		}
		clear
		use "`t0'", clear
		forv f = 1/`nrows' {
			append using "`t`f''"
		}
	}
end

// Apoorva Lal
// V 1.0
program define freq_table
	version 10
	/*
	SYNTAX:
	freq_table var1 var2 i.var3 i.var4#i.var4
	*/
	syntax anything // not quite, but easist to handle programmatically this way

	loc codedir "`c(pwd)'"
	loc temp "`c(tmpdir)'"
	mkdir "`temp'\\freq_table_temp\"
	qui cap cd "`temp'\\freq_table_temp\"
	if _rc {
		noi di as error "Unable to create `temp'\\freq_table_temp\ folder; pelase create manually before running"
		error
	}
	else !erase *.dta
	qui cd "`codedir'"

	loc temp1 : list clean anything
	loc temp2 : list uniq temp1
	loc var: list clean temp2
	loc masterdata "`c(filename)'"
	qui count
	sca denom = `r(N)'

	tokenize `var'
	loc n: word count `var'

	preserve
	qui {
		clear
		set obs 1
		g raw = "ALL"
		g label = "ALL"
		g count = denom
		qui compress
		save "`temp'\\freq_table_temp\\row_0", replace
	}
	restore
	qui {
		forv i = 1/`n' {
			preserve
			loc prefix = substr("``i''",1,2)

			if ("`prefix'" == "i.") {			// stata's dummy notation
				loc interaction_split1 = strpos("``i''","*")
				loc interaction_split2 = strpos("``i''","#")
				loc interaction_split = max(`interaction_split1',`interaction_split2')
				if `interaction_split' != 0 {
					loc suffix1 = substr("``i''",3,`interaction_split'-3)
					cap conf var `suffix1'
					if _rc {
						noi di as error "`suffix1' does not exist."
						error
					}
					loc suffix2 = substr("``i''",`interaction_split'+3,.)
					cap conf var `suffix2'
					if _rc {
						noi di as error "`suffix2' does not exist"
						error
					}
					forv x = 1/2 {
						qui levelsof `suffix`x'', clean loc(`suffix`x''_levels) miss
						loc n_var`x': list sizeof `suffix`x''_levels
						loc lab`x' : var l `suffix`x''
						if "`lab`x''" == loc lab`x' `suffix`x''
						loc vlab`x' : val l `suffix`x''
						cap conf string var `suffix`x''
						if !_rc {
							loc counter_type`x' string
						}
						else {
							loc counter_type`x' numeric
						}
					}

					loc x = 1
					foreach a in ``suffix1'_levels' {
						foreach b in ``suffix2'_levels' {
							if "`vlab1'" != "" loc val_lab1: lab `vlab1' `a'
							else loc val_lab1 "`a'"
							if "`vlab2'" != "" loc val_lab2: lab `vlab2' `a'
							else loc val_lab2 "`b'"

							loc raw`x' = "`suffix1' == `a' X `suffix2' == `b' "
							loc labels`x' = "`lab1' == `val_lab1' X `lab2' == `val_lab2'"
							if "`counter_type1'" == "string" & ///
								 "`counter_type2'" == "string" {
								qui count if `suffix1' == "`a'" & `suffix2' == `b'
							}
							else if "`counter_type1'" == "numeric" & ///
								 "`counter_type2'" == "string" {
								 qui count if `suffix1' == `a' & `suffix2' == "`b'"
							}
							else if "`counter_type1'" == "string" & ///
								 "`counter_type2'" == "numeric" {
								 qui count if `suffix1' == "`a'" & `suffix2' == `b'
							}
							else if "`counter_type1'" == "numeric" & ///
								 "`counter_type2'" == "numeric" {
								 qui count if `suffix1' == `a' & `suffix2' == `b'
							}
							sca N_`x' = `r(N)'
							loc ++x
						}
					}
				}
				else {
					loc suffix1 = substr("``i''",3,.)
					cap conf var `suffix1'
					if _rc {
						noi di as error "`suffix1' does not exist."
						error
					}
					loc x 1
					qui levelsof `suffix`x'', clean loc(`suffix`x''_levels) miss
					loc n_var`x': list sizeof `suffix`x''_levels
					loc lab`x' : var l `suffix`x''
					if "`lab`x''" == loc lab`x' `suffix`x''
					loc vlab`x' : val l `suffix`x''
					cap conf string var `suffix`x''
					if !_rc {
						loc counter_type`x' string
					}
					else {
						loc counter_type`x' numeric
					}
					loc x = 1
					foreach a in ``suffix1'_levels' {
						if "`vlab1'" != "" loc val_lab1: lab `vlab1' `a'
						else loc val_lab1 "`a'"
						loc raw`x' = "`suffix1' == `a'"
						loc labels`x' = "`lab1' == `val_lab1'"
						if "`counter_type1'" == "string" {
							qui count if `suffix1' == "`a'"
						}
						else if "`counter_type1'" == "numeric" {
							qui count if `suffix1' == `a'
						}
						sca N_`x' = `r(N)'
						loc ++x
					}
				}

			}
			else {
				cap conf var ``i''
				if !_rc {
					loc lab : var l ``i''
					if "`lab'" == "" loc lab ``i''
					loc raw1 = "``i''"
					loc labels1 = "`lab' == 1"
					qui count if ``i'' & !mi(``i'')
					sca N_1 `r(N)'
					loc x = 2
				}
				else {
					di as error "``i'' does not exist"
					error
				}
			}
			clear
			set obs `=`x'-1'
			g raw = ""
			g label = ""
			g count = 0
			forv r = 1/`=`x'-1' {
				replace raw = "`raw`r''" if _n == `r'
				replace label = "`labels`r''" if _n == `r'
				replace count = N_`r' if _n == `r'
			}
			g pct = count/denom
			save "`temp'\\freq_table_temp\\row_`i'", replace
			restore
		}
		loc files : dir "`temp'\\freq_table_temp\\" files "row_*.dta"
		clear
		foreach f in `files' {
			append using "`temp'\\freq_table_temp\\`f'"
		}
	}
end


ieboilstart : remove do file method

ieboilstart : remove do file method

This commit removes the do-file option and simply set the settings
inside the command.

	*! version 2.1 1Nov2016  Kristoffer Bjarkefur kbjarkefur@worldbank.org

	capture program drop ieboilstart2
	program ieboilstart2 , rclass

	noi di "command works!"

	qui {

		syntax ,  Versionnumber(string) [noclear maxvar(numlist) matsize(numlist) QUIetly veryquietly CUSTom(string) setmem(string) ]

		noi di "syntax works!"

		version 11.0

		/*********************************

			Check user specifed version
			and apply it is valid

		*********************************/

		local stata_versions "11.0 11.1 11.2 12.0 12.1 13.0 13.1 14.0 14.1"

		if `:list versionnumber in stata_versions' == 0 {

			di as error "{phang}Only recent major releases are allowed. One decimal must always be included. The releases currently allowed are:{break}`stata_versions'{p_end}"
			di ""
			error 198
			exit
		}

		*Return the value
		return local version "version `versionnumber'"

		*Set the version specfied in the command
		version `versionnumber'

		/*********************************

			Check settings related to older
			versions and Stata IC

		*********************************/

		*Test that maxvar is not set in combination when using Stata IC
		if 	!(c(MP) == 1 | c(SE) == 1) & "`maxvar'" != "" {

			di as error "{phang}In Stata IC the maximum number of variables allowed is fixed at 2,047 and maxvar() is therefore not allowed.{p_end}"
			error 198
		}

		if 	"`setmem'" != ""	{

			if `versionnumber' >= 12  {

				di as error "{phang}Option setmem() is only allowed when setting the version number to 11. Setmem() is only applicable in Stata 11 or eralier, but those versions wont be able to run this file as the version number is set to 12 or higher.{p_end}"
				error 198
			}

			*Split the memory setting string into value and byte unit
			local mem_strlen 	= strlen("`setmem'") -1
			local mem_number	= substr("`setmem'",  1 , `mem_strlen')
			local mem_bytetype	= substr("`setmem'", -1 , 1)

			*Test that the number part is a number and a integer
			cap confirm integer number `mem_number'
			if _rc {

				di as error "{phang}The value in setmem(`setmem') must be a number (followed by either b, k, m or g). See {help memory} for details.{p_end}"
				error _rc
			}

			*Test that the byte type is not anything else than b, k, m or g
			if !`=inlist(lower("`mem_bytetype'"), "b", "k", "m", "g")' {

				di as error "{phang}The last character in setmem(`setmem') must be either b, k, m or g. See {help memory} for details.{p_end}""
				error 7

			}
		}
		else {

			*Default value is 50M. This is probably too little, but
			*it will safely work. And users can manually increase this
			local setmem "50M"

		}

		/*********************************

			Check input for maxvar and matsize
			if specified, other wise set
			maximum value allowed.

		*********************************/

		*Setting maxvar requires a cleared memory. Therefore
		*maxvar() and noclear cannot be used at the same time.
		if "`maxvar'" != "" & "`clear'" != "" {

			di as error "{phang}It is not possible to set allowed maximum numbers of variables without clearing the data. noclear and maxvar() can therefore not be specified at the same time{p_end}"
			di ""
			error 198
			exit
		}

		foreach maxlocal in maxvar matsize {

			*Set locals with the max and min values fox maxvar and matsize
			if "`maxlocal'" == "maxvar" {
				local max 32767
				local min 2048
			}
			if "`maxlocal'" == "matsize" {
				local max 11000
				local min 10
			}

			*Test if user set a value for this value
			if "``maxlocal''" != "" {

				*If user specified a value, test that it is between the min and the max
				if !(``maxlocal'' >= `min' & ``maxlocal'' <= `max') {

					di as error "{phang}`maxlocal' must be between `min' and `max' (inclusive) if you are using Stata SE and Stata MP. You entered ``maxlocal''.{p_end}"
					di ""
					*Throw appropriate error if below or above error
					if ``maxlocal'' < `min' error 910
					if ``maxlocal'' > `max' error 912
					exit
				}
			}

			else {
				*User did not specify value, use ieboilstart's defaults:

				if "`maxlocal'" == "maxvar" {
					*Set maxvar to max value allowed as this is often an issue when working with large survey data
					local `maxlocal' `max'
				}
				if "`maxlocal'" == "matsize" {
					*Set to the default as the maximum is rarely requered.
					local `maxlocal' 400
				}
			}
		}

		/*********************************

			Set the settings

		*********************************/

		local setDispLocal "{col 5}{ul:Settings set by this command:}"

		*Set basic memory limits
		if "`clear'" == "" {

			*Setting
			clear all
			local setDispLocal "`setDispLocal'{break}{col 5}clear all"

			**Setting maxvar not allowed in Stata IC.
			if 	(c(MP) == 1 | c(SE) == 1) {

				*Setting
				set maxvar `maxvar'
				local setDispLocal "`setDispLocal'{break}{col 5}set maxvar {col 22}`maxvar'"
			}
		}

		*Setting
		set matsize `matsize'
		local setDispLocal "`setDispLocal'{break}{col 5}set matsize {col 22}`matsize'"

		****************
		*Memory	settings

		*For compatibility with Stata 11 the do file includes and if/else
		*condition testing version number. The memoroy settings introduced
		*in Stata 12 will be applied to Stata version more recent than
		*Stata 11, and set memory will be applied to Stata 11.
		if c(stata_version) >= 12 {

			*Setting
			set niceness 5
			local setDispLocal "`setDispLocal'{break}{col 5}set niceness{col 22}5"

			*These settings cannot be modified with data in memory
			if "`clear'" == "" {

				*Settings
				set min_memory 0
				set max_memory .
				local setDispLocal "`setDispLocal'{break}{col 5}set min_memory {col 22}0{break}{col 5}set max_memory {col 22}."

				*Set segment size to the largest value allowed by the operative system
				if c(bit) == 64 {
					*Setting
					set segmentsize	32m
					local setDispLocal "`setDispLocal'{break}{col 5}set segmentsize	{col 22}32m"
				}
				else {
					*Setting
					set segmentsize	16m
					local setDispLocal "`setDispLocal'{break}{col 5}set segmentsize	{col 22}16m"
				}
			}
		}
		else {

			*If this dofile is generated in Stata 11 then only the old
			*way of setting memory is included. This will be ignored by
			*more recent versios of Stata

			*Setting
			set memory `setmem'
			local setDispLocal "`setDispLocal'{break}{col 5}set memory {col 22}`setmem'"

		}

		*********************
		*Set default settings
		set more 		off
		pause 			on
		set varabbrev 	off
		local setDispLocal "`setDispLocal'{break}{col 5}set more {col 22}off"
		local setDispLocal "`setDispLocal'{break}{col 5}pause {col 22}on"
		local setDispLocal "`setDispLocal'{break}{col 5}set varabbrev {col 22}off"

		/*********************************

			Add custom lines of code

		*********************************/

		if `"`custom'"' != "" {

			local setDispLocal `"`setDispLocal'{break} {break}{col 5}{ul:User specified settings:}"'

			*Create a local with the rowlabel input to be tokenized
			local custom_code_lines `custom'

			while `"`custom_code_lines'"' != "" {

				*Parsing name and label pair
				gettoken code_line custom_code_lines : custom_code_lines, parse("@")

				*Removing leadning or trailing spaces
				local code_line = trim(`"`code_line'"')

				*Set custom setting
				local setDispLocal `"`setDispLocal'{break}{col 5}`code_line'"'
				`code_line'

				*Parse char is not removed by gettoken
				local custom_code_lines = subinstr(`"`custom_code_lines'"' ,"@","",1)
			}
		}

		/*********************************

			Create return value and output message

		*********************************/

		if "`quietly'" == "" & "`veryquietly'" == "" {

			noi di ""
			noi di "{phang}{err:DISCLAIMER:} Due to how settings works in Stata, this command can only attempt to harmonize settings as much as possible across users, but no guarantee can be given that all commands will always behave identical unless the exact same version and type of Stata is used.{p_end}"
			noi di ""
			noi di `"`setDispLocal'"'
		}

		if  "`veryquietly'" == "" {

			noi di ""
			noi di "{phang}{err:IMPORTANT:} The most important setting of this command cannot be set inside the command due to technical reasons. The setting has been prepared by this command, and you only need to write \`r(version)' after this command (include the apostrophes).{p_end}"

		}

	}
	end

	*! version 2.1 1Nov2016  Kristoffer Bjarkefur kbjarkefur@worldbank.org

	capture program drop ieboilstart2
	program ieboilstart2 , rclass

	noi di "Syntax works!"

	qui {

		*This is used to indicate in the do file how the command was specified
		local full_specification `0'

		syntax using ,  Versionnumber(string) [noclear maxvar(numlist) matsize(numlist) QUIetdo CUSTom(string) setmem(string) ]

		version 11.0

		/*********************************

			Check user specifed version
			and apply it is valid

		*********************************/

		local stata_versions "11.0 11.1 11.2 12.0 12.1 13.0 13.1 14.0 14.1"

		if `:list versionnumber in stata_versions' == 0 {

			di as error "{phang}Only recent major releases are allowed. One decimal must always be included. The releases currently allowed are:{break}`stata_versions'{p_end}"
			di ""
			error 198
			exit
		}

		*Set the version specfied in the command
		version `versionnumber'

		/*********************************

			Check file name

		*********************************/

		*Parse out the file name
		tokenize `"`using'"'
		local 		textfile "`2'"

		*Find index for where the file type suffix start
		local dot_index 	= strpos("`textfile'",".")

		*Extract the file index
		local file_suffix 	= substr("`textfile'", `dot_index', .)

		*If no file format suffix is specified, use the default .xlsx
		if "`file_suffix'" == "" {

			local textfile `"`textfile'.do"'
		}

		*If a file format suffix is specified make sure that it is one of the two allowed.
		else if "`file_suffix'" != ".do" {

			noi display as error "{phang}The file format specified in using is other than .do. Only the .do format is allowed. If no format is specified .do is assumed.{p_end}"
			error 198
		}

		/*********************************

			Check settings related to older
			versions and Stata IC

		*********************************/

		*Test that maxvar is not set in combination when using Stata IC
		if 	!(c(MP) == 1 | c(SE) == 1) & "`maxvar'" != "" {

			di as error "{phang}In Stata IC the maximum number of variables allowed is fixed at 2,047 and maxvar() is therefore not allowed.{p_end}"
			error 198
		}

		if 	"`setmem'" != ""	{

			if `versionnumber' >= 12  {

				di as error "{phang}Option setmem() is only allowed when setting the version number to 11. Setmem() is only applicable in Stata 11 or eralier, but those versions wont be able to run this file as the version number is set to 12 or higher.{p_end}"
				error 198
			}

			*Split the memory setting string into value and byte unit
			local mem_strlen 	= strlen("`setmem'") -1
			local mem_number	= substr("`setmem'",  1 , `mem_strlen')
			local mem_bytetype	= substr("`setmem'", -1 , 1)

			*Test that the number part is a number and a integer
			cap confirm integer number `mem_number'
			if _rc {

				di as error "{phang}The value in setmem(`setmem') must be a number (followed by either b, k, m or g). See {help memory} for details.{p_end}"
				error _rc
			}

			*Test that the byte type is not anything else than b, k, m or g
			if !`=inlist(lower("`mem_bytetype'"), "b", "k", "m", "g")' {

				di as error "{phang}The last character in setmem(`setmem') must be either b, k, m or g. See {help memory} for details.{p_end}""
				error 7

			}
		}
		else {

			*Default value is 50M. This is probably too little, but
			*it will safely work. And users can manually increase this
			local setmem "50M"

		}

		/*********************************

			Check input for maxvar and matsize
			if specified, other wise set
			maximum value allowed.

		*********************************/

		*Setting maxvar requires a cleared memory. Therefore
		*maxvar() and noclear cannot be used at the same time.
		if "`maxvar'" != "" & "`clear'" != "" {

			di as error "{phang}It is not possible to set allowed maximum numbers of variables without clearing the data. noclear and maxvar() can therefore not be specified at the same time{p_end}"
			di ""
			error 198
			exit
		}

		foreach maxlocal in maxvar matsize {

			*Set locals with the max and min values fox maxvar and matsize
			if "`maxlocal'" == "maxvar" {
				local max 32767
				local min 2048
			}
			if "`maxlocal'" == "matsize" {
				local max 11000
				local min 10
			}

			*Test if user set a value for this value
			if "``maxlocal''" != "" {

				*If user specified a value, test that it is between the min and the max
				if !(``maxlocal'' >= `min' & ``maxlocal'' <= `max') {

					di as error "{phang}`maxlocal' must be between `min' and `max' (inclusive) if you are using Stata SE and Stata MP. You entered ``maxlocal''.{p_end}"
					di ""
					*Throw appropriate error if below or above error
					if ``maxlocal'' < `min' error 910
					if ``maxlocal'' > `max' error 912
					exit
				}
			}

			else {
				*User did not specify value, use ieboilstart's defaults:

				if "`maxlocal'" == "maxvar" {
					*Set maxvar to max value allowed as this is often an issue when working with large survey data
					local `maxlocal' `max'
				}
				if "`maxlocal'" == "matsize" {
					*Set to the default as the maximum is rarely requered.
					local `maxlocal' 400
				}
			}
		}

		/*********************************

			Prepare a the dofile strings with
			all code (settings and comments)
			that is going in to the dofile

		*********************************/

		*******************************
		*Messages in dofiles generated
		*in Stata 11 and in Stata IC

		*Stata IC
		local dofile_string1 ""
		if 	!(c(MP) == 1 | c(SE) == 1) {
			local dofile_string1 `dofile_string1' **This dofile does not inlcude the setting maxvar as it ;
			local dofile_string1 `dofile_string1' * was generated in Stata IC or Small. If the dofile would ;
			local dofile_string1 `dofile_string1' * be generated in Stata MP or SE, then then maxvar is ;
			local dofile_string1 `dofile_string1' * included but for IC and Small it is ignored. ;
			local dofile_string1 `dofile_string1' * ;
		}

		*Stata 11
		local dofile_string2 ""
		if 	c(stata_version) < 12 {
			local dofile_string2 `dofile_string2' **This dofile does not inlcude modern memory settings as it ;
			local dofile_string2 `dofile_string2' * was generated in Stata 11. It only includes set memory that ;
			local dofile_string2 `dofile_string2' * is ignored by Stata >=12. Dofiles generated in Stata >=12 ;
			local dofile_string2 `dofile_string2' * includes both. ;
			local dofile_string2 `dofile_string2' * ;
		}

		*******************
		*Maxvar and matsize

		local dofile_string3 ""

		*Set verison number
		local dofile_string3 `dofile_string3' version `versionnumber' ;

		*Set basic memory limits
		if "`clear'" == "" {
			local dofile_string3 `dofile_string3' clear all ;

			**Setting maxvar not allowed in Stata IC. If Stata IC creates the file these lines
			* are excluded. The "if (c(MP) == 1 | c(SE) == 1) {" is still included in the dofile
			* so that a dofile generated by MP or SE still works on IC. In such a case you are
			* likely to run in to the varaible count limit in Stata IC, but that error message
			* is more informative then when trying to set maxvar in Stata IC.
			if 	(c(MP) == 1 | c(SE) == 1) {
				local dofile_string3 `dofile_string3' if (c(MP) == 1 | c(SE) == 1) { ;
				local dofile_string3 `dofile_string3' set maxvar 	`maxvar' ;
				local dofile_string3 `dofile_string3' } ;
			}
		}

		local dofile_string3 `dofile_string3'  set matsize 	`matsize' ;

		****************
		*Memory	settings

		local dofile_string4 ""

		*New memory settings was introduced in Stata 12.
		if c(stata_version) >= 12 {

			*For compatibility with Stata 11 the do file includes and if/else
			*condition testing version number. The memoroy settings introduced
			*in Stata 12 will be applied to Stata version more recent than
			*Stata 11, and set memory will be applied to Stata 11.
			local dofile_string4 `dofile_string4'  if c(stata_version) >= 12 { ;

			*Set advanced memory limits
			local dofile_string4 `dofile_string4' set niceness	5 ;

			if "`clear'" == "" {
				*These settings cannot be modified with data in memory
				local dofile_string4 `dofile_string4' set min_memory	0 ;
				local dofile_string4 `dofile_string4' set max_memory	. ;

				*Set segment size to the largest value allowed by the operative system
				if c(bit) == 64 {
					local dofile_string4 `dofile_string4' set segmentsize	32m ;
				}
				else {
					local dofile_string4 `dofile_string4' set segmentsize	16m ;
				}
			}
			local dofile_string4 `dofile_string4'  } ;

			*Inlcude an else statement that allows compatibility with Stata 11. This
			*might cause errors when running the rest of the code for this project,
			*but by using setmem() it might be possible to solve those errors.
			local dofile_string4 `dofile_string4'  else { ;
			local dofile_string4 `dofile_string4'  set memory `setmem' ;
			local dofile_string4 `dofile_string4'  } ;

		}
		else {

			*If this dofile is generated in Stata 11 then only the old
			*way of setting memory is included. This will be ignored by
			*more recent versios of Stata
			local dofile_string4 `dofile_string4'  set memory `setmem' ;

		}

		*********************
		*Set default settings
		local dofile_string4 `dofile_string4' set more 			off ;
		local dofile_string4 `dofile_string4' pause 			on 	;
		local dofile_string4 `dofile_string4' set varabbrev 	off ;

		/*********************************

			Add custom lines of code

		*********************************/

		if "`custom'" != "" {

			local dofile_string5 ""

			*Add a comment that list where
			local dofile_string5 `dofile_string5' **The code below was added manually by the user to iboilstart. If ;
			local dofile_string5 `dofile_string5' * there are any issues please reconfigure ieboilstart. There is a ;
			local dofile_string5 `dofile_string5' * limit of less than 244 characters in the code in Stata 11, and ;
			local dofile_string5 `dofile_string5' * 512 in later versions.;

			*di "`dofile_string5'"
			*di `=strlen("`dofile_string5'")'

			*Create a local with the rowlabel input to be tokenized
			local custom_code_lines `custom'
			local dofile_string6 ""

			while "`custom_code_lines'" != "" {

				*Parsing name and label pair
				gettoken code_line custom_code_lines : custom_code_lines, parse("@")

				*Removing leadning or trailing spaces
				local code_line = trim("`code_line'")

				local dofile_string6 `dofile_string6' `code_line';

				*Parse char is not removed by gettoken
				local custom_code_lines = subinstr("`custom_code_lines'" ,"@","",1)
			}
		}

		/*********************************

			Create the dofile

		*********************************/

		*Create a local with the date
		local		date = subinstr(c(current_date)," ","",.)

		*Write the title rows defined above
		capture file close settingsfile

		*Set up the file to be written
		file open  settingsfile using "`textfile'", text write replace

		*Suppress output from dofile if option qietdo is applied
		if "`quietdo'" != "" file write settingsfile "quietly {" _n

		*Write the standard message in the opening file.
		file write settingsfile ///
																					_n ///
			`" 	/*************************************************"' 				_n ///
			`" 	This file is was prepared by command ieboilstart "' 				_n ///
			`" 	on `date' by username: `c(username)'. "'							_n ///
																					_n ///
			`" 	Run this file to set the recommended settings for "' 				_n ///
			`" 	this project to your instance of Stata"' 							_n ///
			`" 	*************************************************/"' 				_n ///
																					_n ///
			`"	*This dofile was generated by specifying ieboilstart like this:"' 	_n ///
			`"	*ieboilstart `full_specification' "'								_n ///
																					_n ///
			"	*See help file for ieboilstart for details on these settings"		_n ///
			_n

		** Repeat through the possible 6 strings from above. The reason multiple strings
		*  are created is that in older versions of Stata, the maximum string length
		*  used in a string operation, like gettoken or subinstr(), is limited to 214.
		forvalues strNum = 1/6 {

			** Write each setting prepared above to the dofile.
			while ("`dofile_string`strNum''" != "") {

				*noi di `=strlen("`dofile_string`strNum''")'
				*noi di "`dofile_string`strNum''"

				local tempstring `dofile_string`strNum''

				*Tokenize the first setting
				gettoken setting tempstring : tempstring, parse(";")

				*Write it to file
				file write settingsfile "	`setting'" _n

				*Remove the parse character
				local dofile_string`strNum' = subinstr("`tempstring'",";","",1)
			}
		}

		*Close quiet bracket if option quietdo is used.
		if "`quietdo'" != "" file write settingsfile "}" _n

		*Close file
		file close settingsfile

		/*********************************

			Create return value and output message

		*********************************/

		return local ieboil_dofile "`textfile'"

		noi di ""
		noi di "{phang}{err:DISCLAIMER:} Due to how settings works in Stata, it is impossible to create the exact same coding environment for all users regardless of what version of Stata each user is using. Hence, this command only attempts to harmonize the settings as much as possible across users collaborating on the same code, but no guarantee can be given that all commands will always behave identical unless the exact same version of Stata is used.{p_end}"
		noi di ""
		noi di "{phang}For a similar reason, ieboilstart only prepares a dofile with all recommended settings. This dofile {ul:MUST} be executed using the command {it:include} and not with {it:do}, otherwise not all settings will be set properly. See {help ieboilstart :help ieboilstart} for more details on how to correctly execute the dofile ieboilstart has prepared. If you do not run the dofile, ieboilstart does nothing to harmonize your settings.{p_end}"
		noi di ""
		noi di "{phang}dofile with all settings saved to: `textfile'{p_end}"
	}
	end


Changes all.do to not use bash-specific constructs.

Ubuntu uses /bin/sh -> /bin/dash since version 6.10. And, as ./do uses /bin/sh
in its shebang, we shouldn't rely on bash-specific constructs in any of our
*.do files.

https://wiki.ubuntu.com/DashAsBinSh/

for f in $(./libfiles .js); do
  if [ $(echo "$f" | grep -c .min.js$) -eq 0 ]; then
    echo ${f%.*}.min.js
  fi
done |
xargs redo-ifchange

for f in $(./libfiles .js); do
  if [[ ! "$f" =~ .min.js$ ]]; then
    echo ${f%.*}.min.js
  fi
done |
xargs redo-ifchange


cl added

cl added

clear all
set more off
set matsize 11000
program drop _all

qui adopath + "/mnt/nfs6/wikipedia.proj/jmp/jmpscripts/stata/ado"
cd ${path}

// this declares global vars
declare_global
global time "quarter"

// 0. Data

// 0.1 make cleancnty, changesets data
clear
preparebasic

// 0.2 make relevant dependent variables
program drop _all
use ${stash}cleanchangeset1, clear
makedv_uid fips
balancepanel_uid fips
maketime
mergebasic
qui destring, replace
save ${stash}panelfips, replace

// 0.3 make node and way datasets producing mergemaster_x datasets

preparenode
prepareway

// append the two datasets
use ${stash}mergemaster_way, clear
append using ${stash}mergemaster_node, gen(isnode)
save ${stash}mergemaster_way, replace

// make the data and balance
use ${stash}mergemaster_way, clear
makedv_wn fips
save ${stash}wn_stash, replace

use ${stash}wn_stash, clear
balancepanel fips node
maketime
mergebasic
qui destring, replace
save ${stash}panel_wn, replace

////////////////////////////////
//// ANALYSIS

// 1. Summary Stats
// Panel A: By county Level
program drop _all
use ${stash}panelfips, clear
bysort fips: drop if _n > 1
global tabname "summary_county"
local vars "treat aland_sqmi emp_earnings age_median educ_college emp_computer"
makesummary "`vars'"

// Panel B: By county/time Level
use ${stash}panelfips, clear
global tabname "summary_panel"
local vars "year pop_year numcontrib numusers numnewusers numserious18 numnewusers_t2 numnewusers_c18"
makesummary "`vars'"

// 2. Treatment Control Balance
use ${stash}panelfips, clear
makehist cntypop kdensity "Population"
makehist emp_earnings kdensity "Earnings"
makehist age_median kdensity "Median Age"
makehist educ_college kdensity "Num. College Educated"

// 2. TODO: TTest or regressions treatment / control

//  3.1 Meanline charts (raw data)
makemeanline numcontrib time 2011 "Contributions"
makemeanline numusers time 2011 "Users"
makemeanline numserious18 time 2011 "Super Users"

// 3.2 Cross Sectional Regressions
//crossreg

// 3.2 Diff in Diff --- Baseline
local outcomes "numcontrib numusers"
dd_simple "`outcomes'" run
dd_simple "`outcomes'" write ddsimple_baseline

// 3.3 Diff in Diff Picture
ddchart numuser 92 write
ddchart numserious56 92 write
ddchart numnewusers_t2

// 3.4 Diff in Diff --- Additional
local outcomes "numnewusers numnewusers_t2 numnewusers90"
dd_simple "`outcomes'" run xtreg
dd_simple "`outcomes'" run xtpoisson
dd_simple "`outcomes'" write ddsimple_additional

// 3.5 Impact on new users of different types
newusers_reg run
newusers_reg write

// 3.6 Impact on Existing Users

// 3.7 Impact on Attribute level data

// 4.0 Map Quality

// 4.1 Battlegrid
reg_battle

// 4.2 Craigslist
reg_cl

// 3.7 Impact on individuals

// next steps
//1. individual facts
//2. other layers
//3. completeness / quality
//4. model, predictions and whether they confirm?

// 4. How do results vary?
// takes a while
makehetero

// 5. Individual Level Data

// 5.0 create data
makeperson

//5.1 Summary
use ${stash}paneluid, clear

label variable treat "TIGER"
label variable year "YEAR"
label variable numcontrib "Contribs"
label variable numcontrib_home "Home County"
label variable numcontrib_treat "TIGER"
label variable numcontrib_notreat "Non-TIGER"
label variable numcontrib_statenotreat "State(Control)"
label variable numcounties "Num. Counties"

local vars "treat year numcontrib numcontrib_home numcontrib_treat numcontrib_notreat numcontrib_statenotreat numcounties"
global tabname "summary_uid"
makesummary "`vars'"

//Regressions
ddperson maketables
ddperson makechart

// 6. Map Quality

//6.0 summary
use ${stash}panelotherlayers, clear

label variable treat "TIGER"
label variable year "Year"
label variable otherlayer "NonStreet Layers"
label variable amenities "Amenities"
label variable bld_addr "Building/Addresses"
label variable numclass4 "Trails/Bikepaths"

label variable numattrib1 "Attributes:Major Highway"
label variable numattrib2 "Attributes:Class2"
label variable numattrib3 "Attributes:Class3"
label variable numattrib4 "Attributes:Trails etc."

local vars "treat year otherlayer amenities bld_addr numclass4 numattrib1 numattrib2 numattrib3 numattrib4"
global tabname "summary_otherl"
makesummary "`vars'"

// 6.1 Street data completeness

// 6.2 Other layers
reg_layers

// 6.3 Highway level regressions

// 7. Craiglist

////////// OLD //
 ** Analysis

// 0. Data

// 0.1 make fips dataset
use ${stash}mergemaster1, clear

balancepanel fips
save ${stash}panelfips, replace

// 0.2 make node dataset
preparenode

use ${stash}mergemaster_node, clear
makedv fips node
balancepanel fips node
save ${stash}panelnode, replace

// 0.3 make way dataset
prepareway
makedv fips way
balancepanel fips way
save ${stash}panelway, replace

// 3. Baseline effects (xtpoisson, cluster at unit level)

// 3.2 FIPS Sample -- DD

/// 3.2.1 -- contrib, users, superu
program drop _all

clear
local dv "numcontrib numuser numserious90"
local unit fips
rundd panelfips `unit' "`dv'"
diffindiff xtpoisson "`dv'" `unit' 2014 write tab_3.2.1 "Contributions" "Users" "Super Users"

/// 3.2.2 -- newusers, newusers6, newuserssuper
clear
local dv "numnewusers numnewusers6 numnewusers90"
local unit fips
rundd panelfips `unit' "`dv'"
diffindiff xtpoisson "`dv'" `unit' 2014 write tab_3.2.2 "New Users" "New Users(6+)" "New Super Users"

/// 3.2.3 -- streetcontrib, nonstreet contrib, amenities
// TODO

use ${stash}panelnode, clear
program drop _all

diffindiff xtpoisson fips run tab_test numcontrib numuser

/// 3.2.4 -- completeness
// TODO

// 3.2.5 -- Diff in Diff charts
// TODO : make this more flexible
// TODO : this works only for fips
ddchart

// 3.3 Robustness : Drop empty counties
// TODO
//  e. Repeat (a) by dropping empty counties
//  f. Repeat (a) by dropping west
//  g. Repeat (a) for Tile Sample, MSA sample
//  h. Repeat (a) with matched estimators
//  i. Repeat (a) with spatially clustered se, Region X Time trends

// 4 Heterogenous Effects

// 1. Large vs. small
use ${stash}panelfips, clear
gen large = (cntypop > 100000)

diffindiff2 large LARGE tab_4.1 run numcontrib numuser numserious90 numnewusers6

diffindiff2 large LARGE tab_4.1 write numcontrib numuser numserious90 numnewusers6

// 2. engaged vs. non-engaged county

use ${stash}panelfips, clear

bysort fips post: egen tmp = total(numcontrib > 0)
replace tmp = . if post == 1
bysort fips: egen precontrib = max(tmp)
bysort fips: egen totcontrib = total(numcontrib)
drop tmp

drop if precontrib == 0
gen active = (prec > 1)

diffindiff2 active ACTIVE tab_4.2 run numcontrib numuser numserious90 numnewusers6

diffindiff2 active ACTIVE tab_4.2 write numcontrib numuser numserious90 numnewusers6

// 3. populated vs. not

use ${stash}panelfips, clear

gen iseast = (region==1 | region == 2)

diffindiff2 iseast EAST tab_4.3 run numcontrib numuser numserious90 numnewusers6

diffindiff2 iseast EAST tab_4.3 write numcontrib numuser numserious90 numnewusers6

// 3. urban vs rural
// 4. university vs non university
// 5. rich vs poor

// 5 Street Level Regressions

// 6 Person Level Regressions

// [A] outcome vars: County sample -- DD spec

   // a. Contributions
   // b. Street, Non Street
   // b. TIGER vs. Non Tiger
   // d. Home county, Num Counties

// [B] heterogenous

   // a. Junior vs Senior

// 7 Impacts on Craigslist

preparecl

// PENDING

// GEOID Level

// TASKS for tomorrow

// STASH

 // 1.1 Mean Charts

 // 1.2 Produce Diff in diff Latex tables
 // rundd -> batchreg.sh -> runreg.do -> diffindiff.ado
 program drop _all

 // TODO: fix manual process
 // have to do this manually

 // 1.3 Produce Diff in diff Pictures

 ** person level regressions
 // person level

// TODOs for Tuesday Jun 3
// GEOID dataset
// TILEID diff in diff
// What happened to previous people?
// What happened at the street/amenity level?
// Urban vs Rural (framework for third level diff)

// 1.4 Robust: product diff in diff for GEOID

use ${stash}panelfips_geoid10, clear

** Analysis Stage 2
//

// PROGRAMS

clear all
set more off
set matsize 11000
program drop _all

qui adopath + "/mnt/nfs6/wikipedia.proj/jmp/jmpscripts/stata/ado"
cd ${path}

// this declares global vars
declare_global
global time "quarter"

// 0. Data

// 0.1 make cleancnty, changesets data
clear
preparebasic

// 0.2 make relevant dependent variables
program drop _all
use ${stash}cleanchangeset1, clear
makedv_uid fips
balancepanel_uid fips
maketime
mergebasic
qui destring, replace
save ${stash}panelfips, replace

// 0.3 make node and way datasets producing mergemaster_x datasets

preparenode
prepareway

// append the two datasets
use ${stash}mergemaster_way, clear
append using ${stash}mergemaster_node, gen(isnode)
save ${stash}mergemaster_way, replace

// make the data and balance
use ${stash}mergemaster_way, clear
makedv_wn fips
save ${stash}wn_stash, replace

use ${stash}wn_stash, clear
balancepanel fips node
maketime
mergebasic
qui destring, replace
save ${stash}panel_wn, replace

////////////////////////////////
//// ANALYSIS

// 1. Summary Stats
// Panel A: By county Level
program drop _all
use ${stash}panelfips, clear
bysort fips: drop if _n > 1
global tabname "summary_county"
local vars "treat aland_sqmi emp_earnings age_median educ_college emp_computer"
makesummary "`vars'"

// Panel B: By county/time Level
use ${stash}panelfips, clear
global tabname "summary_panel"
local vars "year pop_year numcontrib numusers numnewusers numserious18 numnewusers_t2 numnewusers_c18"
makesummary "`vars'"

// 2. Treatment Control Balance
use ${stash}panelfips, clear
makehist cntypop kdensity "Population"
makehist emp_earnings kdensity "Earnings"
makehist age_median kdensity "Median Age"
makehist educ_college kdensity "Num. College Educated"

// 2. TODO: TTest or regressions treatment / control

//  3.1 Meanline charts (raw data)
makemeanline numcontrib time 2011 "Contributions"
makemeanline numusers time 2011 "Users"
makemeanline numserious18 time 2011 "Super Users"

// 3.2 Cross Sectional Regressions
//crossreg

// 3.2 Diff in Diff --- Baseline
local outcomes "numcontrib numusers"
dd_simple "`outcomes'" run
dd_simple "`outcomes'" write ddsimple_baseline

// 3.3 Diff in Diff Picture
ddchart numuser 92 write
ddchart numserious56 92 write
ddchart numnewusers_t2

// 3.4 Diff in Diff --- Additional
local outcomes "numnewusers numnewusers_t2 numnewusers90"
dd_simple "`outcomes'" run xtreg
dd_simple "`outcomes'" run xtpoisson
dd_simple "`outcomes'" write ddsimple_additional

// 3.5 Impact on new users of different types
newusers_reg run
newusers_reg write

// 4.0 Map Quality

// 4.1 Battlegrid

reg_battle

// 4.2 Craigslist

// 3.7 Impact on Attribute level data

// 3.7 Impact on individuals

// next steps
//1. individual facts
//2. other layers
//3. completeness / quality
//4. model, predictions and whether they confirm?

// 4. How do results vary?
// takes a while
makehetero

// 5. Individual Level Data

// 5.0 create data
makeperson

//5.1 Summary
use ${stash}paneluid, clear

label variable treat "TIGER"
label variable year "YEAR"
label variable numcontrib "Contribs"
label variable numcontrib_home "Home County"
label variable numcontrib_treat "TIGER"
label variable numcontrib_notreat "Non-TIGER"
label variable numcontrib_statenotreat "State(Control)"
label variable numcounties "Num. Counties"

local vars "treat year numcontrib numcontrib_home numcontrib_treat numcontrib_notreat numcontrib_statenotreat numcounties"
global tabname "summary_uid"
makesummary "`vars'"

//Regressions
ddperson maketables
ddperson makechart

// 6. Map Quality

//6.0 summary
use ${stash}panelotherlayers, clear

label variable treat "TIGER"
label variable year "Year"
label variable otherlayer "NonStreet Layers"
label variable amenities "Amenities"
label variable bld_addr "Building/Addresses"
label variable numclass4 "Trails/Bikepaths"

label variable numattrib1 "Attributes:Major Highway"
label variable numattrib2 "Attributes:Class2"
label variable numattrib3 "Attributes:Class3"
label variable numattrib4 "Attributes:Trails etc."

local vars "treat year otherlayer amenities bld_addr numclass4 numattrib1 numattrib2 numattrib3 numattrib4"
global tabname "summary_otherl"
makesummary "`vars'"

// 6.1 Street data completeness

// 6.2 Other layers
reg_layers

// 6.3 Highway level regressions

// 7. Craiglist
reg_cl

////////// OLD //
 ** Analysis

// 0. Data

// 0.1 make fips dataset
use ${stash}mergemaster1, clear

balancepanel fips
save ${stash}panelfips, replace

// 0.2 make node dataset
preparenode

use ${stash}mergemaster_node, clear
makedv fips node
balancepanel fips node
save ${stash}panelnode, replace

// 0.3 make way dataset
prepareway
makedv fips way
balancepanel fips way
save ${stash}panelway, replace

// 3. Baseline effects (xtpoisson, cluster at unit level)

// 3.2 FIPS Sample -- DD

/// 3.2.1 -- contrib, users, superu
program drop _all

clear
local dv "numcontrib numuser numserious90"
local unit fips
rundd panelfips `unit' "`dv'"
diffindiff xtpoisson "`dv'" `unit' 2014 write tab_3.2.1 "Contributions" "Users" "Super Users"

/// 3.2.2 -- newusers, newusers6, newuserssuper
clear
local dv "numnewusers numnewusers6 numnewusers90"
local unit fips
rundd panelfips `unit' "`dv'"
diffindiff xtpoisson "`dv'" `unit' 2014 write tab_3.2.2 "New Users" "New Users(6+)" "New Super Users"

/// 3.2.3 -- streetcontrib, nonstreet contrib, amenities
// TODO

use ${stash}panelnode, clear
program drop _all

diffindiff xtpoisson fips run tab_test numcontrib numuser

/// 3.2.4 -- completeness
// TODO

// 3.2.5 -- Diff in Diff charts
// TODO : make this more flexible
// TODO : this works only for fips
ddchart

// 3.3 Robustness : Drop empty counties
// TODO
//  e. Repeat (a) by dropping empty counties
//  f. Repeat (a) by dropping west
//  g. Repeat (a) for Tile Sample, MSA sample
//  h. Repeat (a) with matched estimators
//  i. Repeat (a) with spatially clustered se, Region X Time trends

// 4 Heterogenous Effects

// 1. Large vs. small
use ${stash}panelfips, clear
gen large = (cntypop > 100000)

diffindiff2 large LARGE tab_4.1 run numcontrib numuser numserious90 numnewusers6

diffindiff2 large LARGE tab_4.1 write numcontrib numuser numserious90 numnewusers6

// 2. engaged vs. non-engaged county

use ${stash}panelfips, clear

bysort fips post: egen tmp = total(numcontrib > 0)
replace tmp = . if post == 1
bysort fips: egen precontrib = max(tmp)
bysort fips: egen totcontrib = total(numcontrib)
drop tmp

drop if precontrib == 0
gen active = (prec > 1)

diffindiff2 active ACTIVE tab_4.2 run numcontrib numuser numserious90 numnewusers6

diffindiff2 active ACTIVE tab_4.2 write numcontrib numuser numserious90 numnewusers6

// 3. populated vs. not

use ${stash}panelfips, clear

gen iseast = (region==1 | region == 2)

diffindiff2 iseast EAST tab_4.3 run numcontrib numuser numserious90 numnewusers6

diffindiff2 iseast EAST tab_4.3 write numcontrib numuser numserious90 numnewusers6

// 3. urban vs rural
// 4. university vs non university
// 5. rich vs poor

// 5 Street Level Regressions

// 6 Person Level Regressions

// [A] outcome vars: County sample -- DD spec

   // a. Contributions
   // b. Street, Non Street
   // b. TIGER vs. Non Tiger
   // d. Home county, Num Counties

// [B] heterogenous

   // a. Junior vs Senior

// 7 Impacts on Craigslist

preparecl

// PENDING

// GEOID Level

// TASKS for tomorrow

// STASH

 // 1.1 Mean Charts

 // 1.2 Produce Diff in diff Latex tables
 // rundd -> batchreg.sh -> runreg.do -> diffindiff.ado
 program drop _all

 // TODO: fix manual process
 // have to do this manually

 // 1.3 Produce Diff in diff Pictures

 ** person level regressions
 // person level

// TODOs for Tuesday Jun 3
// GEOID dataset
// TILEID diff in diff
// What happened to previous people?
// What happened at the street/amenity level?
// Urban vs Rural (framework for third level diff)

// 1.4 Robust: product diff in diff for GEOID

use ${stash}panelfips_geoid10, clear

** Analysis Stage 2
//

// PROGRAMS


commented dot_product

commented dot_product

* v1.0 01062017 , Apoorva Lal

/*
	Calculates the variable `Y = XB` where X is a subset of N variables in the currently loaded dataset, B is an arbitrary column vector (NX1 matrix). Basically a way to construct predicted values from a regression when the coefficients have been stored in a matrix / read in from elsewhere. Produces identical results to `predict` when used with the postestimation `e(b)` coefficient vector.

*/

program define dot_product, rclass
	// syntax : dot_product <projection> <coefficients> <variables>
	syntax namelist(min=3)
	loc targetvar `: word 1 of `namelist''
	loc vector `: word 2 of `namelist''
	cap conf matrix `vector'
	if _rc {
		noi di as error "matrix `vector' not found"
		exit
	}
	loc firsttwo `vector' `targetvar'
	loc xvars : list namelist - firsttwo
	loc ndim : list sizeof xvars
	loc rows `=rowsof(`vector')'
	loc cols `=colsof(`vector')'
	// checks conformability, allows for coefficients to be a row rather than column vector
	if `rows' != `ndim' {
		noi di as result "`vector' needs to be a column vector (N X 1) matrix where N == number of covars"
		if `cols' == `ndim' {
			noi di as result "Using transpose of `vector' instead"
			mat `vector'_t = `vector''
			loc coeffVector `vector'_t
		}
		else {
			noi di as error "Conformability error : coefficient vector not the same size as design matrix"
			exit
		}
	}
	else {
		loc coeffVector `vector'
	}
	mat coeffs = `coeffVector'
	qui mata: matrix_multiply("`xvars'","coeffs","`targetvar'")
end
// main mata subroutine
mata:
void matrix_multiply(string scalar varlist, string scalar CoeffMatrix, ///
	 			string scalar newvar) {
	real matrix X
	B = st_matrix(CoeffMatrix)
	st_view(X=.,.,tokens(varlist),)
	product = X*B
	st_addvar("double",(newvar))
	st_store(.,newvar,product)
}
end

capture program drop dot_product
program define dot_product, rclass
	syntax namelist(min=3)
	loc targetvar `: word 1 of `namelist''
	loc vector `: word 2 of `namelist''
	cap conf matrix `vector'
	if _rc {
		noi di as error "matrix `vector' not found"
		exit
	}
	loc firsttwo `vector' `targetvar'
	loc xvars : list namelist - firsttwo
	loc ndim : list sizeof xvars
	loc rows `=rowsof(`vector')'
	loc cols `=colsof(`vector')'
	if `rows' != `ndim' {
		noi di as result "`vector' needs to be a column vector (N X 1) matrix where N == number of covars"
		if `cols' == `ndim' {
			noi di as result "Using transpose of `vector' instead"
			mat `vector'_t = `vector''
			loc coeffVector `vector'_t
		}
		else {
			noi di as error "Conformability error : coefficient vector not the same size as design matrix"
			exit
		}
	}
	else {
		loc coeffVector `vector'
	}
	mat coeffs = `coeffVector'
	qui mata: matrix_multiply("`xvars'","coeffs","`targetvar'")
end

mata:
void matrix_multiply(string scalar varlist, string scalar CoeffMatrix, ///
	 			string scalar newvar) {
	real matrix X
	B = st_matrix(CoeffMatrix)
	st_view(X=.,.,tokens(varlist),)
	product = X*B
	st_addvar("double",(newvar))
	st_store(.,newvar,product)
}
end


iefolder : remove deletefolder reference

	*Set standardized settings
	ieboilstart, v(11.1)
	`r(version)'

	*global to the exercise folder
	global ief 	"C:\Users\WB462869\Documents\GitHub\ietoolkit"

	*Load the command
	qui do 		"$ief\iefolder.do"

	*Global to the mock project folder you created
	global projectfolder	""

******************************************************************************

	*Set standardized settings
	ieboilstart, v(11.1)
	`r(version)'

	*global to the exercise folder
	global ief 	"C:\Users\WB462869\Documents\GitHub\ietoolkit"

	*Load the command
	qui do 		"$ief\iefolder.do"

	*Global to the mock project folder you created
	global projectfolder	""

	*(Ask me before using this two lines of code)
	*Clean up the folder fomr last run
	*do "$ief\deletefolder.do"
	*cap deleteFolder "$projectfolder\DataWork"

******************************************************************************