iematch : test for more target obs than base obs

Implemented a test that makes sure that the number of target
observations are more than the number of base observations

	cap program drop 	iematch
	    program define	iematch

		noi di "Command OK!"

		syntax  [if] [in]  , 					///
			GRPdummy(varname) 					///
			MATCHvar(varname) 					///
			[									///
			IDvar(varname)						///
			m1									///
			maxdiff(numlist max = 1 min = 1) 	///
			seedok						 		///
			MATCHIDname(string)					///
			MATCHDIffname(string)				///
			MATCHREsultname(string)				///
			MATCHCOuntname(string)				///
			]

		*****
		*1 format errors as smcl
		*2 option to disable countdown
		*3 test not matchcount without m1
		*5 a table with results outputted

		noi di "Syntax OK!"

		pause on

		qui {

		********************************
		*
		*	Gen temp sort and merge var
		*
		********************************

		tempvar originalSort

		gen `originalSort' = _n

		preserve

*Testing input
if 1 {

			if "`if'`in'" != "" keep `if'`in'

			********************************
			*
			*	Checking group dummy
			*
			********************************

			*Test that the group dummy is 1, 0 or missing
			cap assert inlist(`grpdummy', 1, 0, .) | `grpdummy' > .
			if _rc != 0 {

				di as error "The variable in grpdummy(`grpdummy') is not a dummy variable. The variable is only allowed to have the values 1, 0 or missing. Observations with missing varaibles in the grpdummy are ignored by this command."
				error _rc
			}

			**Exclude obs with missing value in groupdummy

			*Count number of obs to be dropped and and output that number if more than zero
			count if missing(`grpdummy')
			if `r(N)' > 0 {
				*This is not an error just outputting the number
				noi di "{pstd}`r(N)' observation(s) were excluded due to missing value in group dummy{p_end}"
			}
			*Drop obs with missing
			drop if missing(`grpdummy')

			********************************
			*
			*	Checking match continous var
			*
			********************************

			*Test that the continous is numeric
			cap confirm numeric variable `matchvar'
			if _rc != 0 {

				noi di as error "The variable to match on specified in matchvar(`matchvar') is not a numeric variable."
				error 109

			}

			**All variables with value 1 or 0 in the group dummy and
			* not excluded by if/in must have a value in matchvar
			count if missing(`matchvar')
			if `r(N)' > 0 {
				noi di as error  "{pstd}`r(N)' observation(s) with either value 1 or 0 in grpdummy(`grpdummy') do not have a non-missing value in matchvar(`matchvar'). Either update the match variable or exclude those observations using {inp:if} or {inp:in}.{p_end}"
				error 416
			}

			********************************
			*
			*	Checking match var and group dummy are unique
			*
			********************************

			if "`seedok'" == "" {

				**test that there are no duplicates in match
				* var within each type of observation
				cap isid `matchvar' `grpdummy'

				if _rc != 0 {

					noi di as error "{pstd}There are base observations or target observations with duplicate values in matchvar(`matchvar'). To guarantee a replicable match you must {help seed:set a seed}. To supress this error message after you have set a the seed, or if a replicable match is not important to you, use option {inp:seedok}{p_end}"
					error _rc

				}
			}

			********************************
			*
			*	Checking that there are more target
			*	obs than base obs in a 1-to-1 match
			*
			********************************

			if "`m1'" == "" {

				*Count number of base vars
				count if `grpdummy' == 1
				local numBaseObs `r(N)'

				*Count number of target vars
				count if `grpdummy' == 0
				local numTrgtObs `r(N)'

				*Test that there are more target
				cap assert `numTrgtObs' >= `numBaseObs'

				if _rc != 0 {

					noi di as error "{pstd}There are more base observations than target observations. This is not allowed in a one-to-one match. See option {inp:m1} for an alternaitve matching where it is allowed to have more base observations than target ovbservations.{p_end}"
					error _rc
				}
			}

			********************************
			*
			*	Checking duplicates in variable names in options
			*
			********************************

			*Create a local of all varnames used in any option
			local allnewvars `idvar' `matchidname' `matchdiffname' `matchresult'

			*Check if there are any duplicates in the local of all varnames in options
			local dupnewvars : list dups allnewvars

			*Throw error if there were any duplicates
			if "`dupnewvars'" != "" {

				di as error "The same new variable name was used twice or more in the options generating a new variable. Go back and check syntax"
				error 198

			}

			********************************
			*
			*	Checking ID var
			*
			********************************

			*Checking set-up for the ID var used for _matchID
			if "`idvar'" == "" {

				*Copy this name
				local idvar _ID

				*Make sure that the default name _ID is not already used
				cap confirm variable `idvar'
				if _rc == 0 {

					di as error "A variable with name `idvar' is already defined. Either drop this variable or specify a variable using idvar() that fully and uniquely identifies the data set."
					error 110

				}

				*Generate the ID var from row number
				gen `idvar' = _n

				*Store local indicating ID var generated has type numeric
				local IDtypeNumeric 1

			}
			else {

				*Make sure that user specified ID is uniquely and fully identifying the observations
				cap isid `idvar'
				if _rc != 0 {

					di as error "Variable `idvar' specified in idvar() does not fully and uniquely identify the observations in the data set. Meaning that `idvar' is not allowed to have any duplicates or missing values. Make `idvar' uniquly and fully idnetifying or exclude the varaibleas using {inp:if} or {inp:in}."
					error 450

				}

				*Test if ID var is string or numeric
				cap confirm numeric variable `idvar'
				if _rc == 0 {

					*Store local indicating ID var type is numeric
					local IDtypeNumeric 1
				}
				else {

					*Store local indicating ID var type is numeric
					local IDtypeNumeric 0
				}
			}

			********************************
			*
			*	Checking and generating the MatchID and the MatchDiff var
			*
			********************************

			*Test input from user or test if deafault value is valiable
			iematchMatchVarCheck _matchID `matchidname'
			local matchIDname "`r(validVarName)'"

			*Allow the ID var used to be string
			if `IDtypeNumeric' == 1 {
				gen  `matchIDname' = .
			}
			else {
				gen  `matchIDname' = ""
			}

			*If many-to-one match used
			if "`m1'" != "" {
				iematchMatchVarCheck _matchCount `matchcount'
				local matchCountName "`r(validVarName)'"
				gen `matchCountName' = .
			}

			iematchMatchVarCheck _matchDiff `matchdiffname'
			local matchDiffName "`r(validVarName)'"
			gen `matchDiffName' = .

			iematchMatchVarCheck _matchOrReason `matchresult'
			local matchReasonName "`r(validVarName)'"
			gen `matchReasonName' = .

			*Create a label for missing values that explains no match
			label define matchLabel 0 "Not Matched" 1 "Matched" .n "Obs not included in match" .m "No match within maxdiff"

			*Apply the label
			label value `matchReasonName' matchLabel

}

		********************************
		*	Start matching
		********************************

			********************************
			*
			*	Creating tempvar used
			*
			********************************

			*Initiate the temporary variables used by this command
			tempvar prefID prefDiff matched

			gen `prefDiff' 		= .
			gen byte `matched' 	= 0

			*Allow the ID var used to be string
			if `IDtypeNumeric' == 1 {
				gen  `prefID' = .
			}
			else {
				gen  `prefID' = ""
			}

			** Generate the inverse of the matchvar to sort descending (gsort is too slow),
			*  a random var to seperate two values with the same match var, and the inverse
			*  of the random var for when sorting descending.
			tempvar rand invsort invrand

			sort `matchvar'
			gen  `invsort' = -1 * `matchvar'

			gen    `rand' = uniform()
			gen `invrand' = -1 * `rand'

			***********
			*Tempvars for matching

			*Diffvars, they are always numeric
			tempvar					 diffup   diffdo   valUp_0   valDo_0   valUp_1   valDo_1
			local 	 updownTempVars `diffup' `diffdo' `valUp_0' `valDo_0' `valUp_1' `valDo_1'

			foreach tempVar of local updownTempVars {

				gen `tempVar' = .
			}

			*ID vars, allowed to be both numeric and string
			tempvar					  	IDup   IDdo   IDup_0   IDdo_0   IDup_1   IDdo_1
			local 	 updownIDTempVars  `IDup' `IDdo' `IDup_0' `IDdo_0' `IDup_1' `IDdo_1'

			foreach tempVar of local updownIDTempVars {

				*Allow the ID var used to be string
				if `IDtypeNumeric' == 1 {
					gen  `tempVar' = .
				}
				else {
					gen  `tempVar' = ""
				}
			}

			***************************
			*
			*	One to one match
			*
			***************************

			if "`m1'" == "" {

				noi di ""
				noi di "{pstd}Base observations left to match: {p_end}"
				count if `grpdummy' == 1 & `matched' == 0
				local left2Match = `r(N)'
				noi di "{pstd}`left2Match' " _c

				*di "`idvar'"
				*di "`IDtypeNumeric'"
				*pause on
				*pause

				while (`left2Match' > 0) {

					*noi di 0

					**For all observations still to be matched, assign the preferred
					* match among the other unmatched observations
					qui updatePrefDiffPreffID `prefID' `prefDiff' `matchvar' `invsort' `idvar' `grpdummy' `matched' `rand' `invrand' `updownTempVars' `updownIDTempVars'

					*noi di 1

					if "`maxdiff'" != "" {

						replace `matched' 			= 1		if `prefDiff' > `maxdiff' & `grpdummy' == 1
						replace `matchReasonName' 	= .m 	if `prefDiff' > `maxdiff' & `grpdummy' == 1

						*Allow the ID var used to be string
						if `IDtypeNumeric' == 1 {
							replace `prefID'		= .		if `prefDiff' > `maxdiff'
						}
						else {
							replace `prefID'		= ""	if `prefDiff' > `maxdiff'
						}
					}

					*noi di 2

					replace `matched' = 1 if `matched' == 0 & `prefID' == `idvar'[_n-1] & `prefID'[_n-1] == `idvar'
					replace `matched' = 1 if `matched' == 0 & `prefID' == `idvar'[_n+1] & `prefID'[_n+1] == `idvar'

					*noi di 3

					count if `grpdummy' == 1 & `matched' == 0
					local left2Match = `r(N)'
					noi di "`left2Match' " _c

				}
				noi di "{p_end}" _c

			}

			***************************
			*
			*	Many to one match
			*
			***************************
			else {

				updatePrefDiffPreffID `prefID' `prefDiff' `matchvar' `invsort' `idvar' `grpdummy' `matched' `rand' `invrand' `updownTempVars' `updownIDTempVars'

				if "`maxdiff'" != "" {

					replace `matched' 			= 1		if `prefDiff' > `maxdiff' & `grpdummy' == 1
					replace `matchReasonName' 	= .m 	if `prefDiff' > `maxdiff' & `grpdummy' == 1

					*Allow the ID var used to be string
					if `IDtypeNumeric' == 1 {
						replace `prefID'		= .		if `prefDiff' > `maxdiff'
					}
					else {
						replace `prefID'		= ""	if `prefDiff' > `maxdiff'
					}

				}

				replace `prefID'			=  `idvar' if `grpdummy' == 0
				bys 	`prefID' 			:   replace `matchCountName' = _N - 1

				**Replace prefID to missing for target obs that had no base
				* obs matched to it. T
				if `IDtypeNumeric' == 1 {
					*Id var is numeric
					replace `prefID'		= .		if `matchCountName' == 0
				}
				else {
					*id var is string
					replace `prefID'		= ""	if `matchCountName' == 0
				}

				*Only target obs with base obs prefering it are matched
				replace	 `matched' 			= 1		if `matchCountName' != 0

				*Remove values for target obs that were not matched
				replace  `matchCountName'	= . 	if `matchCountName' == 0
			}

			*Return vars
			replace `matchDiffName' 	= `prefDiff'	if `matched' == 1
			replace `matchIDname' 		= `idvar' 		if `matched' == 1 & `grpdummy' == 0
			replace `matchIDname' 		= `prefID' 		if `matched' == 1 & `grpdummy' == 1

			*Target obs not used
			replace `matchReasonName' = 0 if `matched' == 0 & `grpdummy' == 0

			*only keep output vars
			keep `matchIDname' `matchDiffName' `idvar' `originalSort' `matchReasonName' `matchCountName'

			tempfile mergefile

			save `mergefile'

		restore

		tempvar mergevar

		merge 1:1  `originalSort' using `mergefile', gen(`mergevar')

		replace `matchReasonName' = .n if `mergevar' == 1
		replace `matchReasonName' = 1 if `matchReasonName' == .

		*comress the variables generated
		compress  `matchIDname' `matchDiffName' `matchReasonName' `matchCountName'

		*Output result to user
		noi tab `matchReasonName' `grpdummy', m

		sort `originalSort'

	}

	end

*Testing input (functions)
 if 1 {
	*Cheack matchVar input
	cap program drop 	iematchMatchVarCheck
		program define 	iematchMatchVarCheck  , rclass

		args defaultName userName

		if "`defaultName'" == "_matchID" 		local optionName matchidname
		if "`defaultName'" == "_matchDiff" 		local optionName matchdiffname
		if "`defaultName'" == "_matchOrReason" 	local optionName matchresult
		if "`defaultName'" == "_matchCount" 	local optionName matchcount

		*All the default names
		local dfltNms _noMatchReason _matchDiff _matchOrReason _matchCount

		*The other two default names
		local othDfltNms : list dfltNms - defaultName

		*Creating two locals with the deafult name in each local
		local Oth1 : word 1 of `othDfltNms'
		local Oth2 : word 2 of `othDfltNms'
		local Oth3 : word 3 of `othDfltNms'

		*Testing that the option name is not the same as the other variable's default name.
		if ("`userName'" == "`Oth1'" | "`userName'" == "`Oth2'" | "`userName'" == "`Oth3'" | "`userName'" == "_ID")  {

			noi di as error "{phang}The new name specified in `optionName'(`userName') is not allowed to be _ID, `Oth1', `Oth2', or `Oth3'{p_end}"
			error 198
		}

		*Checking set-up for the ID var used for _matchID
		if "`userName'" == "" {

			*CSet the default name
			local validMatchVarname `defaultName'

			*Make sure that the default name _ID is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				noi di as error "A variable with name `validMatchVarname' is already defined. Either drop this variable or specify a new variable name using `optionName'()."
				error 110

			}

		}

		else {

			*CSet the default name
			local validMatchVarname `userName'

			*Make sure that the default name _ID is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				noi di as error "The variable name specified in `optionName'(`userName') is already defined in the data set. Either drop this variable or specify a another variable name using `optionName'()."
				error 110

			}

		}

		*Testing that the new name is valid by creating a variable that is dropped immedeatly afterwards
		cap gen `validMatchVarname' = .

		if _rc != 0 {

			noi di as error "The variable name specified in `optionName'(`userName') is not a valid variable name."
			error _rc

		}
		drop `validMatchVarname'

		return local validVarName 	"`validMatchVarname'"

	end
}

*Matching functions
if 1 {

	** This function is called to assign the preferred match for all unmatched
	*  observations to any other unmatched observation. Returns a variable pref that is the
	cap program drop 	updatePrefDiffPreffID
		program define 	updatePrefDiffPreffID

		args prefID prefDiff matchvar invsort idvar grpvar matched rand invrand diffup diffdo valUp_0 valDo_0 valUp_1 valDo_1 IDup IDdo IDup_0 IDdo_0 IDup_1 IDdo_1

		*****************
		*Update possible matches with higher rank value
		sort `matched' `matchvar' `rand'

		updateBestValID 0 `matchvar' `idvar' `grpvar' `matched' `valUp_0' `IDup_0'
		updateBestValID 1 `matchvar' `idvar' `grpvar' `matched' `valUp_1' `IDup_1'

		*****************
		*Update possible matches with lower rank value
		sort `matched' `invsort' `invrand'

		updateBestValID 0 `matchvar' `idvar' `grpvar' `matched' `valDo_0' `IDdo_0'
		updateBestValID 1 `matchvar' `idvar' `grpvar' `matched' `valDo_1' `IDdo_1'

		*****************
		*Create one varaible common for both groups with diff upwards
		replace `diffup' 	= abs(`matchvar' - `valUp_1') if `grpvar' == 1
		replace `diffup' 	= abs(`matchvar' - `valUp_0') if `grpvar' == 0

		*Create one varaible common for both groups with diff downwards
		replace `diffdo' 	= abs(`matchvar' - `valDo_1') if `grpvar' == 1
		replace `diffdo' 	= abs(`matchvar' - `valDo_0') if `grpvar' == 0

		*Create one varaible common for both groups with ID upwards
		replace `IDup' 		= `IDup_1' 	if `grpvar' == 1
		replace `IDup' 		= `IDup_0' 	if `grpvar' == 0

		*Create one varaible common for both groups with ID downwards
		replace `IDdo' 		= `IDdo_1' 	if `grpvar' == 1
		replace `IDdo' 		= `IDdo_0' 	if `grpvar' == 0

		replace `prefID' 	= `IDdo' 	if `matched' == 0 & `diffdo' <= `diffup'
		replace `prefID' 	= `IDup' 	if `matched' == 0 & `diffdo' >  `diffup'

		replace `prefDiff' 	= `diffdo' 	if `matched' == 0 & `diffdo' <= `diffup'
		replace `prefDiff' 	= `diffup' 	if `matched' == 0 & `diffdo' >  `diffup'

	end

	** This function goes through the observatioins from top to down and copying
	*  the match value of the closest observation of the other group above to
	*  (bestVal) and the ccoprresponding ID to (bestID) for obe group at the time.
	*  To go from bottuom to top sort with invsort and apply this command.
	cap program drop 	updateBestValID
		program define 	updateBestValID

		args grpvl matchval idvar grpvar matched bestVal bestID

		*Test if ID var is string or numeric
		local IDNumeric 0
		cap confirm numeric variable `idvar'
		if _rc == 0 local IDNumeric 1

		*Reset all values
		replace `bestVal' 	= .
		if `IDNumeric' == 1 replace `bestID' 	= .
		if `IDNumeric' == 0 replace `bestID' 	= ""

		*Set the match value and ID of observations that are in the other group
		replace `bestVal'   = `matchval' 		if `grpvar' != `grpvl' & `matched' == 0
		replace `bestID'  	= `idvar' 			if `grpvar' != `grpvl' & `matched' == 0

		*Fill in that value from the observation in the other group until getting to another observation of the other group
							replace `bestVal'   = `bestVal'[_n-1]	if `bestVal' == .  & `matched' == 0
		if `IDNumeric' == 1 replace `bestID'  	= `bestID'[_n-1] 	if `bestID'	 == .  & `matched' == 0
		if `IDNumeric' == 0 replace `bestID'  	= `bestID'[_n-1] 	if `bestID'	 == "" & `matched' == 0

	end

}

	cap program drop 	iematch
	    program define	iematch

		noi di "Command OK!"

		syntax  [if] [in]  , 					///
			GRPdummy(varname) 					///
			MATCHvar(varname) 					///
			[									///
			IDvar(varname)						///
			m1									///
			maxdiff(numlist max = 1 min = 1) 	///
			seedok						 		///
			MATCHIDname(string)					///
			MATCHDIffname(string)				///
			MATCHREsultname(string)				///
			MATCHCOuntname(string)				///
			]

		*****
		*1 format errors as smcl
		*2 option to disable countdown
		*3 test not matchcount without m1
		*4 test that tmt are not more obs than ctrl if one to one
		*5 a table with results outputted

		noi di "Syntax OK!"

		pause on

		qui {

		********************************
		*
		*	Gen temp sort and merge var
		*
		********************************

		tempvar originalSort

		gen `originalSort' = _n

		preserve

*Testing input
if 1 {

			if "`if'`in'" != "" keep `if'`in'

			********************************
			*
			*	Checking group dummy
			*
			********************************

			*Test that the group dummy is 1, 0 or missing
			cap assert inlist(`grpdummy', 1, 0, .) | `grpdummy' > .
			if _rc != 0 {

				di as error "The variable in grpdummy(`grpdummy') is not a dummy variable. The variable is only allowed to have the values 1, 0 or missing. Observations with missing varaibles in the grpdummy are ignored by this command."
				error _rc
			}

			**Exclude obs with missing value in groupdummy

			*Count number of obs to be dropped and and output that number if more than zero
			count if missing(`grpdummy')
			if `r(N)' > 0 {
				*This is not an error just outputting the number
				noi di "{pstd}`r(N)' observation(s) were excluded due to missing value in group dummy{p_end}"
			}
			*Drop obs with missing
			drop if missing(`grpdummy')

			********************************
			*
			*	Checking match continous var
			*
			********************************

			*Test that the continous is numeric
			cap confirm numeric variable `matchvar'
			if _rc != 0 {

				noi di as error "The variable to match on specified in matchvar(`matchvar') is not a numeric variable."
				error 109

			}

			**All variables with value 1 or 0 in the group dummy and
			* not excluded by if/in must have a value in matchvar
			count if missing(`matchvar')
			if `r(N)' > 0 {
				noi di as error  "{pstd}`r(N)' observation(s) with either value 1 or 0 in grpdummy(`grpdummy') do not have a non-missing value in matchvar(`matchvar'). Either update the match variable or exclude those observations using {inp:if} or {inp:in}.{p_end}"
				error 416
			}

			********************************
			*
			*	Checking match var and group dummy are unique			*
			********************************

			if "`seedok'" == "" {

				cap isid `matchvar' `grpdummy'

				if _rc != 0 {

					noi di as error "{pstd}There are base observations or target observations with duplicate values in matchvar(`matchvar'). To guarantee a replicable match you must {help seed:set a seed}. To supress this error message after you have set a the seed, or if a replicable match is not important to you, use option {inp:seedok}{p_end}"
					error _rc

				}
			}

			********************************
			*
			*	Checking duplicates in variable names in options
			*
			********************************

			*Create a local of all varnames used in any option
			local allnewvars `idvar' `matchidname' `matchdiffname' `matchresult'

			*Check if there are any duplicates in the local of all varnames in options
			local dupnewvars : list dups allnewvars

			*Throw error if there were any duplicates
			if "`dupnewvars'" != "" {

				di as error "The same variable name was used twice or more in the options. Go back and check syntax"
				error 198

			}

			********************************
			*
			*	Checking ID var
			*
			********************************

			*Checking set-up for the ID var used for _matchID
			if "`idvar'" == "" {

				*Copy this name
				local idvar _ID

				*Make sure that the default name _ID is not already used
				cap confirm variable `idvar'
				if _rc == 0 {

					di as error "A variable with name `idvar' is already defined. Either drop this variable or specify a variable using idvar() that fully and uniquely identifies the data set."
					error 110

				}

				*Generate the ID var from row number
				gen `idvar' = _n

				*Store local indicating ID var generated has type numeric
				local IDtypeNumeric 1

			}
			else {

				*Make sure that user specified ID is uniquely and fully identifying the observations
				cap isid `idvar'
				if _rc != 0 {

					di as error "Variable `idvar' specified in idvar() does not fully and uniquely identify the observations in the data set. Meaning that `idvar' is not allowed to have any duplicates or missing values. Make `idvar' uniquly and fully idnetifying or exclude the varaibleas using {inp:if} or {inp:in}."
					error 450

				}

				*Test if ID var is string or numeric
				cap confirm numeric variable `idvar'
				if _rc == 0 {

					*Store local indicating ID var type is numeric
					local IDtypeNumeric 1
				}
				else {

					*Store local indicating ID var type is numeric
					local IDtypeNumeric 0
				}
			}

			********************************
			*
			*	Checking and generating the MatchID and the MatchDiff var
			*
			********************************

			*Test input from user or test if deafault value is valiable
			iematchMatchVarCheck _matchID `matchidname'
			local matchIDname "`r(validVarName)'"

			*Allow the ID var used to be string
			if `IDtypeNumeric' == 1 {
				gen  `matchIDname' = .
			}
			else {
				gen  `matchIDname' = ""
			}

			*If many-to-one match used
			if "`m1'" != "" {
				iematchMatchVarCheck _matchCount `matchcount'
				local matchCountName "`r(validVarName)'"
				gen `matchCountName' = .
			}

			iematchMatchVarCheck _matchDiff `matchdiffname'
			local matchDiffName "`r(validVarName)'"
			gen `matchDiffName' = .

			iematchMatchVarCheck _matchOrReason `matchresult'
			local matchReasonName "`r(validVarName)'"
			gen `matchReasonName' = .

			*Create a label for missing values that explains no match
			label define matchLabel 0 "Not Matched" 1 "Matched" .n "Obs not included in match" .m "No match within maxdiff"

			*Apply the label
			label value `matchReasonName' matchLabel

}

		********************************
		*	Start matching
		********************************

			********************************
			*
			*	Creating tempvar used
			*
			********************************

			*Initiate the temporary variables used by this command
			tempvar prefID prefDiff matched

			gen `prefDiff' 		= .
			gen byte `matched' 	= 0

			*Allow the ID var used to be string
			if `IDtypeNumeric' == 1 {
				gen  `prefID' = .
			}
			else {
				gen  `prefID' = ""
			}

			** Generate the inverse of the matchvar to sort descending (gsort is too slow),
			*  a random var to seperate two values with the same match var, and the inverse
			*  of the random var for when sorting descending.
			tempvar rand invsort invrand

			sort `matchvar'
			gen  `invsort' = -1 * `matchvar'

			gen    `rand' = uniform()
			gen `invrand' = -1 * `rand'

			***********
			*Tempvars for matching

			*Diffvars, they are always numeric
			tempvar					 diffup   diffdo   valUp_0   valDo_0   valUp_1   valDo_1
			local 	 updownTempVars `diffup' `diffdo' `valUp_0' `valDo_0' `valUp_1' `valDo_1'

			foreach tempVar of local updownTempVars {

				gen `tempVar' = .
			}

			*ID vars, allowed to be both numeric and string
			tempvar					  	IDup   IDdo   IDup_0   IDdo_0   IDup_1   IDdo_1
			local 	 updownIDTempVars  `IDup' `IDdo' `IDup_0' `IDdo_0' `IDup_1' `IDdo_1'

			foreach tempVar of local updownIDTempVars {

				*Allow the ID var used to be string
				if `IDtypeNumeric' == 1 {
					gen  `tempVar' = .
				}
				else {
					gen  `tempVar' = ""
				}
			}

			***************************
			*
			*	One to one match
			*
			***************************

			if "`m1'" == "" {

				noi di ""
				noi di "{pstd}Base observations left to match: {p_end}"
				count if `grpdummy' == 1 & `matched' == 0
				local left2Match = `r(N)'
				noi di "{pstd}`left2Match' " _c

				*di "`idvar'"
				*di "`IDtypeNumeric'"
				*pause on
				*pause

				while (`left2Match' > 0) {

					*noi di 0

					**For all observations still to be matched, assign the preferred
					* match among the other unmatched observations
					qui updatePrefDiffPreffID `prefID' `prefDiff' `matchvar' `invsort' `idvar' `grpdummy' `matched' `rand' `invrand' `updownTempVars' `updownIDTempVars'

					*noi di 1

					if "`maxdiff'" != "" {

						replace `matched' 			= 1		if `prefDiff' > `maxdiff' & `grpdummy' == 1
						replace `matchReasonName' 	= .m 	if `prefDiff' > `maxdiff' & `grpdummy' == 1

						*Allow the ID var used to be string
						if `IDtypeNumeric' == 1 {
							replace `prefID'		= .		if `prefDiff' > `maxdiff'
						}
						else {
							replace `prefID'		= ""	if `prefDiff' > `maxdiff'
						}
					}

					*noi di 2

					replace `matched' = 1 if `matched' == 0 & `prefID' == `idvar'[_n-1] & `prefID'[_n-1] == `idvar'
					replace `matched' = 1 if `matched' == 0 & `prefID' == `idvar'[_n+1] & `prefID'[_n+1] == `idvar'

					*noi di 3

					count if `grpdummy' == 1 & `matched' == 0
					local left2Match = `r(N)'
					noi di "`left2Match' " _c

				}
				noi di "{p_end}" _c

			}

			***************************
			*
			*	Many to one match
			*
			***************************
			else {

				updatePrefDiffPreffID `prefID' `prefDiff' `matchvar' `invsort' `idvar' `grpdummy' `matched' `rand' `invrand' `updownTempVars' `updownIDTempVars'

				if "`maxdiff'" != "" {

					replace `matched' 			= 1		if `prefDiff' > `maxdiff' & `grpdummy' == 1
					replace `matchReasonName' 	= .m 	if `prefDiff' > `maxdiff' & `grpdummy' == 1

					*Allow the ID var used to be string
					if `IDtypeNumeric' == 1 {
						replace `prefID'		= .		if `prefDiff' > `maxdiff'
					}
					else {
						replace `prefID'		= ""	if `prefDiff' > `maxdiff'
					}

				}

				replace `prefID'			=  `idvar' if `grpdummy' == 0
				bys 	`prefID' 			:   replace `matchCountName' = _N - 1

				**Replace prefID to missing for target obs that had no base
				* obs matched to it. T
				if `IDtypeNumeric' == 1 {
					*Id var is numeric
					replace `prefID'		= .		if `matchCountName' == 0
				}
				else {
					*id var is string
					replace `prefID'		= ""	if `matchCountName' == 0
				}

				*Only target obs with base obs prefering it are matched
				replace	 `matched' 			= 1		if `matchCountName' != 0

				*Remove values for target obs that were not matched
				replace  `matchCountName'	= . 	if `matchCountName' == 0
			}

			*Return vars
			replace `matchDiffName' 	= `prefDiff'	if `matched' == 1
			replace `matchIDname' 		= `idvar' 		if `matched' == 1 & `grpdummy' == 0
			replace `matchIDname' 		= `prefID' 		if `matched' == 1 & `grpdummy' == 1

			*Target obs not used
			replace `matchReasonName' = 0 if `matched' == 0 & `grpdummy' == 0

			*only keep output vars
			keep `matchIDname' `matchDiffName' `idvar' `originalSort' `matchReasonName' `matchCountName'

			tempfile mergefile

			save `mergefile'

		restore

		tempvar mergevar

		merge 1:1  `originalSort' using `mergefile', gen(`mergevar')

		replace `matchReasonName' = .n if `mergevar' == 1
		replace `matchReasonName' = 1 if `matchReasonName' == .

		*comress the variables generated
		compress  `matchIDname' `matchDiffName' `matchReasonName' `matchCountName'

		*Output result to user
		noi tab `matchReasonName' `grpdummy', m

		sort `originalSort'

	}

	end

*Testing input (functions)
 if 1 {
	*Cheack matchVar input
	cap program drop 	iematchMatchVarCheck
		program define 	iematchMatchVarCheck  , rclass

		args defaultName userName

		if "`defaultName'" == "_matchID" 		local optionName matchidname
		if "`defaultName'" == "_matchDiff" 		local optionName matchdiffname
		if "`defaultName'" == "_matchOrReason" 	local optionName matchresult
		if "`defaultName'" == "_matchCount" 	local optionName matchcount

		*All the default names
		local dfltNms _noMatchReason _matchDiff _matchOrReason _matchCount

		*The other two default names
		local othDfltNms : list dfltNms - defaultName

		*Creating two locals with the deafult name in each local
		local Oth1 : word 1 of `othDfltNms'
		local Oth2 : word 2 of `othDfltNms'
		local Oth3 : word 3 of `othDfltNms'

		*Testing that the option name is not the same as the other variable's default name.
		if ("`userName'" == "`Oth1'" | "`userName'" == "`Oth2'" | "`userName'" == "`Oth3'" | "`userName'" == "_ID")  {

			noi di as error "{phang}The new name specified in `optionName'(`userName') is not allowed to be _ID, `Oth1', `Oth2', or `Oth3'{p_end}"
			error 198
		}

		*Checking set-up for the ID var used for _matchID
		if "`userName'" == "" {

			*CSet the default name
			local validMatchVarname `defaultName'

			*Make sure that the default name _ID is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				noi di as error "A variable with name `validMatchVarname' is already defined. Either drop this variable or specify a new variable name using `optionName'()."
				error 110

			}

		}

		else {

			*CSet the default name
			local validMatchVarname `userName'

			*Make sure that the default name _ID is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				noi di as error "The variable name specified in `optionName'(`userName') is already defined in the data set. Either drop this variable or specify a another variable name using `optionName'()."
				error 110

			}

		}

		*Testing that the new name is valid by creating a variable that is dropped immedeatly afterwards
		cap gen `validMatchVarname' = .

		if _rc != 0 {

			noi di as error "The variable name specified in `optionName'(`userName') is not a valid variable name."
			error _rc

		}
		drop `validMatchVarname'

		return local validVarName 	"`validMatchVarname'"

	end
}

*Matching functions
if 1 {

	** This function is called to assign the preferred match for all unmatched
	*  observations to any other unmatched observation. Returns a variable pref that is the
	cap program drop 	updatePrefDiffPreffID
		program define 	updatePrefDiffPreffID

		args prefID prefDiff matchvar invsort idvar grpvar matched rand invrand diffup diffdo valUp_0 valDo_0 valUp_1 valDo_1 IDup IDdo IDup_0 IDdo_0 IDup_1 IDdo_1

		*****************
		*Update possible matches with higher rank value
		sort `matched' `matchvar' `rand'

		updateBestValID 0 `matchvar' `idvar' `grpvar' `matched' `valUp_0' `IDup_0'
		updateBestValID 1 `matchvar' `idvar' `grpvar' `matched' `valUp_1' `IDup_1'

		*****************
		*Update possible matches with lower rank value
		sort `matched' `invsort' `invrand'

		updateBestValID 0 `matchvar' `idvar' `grpvar' `matched' `valDo_0' `IDdo_0'
		updateBestValID 1 `matchvar' `idvar' `grpvar' `matched' `valDo_1' `IDdo_1'

		*****************
		*Create one varaible common for both groups with diff upwards
		replace `diffup' 	= abs(`matchvar' - `valUp_1') if `grpvar' == 1
		replace `diffup' 	= abs(`matchvar' - `valUp_0') if `grpvar' == 0

		*Create one varaible common for both groups with diff downwards
		replace `diffdo' 	= abs(`matchvar' - `valDo_1') if `grpvar' == 1
		replace `diffdo' 	= abs(`matchvar' - `valDo_0') if `grpvar' == 0

		*Create one varaible common for both groups with ID upwards
		replace `IDup' 		= `IDup_1' 	if `grpvar' == 1
		replace `IDup' 		= `IDup_0' 	if `grpvar' == 0

		*Create one varaible common for both groups with ID downwards
		replace `IDdo' 		= `IDdo_1' 	if `grpvar' == 1
		replace `IDdo' 		= `IDdo_0' 	if `grpvar' == 0

		replace `prefID' 	= `IDdo' 	if `matched' == 0 & `diffdo' <= `diffup'
		replace `prefID' 	= `IDup' 	if `matched' == 0 & `diffdo' >  `diffup'

		replace `prefDiff' 	= `diffdo' 	if `matched' == 0 & `diffdo' <= `diffup'
		replace `prefDiff' 	= `diffup' 	if `matched' == 0 & `diffdo' >  `diffup'

	end

	** This function goes through the observatioins from top to down and copying
	*  the match value of the closest observation of the other group above to
	*  (bestVal) and the ccoprresponding ID to (bestID) for obe group at the time.
	*  To go from bottuom to top sort with invsort and apply this command.
	cap program drop 	updateBestValID
		program define 	updateBestValID

		args grpvl matchval idvar grpvar matched bestVal bestID

		*Test if ID var is string or numeric
		local IDNumeric 0
		cap confirm numeric variable `idvar'
		if _rc == 0 local IDNumeric 1

		*Reset all values
		replace `bestVal' 	= .
		if `IDNumeric' == 1 replace `bestID' 	= .
		if `IDNumeric' == 0 replace `bestID' 	= ""

		*Set the match value and ID of observations that are in the other group
		replace `bestVal'   = `matchval' 		if `grpvar' != `grpvl' & `matched' == 0
		replace `bestID'  	= `idvar' 			if `grpvar' != `grpvl' & `matched' == 0

		*Fill in that value from the observation in the other group until getting to another observation of the other group
							replace `bestVal'   = `bestVal'[_n-1]	if `bestVal' == .  & `matched' == 0
		if `IDNumeric' == 1 replace `bestID'  	= `bestID'[_n-1] 	if `bestID'	 == .  & `matched' == 0
		if `IDNumeric' == 0 replace `bestID'  	= `bestID'[_n-1] 	if `bestID'	 == "" & `matched' == 0

	end

}


Create blank.do

Create blank.do

error: pathspec 'do/blank.do' did not match any file(s) known to git


iematch: algortihtm v2 first commit

iematch: algortihtm v2 first commit

New version of the algorithm. This versions guarantees that there will
be no cases where a match is missed becasue the preferred option was
more than one step away. This algoritm is also very easy to converst
into a 1:m match.

fatal: unable to access 'https://github.com/worldbank/ietoolkit.git/': The requested URL returned error: 403


all.do

all.do

exec 1>&2
agda-pkg -pstdlib -i. -ilib agda-nplib.agda

error: pathspec 'all.do' did not match any file(s) known to git


iedropone : start helpfile

iedropone : start helpfile

{smcl}
{* 26 Dec 2016}{...}
{hline}
help for {hi:iedropone}
{hline}

{title:Title}

{phang2}{cmdab:iedropone} {hline 2} Same function as {help drop} but prevents that additional observation are unintentionally dropped

{title:Syntax}

{phang2}
{cmdab:iedropone} [if] , {cmdab:n:umobs(}{it:integer}{cmd:)}
	{cmdab:mvar(}{it:varname}{cmd:)} {cmdab:mval(}{it:list of values}{cmd:)}
	{cmd:zerook}]

{marker opts}{...}
{synoptset 22}{...}
{synopthdr:options}
{synoptline}
{synopt :{cmdab:n:umobs(}{it:integer}{cmd:)}}Number of observations that is allowed to be dropped. Default is 1.{p_end}
{synopt :{cmdab:mvar(}{it:varname}{cmd:)}}Variable for which multiple values should be dropped. Must be used together with {cmdab:mval()}.{p_end}
{synopt :{cmdab:mval(}{it:list of values}{cmd:)}}The list of values in {cmdab:mvar()} that should be dropped. Must be used together with {cmdab:mvar()}.{p_end}
{synopt :{cmd:zerook}}Allows that no observation is dropped.{p_end}
{synoptline}

{marker desc}
{title:Description}

{pstd}{cmdab:iedropone} has the identical purpose as {drop help} when dropping
	observations. However, {cmdab:iedropone} safeguards that no additional
	observations are unintentionally dropped, or that changes are made to that
	data that observations that are suppoed to be dropped are no longer dropped.

{pstd}{cmdab:iedropone} checks that no more or fewer observations than intended are
	dropped. For example, in the case that one observation has been identified to
	be dropped, then we want to make sure that when re-running the do-file only
	that observations is dropped even if more observations are added to that data
	set or changed in any other way.

{pstd}While the default is 1, {cmdab:iedropone} allows the user to set another number
	of observation that should exactly be dropped. It also alows for dropping an exact
	number of observations for multiple values in a variable.

{marker optslong}
{title:Options}

{phang}{cmdab:n:umobs(}{it:integer}{cmd:)}} this allows the user to set the
	number of observation that should be dropped. The default is 1 but any
	positive integer can be used. The command throws an error if any other number
	of observations are dropped.

{phang}{cmdab:mvar(}{it:varname}{cmd:)} indicates a variables where multiple
	values should be dropped. It must be exactly one observation for each value that is dropped, unless {cmd:

{phang}{cmdab:mval(}{it:list of values}{cmd:)}

{pmore}{cmd:zerook} allows that no observations are dropped. The default is that an error is thrown if no observations are dropped.

{title:Examples}

{pstd} {hi:Example 1.}

{pmore}{inp:iematch , grpdummy({it:tmt}) matchvar({it:p_hat})}

{pmore}In the example above, the observations with value 1 in {it:tmt} will be matched
	towards the nearest, in terms of {it:p_hat}, observations with value 0 in {it:tmt}.

{pstd} {hi:Example 2.}

{pmore}{inp:iematch if {it:baseline} == 1  , grpdummy({it:tmt}) matchvar({it:p_hat}) maxdiff(.001)}

{pmore}In the example above, the observations with value 1 in {it:tmt} will be matched
	towards the nearest, in terms of {it:p_hat}, observations with value 0 in {it:tmt} as
	long as the difference in {it:p_hat} is less than .001. Only observations that has the
	value 1 in variable {it:baseline} will be included in the match.

{title:Acknowledgements}

{phang}I would like to acknowledge the help in testing and proofreading I received in relation to this command and help file from (in alphabetic order):{p_end}
{pmore}Mrijan Rimal{break}

{title:Author}

{phang}Kristoffer Bjarkefur, The World Bank, DECIE

{phang}Please send bug-reports, suggestions and requests for clarifications
		 writing "ietools iematch" in the subject line to:{break}
		 kbjarkefur@worldbank.org

{phang}You can also see the code, make comments to the code, see the version
		 history of the code, and submit additions or edits to the code through
		 the github repository of ietoolkit:{break}
		 {browse "https://github.com/worldbank/ietoolkit"}

error: pathspec 'iedropone.sthlp' did not match any file(s) known to git


(misc) montecarlo code with nested bug in xtreg

clear all
set more off
cls

* Params
	local N 100
	local reps 2

	local dtasize = max(`N', `reps')

* Create dataset
	set obs `N'
	gen id = _n in 1/`N'
	replace id = (id - 1) if mod(id,2)==0 & id<=10 in 1/`N'
	bys id: gen t = _n if id<.
	xtset id t
	by id: gen N = _N if id<.
	tab N
	gen byte singleton = N==1

	* X = xtreg_fe A = areg B = xtreg_fe
	* C=cluster O=ols B=bootstrap
	* F=Full Sample S = Small Sample
	gen pvalue_AO = .
	gen pvalue_XO = .
	gen pvalue_ACF = .
	gen pvalue_XCF = .
	gen pvalue_ACS = .
	gen pvalue_XCS = .
	gen pvalue_XBF = .
	gen y = .
	gen x = .
	gen alpha = .
	gen e = .

forval rep = 1/`reps' {

	* Update data
	replace x = rnormal() in 1/`N'
	replace alpha = rnormal() in 1/`N'
	by id: replace alpha = alpha[1] if id<.
	replace e = rnormal() // can add AR/MA terms within cluster
	replace y = alpha + 0 * x + e in 1/`N'

	local vce  // Cluster by ID

	* AREG OLS all obs
	cap noi areg y x in 1/`N', absorb(id)
	test x // avoids writing the formula for the pvalue
	replace pvalue_AO = r(p)

	* XTREG OLS all obs
	cap noi xtreg y x in 1/`N', fe vce(conventional)
	test x // avoids writing the formula for the pvalue
	replace pvalue_XO = r(p)

	* AREG CLUSTER all obs
	cap noi areg y x in 1/`N', absorb(id) vce(cluster id)
	test x // avoids writing the formula for the pvalue
	replace pvalue_ACF = r(p)

	* AREG CLUSTER useful obs
	cap noi areg y x if !singleton in 1/`N', absorb(id) vce(cluster id)
	test x // avoids writing the formula for the pvalue
	replace pvalue_ACS = r(p)

	* XTREG CLUSTER all obs
	cap noi xtreg y x in 1/`N', fe vce(cluster id)
	test x // avoids writing the formula for the pvalue
	replace pvalue_XCF = r(p)

	* XTREG CLUSTER useful obs
	cap noi xtreg y x if !singleton in 1/`N', fe vce(cluster id)
	test x // avoids writing the formula for the pvalue
	replace pvalue_XCS = r(p)

	* XTREG Bootstrap all obs
	cap noi xtreg y x in 1/`N', fe vce(boot, reps(400) seed(10101))
	test x // avoids writing the formula for the pvalue
	replace pvalue_XBF = r(p)
}

su pvalue*, sep(2)
collapse (mean) pvalue*, fast
save "example_nested_bug.dta", replace

exit

cls
clear all
set more off
sysuse auto

gen id = _n
replace id = 1 if id==2
replace id = 3 if id==4
replace id = 5 if id==6
bys id: gen t = _n

xtset id t
xtreg price weight length if id<=6, fe vce(cluster id)

global vce vce(cluster id) // vce(robust) vce(cluster t)
cap noi xtreg price weight length, fe $vce // dfadj
cap noi areg price weight length, absorb(id) $vce
cap noi reghdfe price weight length, absorb(id) $vce

exit

cap noi xtreg price weight length, fe vce(boot, reps(500) seed(10101))

error: pathspec 'misc/example_nested_bug.do' did not match any file(s) known to git