Quarter work

Quarter work

cap	program drop	iegraph
	program define 	iegraph
	preserve
	syntax varlist, [noconfbars TItle(string) save(string) confbarsnone(varlist) yzero *]
	mat beta_ = e(b)

	local counter = 0
	//local textfile = "Mrijan"

	qui{
		if "`confbars'" 			!= "" {
		local CONFINT_BAR 	= 0
		}
		else if "`confbars'" 			== "" {
		local CONFINT_BAR 	= 1
		}

		local varTest : list confbarsnone in varlist

		if `varTest' == 0 {
					noi display as error "{phang} Variables defined in confbarsnone cannot be found in the graph variable list. {p_end}"
					noi display ""
					error 111
					 }

		foreach var of local varlist{
			cap assert inlist(`var',0,1) | missing(`var')
				if _rc {
					noi display as error "{phang} The variable `var' is not a dummy. Treatment variable needs to be a dummy(0 or 1) variable. {p_end}"
					noi display ""
					error 149
					}
				else {
					}

		local counter = `counter' + 1
		di `counter'

		scalar coeff_`var' = _b[`var']
		//noi display coeff_`var'

		scalar coeff_se_`var' = _se[`var']
		//noi display coeff_se_`var'

		count if e(sample) == 1 & `var' == 1 //Count one tmt group at the time
		scalar obs_`var' = r(N)
		//noi display obs_`var'

	}

	local count: word count `varlist'
	local graphCount = `count' + 1
	//tokenize "`varlist'"
	//noi di `count'

	//Make all vars tempvars (maybe do later)
	//Make sure that missing is properly handled
	tempvar anyTMT
	egen `anyTMT' = rowmax(`varlist') if e(sample) == 1
	tempvar control
	gen `control' = (`anyTMT' == 0) if !missing(`anyTMT')

	sum `e(depvar)' if e(sample) == 1 & `control' == 1
	scalar ctl_N		= r(N)
	scalar ctl_mean	  	= r(mean)
	scalar ctl_mean_sd 	= r(sd)

		foreach var of local varlist {
			scalar  conf_int_min_`var'   =	coeff_`var'-(1.96*coeff_se_`var') + ctl_mean
			scalar  conf_int_max_`var'   =	coeff_`var'+(1.96*coeff_se_`var') + ctl_mean

			test `var'
			local star_`var' " "

			scalar  pvalue =r(p)
			if pvalue < 0.10 {
				local star_`var' "*"
			}
			if pvalue < 0.05 {
				local star_`var' "**"
			}
			if pvalue < 0.01 {
				local star_`var' "***"
			}
			*noi display "`star_`var''"
			scalar tmt_mean_`var' = ctl_mean + coeff_`var'
		}

	tempname 	newHandle
	tempfile	newTextFile
	cap file close 	`newHandle'
	file open  	`newHandle' using "`newTextFile'", text write append

	*Write headers and control value
	file write `newHandle' ///
		"order" 	_tab "xLabeled"	_tab "mean" _tab "coeff" _tab "conf_int_min" _tab "conf_int_max" _tab "obs" _tab "star" _n 	///
		%9.3f (1) _tab "Control"  _tab %9.3f (ctl_mean)      _tab  		 			_tab 		 			  	  _tab 			 				_tab %9.3f (ctl_N) 	   _tab			  	  _n

	tempvar newCounter
	gen `newCounter' = 2  //First tmt group starts at 2 (1 is control)

	foreach var in `varlist' {
	     if `: list var in confbarsnone' {
		file write `newHandle' ///
			%9.3f (`newCounter') _tab "`var'" _tab %9.3f (tmt_mean_`var') _tab  %9.3f (coeff_`var')  _tab %9.3f  _tab _tab %9.3f (obs_`var')   _tab "`star_`var''"  _n

		replace `newCounter' = `newCounter' + 1
			}
		else {
		file write `newHandle' ///
			%9.3f (`newCounter') _tab "`var'" _tab %9.3f (tmt_mean_`var') _tab  %9.3f (coeff_`var')  _tab %9.3f (conf_int_min_`var') _tab %9.3f (conf_int_max_`var') _tab %9.3f (obs_`var')   _tab "`star_`var''"  _n

		replace `newCounter' = `newCounter' + 1
		}
	}

	file close `newHandle'

	copy "`newTextFile'"  "mainMasterDofile.txt" , replace

	/*************************************

		Create the graph

	*************************************/

	insheet using mainMasterDofile.txt, clear

	local tmtGroupBars 	""
	local xAxisLabels 	`"xlabel( "'
	local legendLabels	`"lab(1 "`obsLabelControl'")"'
	local legendNumbers	""

	forval tmtGroupCount = 1/`graphCount' {

		************
		*Create the bar for this group

		colorPicker `tmtGroupCount' `graphCount'

		local tmtGroupBars `"`tmtGroupBars' (bar mean order if order == `tmtGroupCount', color("`r(color)'")) "'

		************
		*Create labels etc. for this group

		local obs 			= obs[`tmtGroupCount']
		local stars 		= star[`tmtGroupCount']
		local legendLabel 	= xlabeled[`tmtGroupCount']

		local xAxisLabels 	`"`xAxisLabels' `tmtGroupCount' "(N = `obs') `stars'" "'
		local legendLabels 	`"`legendLabels' lab(`tmtGroupCount' "`legendLabel'") "'
		local legendNumbers	`"`legendNumbers' `tmtGroupCount'"'
	}

	*Close or comple some strings
	local xAxisLabels `"`xAxisLabels' ,noticks labsize(medsmall)) "'
	local legendOption `"legend(order(`legendNumbers') `legendLabels')"'

	*Create the confidence interval bars
	if `CONFINT_BAR' == 0 {
		//local confIntGraph == `"(scatter mean order,  msym(none)  mlabs(medium) mlabpos(10) mlabcolor(black)), xtitle("") ytitle("`e(depvar)'") "'
		local confIntGraph = ""
	}
		else if `CONFINT_BAR' == 1 {
		local confIntGraph = `"(rcap conf_int_max conf_int_min order, lc(gs)) (scatter mean order,  msym(none)  mlabs(medium) mlabpos(10) mlabcolor(black))"'
	}

	local titleOption `" , xtitle("") ytitle("`e(depvar)'") "'

	if "`save'" != "" {
		local saveOption saving("`save'", replace)
	}

	if "`yzero'" != "" {

		gen maxvalue = max(mean , conf_int_max)

		sum maxvalue
		local max = `r(max)'

		local logmax = log10(`max')

		local logmax = round(`logmax') - 1

		local tenpower = 10 ^ (`logmax')

		local up = `tenpower' * ceil(`max' / `tenpower')

		local quarter = (`up') / 4

		noi di "up: `up'"
		noi di "quarter: `quarter'"

		local yzero_option ylabel(0(`quarter')`up')
	}

	noi di `" graph twoway `tmtGroupBars' `confIntGraph' `titleOption' `options' `legendOption' `xAxisLabels' `saveOption' title("`title'") `yzero_option'  "'
	graph twoway `tmtGroupBars' `confIntGraph' `titleOption' `options' `legendOption' `xAxisLabels' `saveOption' title("`title'") `yzero_option'
	//noi di 	`"graph twoway `tmtGroupBars' `legendOption' (scatter mean order,  msym(none)  mlabs(medium) mlabpos(10) mlabcolor(black))), xtitle("") ytitle("`e(depvar)'") `xAxisLabels' `saveOption' title("`title'")  "'

	restore
}

end

cap	program drop	colorPicker
	program define 	colorPicker , rclass

	args groupCount totalNumGroups

	if `totalNumGroups' == 2 {

		if `groupCount' == 1 return local color "215 25 28"
		if `groupCount' == 2 return local color "43 123 182"
	}
	else if `totalNumGroups' == 3 {

		if `groupCount' == 1 return local color "215 25 28"
		if `groupCount' == 2 return local color "255 255 191"
		if `groupCount' == 3 return local color "43 123 182"
	}
	else if `totalNumGroups' == 4 {

		if `groupCount' == 1 return local color "215 25 28"
		if `groupCount' == 2 return local color "255 255 191"
		if `groupCount' == 3 return local color "171 217 233"
		if `groupCount' == 4 return local color "43 123 182"
	}
	else {

		*For five or more colors we repeat the same pattern

		local colourNum = mod(`groupCount', 5)
		if `colourNum' == 1 return local color "215 25 28"
		if `colourNum' == 2 return local color "253 174 93"
		if `colourNum' == 3 return local color "255 255 191"
		if `colourNum' == 4 return local color "171 217 233"
		if `colourNum' == 0 return local color "43 123 182"

	}

end

cap	program drop	iegraph
	program define 	iegraph
	preserve
	syntax varlist, [noconfbars TItle(string) save(string) confbarsnone(varlist) yzero *]

	mat beta_ = e(b)

	local counter = 0
	//local textfile = "Mrijan"

	qui{
		if "`confbars'" 			!= "" {
		local CONFINT_BAR 	= 0
		}
		else if "`confbars'" 			== "" {
		local CONFINT_BAR 	= 1
		}

		local varTest : list confbarsnone in varlist

		if `varTest' == 0 {
					noi display as error "{phang} Variables defined in confbarsnone cannot be found in the graph variable list. {p_end}"
					noi display ""
					error 111
					 }

		foreach var of local varlist{
			cap assert inlist(`var',0,1) | missing(`var')
				if _rc {
					noi display as error "{phang} The variable `var' is not a dummy. Treatment variable needs to be a dummy(0 or 1) variable. {p_end}"
					noi display ""
					error 149
					}
				else {
					}

		local counter = `counter' + 1
		di `counter'

		scalar coeff_`var' = _b[`var']
		//noi display coeff_`var'

		scalar coeff_se_`var' = _se[`var']
		//noi display coeff_se_`var'

		count if e(sample) == 1 & `var' == 1 //Count one tmt group at the time
		scalar obs_`var' = r(N)
		//noi display obs_`var'

	}

	local count: word count `varlist'
	local graphCount = `count' + 1
	//tokenize "`varlist'"
	//noi di `count'

	//Make all vars tempvars (maybe do later)
	//Make sure that missing is properly handled
	tempvar anyTMT
	egen `anyTMT' = rowmax(`varlist') if e(sample) == 1
	tempvar control
	gen `control' = (`anyTMT' == 0) if !missing(`anyTMT')

	sum `e(depvar)' if e(sample) == 1 & `control' == 1
	scalar ctl_N		= r(N)
	scalar ctl_mean	  	= r(mean)
	scalar ctl_mean_sd 	= r(sd)

		foreach var of local varlist {
			scalar  conf_int_min_`var'   =	coeff_`var'-(1.96*coeff_se_`var') + ctl_mean
			scalar  conf_int_max_`var'   =	coeff_`var'+(1.96*coeff_se_`var') + ctl_mean

			test `var'
			local star_`var' " "

			scalar  pvalue =r(p)
			if pvalue < 0.10 {
				local star_`var' "*"
			}
			if pvalue < 0.05 {
				local star_`var' "**"
			}
			if pvalue < 0.01 {
				local star_`var' "***"
			}
			*noi display "`star_`var''"
			scalar tmt_mean_`var' = ctl_mean + coeff_`var'
		}

	tempname 	newHandle
	tempfile	newTextFile
	cap file close 	`newHandle'
	file open  	`newHandle' using "`newTextFile'", text write append

	*Write headers and control value
	file write `newHandle' ///
		"order" 	_tab "xLabeled"	_tab "mean" _tab "coeff" _tab "conf_int_min" _tab "conf_int_max" _tab "obs" _tab "star" _n 	///
		%9.3f (1) _tab "Control"  _tab %9.3f (ctl_mean)      _tab  		 			_tab 		 			  	  _tab 			 				_tab %9.3f (ctl_N) 	   _tab			  	  _n

	tempvar newCounter
	gen `newCounter' = 2  //First tmt group starts at 2 (1 is control)

	foreach var in `varlist' {
	     if `: list var in confbarsnone' {
		file write `newHandle' ///
			%9.3f (`newCounter') _tab "`var'" _tab %9.3f (tmt_mean_`var') _tab  %9.3f (coeff_`var')  _tab %9.3f  _tab _tab %9.3f (obs_`var')   _tab "`star_`var''"  _n

		replace `newCounter' = `newCounter' + 1
			}
		else {
		file write `newHandle' ///
			%9.3f (`newCounter') _tab "`var'" _tab %9.3f (tmt_mean_`var') _tab  %9.3f (coeff_`var')  _tab %9.3f (conf_int_min_`var') _tab %9.3f (conf_int_max_`var') _tab %9.3f (obs_`var')   _tab "`star_`var''"  _n

		replace `newCounter' = `newCounter' + 1
		}
	}

	file close `newHandle'

	copy "`newTextFile'"  "mainMasterDofile.txt" , replace

	/*************************************

		Create the graph

	*************************************/

	insheet using mainMasterDofile.txt, clear

	local tmtGroupBars 	""
	local xAxisLabels 	`"xlabel( "'
	local legendLabels	`"lab(1 "`obsLabelControl'")"'
	local legendNumbers	""

	forval tmtGroupCount = 1/`graphCount' {

		************
		*Create the bar for this group

		colorPicker `tmtGroupCount' `graphCount'

		local tmtGroupBars `"`tmtGroupBars' (bar mean order if order == `tmtGroupCount', color("`r(color)'")) "'

		************
		*Create labels etc. for this group

		local obs 			= obs[`tmtGroupCount']
		local stars 		= star[`tmtGroupCount']
		local legendLabel 	= xlabeled[`tmtGroupCount']

		local xAxisLabels 	`"`xAxisLabels' `tmtGroupCount' "(N = `obs') `stars'" "'
		local legendLabels 	`"`legendLabels' lab(`tmtGroupCount' "`legendLabel'") "'
		local legendNumbers	`"`legendNumbers' `tmtGroupCount'"'
	}

	*Close or comple some strings
	local xAxisLabels `"`xAxisLabels' ,noticks labsize(medsmall)) "'
	local legendOption `"legend(order(`legendNumbers') `legendLabels')"'

	*Create the confidence interval bars
	if `CONFINT_BAR' == 0 {
		//local confIntGraph == `"(scatter mean order,  msym(none)  mlabs(medium) mlabpos(10) mlabcolor(black)), xtitle("") ytitle("`e(depvar)'") "'
		local confIntGraph = ""
	}
		else if `CONFINT_BAR' == 1 {
		local confIntGraph = `"(rcap conf_int_max conf_int_min order, lc(gs)) (scatter mean order,  msym(none)  mlabs(medium) mlabpos(10) mlabcolor(black))"'
	}

	local titleOption `" , xtitle("") ytitle("`e(depvar)'") "'

	if "`save'" != "" {
		local saveOption saving("`save'", replace)
	}

	if "`yzero'" != "" {

		gen maxvalue = max(mean , conf_int_max)

		sum maxvalue
		local max = `r(max)'

		local logmax = log10(`max')

		local logmax = round(`logmax') - 1

		local tenpower = 10 ^ (`logmax')

		local up = `tenpower' * ceil(`max' / `tenpower')

		local half = (`up') / 2

		noi di "up: `up'"
		noi di "half: `half'"

		local yzero_option ylabel(0(`half')`up')
	}

	noi di `" graph twoway `tmtGroupBars' `confIntGraph' `titleOption' `options' `legendOption' `xAxisLabels' `saveOption' title("`title'") `yzero_option'  "'
	graph twoway `tmtGroupBars' `confIntGraph' `titleOption' `options' `legendOption' `xAxisLabels' `saveOption' title("`title'") `yzero_option'
	//noi di 	`"graph twoway `tmtGroupBars' `legendOption' (scatter mean order,  msym(none)  mlabs(medium) mlabpos(10) mlabcolor(black))), xtitle("") ytitle("`e(depvar)'") `xAxisLabels' `saveOption' title("`title'")  "'

	restore
}

end

cap	program drop	colorPicker
	program define 	colorPicker , rclass

	args groupCount totalNumGroups

	if `totalNumGroups' == 2 {

		if `groupCount' == 1 return local color "215 25 28"
		if `groupCount' == 2 return local color "43 123 182"
	}
	else if `totalNumGroups' == 3 {

		if `groupCount' == 1 return local color "215 25 28"
		if `groupCount' == 2 return local color "255 255 191"
		if `groupCount' == 3 return local color "43 123 182"
	}
	else if `totalNumGroups' == 4 {

		if `groupCount' == 1 return local color "215 25 28"
		if `groupCount' == 2 return local color "255 255 191"
		if `groupCount' == 3 return local color "171 217 233"
		if `groupCount' == 4 return local color "43 123 182"
	}
	else {

		*For five or more colors we repeat the same pattern

		local colourNum = mod(`groupCount', 5)
		if `colourNum' == 1 return local color "215 25 28"
		if `colourNum' == 2 return local color "253 174 93"
		if `colourNum' == 3 return local color "255 255 191"
		if `colourNum' == 4 return local color "171 217 233"
		if `colourNum' == 0 return local color "43 123 182"

	}

end


added keepvar program

added keepvar program

clear
set more off

global path "/mnt/nfs6/wikipedia.proj/jmp/"
global rawosm "/mnt/nfs6/wikipedia.proj/jmp/rawdata/osm/"
global rawmaps "/mnt/nfs6/wikipedia.proj/jmp/rawdata/maps/"
global rawtrips "/mnt/nfs6/wikipedia.proj/jmp/rawdata/trips/"
global stash "/mnt/nfs6/wikipedia.proj/jmp/rawdata/stash/"
global myestimates "/mnt/nfs6/wikipedia.proj/jmp/jmpscripts/stata/estimates/"
global tables "/mnt/nfs6/wikipedia.proj/jmp/jmpscripts/stata/tables/"

cd `path'

********************************************
*************** datasets ********************
********************************************

insheet using ${rawosm}ne_final_head.csv, clear
insheet using ${rawosm}ne_final_sed.csv, clear

renamevar

keepvar

gennewvar

//// programlib
program drop keepvar
program keepvar

// drop non US
drop if fips == "NA" & geoid10 == "NA"
drop if otype == "v1"

gen keeptag = 0

local keeplist "building amenity highway maxspeed gnisfcode gnisfeatureid oneway waterway natural place leisure foot access wheelchair cycleway addr sidewalk import_uuid"

foreach x in `keeplist' {
    capture replace keeptag = 1 if `x' != ""
    capture replace keeptag = 1 if `x' != .
}

drop if keeptag == 0
drop keeptag

end

program drop renamevar
program renamevar
rename v1 otype
rename v2 id
rename v3 lon
rename v4 lat
rename v5 version
rename v6 timestamp
rename v7 changeset
rename v8 uid
rename v9 user
rename v10 highway
rename v11 name
rename v12 amenity
rename v13 building
rename v14 waterway
rename v15 natural
rename v16 oneway
rename v17 maxspeed
rename v18 place
rename v19 leisure
rename v20 foot
rename v21 access
rename v22 wheelchair
rename v23 cycleway
rename v24 gnisfeatureid
rename v25 gnisfcode
rename v26 nhdfcode
rename v27 addr
rename v28 import_uuid
rename v29 sidewalk
rename v30 lon_tmp
rename v31 lat_tmp
rename v32 tilename
rename v33 geoid10
rename v34 msaname
rename v35 x_tmp
rename v36 y_tmp
rename v37 fips
rename v38 countyname
rename v39 class
rename v40 mergevar
end

clear
set more off

global path "/mnt/nfs6/wikipedia.proj/jmp/"
global rawosm "/mnt/nfs6/wikipedia.proj/jmp/rawdata/osm/"
global rawmaps "/mnt/nfs6/wikipedia.proj/jmp/rawdata/maps/"
global rawtrips "/mnt/nfs6/wikipedia.proj/jmp/rawdata/trips/"
global stash "/mnt/nfs6/wikipedia.proj/jmp/rawdata/stash/"
global myestimates "/mnt/nfs6/wikipedia.proj/jmp/jmpscripts/stata/estimates/"
global tables "/mnt/nfs6/wikipedia.proj/jmp/jmpscripts/stata/tables/"

cd `path'

********************************************
*************** datasets ********************
********************************************

insheet using ${rawosm}ne_final.csv, clear


iefolder - adding abbreviations to options

cap program drop 	iefolder
	program define	iefolder

	syntax anything, PROJectfolder(string) [ABBreviation(string)]

	***Todo
	*Test that a folder called DataWork already exist (unless new project)
	*Test that old master do file exist
	*allow to specifiy other place for master do file

	/***************************************************

		Parse input

	***************************************************/

	*Parse out the sub-command
	gettoken subcommand item : anything

	*Parse out the item type and the item name
	gettoken itemType itemName : item

	*Make sure that the item name is only one word
	gettoken itemName rest : itemName

	*Clean up the input
	local subcomand = trim("`subcommand'")
	local itemType 	= trim("`itemType'")
	local itemName 	= trim("`itemName'")

	di "SubCommand 	`subcommand'"
	di "ItemType 	`itemType'"
	di "ItemName	`itemName'"

	if "`rest'" != "" {

		di as error "You have specified to many words. Spaces are not allowed."
		error 198
	}

	 /***************************************************

		Test input

	***************************************************/

	local sub_commands "new"
	local itemTypes "project round master unitofobs"

	*Test if subcommand is valid
	if `:list subcommand in sub_commands' == 0 {

		di as error "{phang}You have not used a valid subcommand. See help file for details.{p_end}"
		error 198
	}

	*Test if item type is valid
	if `:list itemType in itemTypes' == 0 {

		di as error "{phang}You have not used a valid item type. See help file for details.{p_end}"
		error 198
	}

	*Test that item name is used when item type is anything but project
	else if ("`itemType'" != "project" & "`itemName'" == "" ) {

		di as error "{phang}You must specify a name of the `itemType'. See help file for details.{p_end}"
		error 198
	}

	di "All tests ok"

	/***************************************************

		Start making updates to the project folder

	***************************************************/

	*Create a temporary textfile
	tempname 	newHandle
	tempfile	newTextFile

	cap file close 	`newHandle'
		file open  	`newHandle' using "`newTextFile'", text write append

	*Create a global pointing to the main data folder
	global dataWorkFolder 	"`projectfolder'/DataWork"
	global encryptFolder 	"$dataWorkFolder/EncryptedData"

	if "`subcommand'" == "new" {

		di "Subcommand: New"

		*Creating a new project
		if "`itemType'" == "project" {

			di "ItemType: Project"
			iefolder_newProject "`projectfolder'" `newHandle'
		}
		*Creating a new round
		else if "`itemType'" == "round" {

			*Use the full item name if abbrevaition was not specified
			if "`abb'" == "" local abb "`itemName'"

			di "ItemType: round"
			iefolder_newRound `newHandle' "`itemName'" "`abb'"
		}
		*Creating a new level of observation for master data set
		else if "`itemType'" == "master" | "`itemType'" == "unitofobs" {

			di "ItemType: master/unitofobs"
			di `"iefolder_newMaster	`newHandle' "`itemName'""'
			iefolder_newMaster	`newHandle' "`itemName'"
		}
	}

	*Closing the new main master dofile handle
	file close 		`newHandle'

	*Copy the new master dofile from the tempfile to the original position
	copy "`newTextFile'"  "$dataWorkFolder/Project_MasterDofile.do" , replace

end

	/***************************************************

		Here are all functions for combination of
		subcommand and item type

	***************************************************/

cap program drop 	iefolder_newProject
	program define	iefolder_newProject

		args projectfolder newHandle

		*Test if folder where to create new folder exist
		checkFolderExists "`projectfolder'" "parent"

		*Test that the new folder does not already exist
		checkFolderExists "$dataWorkFolder" "new"

		mkdir "$dataWorkFolder"

		******************************
		*Writing master do file header
		******************************

		*Write intro part with description of project,
		mdofle_p0 `newHandle' project

		*Write flolder globals section header and the root folders
		mdofle_p1 `newHandle' "`projectfolder'"

		*Write globals section header and the root folders
		mdofle_p2 `newHandle'

		*Write section that runs sub-master dofiles
		mdofle_p3 `newHandle' project

end

cap program drop 	iefolder_newRound
	program define	iefolder_newRound

	args subHandle rndName rndAbb

	di "new round command"

	*Test if folder where to create new folder exist
	checkFolderExists "$dataWorkFolder" "parent"

	*Test that the new folder does not already exist
	checkFolderExists "$dataWorkFolder/`rndName'" "new"

	*Old file reference
	tempname 	oldHandle
	local 		oldTextFile 	"$dataWorkFolder/Project_MasterDofile.do"

	file open `oldHandle' using `"`oldTextFile'"', read
	file read `oldHandle' line

	*Locals needed for the section devider
	local partNum 		= 0 //Keeps track of the part number

	while r(eof)==0 {

		*Do not interpret macros
		local line : subinstr local line "\$"  "\\$"
		local line : subinstr local line "\`"  "\\`"

		parseReadLine `"`line'"'

		if `r(ief_line)' == 0 {

			*This is a regular line of code. It should be copied as-is. No action needed.
			file write	`subHandle' `"`line'"' _n

		}
		else if `r(ief_line)' == 1 {

			**This is a line of code with a devisor written by
			* iefodler. New additions to a section are always added right before an end of section line.

			*return list

			**This is NOT an end of section line. Nothing will be written here
			* but we test that there is no confict with previous names
			if `r(ief_end)' == 0 {

					*Test if it is an end of globals section as we are writing a new
					if "`r(sectionName)'" == "monitor" {

					*Write devisor for this section
					writeDevisor 			`subHandle' 1 RoundGlobals rounds `rndName' `rndAbb'

					*Write the globals to the master do file and create the folders
					newRndFolderAndGlobals 	`rndName' `rndAbb' "dataWorkFolder" `subHandle'

					*Write an empty line before the end devisor
					file write		`subHandle' "" _n

				}
				else {

					*Test that the folder name about to be used is not already in use
					if "`r(itemName)'" == "`rndName'" {
						di as error "{phang}The new round name `rndName' is already in use. No new folders are creaetd and the master do-files has not been changed.{p_end}"
						error 507
					}

					*Test that the folder name about to be used is not already in use
					if "`r(itemAbb)'" == "`rndAbb'" {
						di as error "{phang}The new round abbreviation `rndAbb' is already in use. No new folders are creaetd and the master do-files has not been changed.{p_end}"
						error 507
					}

				}

				*Copy the line as is
				file write		`subHandle' `"`line'"' _n
			}

			**This is an end of section line. We will add the new content here
			* before writing the end of section line
			else if `r(ief_end)' == 1 {

				*Test if it is an end of globals section as we are writing a new
				if "`r(partName)'" == "End_RunDofiles" {

					*Write devisor for this section
					writeDevisor 			`subHandle' 3 RunDofiles `rndName' `rndAbb'

					*Write the
					file write  `subHandle' ///
						_col(4)"if (0) { //Change the 0 to 1 to run the `rndName' master dofile" _n ///
						_col(8) `"do ""' _char(36) `"`rndAbb'/`rndName'_master_dofile.do" "' _n ///
						_col(4)"}" ///
						_n

					*Write an empty line before the end devisor
					file write		`subHandle' "" _n

				}

				*Write the end of section line
				file write		`subHandle' `"`line'"' _n
			}
		}

		*Read next file and repeat the while loop
		file read `oldHandle' line
	}

	*Close the old file.
	file close `oldHandle'

end

cap program drop 	iefolder_newMaster
	program define	iefolder_newMaster

	args subHandle obsName

	di "new master command"

	*************
	*create folder in encrypred ID key master

	*Test if folder where to create new folder exist
	checkFolderExists "$dataWorkFolder/EncryptedData/IDMasterKey" "parent"

	*Test that the new folder does not already exist
	checkFolderExists "$dataWorkFolder/EncryptedData/IDMasterKey/`obsName'" "new"

	*Create the folder
	mkdir "$dataWorkFolder/EncryptedData/IDMasterKey/`obsName'"

	*************
	*create folder in masterdata
	global mastData 	"$dataWorkFolder/MasterData"

	*Old file reference
	tempname 	oldHandle
	local 		oldTextFile 	"$dataWorkFolder/Project_MasterDofile.do"

	file open `oldHandle' using `"`oldTextFile'"', read
	file read `oldHandle' line

	*Locals needed for the section devider
	local partNum 		= 0 //Keeps track of the part number

	while r(eof)==0 {

		*Do not interpret macros
		local line : subinstr local line "\$"  "\\$"
		local line : subinstr local line "\`"  "\\`"

		parseReadLine `"`line'"'

		if `r(ief_line)' == 1 & `r(ief_line)' == 1 & `r(ief_end)' == 0  & "`r(sectionName)'" == "rawData" {

			*Create master data folder and add global to folder in master do file
			cap createFolderWriteGlobal "`obsName'"  "mastData"  	mastData_`obsName'	`subHandle'

			if _rc == 693 {

				*Test if folder where to create new fodler exist
				checkFolderExists "$mastData" "parent"

				*Test that the new folder does not already exist
				checkFolderExists "$mastData\`obsName'" "new"

				*If that folder exist, problem
				di as error "{phang}could not create new folder, folder name might already exist "
				error _rc

			}
			else if _rc != 0 {

				*Unexpected error, run the exact command name again and throw the error to user
				createFolderWriteGlobal "`obsName'"  "mastData"  	mastData_`obsName'	`subHandle'
			}

			file write		`subHandle' `"`line'"' _n

		}
		else {

			*Copy the line as is
			file write		`subHandle' `"`line'"' _n

		}

		*Read next file and repeat the while loop
		file read `oldHandle' line
	}

	*Close the old file.
	file close `oldHandle'

	*Create master data subfolders
	createFolderWriteGlobal "MasterDataSets"  	"mastData_`obsName'"  masterDataSets
	createFolderWriteGlobal "Dofiles"  			"mastData_`obsName'"  mastDataDo
	createFolderWriteGlobal "Sampling"  		"mastData_`obsName'"  mastDataSamp
	createFolderWriteGlobal "Treatment"  		"mastData_`obsName'"  mastDataTreat

end

cap program drop 	iefolder
	program define	iefolder

	syntax anything, projectfolder(string) [abb(string)]

	***Todo
	*Test that a folder called DataWork already exist (unless new project)
	*Test that old master do file exist
	*allow to specifiy other place for master do file

	/***************************************************

		Parse input

	***************************************************/

	*Parse out the sub-command
	gettoken subcommand item : anything

	*Parse out the item type and the item name
	gettoken itemType itemName : item

	*Make sure that the item name is only one word
	gettoken itemName rest : itemName

	*Clean up the input
	local subcomand = trim("`subcommand'")
	local itemType 	= trim("`itemType'")
	local itemName 	= trim("`itemName'")

	di "SubCommand 	`subcommand'"
	di "ItemType 	`itemType'"
	di "ItemName	`itemName'"

	if "`rest'" != "" {

		di as error "You have specified to many words. Spaces are not allowed."
		error 198
	}

	 /***************************************************

		Test input

	***************************************************/

	local sub_commands "new"
	local itemTypes "project round master unitofobs"

	*Test if subcommand is valid
	if `:list subcommand in sub_commands' == 0 {

		di as error "{phang}You have not used a valid subcommand. See help file for details.{p_end}"
		error 198
	}

	*Test if item type is valid
	if `:list itemType in itemTypes' == 0 {

		di as error "{phang}You have not used a valid item type. See help file for details.{p_end}"
		error 198
	}

	*Test that item name is used when item type is anything but project
	else if ("`itemType'" != "project" & "`itemName'" == "" ) {

		di as error "{phang}You must specify a name of the `itemType'. See help file for details.{p_end}"
		error 198
	}

	di "All tests ok"

	/***************************************************

		Start making updates to the project folder

	***************************************************/

	*Create a temporary textfile
	tempname 	newHandle
	tempfile	newTextFile

	cap file close 	`newHandle'
		file open  	`newHandle' using "`newTextFile'", text write append

	*Create a global pointing to the main data folder
	global dataWorkFolder 	"`projectfolder'/DataWork"
	global encryptFolder 	"$dataWorkFolder/EncryptedData"

	if "`subcommand'" == "new" {

		di "Subcommand: New"

		*Creating a new project
		if "`itemType'" == "project" {

			di "ItemType: Project"
			iefolder_newProject "`projectfolder'" `newHandle'
		}
		*Creating a new round
		else if "`itemType'" == "round" {

			*Use the full item name if abbrevaition was not specified
			if "`abb'" == "" local abb "`itemName'"

			di "ItemType: round"
			iefolder_newRound `newHandle' "`itemName'" "`abb'"
		}
		*Creating a new level of observation for master data set
		else if "`itemType'" == "master" | "`itemType'" == "unitofobs" {

			di "ItemType: master/unitofobs"
			di `"iefolder_newMaster	`newHandle' "`itemName'""'
			iefolder_newMaster	`newHandle' "`itemName'"
		}
	}

	*Closing the new main master dofile handle
	file close 		`newHandle'

	*Copy the new master dofile from the tempfile to the original position
	copy "`newTextFile'"  "$dataWorkFolder/Project_MasterDofile.do" , replace

end

	/***************************************************

		Here are all functions for combination of
		subcommand and item type

	***************************************************/

cap program drop 	iefolder_newProject
	program define	iefolder_newProject

		args projectfolder newHandle

		*Test if folder where to create new folder exist
		checkFolderExists "`projectfolder'" "parent"

		*Test that the new folder does not already exist
		checkFolderExists "$dataWorkFolder" "new"

		mkdir "$dataWorkFolder"

		******************************
		*Writing master do file header
		******************************

		*Write intro part with description of project,
		mdofle_p0 `newHandle' project

		*Write flolder globals section header and the root folders
		mdofle_p1 `newHandle' "`projectfolder'"

		*Write globals section header and the root folders
		mdofle_p2 `newHandle'

		*Write section that runs sub-master dofiles
		mdofle_p3 `newHandle' project

end

cap program drop 	iefolder_newRound
	program define	iefolder_newRound

	args subHandle rndName rndAbb

	di "new round command"

	*Test if folder where to create new folder exist
	checkFolderExists "$dataWorkFolder" "parent"

	*Test that the new folder does not already exist
	checkFolderExists "$dataWorkFolder/`rndName'" "new"

	*Old file reference
	tempname 	oldHandle
	local 		oldTextFile 	"$dataWorkFolder/Project_MasterDofile.do"

	file open `oldHandle' using `"`oldTextFile'"', read
	file read `oldHandle' line

	*Locals needed for the section devider
	local partNum 		= 0 //Keeps track of the part number

	while r(eof)==0 {

		*Do not interpret macros
		local line : subinstr local line "\$"  "\\$"
		local line : subinstr local line "\`"  "\\`"

		parseReadLine `"`line'"'

		if `r(ief_line)' == 0 {

			*This is a regular line of code. It should be copied as-is. No action needed.
			file write	`subHandle' `"`line'"' _n

		}
		else if `r(ief_line)' == 1 {

			**This is a line of code with a devisor written by
			* iefodler. New additions to a section are always added right before an end of section line.

			*return list

			**This is NOT an end of section line. Nothing will be written here
			* but we test that there is no confict with previous names
			if `r(ief_end)' == 0 {

					*Test if it is an end of globals section as we are writing a new
					if "`r(sectionName)'" == "monitor" {

					*Write devisor for this section
					writeDevisor 			`subHandle' 1 RoundGlobals rounds `rndName' `rndAbb'

					*Write the globals to the master do file and create the folders
					newRndFolderAndGlobals 	`rndName' `rndAbb' "dataWorkFolder" `subHandle'

					*Write an empty line before the end devisor
					file write		`subHandle' "" _n

				}
				else {

					*Test that the folder name about to be used is not already in use
					if "`r(itemName)'" == "`rndName'" {
						di as error "{phang}The new round name `rndName' is already in use. No new folders are creaetd and the master do-files has not been changed.{p_end}"
						error 507
					}

					*Test that the folder name about to be used is not already in use
					if "`r(itemAbb)'" == "`rndAbb'" {
						di as error "{phang}The new round abbreviation `rndAbb' is already in use. No new folders are creaetd and the master do-files has not been changed.{p_end}"
						error 507
					}

				}

				*Copy the line as is
				file write		`subHandle' `"`line'"' _n
			}

			**This is an end of section line. We will add the new content here
			* before writing the end of section line
			else if `r(ief_end)' == 1 {

				*Test if it is an end of globals section as we are writing a new
				if "`r(partName)'" == "End_RunDofiles" {

					*Write devisor for this section
					writeDevisor 			`subHandle' 3 RunDofiles `rndName' `rndAbb'

					*Write the
					file write  `subHandle' ///
						_col(4)"if (0) { //Change the 0 to 1 to run the `rndName' master dofile" _n ///
						_col(8) `"do ""' _char(36) `"`rndAbb'/`rndName'_master_dofile.do" "' _n ///
						_col(4)"}" ///
						_n

					*Write an empty line before the end devisor
					file write		`subHandle' "" _n

				}

				*Write the end of section line
				file write		`subHandle' `"`line'"' _n
			}
		}

		*Read next file and repeat the while loop
		file read `oldHandle' line
	}

	*Close the old file.
	file close `oldHandle'

end

cap program drop 	iefolder_newMaster
	program define	iefolder_newMaster

	args subHandle obsName

	di "new master command"

	*************
	*create folder in encrypred ID key master

	*Test if folder where to create new folder exist
	checkFolderExists "$dataWorkFolder/EncryptedData/IDMasterKey" "parent"

	*Test that the new folder does not already exist
	checkFolderExists "$dataWorkFolder/EncryptedData/IDMasterKey/`obsName'" "new"

	*Create the folder
	mkdir "$dataWorkFolder/EncryptedData/IDMasterKey/`obsName'"

	*************
	*create folder in masterdata
	global mastData 	"$dataWorkFolder/MasterData"

	*Old file reference
	tempname 	oldHandle
	local 		oldTextFile 	"$dataWorkFolder/Project_MasterDofile.do"

	file open `oldHandle' using `"`oldTextFile'"', read
	file read `oldHandle' line

	*Locals needed for the section devider
	local partNum 		= 0 //Keeps track of the part number

	while r(eof)==0 {

		*Do not interpret macros
		local line : subinstr local line "\$"  "\\$"
		local line : subinstr local line "\`"  "\\`"

		parseReadLine `"`line'"'

		if `r(ief_line)' == 1 & `r(ief_line)' == 1 & `r(ief_end)' == 0  & "`r(sectionName)'" == "rawData" {

			*Create master data folder and add global to folder in master do file
			cap createFolderWriteGlobal "`obsName'"  "mastData"  	mastData_`obsName'	`subHandle'

			if _rc == 693 {

				*Test if folder where to create new fodler exist
				checkFolderExists "$mastData" "parent"

				*Test that the new folder does not already exist
				checkFolderExists "$mastData\`obsName'" "new"

				*If that folder exist, problem
				di as error "{phang}could not create new folder, folder name might already exist "
				error _rc

			}
			else if _rc != 0 {

				*Unexpected error, run the exact command name again and throw the error to user
				createFolderWriteGlobal "`obsName'"  "mastData"  	mastData_`obsName'	`subHandle'
			}

			file write		`subHandle' `"`line'"' _n

		}
		else {

			*Copy the line as is
			file write		`subHandle' `"`line'"' _n

		}

		*Read next file and repeat the while loop
		file read `oldHandle' line
	}

	*Close the old file.
	file close `oldHandle'

	*Create master data subfolders
	createFolderWriteGlobal "MasterDataSets"  	"mastData_`obsName'"  masterDataSets
	createFolderWriteGlobal "Dofiles"  			"mastData_`obsName'"  mastDataDo
	createFolderWriteGlobal "Sampling"  		"mastData_`obsName'"  mastDataSamp
	createFolderWriteGlobal "Treatment"  		"mastData_`obsName'"  mastDataTreat

end


iematch : remove match info if maxdiff()

Observations that did not have a match withing maxdiff() still had
results in _matchDiff (the best match) and _matchCount(the number of obs
that did not match). After this, these values are not missing

	*! version 3.0 26DEC2016  Kristoffer Bjarkefur kbjarkefur@worldbank.org

	cap program drop 	iematch
	    program define	iematch

		syntax  [if] [in]  , 					///
			GRPdummy(varname) 					///
			MATCHvar(varname) 					///
			[									///
			IDvar(varname)						///
			m1									///
			maxdiff(numlist max = 1 min = 1) 	///
			seedok						 		///
			MATCHIDname(string)					///
			MATCHDIffname(string)				///
			MATCHREsultname(string)				///
			MATCHCOuntname(string)				///
			]

		*****

	qui {

		* Set version
		version 11.0

		********************************
		*
		*	Gen temp sort and merge var
		*
		********************************

		tempvar originalSort ifinvar

		**Generate a variable that is used
		* to restore original sort.
		gen `originalSort' = _n

		preserve

			* String used to store why and how
			*  many observations are excluded.
			local obsexcludstring ""

			*Deal with if and in and display info.
			if "`if'`in'" != "" {

				* Gen dummy that is 1 when if/in is true. Then
				* count when this dummy is not 1. Ther is no way
				* to negate `if'`in'
				gen 	 `ifinvar'  = 1 `if'`in'
				count if `ifinvar' != 1
				if `r(N)' > 0 {

					*This is not an error just outputting the number
					local obsexcludstring "`obsexcludstring'`r(N)' observation(s) were excluded in {inp:if}/{inp:in}: (`if'`in').{break}"

				}

				*Drop variables excluded by `if'`in'
				keep `if'`in'
			}

			********************************
			*
			*	Keep only relevant vars
			*
			********************************

			*Keep only input vars
			keep  `grpdummy' `idvar' `matchvar'	`originalSort'

			********************************
			*
			*	Checking group dummy
			*
			********************************

			*Test that the group dummy is 1, 0 or missing
			cap assert inlist(`grpdummy', 1, 0, .) | `grpdummy' > .
			if _rc != 0 {

				di as error "{pstd}The variable in grpdummy(`grpdummy') is not a dummy variable. The variable is only allowed to have the values 1, 0 or missing. Observations with missing varaibles in the grpdummy are ignored by this command.{p_end}"
				error _rc
			}

			**********
			**Exclude obs with missing value in groupdummy

			*Count number of obs to be dropped and and output that number if more than zero
			count if missing(`grpdummy')
			if `r(N)' > 0 {

				*Prepare the local to be outputted with info on observations excluded
				local obsexcludstring  "`obsexcludstring'`r(N)' observation(s) were excluded due to missing value in  grpdummy(`grpdummy').{break}"
			}

			*Drop obs with missing value in groupvar
			drop if missing(`grpdummy')

			********************************
			*
			*	Checking match continous var
			*
			********************************

			*Test that the continous is numeric
			cap confirm numeric variable `matchvar'
			if _rc != 0 {

				noi di as error "{pstd}The variable to match on specified in matchvar(`matchvar') is not a numeric variable.{p_end}"
				error 109
			}

			**All variables with value 1 or 0 in the group dummy and
			* not excluded by if/in must have a value in matchvar
			count if missing(`matchvar')
			if `r(N)' > 0 {

				*Prepare the local to be outputted with info on observations excluded
				local obsexcludstring  "`obsexcludstring'`r(N)' observation(s) were excluded due to missing value in matchvar(`matchvar').{break}"
			}

			*Drop obs with missing values in match var
			drop if missing(`matchvar')

			********************************
			*
			*	Checking match var and group dummy are unique
			*
			********************************

			if "`seedok'" == "" {

				**test that there are no duplicates in match
				* var within each type of observation
				cap isid `matchvar' `grpdummy'

				if _rc != 0 {

					*Output the error message
					noi di as error "{pstd}There are base observations or target observations {...}"
					noi di as error "with duplicate values in matchvar(`matchvar'). To guarantee {...}"
					noi di as error "a replicable match you must {help seed:set a seed}. To supress {...}"
					noi di as error "this error message after you have set a the seed, or if a {...}"
					noi di as error "replicable match is not important to you, use option {inp:seedok}{p_end}"
					error 198

				}
			}

			********************************
			*
			*	Checking that there are more target
			*	obs than base obs in a 1-to-1 match
			*
			********************************

			if "`m1'" == "" {

				*Count number of base vars
				count if `grpdummy' == 1
				local numBaseObs `r(N)'

				*Count number of target vars
				count if `grpdummy' == 0
				local numTrgtObs `r(N)'

				*Test that there are more target
				cap assert `numTrgtObs' >= `numBaseObs'

				if _rc != 0 {

					noi di as error "{pstd}There are more base observations than target observations. This is not allowed in a one-to-one match. See option {inp:m1} for an alternaitve matching where it is allowed to have more base observations than target ovbservations.{p_end}"
					error _rc
				}
			}

			********************************
			*
			*	Checking duplicates in variable names in options
			*
			********************************

			*Create a local of all varnames used in any option
			local allnewvars `idvar' `matchidname' `matchdiffname' `matchresult'

			*Check if there are any duplicates in the local of all varnames in options
			local dupnewvars : list dups allnewvars

			*Throw error if there were any duplicates
			if "`dupnewvars'" != "" {

				di as error "{pstd}The same new variable name was used twice or more in the options generating a new variable. Go back and check syntax.{p_end}"
				error 198
			}

			********************************
			*
			*	Checking ID var
			*
			********************************

			*Checking set-up for the ID var used for _matchID
			if "`idvar'" == "" {

				*Copy this name
				local idvar _ID

				*Make sure that the default name _ID is not already used
				cap confirm variable `idvar'
				if _rc == 0 {

					di as error "{pstd}A variable with name `idvar' is already defined. Either drop this variable or specify a variable using idvar() that fully and uniquely identifies the data set.{p_end}"
					error 110
				}

				*Generate the ID var from row number
				gen `idvar' = _n

				*Store local indicating ID var generated has type numeric
				local IDtypeNumeric 1

			}
			else {

				*Make sure that user specified ID is uniquely and fully identifying the observations
				cap isid `idvar'
				if _rc != 0 {

					di as error "{pstd}Variable `idvar' specified in idvar() does not fully and uniquely identify the observations in the data set. Meaning that `idvar' is not allowed to have any duplicates or missing values. Make `idvar' uniquly and fully idnetifying or exclude the varaibleas using {inp:if} or {inp:in}.{p_end}"
					error 450
				}

				*Test if ID var is string or numeric
				cap confirm numeric variable `idvar'
				if _rc == 0 {

					*Store local indicating ID var type is numeric
					local IDtypeNumeric 1
				}
				else {

					*Store local indicating ID var type is numeric
					local IDtypeNumeric 0
				}
			}

			********************************
			*
			*	Checking user defined names
			*	and then create the result
			*	varaibles
			*
			********************************

			*******
			* Result var

			iematchMatchVarCheck _matchResult `matchresultname'
			local 	 		 matchResultName "`r(validVarName)'"
			gen 			`matchResultName' = .
			label variable	`matchResultName' "Matched obs = 1, not mathched = 0, all other different missing values"

			*Create a label for missing values that explains no match
			label define matchLabel 0 "Not Matched" 1 "Matched" .i "Obs excluded due to if/in" .g "Missing value in `grpdummy'" .m "Missing value in `matchvar'" .d "No match within maxdiff"

			*Apply the label
			label value `matchResultName' matchLabel

			*******
			*ID var

			iematchMatchVarCheck _matchID `matchidname'
			local matchIDname "`r(validVarName)'"

			*Allow the ID var used to be string
			if `IDtypeNumeric' == 1 {
				gen  `matchIDname' = .
			}
			else {
				gen  `matchIDname' = ""
			}

			*This variable is slightly differnt between a many-one and one-one match
			if "`m1'" != "" {
				label variable	`matchIDname' "The ID of the target var in each matched group"
			}
			else {
				label variable	`matchIDname' "The ID of the target var in each matched pair"
			}

			*******
			*Diff var

			iematchMatchVarCheck _matchDiff `matchdiffname'
			local 	 matchDiffName "`r(validVarName)'"
			gen 	`matchDiffName' = .

			*This variable is slightly differnt between a many-one and one-one match
			if "`m1'" != "" {
				label variable	`matchDiffName' "The difference in matchvar() between base obs and the target obs it matched with"
			}
			else {
				label variable	`matchDiffName' "The difference in matchvar() between base and target obs in each pair"
			}

			*******
			*Count var

			*If many-to-one match used
			if "`m1'" != "" {
				iematchMatchVarCheck _matchCount `matchcountname'
				local 			 matchCountName "`r(validVarName)'"
				gen 			`matchCountName' = .
				label variable	`matchDiffName' "The number of base obs this target obs matched with"

			}
			else {

				*Option match count name is not allowed if it is a one-to-one match
				if "`matchcountname'" != "" {

					di as error "{pstd}Option {inp:matchcountname()} is only allowed in combination with option {inp:m1}.{p_end}"
					error 198
				}
			}

		********************************
		*
		*	Output exclude string
		*
		********************************

			*The exclude string is created above but only displayed
			*after all testing of input is done.

			if "`obsexcludstring'" != "" {

				noi di ""
				noi di "{hline}"
				noi di ""
				noi di "{pstd}{ul:Observations are excluded from the matching for the following reasons:}{break}"
				noi di "`obsexcludstring'{p_end}"

			}

	********************************
	*	Start matching
	********************************

		********************************
		*
		*	Creating tempvar used in matching
		*
		********************************

		*Initiate the temporary variables used by this command
		tempvar prefID prefDiff matched

		gen `prefDiff' 		= .
		gen byte `matched' 	= 0

		*Allow the ID var used to be string
		if `IDtypeNumeric' == 1 {
			gen  `prefID' = .
		}
		else {
			gen  `prefID' = ""
		}

		** Generate the inverse of the matchvar to sort descending (gsort is too slow),
		*  a random var to seperate two values with the same match var, and the inverse
		*  of the random var for when sorting descending.
		tempvar rand invsort invrand

		sort `matchvar'
		gen  `invsort' = -1 * `matchvar'

		gen    `rand' = uniform()
		gen `invrand' = -1 * `rand'

		***********
		*Tempvars for matching

		*Diffvars, they are always numeric
		tempvar					 diffup   diffdo   valUp_0   valDo_0   valUp_1   valDo_1
		local 	 updownTempVars `diffup' `diffdo' `valUp_0' `valDo_0' `valUp_1' `valDo_1'

		foreach tempVar of local updownTempVars {

			gen `tempVar' = .
		}

		*ID vars, allowed to be both numeric and string
		tempvar					  	IDup   IDdo   IDup_0   IDdo_0   IDup_1   IDdo_1
		local 	 updownIDTempVars  `IDup' `IDdo' `IDup_0' `IDdo_0' `IDup_1' `IDdo_1'

		foreach tempVar of local updownIDTempVars {

			*Allow the ID var used to be string
			if `IDtypeNumeric' == 1 {
				gen  `tempVar' = .
			}
			else {
				gen  `tempVar' = ""
			}
		}

		***************************
		*
		*	One to one match
		*
		***************************

		if "`m1'" == "" {

			*Start outputting the countdown
			noi di ""
			noi di "{hline}{break}"
			noi di "{pstd}{ul:Matching one-to-one. Base observations left to match:}{p_end}"
			count if `grpdummy' == 1 & `matched' == 0

			*Create local to display "obs left to match" and to use in while loop
			local left2Match = `r(N)'
			noi di "{pstd}`left2Match' " _c

			*Match until no more observations to match.
			while (`left2Match' > 0) {

				**For all observations still to be matched, assign the preferred
				* match among the other unmatched observations
				qui updatePrefDiffPreffID `prefID' `prefDiff' `matchvar' `invsort' `idvar' `grpdummy' `matched' `rand' `invrand' `updownTempVars' `updownIDTempVars'

				*Restrict matches to within maxdiff() if that option is used.
				if "`maxdiff'" != "" {

					*Omit base observation from matching if diff is to big
					replace `matched' 			= 1		if `prefDiff' > `maxdiff' & `grpdummy' == 1

					*Indicate in result var that this obs did not have valid match within maxdiff()
					replace `matchResultName' 	= .d 	if `prefDiff' > `maxdiff' & `grpdummy' == 1

					*Removed preferred match
					if `IDtypeNumeric' == 1 {
						*IDvar is numeric
						replace `prefID'		= .		if `prefDiff' > `maxdiff'
					}
					else {
						*IDvar is string
						replace `prefID'		= ""	if `prefDiff' > `maxdiff'
					}
				}

				*If two observations mutually prefer each other, then indicate both of them as matched.
				replace `matched' = 1 if `matched' == 0 & `prefID' == `idvar'[_n-1] & `prefID'[_n-1] == `idvar'
				replace `matched' = 1 if `matched' == 0 & `prefID' == `idvar'[_n+1] & `prefID'[_n+1] == `idvar'

				*Update local to display "obs left to match" and to use in while loop
				count if `grpdummy' == 1 & `matched' == 0
				local left2Match = `r(N)'
				noi di "`left2Match' " _c

			}

			*End formatting for the "base obs left to match"
			noi di "{p_end}" _c
		}

		***************************
		*
		*	Many to one match
		*
		***************************
		else {

			**For all observations to be matched, assign the preferred
			* match among the other unmatched observations
			updatePrefDiffPreffID `prefID' `prefDiff' `matchvar' `invsort' `idvar' `grpdummy' `matched' `rand' `invrand' `updownTempVars' `updownIDTempVars'

			*Restrict matches to within maxdiff() if that option is used.
			if "`maxdiff'" != "" {

				*Indicate in result var that this obs did not have valid match within maxdiff()
				replace `matchResultName' 	= .d 	if `prefDiff' > `maxdiff' & `grpdummy' == 1

				*Removed preferred match
				if `IDtypeNumeric' == 1 {
					*IDvar is numeric
					replace `prefID'		= .		if `prefDiff' > `maxdiff'
				}
				else {
					*IDvar is string
					replace `prefID'		= ""	if `prefDiff' > `maxdiff'
				}
			}

			*Assign it's own ID as pref ID for all target vars
			replace `prefID'			=  `idvar' if `grpdummy' == 0

			* Replace the _matchCount var with number of base observations in each
			* match group. Each group is all base observation plus the target
			* observation, therefore (_N - 1)
			bys 	`prefID' 			:   replace `matchCountName' = _N - 1 if !missing(`prefID')

			**Replace prefID to missing for target obs that had no base
			* obs matched to it. T
			if `IDtypeNumeric' == 1 {
				*IDvar is numeric
				replace `prefID'		= .		if `matchCountName' == 0
			}
			else {
				*IDvar is string
				replace `prefID'		= ""	if `matchCountName' == 0
			}

			*Only target obs with base obs prefering it are matched
			replace	 `matched' 			= 1		if `matchCountName' != 0

			*Remove values for target obs that were not matched
			replace  `matchCountName'	= . 	if `matchCountName' == 0
		}

		*Update all return vars
		replace `matchDiffName' 	= `prefDiff'	if `matched' == 1
		replace `matchIDname' 		= `idvar' 		if `matched' == 1 & `grpdummy' == 0
		replace `matchIDname' 		= `prefID' 		if `matched' == 1 & `grpdummy' == 1

		*Remove the best match value in obs that did not have a match within maxdiff()
		replace `matchDiffName'  == . 				if `matchResultName' == .d

		*Matched observations are give value 1 in result var
		replace `matchResultName' = 1 				if `matched' == 1 & `matchResultName' != .d

		*Target obs not used
		replace `matchResultName' = 0 if `matched' == 0 & `grpdummy' == 0

		*only keep output vars
		keep `matchIDname' `matchDiffName' `originalSort' `matchResultName' `matchCountName'

		*Merge the results back to the original data set
		tempfile mergefile
		save 	`mergefile'

	restore

	***************************
	*
	*	Merge match results to
	*	original data and assign
	*	remaining missing values
	*	to the result var
	*
	***************************

	*Merge the results with the original data
	tempvar mergevar
	merge 1:1  `originalSort' using `mergefile', gen(`mergevar')

	*remaining missing values are listed in ascending order of importance.
	*Meaning that if a variable is both .m and .i then it will be .i as it
	*is assigned afterwards below.

	*Missing matching var
	replace `matchResultName' = .m if missing(`matchvar')

	*Msising dummy var
	replace `matchResultName' = .g if missing(`grpdummy')

	*Excluded in if/in
	if "`if'`in'" != "" {

		tempvar  ifinvar
		gen 	`ifinvar' = 1 `if'`in'
		replace `matchResultName' = .i if `ifinvar' != 1
	}

	*comress the variables generated
	compress  `matchIDname' `matchDiffName' `matchResultName' `matchCountName'

	*Output result table
	noi outputTable `matchResultName' `grpdummy'

	*Restore the oridinal sort
	sort `originalSort'

}

end

	*Check manually entered names for the return vars.
	cap program drop 	iematchMatchVarCheck
		program define 	iematchMatchVarCheck  , rclass

		args defaultName userName

		if "`defaultName'" == "_matchID" 		local optionName matchidname
		if "`defaultName'" == "_matchDiff" 		local optionName matchdiffname
		if "`defaultName'" == "_matchResult" 	local optionName matchresult
		if "`defaultName'" == "_matchCount" 	local optionName matchcount

		*All the default names
		local dfltNms _noMatchReason _matchDiff _matchResult _matchCount

		*The other two default names
		local othDfltNms : list dfltNms - defaultName

		*Creating two locals with the deafult name in each local
		local Oth1 : word 1 of `othDfltNms'
		local Oth2 : word 2 of `othDfltNms'
		local Oth3 : word 3 of `othDfltNms'

		*Testing that the option name is not the same as the other variable's default name.
		if ("`userName'" == "`Oth1'" | "`userName'" == "`Oth2'" | "`userName'" == "`Oth3'" | "`userName'" == "_ID")  {

			noi di as error "{phang}The new name specified in `optionName'(`userName') is not allowed to be _ID, `Oth1', `Oth2', or `Oth3'{p_end}"
			error 198
		}

		*Test if name is set manually
		if "`userName'" == "" {

			*CSet the default name
			local validMatchVarname `defaultName'

			*Make sure that the manually entered name is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				noi di as error "A variable with name `validMatchVarname' is already defined. Either drop this variable or specify a new variable name using `optionName'()."
				error 110
			}
		}
		*Otherwise use default name
		else {

			*CSet the default name
			local validMatchVarname `userName'

			*Make sure that the default name is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				noi di as error "The variable name specified in `optionName'(`userName') is already defined in the data set. Either drop this variable or specify a another variable name using `optionName'()."
				error 110
			}
		}

		*Testing that the new name is valid by creating a variable that is dropped immedeatly afterwards
		cap gen `validMatchVarname' = .

		if _rc != 0 {

			noi di as error "The variable name specified in `optionName'(`userName') is not a valid variable name."
			error _rc
		}
		drop `validMatchVarname'

		return local validVarName 	"`validMatchVarname'"

	end

	** This function is called to assign the preferred match for all unmatched
	*  observations to any other unmatched observation. Returns a variable pref that is the
	cap program drop 	updatePrefDiffPreffID
		program define 	updatePrefDiffPreffID

		args prefID prefDiff matchvar invsort idvar grpvar matched rand invrand diffup diffdo valUp_0 valDo_0 valUp_1 valDo_1 IDup IDdo IDup_0 IDdo_0 IDup_1 IDdo_1

		*****************
		*Update possible matches with higher rank value
		sort `matched' `matchvar' `rand'

		updateBestValID 0 `matchvar' `idvar' `grpvar' `matched' `valUp_0' `IDup_0'
		updateBestValID 1 `matchvar' `idvar' `grpvar' `matched' `valUp_1' `IDup_1'

		*****************
		*Update possible matches with lower rank value
		sort `matched' `invsort' `invrand'

		updateBestValID 0 `matchvar' `idvar' `grpvar' `matched' `valDo_0' `IDdo_0'
		updateBestValID 1 `matchvar' `idvar' `grpvar' `matched' `valDo_1' `IDdo_1'

		*****************
		*Create one varaible common for both groups with diff upwards
		replace `diffup' 	= abs(`matchvar' - `valUp_1') if `grpvar' == 1
		replace `diffup' 	= abs(`matchvar' - `valUp_0') if `grpvar' == 0

		*Create one varaible common for both groups with diff downwards
		replace `diffdo' 	= abs(`matchvar' - `valDo_1') if `grpvar' == 1
		replace `diffdo' 	= abs(`matchvar' - `valDo_0') if `grpvar' == 0

		*Create one varaible common for both groups with ID upwards
		replace `IDup' 		= `IDup_1' 	if `grpvar' == 1
		replace `IDup' 		= `IDup_0' 	if `grpvar' == 0

		*Create one varaible common for both groups with ID downwards
		replace `IDdo' 		= `IDdo_1' 	if `grpvar' == 1
		replace `IDdo' 		= `IDdo_0' 	if `grpvar' == 0

		replace `prefID' 	= `IDdo' 	if `matched' == 0 & `diffdo' <= `diffup'
		replace `prefID' 	= `IDup' 	if `matched' == 0 & `diffdo' >  `diffup'

		replace `prefDiff' 	= `diffdo' 	if `matched' == 0 & `diffdo' <= `diffup'
		replace `prefDiff' 	= `diffup' 	if `matched' == 0 & `diffdo' >  `diffup'

	end

	** This function goes through the observatioins from top to down and copying
	*  the match value of the closest observation of the other group above to
	*  (bestVal) and the ccoprresponding ID to (bestID) for obe group at the time.
	*  To go from bottuom to top sort with invsort and apply this command.
	cap program drop 	updateBestValID
		program define 	updateBestValID

		args grpvl matchval idvar grpvar matched bestVal bestID

		*Test if ID var is string or numeric
		local IDNumeric 0
		cap confirm numeric variable `idvar'
		if _rc == 0 local IDNumeric 1

		*Reset all values
		replace `bestVal' 	= .
		if `IDNumeric' == 1 replace `bestID' 	= .
		if `IDNumeric' == 0 replace `bestID' 	= ""

		*Set the match value and ID of observations that are in the other group
		replace `bestVal'   = `matchval' 		if `grpvar' != `grpvl' & `matched' == 0
		replace `bestID'  	= `idvar' 			if `grpvar' != `grpvl' & `matched' == 0

		*Fill in that value from the observation in the other group until getting to another observation of the other group
							replace `bestVal'   = `bestVal'[_n-1]	if `bestVal' == .  & `matched' == 0
		if `IDNumeric' == 1 replace `bestID'  	= `bestID'[_n-1] 	if `bestID'	 == .  & `matched' == 0
		if `IDNumeric' == 0 replace `bestID'  	= `bestID'[_n-1] 	if `bestID'	 == "" & `matched' == 0

	end

	** This function creates the output table.
	cap program drop 	outputTable
		program define 	outputTable

		qui {
			args resultVar grpdum

			*List the texts used in the first column in the
			local text_1 " 1 Matched"
			local text_0 " 0 Not matched"
			local text_d ".d No match within maxdiff()"
			local text_i ".i Excluded using if/in"
			local text_g ".g Missing grpdummy()"
			local text_m ".m Missing matchvar()"

			*The minimum width of the
			local firstColWidth = max(strlen(" `resultVar'") , strlen(" value and result"))

			levelsof `resultVar' , missing local(resultsUsed)

			foreach result of local resultsUsed {

				*Remove the . from the local to match the string
				local result = subinstr("`result'",".","",1)

				* Test if the text in the first column is longer
				* for this row than previous row
				local firstColWidth = max(`firstColWidth' , strlen("`text_`result''"))
			}

			* Create a local that indicates if there is column
			* for obs with missing value in tmt dummy
			local missingGrpDum 0
			count if missing(`grpdum')
			if `r(N)' > 0 local missingGrpDum 1

			*Set all locals that determins column width

			local C1 "{col 4}{c |}"

			noi di "`firstColWidth'"

			local col1width = `firstColWidth'  + 1
			local hli1 "{hline `col1width'}"
			local cen1 "{center `col1width':"

			local col2border = `firstColWidth' + 6
			local C2  "{col `col2border'}{c |}"
			local C2a "{col `col2border'}"

			local col3border = `firstColWidth' + 17
			local C3 "{col `col3border'}{c |}"

			local col4border = `firstColWidth' + 30
			local C4 "{col `col4border'}{c |}"

			local grpdumCentre 23 //Overspills automatically

			local lastT  	"{c RT}"
			local lastTdown	"{c RT}"
			local lastC  	"{c BRC}"

			if `missingGrpDum' {

				local lastT 		"{c +}{hline 9}{c RT}"
				local lastTdown 	"{c BT}{hline 9}{c RT}"
				local lastC 		"{c BT}{hline 9}{c BRC}"
				local missTitle 	"  missing "
				local grpdumCentre 	33

			}

			noi di ""
			noi di ""
			noi di "{hline}"
			noi di ""
			noi di "{pstd}{ul:Output of Matching Result:}{p_end}"
			noi di ""
			noi di "{col 5}`cen1'`resultVar'}`C2a' {centre `grpdumCentre':`grpdum' }"
			noi di "{col 4}{c LT}`hli1'{c +}{hline `grpdumCentre'}{c RT}"
			noi di "`C1'`cen1'value and result}`C2' 1 (base)   0 (target) `missTitle'{c |}"
			noi di "{col 4}{c LT}`hli1'{c +}{hline 10}{c +}{hline 12}`lastT'"

			*This is the roworder for the output table
			local resultTableOrder 1 0 .d .i .g .m

			foreach result of local resultTableOrder {

				*Test if this result was used
				if `:list result in resultsUsed' {

					*Count base observations in this result type
					qui count if `resultVar' == `result' & `grpdum' == 1
					local numBase = trim("`: display %16.0gc  `r(N)''")

					*Count target observations in this result type
					qui count if `resultVar' == `result' & `grpdum' == 0
					local numTarg = trim("`: display %16.0gc  `r(N)''")

					*Count observations that are neither base or target in this result type
					if `missingGrpDum' {

						qui count if `resultVar' == `result' & missing(`grpdum')
						local numMiss = trim("`: display %16.0gc  `r(N)''")

						local missCol "{ralign 9 :`numMiss' }{c |}"

					}

					*Prepare the local that displays the columns with numbers
					local numCols "`C2'{ralign 10 :`numBase' }`C3'{ralign 12 :`numTarg' }`C4'`missCol'"

					*Remove the . from the local to match the string
					local result = subinstr("`result'",".","",1)

					*Output the table row
					noi di "`C1'`text_`result''`numCols'"

				}
			}

			*Output line before N per group row
			noi di "{col 4}{c LT}`hli1'{c +}{hline 10}{c +}{hline 12}`lastT'"

			*Caculate N per group row for base observations
			qui count if `grpdum' == 1
			local numBase = trim("`: display %16.0gc  `r(N)''")

			*Caculate N per group row for target observations
			qui count if `grpdum' == 0
			local numTarg = trim("`: display %16.0gc  `r(N)''")

			*Caculate N per group row for observations neither base or target
			if `missingGrpDum' {

				qui count if missing(`grpdum')
				local numMiss = trim("`: display %16.0gc  `r(N)''")

				local missCol "{ralign 9 :`numMiss' }{c |}"

			}

			*Output N per group row
			noi di "`C1'`cen1'N per group}`C2'{ralign 10 :`numBase' }`C3'{ralign 12 :`numTarg' }`C4'`missCol'"

			*Output line before total N row
			noi di "{col 4}{c LT}`hli1'{c +}{hline 10}{c BT}{hline 12}`lastTdown'"

			*Count total N
			qui count
			local numTot = trim("`: display %16.0gc  `r(N)''")

			*Output total N row
			noi di "`C1'`cen1'Total N}`C2'{centre `grpdumCentre':`numTot'}`C4'"

			*Output bottom line
			noi di "{col 4}{c BLC}`hli1'{c BT}{hline `grpdumCentre'}{c BRC}"
		}
	end

	*! version 3.0 26DEC2016  Kristoffer Bjarkefur kbjarkefur@worldbank.org

	cap program drop 	iematch
	    program define	iematch

		syntax  [if] [in]  , 					///
			GRPdummy(varname) 					///
			MATCHvar(varname) 					///
			[									///
			IDvar(varname)						///
			m1									///
			maxdiff(numlist max = 1 min = 1) 	///
			seedok						 		///
			MATCHIDname(string)					///
			MATCHDIffname(string)				///
			MATCHREsultname(string)				///
			MATCHCOuntname(string)				///
			]

		*****

	qui {

		* Set version
		version 11.0

		********************************
		*
		*	Gen temp sort and merge var
		*
		********************************

		tempvar originalSort ifinvar

		**Generate a variable that is used
		* to restore original sort.
		gen `originalSort' = _n

		preserve

			* String used to store why and how
			*  many observations are excluded.
			local obsexcludstring ""

			*Deal with if and in and display info.
			if "`if'`in'" != "" {

				* Gen dummy that is 1 when if/in is true. Then
				* count when this dummy is not 1. Ther is no way
				* to negate `if'`in'
				gen 	 `ifinvar'  = 1 `if'`in'
				count if `ifinvar' != 1
				if `r(N)' > 0 {

					*This is not an error just outputting the number
					local obsexcludstring "`obsexcludstring'`r(N)' observation(s) were excluded in {inp:if}/{inp:in}: (`if'`in').{break}"

				}

				*Drop variables excluded by `if'`in'
				keep `if'`in'
			}

			********************************
			*
			*	Keep only relevant vars
			*
			********************************

			*Keep only input vars
			keep  `grpdummy' `idvar' `matchvar'	`originalSort'

			********************************
			*
			*	Checking group dummy
			*
			********************************

			*Test that the group dummy is 1, 0 or missing
			cap assert inlist(`grpdummy', 1, 0, .) | `grpdummy' > .
			if _rc != 0 {

				di as error "{pstd}The variable in grpdummy(`grpdummy') is not a dummy variable. The variable is only allowed to have the values 1, 0 or missing. Observations with missing varaibles in the grpdummy are ignored by this command.{p_end}"
				error _rc
			}

			**********
			**Exclude obs with missing value in groupdummy

			*Count number of obs to be dropped and and output that number if more than zero
			count if missing(`grpdummy')
			if `r(N)' > 0 {

				*Prepare the local to be outputted with info on observations excluded
				local obsexcludstring  "`obsexcludstring'`r(N)' observation(s) were excluded due to missing value in  grpdummy(`grpdummy').{break}"
			}

			*Drop obs with missing value in groupvar
			drop if missing(`grpdummy')

			********************************
			*
			*	Checking match continous var
			*
			********************************

			*Test that the continous is numeric
			cap confirm numeric variable `matchvar'
			if _rc != 0 {

				noi di as error "{pstd}The variable to match on specified in matchvar(`matchvar') is not a numeric variable.{p_end}"
				error 109
			}

			**All variables with value 1 or 0 in the group dummy and
			* not excluded by if/in must have a value in matchvar
			count if missing(`matchvar')
			if `r(N)' > 0 {

				*Prepare the local to be outputted with info on observations excluded
				local obsexcludstring  "`obsexcludstring'`r(N)' observation(s) were excluded due to missing value in matchvar(`matchvar').{break}"
			}

			*Drop obs with missing values in match var
			drop if missing(`matchvar')

			********************************
			*
			*	Checking match var and group dummy are unique
			*
			********************************

			if "`seedok'" == "" {

				**test that there are no duplicates in match
				* var within each type of observation
				cap isid `matchvar' `grpdummy'

				if _rc != 0 {

					*Output the error message
					noi di as error "{pstd}There are base observations or target observations {...}"
					noi di as error "with duplicate values in matchvar(`matchvar'). To guarantee {...}"
					noi di as error "a replicable match you must {help seed:set a seed}. To supress {...}"
					noi di as error "this error message after you have set a the seed, or if a {...}"
					noi di as error "replicable match is not important to you, use option {inp:seedok}{p_end}"
					error 198

				}
			}

			********************************
			*
			*	Checking that there are more target
			*	obs than base obs in a 1-to-1 match
			*
			********************************

			if "`m1'" == "" {

				*Count number of base vars
				count if `grpdummy' == 1
				local numBaseObs `r(N)'

				*Count number of target vars
				count if `grpdummy' == 0
				local numTrgtObs `r(N)'

				*Test that there are more target
				cap assert `numTrgtObs' >= `numBaseObs'

				if _rc != 0 {

					noi di as error "{pstd}There are more base observations than target observations. This is not allowed in a one-to-one match. See option {inp:m1} for an alternaitve matching where it is allowed to have more base observations than target ovbservations.{p_end}"
					error _rc
				}
			}

			********************************
			*
			*	Checking duplicates in variable names in options
			*
			********************************

			*Create a local of all varnames used in any option
			local allnewvars `idvar' `matchidname' `matchdiffname' `matchresult'

			*Check if there are any duplicates in the local of all varnames in options
			local dupnewvars : list dups allnewvars

			*Throw error if there were any duplicates
			if "`dupnewvars'" != "" {

				di as error "{pstd}The same new variable name was used twice or more in the options generating a new variable. Go back and check syntax.{p_end}"
				error 198
			}

			********************************
			*
			*	Checking ID var
			*
			********************************

			*Checking set-up for the ID var used for _matchID
			if "`idvar'" == "" {

				*Copy this name
				local idvar _ID

				*Make sure that the default name _ID is not already used
				cap confirm variable `idvar'
				if _rc == 0 {

					di as error "{pstd}A variable with name `idvar' is already defined. Either drop this variable or specify a variable using idvar() that fully and uniquely identifies the data set.{p_end}"
					error 110
				}

				*Generate the ID var from row number
				gen `idvar' = _n

				*Store local indicating ID var generated has type numeric
				local IDtypeNumeric 1

			}
			else {

				*Make sure that user specified ID is uniquely and fully identifying the observations
				cap isid `idvar'
				if _rc != 0 {

					di as error "{pstd}Variable `idvar' specified in idvar() does not fully and uniquely identify the observations in the data set. Meaning that `idvar' is not allowed to have any duplicates or missing values. Make `idvar' uniquly and fully idnetifying or exclude the varaibleas using {inp:if} or {inp:in}.{p_end}"
					error 450
				}

				*Test if ID var is string or numeric
				cap confirm numeric variable `idvar'
				if _rc == 0 {

					*Store local indicating ID var type is numeric
					local IDtypeNumeric 1
				}
				else {

					*Store local indicating ID var type is numeric
					local IDtypeNumeric 0
				}
			}

			********************************
			*
			*	Checking user defined names
			*	and then create the result
			*	varaibles
			*
			********************************

			*******
			* Result var

			iematchMatchVarCheck _matchResult `matchresultname'
			local 	 		 matchResultName "`r(validVarName)'"
			gen 			`matchResultName' = .
			label variable	`matchResultName' "Matched obs = 1, not mathched = 0, all other different missing values"

			*Create a label for missing values that explains no match
			label define matchLabel 0 "Not Matched" 1 "Matched" .i "Obs excluded due to if/in" .g "Missing value in `grpdummy'" .m "Missing value in `matchvar'" .d "No match within maxdiff"

			*Apply the label
			label value `matchResultName' matchLabel

			*******
			*ID var

			iematchMatchVarCheck _matchID `matchidname'
			local matchIDname "`r(validVarName)'"

			*Allow the ID var used to be string
			if `IDtypeNumeric' == 1 {
				gen  `matchIDname' = .
			}
			else {
				gen  `matchIDname' = ""
			}

			*This variable is slightly differnt between a many-one and one-one match
			if "`m1'" != "" {
				label variable	`matchIDname' "The ID of the target var in each matched group"
			}
			else {
				label variable	`matchIDname' "The ID of the target var in each matched pair"
			}

			*******
			*Diff var

			iematchMatchVarCheck _matchDiff `matchdiffname'
			local 	 matchDiffName "`r(validVarName)'"
			gen 	`matchDiffName' = .

			*This variable is slightly differnt between a many-one and one-one match
			if "`m1'" != "" {
				label variable	`matchDiffName' "The difference in matchvar() between base obs and the target obs it matched with"
			}
			else {
				label variable	`matchDiffName' "The difference in matchvar() between base and target obs in each pair"
			}

			*******
			*Count var

			*If many-to-one match used
			if "`m1'" != "" {
				iematchMatchVarCheck _matchCount `matchcountname'
				local 			 matchCountName "`r(validVarName)'"
				gen 			`matchCountName' = .
				label variable	`matchDiffName' "The number of base obs this target obs matched with"

			}
			else {

				*Option match count name is not allowed if it is a one-to-one match
				if "`matchcountname'" != "" {

					di as error "{pstd}Option {inp:matchcountname()} is only allowed in combination with option {inp:m1}.{p_end}"
					error 198
				}
			}

		********************************
		*
		*	Output exclude string
		*
		********************************

			*The exclude string is created above but only displayed
			*after all testing of input is done.

			if "`obsexcludstring'" != "" {

				noi di ""
				noi di "{hline}"
				noi di ""
				noi di "{pstd}{ul:Observations are excluded from the matching for the following reasons:}{break}"
				noi di "`obsexcludstring'{p_end}"

			}

	********************************
	*	Start matching
	********************************

		********************************
		*
		*	Creating tempvar used in matching
		*
		********************************

		*Initiate the temporary variables used by this command
		tempvar prefID prefDiff matched

		gen `prefDiff' 		= .
		gen byte `matched' 	= 0

		*Allow the ID var used to be string
		if `IDtypeNumeric' == 1 {
			gen  `prefID' = .
		}
		else {
			gen  `prefID' = ""
		}

		** Generate the inverse of the matchvar to sort descending (gsort is too slow),
		*  a random var to seperate two values with the same match var, and the inverse
		*  of the random var for when sorting descending.
		tempvar rand invsort invrand

		sort `matchvar'
		gen  `invsort' = -1 * `matchvar'

		gen    `rand' = uniform()
		gen `invrand' = -1 * `rand'

		***********
		*Tempvars for matching

		*Diffvars, they are always numeric
		tempvar					 diffup   diffdo   valUp_0   valDo_0   valUp_1   valDo_1
		local 	 updownTempVars `diffup' `diffdo' `valUp_0' `valDo_0' `valUp_1' `valDo_1'

		foreach tempVar of local updownTempVars {

			gen `tempVar' = .
		}

		*ID vars, allowed to be both numeric and string
		tempvar					  	IDup   IDdo   IDup_0   IDdo_0   IDup_1   IDdo_1
		local 	 updownIDTempVars  `IDup' `IDdo' `IDup_0' `IDdo_0' `IDup_1' `IDdo_1'

		foreach tempVar of local updownIDTempVars {

			*Allow the ID var used to be string
			if `IDtypeNumeric' == 1 {
				gen  `tempVar' = .
			}
			else {
				gen  `tempVar' = ""
			}
		}

		***************************
		*
		*	One to one match
		*
		***************************

		if "`m1'" == "" {

			*Start outputting the countdown
			noi di ""
			noi di "{hline}{break}"
			noi di "{pstd}{ul:Matching one-to-one. Base observations left to match:}{p_end}"
			count if `grpdummy' == 1 & `matched' == 0

			*Create local to display "obs left to match" and to use in while loop
			local left2Match = `r(N)'
			noi di "{pstd}`left2Match' " _c

			*Match until no more observations to match.
			while (`left2Match' > 0) {

				**For all observations still to be matched, assign the preferred
				* match among the other unmatched observations
				qui updatePrefDiffPreffID `prefID' `prefDiff' `matchvar' `invsort' `idvar' `grpdummy' `matched' `rand' `invrand' `updownTempVars' `updownIDTempVars'

				*Restrict matches to within maxdiff() if that option is used.
				if "`maxdiff'" != "" {

					*Omit base observation from matching if diff is to big
					replace `matched' 			= 1		if `prefDiff' > `maxdiff' & `grpdummy' == 1

					*Indicate in result var that this obs did not have valid match within maxdiff()
					replace `matchResultName' 	= .d 	if `prefDiff' > `maxdiff' & `grpdummy' == 1

					*Removed preferred match
					if `IDtypeNumeric' == 1 {
						*IDvar is numeric
						replace `prefID'		= .		if `prefDiff' > `maxdiff'
					}
					else {
						*IDvar is string
						replace `prefID'		= ""	if `prefDiff' > `maxdiff'
					}
				}

				*If two observations mutually prefer each other, then indicate both of them as matched.
				replace `matched' = 1 if `matched' == 0 & `prefID' == `idvar'[_n-1] & `prefID'[_n-1] == `idvar'
				replace `matched' = 1 if `matched' == 0 & `prefID' == `idvar'[_n+1] & `prefID'[_n+1] == `idvar'

				*Update local to display "obs left to match" and to use in while loop
				count if `grpdummy' == 1 & `matched' == 0
				local left2Match = `r(N)'
				noi di "`left2Match' " _c

			}

			*End formatting for the "base obs left to match"
			noi di "{p_end}" _c
		}

		***************************
		*
		*	Many to one match
		*
		***************************
		else {

			**For all observations to be matched, assign the preferred
			* match among the other unmatched observations
			updatePrefDiffPreffID `prefID' `prefDiff' `matchvar' `invsort' `idvar' `grpdummy' `matched' `rand' `invrand' `updownTempVars' `updownIDTempVars'

			*Restrict matches to within maxdiff() if that option is used.
			if "`maxdiff'" != "" {

				*Indicate in result var that this obs did not have valid match within maxdiff()
				replace `matchResultName' 	= .d 	if `prefDiff' > `maxdiff' & `grpdummy' == 1

				*Removed preferred match
				if `IDtypeNumeric' == 1 {
					*IDvar is numeric
					replace `prefID'		= .		if `prefDiff' > `maxdiff'
				}
				else {
					*IDvar is string
					replace `prefID'		= ""	if `prefDiff' > `maxdiff'
				}
			}

			*Assign it's own ID as pref ID for all target vars
			replace `prefID'			=  `idvar' if `grpdummy' == 0

			* Replace the _matchCount var with number of base observations in each
			* match group. Each group is all base observation plus the target
			* observation, therefore (_N - 1)
			bys 	`prefID' 			:   replace `matchCountName' = _N - 1

			**Replace prefID to missing for target obs that had no base
			* obs matched to it. T
			if `IDtypeNumeric' == 1 {
				*IDvar is numeric
				replace `prefID'		= .		if `matchCountName' == 0
			}
			else {
				*IDvar is string
				replace `prefID'		= ""	if `matchCountName' == 0
			}

			*Only target obs with base obs prefering it are matched
			replace	 `matched' 			= 1		if `matchCountName' != 0

			*Remove values for target obs that were not matched
			replace  `matchCountName'	= . 	if `matchCountName' == 0
		}

		*Update all return vars
		replace `matchDiffName' 	= `prefDiff'	if `matched' == 1
		replace `matchIDname' 		= `idvar' 		if `matched' == 1 & `grpdummy' == 0
		replace `matchIDname' 		= `prefID' 		if `matched' == 1 & `grpdummy' == 1

		*Matched observations are give value 1 in result var
		replace `matchResultName' = 1 				if `matched' == 1 & `matchResultName' != .d

		*Target obs not used
		replace `matchResultName' = 0 if `matched' == 0 & `grpdummy' == 0

		*only keep output vars
		keep `matchIDname' `matchDiffName' `originalSort' `matchResultName' `matchCountName'

		*Merge the results back to the original data set
		tempfile mergefile
		save 	`mergefile'

	restore

	***************************
	*
	*	Merge match results to
	*	original data and assign
	*	remaining missing values
	*	to the result var
	*
	***************************

	*Merge the results with the original data
	tempvar mergevar
	merge 1:1  `originalSort' using `mergefile', gen(`mergevar')

	*remaining missing values are listed in ascending order of importance.
	*Meaning that if a variable is both .m and .i then it will be .i as it
	*is assigned afterwards below.

	*Missing matching var
	replace `matchResultName' = .m if missing(`matchvar')

	*Msising dummy var
	replace `matchResultName' = .g if missing(`grpdummy')

	*Excluded in if/in
	if "`if'`in'" != "" {

		tempvar  ifinvar
		gen 	`ifinvar' = 1 `if'`in'
		replace `matchResultName' = .i if `ifinvar' != 1
	}

	*comress the variables generated
	compress  `matchIDname' `matchDiffName' `matchResultName' `matchCountName'

	*Output result table
	noi outputTable `matchResultName' `grpdummy'

	*Restore the oridinal sort
	sort `originalSort'

}

end

	*Check manually entered names for the return vars.
	cap program drop 	iematchMatchVarCheck
		program define 	iematchMatchVarCheck  , rclass

		args defaultName userName

		if "`defaultName'" == "_matchID" 		local optionName matchidname
		if "`defaultName'" == "_matchDiff" 		local optionName matchdiffname
		if "`defaultName'" == "_matchResult" 	local optionName matchresult
		if "`defaultName'" == "_matchCount" 	local optionName matchcount

		*All the default names
		local dfltNms _noMatchReason _matchDiff _matchResult _matchCount

		*The other two default names
		local othDfltNms : list dfltNms - defaultName

		*Creating two locals with the deafult name in each local
		local Oth1 : word 1 of `othDfltNms'
		local Oth2 : word 2 of `othDfltNms'
		local Oth3 : word 3 of `othDfltNms'

		*Testing that the option name is not the same as the other variable's default name.
		if ("`userName'" == "`Oth1'" | "`userName'" == "`Oth2'" | "`userName'" == "`Oth3'" | "`userName'" == "_ID")  {

			noi di as error "{phang}The new name specified in `optionName'(`userName') is not allowed to be _ID, `Oth1', `Oth2', or `Oth3'{p_end}"
			error 198
		}

		*Test if name is set manually
		if "`userName'" == "" {

			*CSet the default name
			local validMatchVarname `defaultName'

			*Make sure that the manually entered name is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				noi di as error "A variable with name `validMatchVarname' is already defined. Either drop this variable or specify a new variable name using `optionName'()."
				error 110
			}
		}
		*Otherwise use default name
		else {

			*CSet the default name
			local validMatchVarname `userName'

			*Make sure that the default name is not already used
			cap confirm variable `validMatchVarname'
			if _rc == 0 {

				noi di as error "The variable name specified in `optionName'(`userName') is already defined in the data set. Either drop this variable or specify a another variable name using `optionName'()."
				error 110
			}
		}

		*Testing that the new name is valid by creating a variable that is dropped immedeatly afterwards
		cap gen `validMatchVarname' = .

		if _rc != 0 {

			noi di as error "The variable name specified in `optionName'(`userName') is not a valid variable name."
			error _rc
		}
		drop `validMatchVarname'

		return local validVarName 	"`validMatchVarname'"

	end

	** This function is called to assign the preferred match for all unmatched
	*  observations to any other unmatched observation. Returns a variable pref that is the
	cap program drop 	updatePrefDiffPreffID
		program define 	updatePrefDiffPreffID

		args prefID prefDiff matchvar invsort idvar grpvar matched rand invrand diffup diffdo valUp_0 valDo_0 valUp_1 valDo_1 IDup IDdo IDup_0 IDdo_0 IDup_1 IDdo_1

		*****************
		*Update possible matches with higher rank value
		sort `matched' `matchvar' `rand'

		updateBestValID 0 `matchvar' `idvar' `grpvar' `matched' `valUp_0' `IDup_0'
		updateBestValID 1 `matchvar' `idvar' `grpvar' `matched' `valUp_1' `IDup_1'

		*****************
		*Update possible matches with lower rank value
		sort `matched' `invsort' `invrand'

		updateBestValID 0 `matchvar' `idvar' `grpvar' `matched' `valDo_0' `IDdo_0'
		updateBestValID 1 `matchvar' `idvar' `grpvar' `matched' `valDo_1' `IDdo_1'

		*****************
		*Create one varaible common for both groups with diff upwards
		replace `diffup' 	= abs(`matchvar' - `valUp_1') if `grpvar' == 1
		replace `diffup' 	= abs(`matchvar' - `valUp_0') if `grpvar' == 0

		*Create one varaible common for both groups with diff downwards
		replace `diffdo' 	= abs(`matchvar' - `valDo_1') if `grpvar' == 1
		replace `diffdo' 	= abs(`matchvar' - `valDo_0') if `grpvar' == 0

		*Create one varaible common for both groups with ID upwards
		replace `IDup' 		= `IDup_1' 	if `grpvar' == 1
		replace `IDup' 		= `IDup_0' 	if `grpvar' == 0

		*Create one varaible common for both groups with ID downwards
		replace `IDdo' 		= `IDdo_1' 	if `grpvar' == 1
		replace `IDdo' 		= `IDdo_0' 	if `grpvar' == 0

		replace `prefID' 	= `IDdo' 	if `matched' == 0 & `diffdo' <= `diffup'
		replace `prefID' 	= `IDup' 	if `matched' == 0 & `diffdo' >  `diffup'

		replace `prefDiff' 	= `diffdo' 	if `matched' == 0 & `diffdo' <= `diffup'
		replace `prefDiff' 	= `diffup' 	if `matched' == 0 & `diffdo' >  `diffup'

	end

	** This function goes through the observatioins from top to down and copying
	*  the match value of the closest observation of the other group above to
	*  (bestVal) and the ccoprresponding ID to (bestID) for obe group at the time.
	*  To go from bottuom to top sort with invsort and apply this command.
	cap program drop 	updateBestValID
		program define 	updateBestValID

		args grpvl matchval idvar grpvar matched bestVal bestID

		*Test if ID var is string or numeric
		local IDNumeric 0
		cap confirm numeric variable `idvar'
		if _rc == 0 local IDNumeric 1

		*Reset all values
		replace `bestVal' 	= .
		if `IDNumeric' == 1 replace `bestID' 	= .
		if `IDNumeric' == 0 replace `bestID' 	= ""

		*Set the match value and ID of observations that are in the other group
		replace `bestVal'   = `matchval' 		if `grpvar' != `grpvl' & `matched' == 0
		replace `bestID'  	= `idvar' 			if `grpvar' != `grpvl' & `matched' == 0

		*Fill in that value from the observation in the other group until getting to another observation of the other group
							replace `bestVal'   = `bestVal'[_n-1]	if `bestVal' == .  & `matched' == 0
		if `IDNumeric' == 1 replace `bestID'  	= `bestID'[_n-1] 	if `bestID'	 == .  & `matched' == 0
		if `IDNumeric' == 0 replace `bestID'  	= `bestID'[_n-1] 	if `bestID'	 == "" & `matched' == 0

	end

	** This function creates the output table.
	cap program drop 	outputTable
		program define 	outputTable

		qui {
			args resultVar grpdum

			*List the texts used in the first column in the
			local text_1 " 1 Matched"
			local text_0 " 0 Not matched"
			local text_d ".d No match within maxdiff()"
			local text_i ".i Excluded using if/in"
			local text_g ".g Missing grpdummy()"
			local text_m ".m Missing matchvar()"

			*The minimum width of the
			local firstColWidth = max(strlen(" `resultVar'") , strlen(" value and result"))

			levelsof `resultVar' , missing local(resultsUsed)

			foreach result of local resultsUsed {

				*Remove the . from the local to match the string
				local result = subinstr("`result'",".","",1)

				* Test if the text in the first column is longer
				* for this row than previous row
				local firstColWidth = max(`firstColWidth' , strlen("`text_`result''"))
			}

			* Create a local that indicates if there is column
			* for obs with missing value in tmt dummy
			local missingGrpDum 0
			count if missing(`grpdum')
			if `r(N)' > 0 local missingGrpDum 1

			*Set all locals that determins column width

			local C1 "{col 4}{c |}"

			noi di "`firstColWidth'"

			local col1width = `firstColWidth'  + 1
			local hli1 "{hline `col1width'}"
			local cen1 "{center `col1width':"

			local col2border = `firstColWidth' + 6
			local C2  "{col `col2border'}{c |}"
			local C2a "{col `col2border'}"

			local col3border = `firstColWidth' + 17
			local C3 "{col `col3border'}{c |}"

			local col4border = `firstColWidth' + 30
			local C4 "{col `col4border'}{c |}"

			local grpdumCentre 23 //Overspills automatically

			local lastT  	"{c RT}"
			local lastTdown	"{c RT}"
			local lastC  	"{c BRC}"

			if `missingGrpDum' {

				local lastT 		"{c +}{hline 9}{c RT}"
				local lastTdown 	"{c BT}{hline 9}{c RT}"
				local lastC 		"{c BT}{hline 9}{c BRC}"
				local missTitle 	"  missing "
				local grpdumCentre 	33

			}

			noi di ""
			noi di ""
			noi di "{hline}"
			noi di ""
			noi di "{pstd}{ul:Output of Matching Result:}{p_end}"
			noi di ""
			noi di "{col 5}`cen1'`resultVar'}`C2a' {centre `grpdumCentre':`grpdum' }"
			noi di "{col 4}{c LT}`hli1'{c +}{hline `grpdumCentre'}{c RT}"
			noi di "`C1'`cen1'value and result}`C2' 1 (base)   0 (target) `missTitle'{c |}"
			noi di "{col 4}{c LT}`hli1'{c +}{hline 10}{c +}{hline 12}`lastT'"

			*This is the roworder for the output table
			local resultTableOrder 1 0 .d .i .g .m

			foreach result of local resultTableOrder {

				*Test if this result was used
				if `:list result in resultsUsed' {

					*Count base observations in this result type
					qui count if `resultVar' == `result' & `grpdum' == 1
					local numBase = trim("`: display %16.0gc  `r(N)''")

					*Count target observations in this result type
					qui count if `resultVar' == `result' & `grpdum' == 0
					local numTarg = trim("`: display %16.0gc  `r(N)''")

					*Count observations that are neither base or target in this result type
					if `missingGrpDum' {

						qui count if `resultVar' == `result' & missing(`grpdum')
						local numMiss = trim("`: display %16.0gc  `r(N)''")

						local missCol "{ralign 9 :`numMiss' }{c |}"

					}

					*Prepare the local that displays the columns with numbers
					local numCols "`C2'{ralign 10 :`numBase' }`C3'{ralign 12 :`numTarg' }`C4'`missCol'"

					*Remove the . from the local to match the string
					local result = subinstr("`result'",".","",1)

					*Output the table row
					noi di "`C1'`text_`result''`numCols'"

				}
			}

			*Output line before N per group row
			noi di "{col 4}{c LT}`hli1'{c +}{hline 10}{c +}{hline 12}`lastT'"

			*Caculate N per group row for base observations
			qui count if `grpdum' == 1
			local numBase = trim("`: display %16.0gc  `r(N)''")

			*Caculate N per group row for target observations
			qui count if `grpdum' == 0
			local numTarg = trim("`: display %16.0gc  `r(N)''")

			*Caculate N per group row for observations neither base or target
			if `missingGrpDum' {

				qui count if missing(`grpdum')
				local numMiss = trim("`: display %16.0gc  `r(N)''")

				local missCol "{ralign 9 :`numMiss' }{c |}"

			}

			*Output N per group row
			noi di "`C1'`cen1'N per group}`C2'{ralign 10 :`numBase' }`C3'{ralign 12 :`numTarg' }`C4'`missCol'"

			*Output line before total N row
			noi di "{col 4}{c LT}`hli1'{c +}{hline 10}{c BT}{hline 12}`lastTdown'"

			*Count total N
			qui count
			local numTot = trim("`: display %16.0gc  `r(N)''")

			*Output total N row
			noi di "`C1'`cen1'Total N}`C2'{centre `grpdumCentre':`numTot'}`C4'"

			*Output bottom line
			noi di "{col 4}{c BLC}`hli1'{c BT}{hline `grpdumCentre'}{c BRC}"
		}
	end


commented dot_product

commented dot_product

* v1.0 01062017 , Apoorva Lal

/*
	Calculates the variable `Y = XB` where X is a subset of N variables in the currently loaded dataset, B is an arbitrary column vector (NX1 matrix). Basically a way to construct predicted values from a regression when the coefficients have been stored in a matrix / read in from elsewhere. Produces identical results to `predict` when used with the postestimation `e(b)` coefficient vector.

*/

program define dot_product, rclass
	// syntax : dot_product <projection> <coefficients> <variables>
	syntax namelist(min=3)
	loc targetvar `: word 1 of `namelist''
	loc vector `: word 2 of `namelist''
	cap conf matrix `vector'
	if _rc {
		noi di as error "matrix `vector' not found"
		exit
	}
	loc firsttwo `vector' `targetvar'
	loc xvars : list namelist - firsttwo
	loc ndim : list sizeof xvars
	loc rows `=rowsof(`vector')'
	loc cols `=colsof(`vector')'
	// checks conformability, allows for coefficients to be a row rather than column vector
	if `rows' != `ndim' {
		noi di as result "`vector' needs to be a column vector (N X 1) matrix where N == number of covars"
		if `cols' == `ndim' {
			noi di as result "Using transpose of `vector' instead"
			mat `vector'_t = `vector''
			loc coeffVector `vector'_t
		}
		else {
			noi di as error "Conformability error : coefficient vector not the same size as design matrix"
			exit
		}
	}
	else {
		loc coeffVector `vector'
	}
	mat coeffs = `coeffVector'
	qui mata: matrix_multiply("`xvars'","coeffs","`targetvar'")
end
// main mata subroutine
mata:
void matrix_multiply(string scalar varlist, string scalar CoeffMatrix, ///
	 			string scalar newvar) {
	real matrix X
	B = st_matrix(CoeffMatrix)
	st_view(X=.,.,tokens(varlist),)
	product = X*B
	st_addvar("double",(newvar))
	st_store(.,newvar,product)
}
end

capture program drop dot_product
program define dot_product, rclass
	syntax namelist(min=3)
	loc targetvar `: word 1 of `namelist''
	loc vector `: word 2 of `namelist''
	cap conf matrix `vector'
	if _rc {
		noi di as error "matrix `vector' not found"
		exit
	}
	loc firsttwo `vector' `targetvar'
	loc xvars : list namelist - firsttwo
	loc ndim : list sizeof xvars
	loc rows `=rowsof(`vector')'
	loc cols `=colsof(`vector')'
	if `rows' != `ndim' {
		noi di as result "`vector' needs to be a column vector (N X 1) matrix where N == number of covars"
		if `cols' == `ndim' {
			noi di as result "Using transpose of `vector' instead"
			mat `vector'_t = `vector''
			loc coeffVector `vector'_t
		}
		else {
			noi di as error "Conformability error : coefficient vector not the same size as design matrix"
			exit
		}
	}
	else {
		loc coeffVector `vector'
	}
	mat coeffs = `coeffVector'
	qui mata: matrix_multiply("`xvars'","coeffs","`targetvar'")
end

mata:
void matrix_multiply(string scalar varlist, string scalar CoeffMatrix, ///
	 			string scalar newvar) {
	real matrix X
	B = st_matrix(CoeffMatrix)
	st_view(X=.,.,tokens(varlist),)
	product = X*B
	st_addvar("double",(newvar))
	st_store(.,newvar,product)
}
end


Set the gridsearch plot scale explicitly, since it has hard edges of [0,1].

// grid search for good SVM parameters
// by Matthias Schonlau
// enhanced to use cv.ado by Nick Guenther
//
// this is not a function because
// - it is technically difficult:
//   - looping over ranges of arbitrary parameters is muddy (the sklearn GridSearch object has to take a dictionary of parameter->[set of values to try], which just looks bad, but is probably as good as you're going to get
//   - doing that in *Stata* is extra muddy
//   -
// - you always need to inspect the results before going on because there may be multiple equally good regions to search down into

set more off
clockseed

use tests/joe_dutch_merged

///////////////////////////////////////////////////////////
// randomize order
gen u=uniform()
sort u
///////////////////////////////////////////////////////////
// scaling autonormalize
sum _textnum_tokens
gen ntoken_stan= (_textnum_tokens -r(mean)) / r(sd)
///////////////////////////////////////////////////////////

// on top of the train/test splits within cv, hold out some data which never get trained on ever
gen train = _n<=(_N*.70)
local train = "train==1"
local test = "train==0"

// nevermind!
replace train = 1

// these columns are not a part of the dataset
// however Stata only lets us have one dataset
// so we just use this convention: never look in these columns when you mean to look in the others
// entries in them are filled in one
gen accuracy=.
gen C=.
gen gamma=.

// XXX there's a memory leak in _svm.plugin which means that running 5+-fold CV on this dataset crashes before it can finish
local folds = 2

local i = 0
foreach C of numlist 0.01 1 100 10000 100000 {
	foreach G of numlist .0001 .001 .01 .1 1 10 {
	local i = "`++i'"
	di as txt "svm category {indepvars} if `train' , c(`C') gamma(`G') cache(1024)"
	// generate accuracy measurements using 5-fold cross-validation
	cv pred   svm category  ntoken_stan  q*  if `train' , folds(`folds') shuffle est(c(`C') gamma(`G') cache(1024))
	gen acc = pred == category
	qui sum acc
	local accuracy = `r(mean)'

	// save results to our side-table (which is joined to the main table, but don't tell anyone that)
	qui replace C = `C' in `i'
	qui replace gamma = `G' in `i'
	qui replace accuracy = `accuracy' in `i'

	drop pred acc

	list C gamma accuracy in `i'
	di ""
	}
}

list C gamma accuracy in 1/`i'

twoway contour accuracy gamma C, yscale(log) xscale(log) ///
     ylabel(.0001 .001 .01 .1 1 10) xlabel(0.01 1 100 10000 100000 )  ///
	   ccuts(0(0.1)1) zlabel(#10, format(%9.2f))
graph export  svm_contour_cv_`folds'.pdf , replace

// XXX temporary
exit, clear

// grid search for good SVM parameters
// by Matthias Schonlau
// enhanced to use cv.ado by Nick Guenther
//
// this is not a function because
// - it is technically difficult:
//   - looping over ranges of arbitrary parameters is muddy (the sklearn GridSearch object has to take a dictionary of parameter->[set of values to try], which just looks bad, but is probably as good as you're going to get
//   - doing that in *Stata* is extra muddy
//   -
// - you always need to inspect the results before going on because there may be multiple equally good regions to search down into

set more off
clockseed

use tests/joe_dutch_merged

///////////////////////////////////////////////////////////
// randomize order
gen u=uniform()
sort u
///////////////////////////////////////////////////////////
// scaling autonormalize
sum _textnum_tokens
gen ntoken_stan= (_textnum_tokens -r(mean)) / r(sd)
///////////////////////////////////////////////////////////

// on top of the train/test splits within cv, hold out some data which never get trained on ever
gen train = _n<=(_N*.70)
local train = "train==1"
local test = "train==0"

// nevermind!
replace train = 1

// these columns are not a part of the dataset
// however Stata only lets us have one dataset
// so we just use this convention: never look in these columns when you mean to look in the others
// entries in them are filled in one
gen accuracy=.
gen C=.
gen gamma=.

// XXX there's a memory leak in _svm.plugin which means that running 5+-fold CV on this dataset crashes before it can finish
local folds = 2

local i = 0
foreach C of numlist 0.01 1 100 10000 100000 {
	foreach G of numlist .0001 .001 .01 .1 1 10 {
	local i = "`++i'"
	di as txt "svm category {indepvars} if `train' , c(`C') gamma(`G') cache(1024)"
	// generate accuracy measurements using 5-fold cross-validation
	cv pred   svm category  ntoken_stan  q*  if `train' , folds(`folds') shuffle est(c(`C') gamma(`G') cache(1024))
	gen acc = pred == category
	qui sum acc
	local accuracy = `r(mean)'

	// save results to our side-table (which is joined to the main table, but don't tell anyone that)
	qui replace C = `C' in `i'
	qui replace gamma = `G' in `i'
	qui replace accuracy = `accuracy' in `i'

	drop pred acc

	list C gamma accuracy in `i'
	di ""
	}
}

list C gamma accuracy in 1/`i'

twoway contour accuracy gamma C, yscale(log) xscale(log) ///
     ylabel(.0001 .001 .01 .1 1 10) xlabel(0.01 1 100 10000 100000 )  ///
	   levels(10) zlabel(#10, format(%9.2f))
graph export  svm_contour_cv_`folds'.pdf , replace

// XXX temporary
exit, clear