iedup - fix to #146 - merge only on uniquevars

This is the fix to the orignal issue. Unique vars may no longer be time vars.  Remove precision option. Merge report to oringal data only on unique vars

*! version 5.5 26APR2018 DIME Analytics lcardosodeandrad@worldbank.org

	capture program drop ieduplicates
	program ieduplicates , rclass

	qui {

		syntax varname ,  FOLder(string) UNIQUEvars(varlist) [KEEPvars(varlist) tostringok droprest nodaily SUFfix(string)]

		version 11.0

		*Add version of Stata fix
		//Make sure that keepvars are still saved if saved if the duplicates file
		*	is generated on a subset of the data. For example, duplicates from
		*	version 1, 2 . If the command is run on only version 1. Then values
		*	for keeepvars in version 2 and 3 are dropped and not reloaded as those
		*	obs are not in current memory

		**Test that observations have not been deleted from the report before readind
		* it. Deleted in a way that the report does not make sense. Provide an error
		* message to this that is more informative.

		preserve

			/***********************************************************************
			************************************************************************

				Section 1 - Set up locals needed in data

				Saving a version of the data to be used before merging corrections
				back to the original data set and before correcting duplicates

			************************************************************************
			***********************************************************************/

			*Tempfiles to be uses
			tempfile originalData preppedReport datawithreportmerged dataToReturn

			** Save a version of original data that can be brought
			*  back withing the preserve/restore section
			save 	`originalData'

			* Create a local with todays date to date the reports
			local  date = subinstr(c(current_date)," ","",.)

			*Put idvar in a local with a more descriptive name
			local idvar `varlist'

			** Making one macro with all variables that will be
			*  imported and exported from the Excel file
			local argumentVars `idvar' `uniquevars' `keepvars'

			* Create a list of the varaibles created by this command to put in the report
			local excelvars dupListID dateListed dateFixed correct drop newID initials notes

			/***********************************************************************
			************************************************************************

				Section 2 - Test unique vars

				Test the unique vars so that they are identifying the data set and
				are not in a time format that might get corrupted by exporting to
				and importing from Excel. This is needed as the uniquevars are needed
				to merge the correct correction to the correct duplicat

			************************************************************************
			***********************************************************************/

			*Test that the unique vars fully and uniquely identifies the data set
			cap isid `uniquevars'
			if _rc {

				noi display as error "{phang}The variable(s) listed in uniquevars() does not uniquely and fully identifies all observations in the data set.{p_end}"
				isid `uniquevars'
				error 198
				exit
			}

			** Test that no unique vars are time format. Time values might be corrupted
			*  and changed a tiny bit when importing and exporting to Excel, which make merge not possible
			foreach uniquevar of local uniquevars {

				*Test if format is time format
				local format : format `uniquevar'
				if substr("`format'",1,2) == "%t" {

					noi display as error `"{phang}The variable {inp:`uniquevar'} listed in {inp:uniquevars()} is using time format which is not allowed. Stata and Excel stores and displays time slightly differently which can lead to small changes in the value when the value is imported and exported between Stata and Excel, and then the variable can no longer be used to merge the report back to the original data. Use another variable or create a string variable out of your time variable using this code: {inp: generate `uniquevar'_str = string(`uniquevar',"%tc")}.{p_end}"'
					noi di ""
					error 198
					exit
				}
			}

			tempfile restart
			save 	`restart'

			/***********************************************************************
			************************************************************************

				Section 3

				Import and prepare Corrections (if file exists)

			************************************************************************
			***********************************************************************/

			/******************
				Section 3.1
				Check if earlier report exists.
			******************/

			cap confirm file "`folder'/iedupreport`suffix'.xlsx"

			if !_rc {
				local fileExists 1
			}
			else {
				local fileExists 0
			}

			/******************
				Section 3.2
				If report exist, load file and check input
			******************/

			if `fileExists' {

				*Load excel file. Load all vars as string and use meta data from Section 1
				import excel "`folder'/iedupreport`suffix'.xlsx"	, clear firstrow

				** All excelvars but dupListID and newID should be string. dupListID
				*  should be numeric and the type of newID should be based on the user input
				foreach excelvar of local excelvars {
					if !inlist("`excelvar'", "dupListID", "newID") {

						* Make original ID var string
						tostring `excelvar' , replace
						replace  `excelvar' = "" if `excelvar' == "."
					}
				}

				/******************
					Section 3.2.1
					Make sure that the
					ID variable and
					the uniquevars are
					not changed since
					last report.
				******************/

				ds
				local existingexcelvars  `r(varlist)'

				if `:list varlist in existingexcelvars' == 0 {

					noi display as error "{phang}ID variable [`varlist'] does not exist in the previously exported Excle file. If you renamed or changed the ID variable, you need to start over with a new file. Rename or move the already existing file. Create a new file and carefully copy any corrections from the old file to the new.{p_end}"
					noi di ""
					error 111
					exit

				}
				if `:list uniquevars in existingexcelvars' == 0 {

					noi display as error "{phang}One or more unique variables in [`uniquevars'] do not exist in the previously exported Excel file. If you renamed or changed any variable used in uniquevars(), you need to start over with a new file. Rename or move the already existing file. Create a new file and carefully copy any corrections from the old file to the new.{p_end}"
					noi di ""
					error 111
					exit
				}

				/******************
					Section 3.3
					Make sure input is correct
				******************/

				*Temporary variables needed for checking input
				tempvar tmpvar_multiInp tmpvar_inputNotYes tmpvar_maxMultiInp tmpvar_notDrop tempvar_yescorrect tempvar_numcorrect

				*Locals indicating in which ways input is incorrect (if any)
				local local_multiInp 		0
				local local_multiCorr		0
				local local_inputNotYes		0
				local local_notDrop			0

				/******************
					Section 3.3.1
					Make sure there are not too many corrections
				******************/

				* Count the number of corrections (correct drop newID) per
				* observation. Only one correction per observation is allowed.
				egen `tmpvar_multiInp' = rownonmiss(correct drop newID), strok

				*Check that all rows have utmost one correction
				cap assert `tmpvar_multiInp' == 0 | `tmpvar_multiInp' == 1

				if _rc {

					*Error will be outputted below
					local local_multiInp 	1
				}

				/******************
					Section 3.3.2
					Make sure string input is yes or y
				******************/

				* Make string input lower case and change "y" to "yes"
				replace correct = lower(correct)
				replace drop 	= lower(drop)
				replace correct = "yes" if correct 	== "y"
				replace drop 	= "yes" if drop 	== "y"

				*Check that varaibles are wither empty or "yes"
				gen `tmpvar_inputNotYes' = !((correct  == "yes" | correct == "") & (drop  == "yes" | drop == ""))

				cap assert `tmpvar_inputNotYes' == 0
				if _rc {

					*Error will be outputted below
					local local_inputNotYes 	1
				}

				/******************
					Section 3.3.3
					Make sure that either option droprest is specified, or that
					drop was correctly indicated for all observations. i.e.; if
					correct or newID was indicated for at least one duplicate in
					a duplicate group, then all other observations should be
					indicated as drop (unless droprest is specified)

				******************/

				*Check if any other duplicate in duplicate group has at least one correction
				gen `tempvar_yescorrect' = (correct == "yes")
				bys `varlist' : egen `tempvar_numcorrect' =  total(`tempvar_yescorrect')
				count if `tempvar_numcorrect' > 1
				if `r(N)' != 0 local local_multiCorr 1

				*Check if any other duplicate in duplicate group has at least one correction
				bys `varlist' : egen `tmpvar_maxMultiInp' = max(`tmpvar_multiInp')

				*Check that drops are explicitly indicated
				gen `tmpvar_notDrop' = (`tmpvar_multiInp' == 0 & `tmpvar_maxMultiInp' > 0)

				* Check if option droprest is specified
				if "`droprest'" != "" {
					cap assert `tmpvar_notDrop' == 0
					if _rc {

						*Error will be outputted below
						local local_notDrop 	1
					}
				}
				else {

					** Option -droprest- specified. Drop will be changed to yes
					*  for any observations without drop or any other correction
					*  explicitly specified if the observation is in a duplicate
					*  group with at least one observation has a correction
					replace drop 	= "yes" if `tmpvar_notDrop' 	== 1

				}

				/******************
					Section 3.4
					Throw errors if any of the tests were not passed
				******************/

				*Was any error detected
				if `local_multiInp' == 1 | `local_inputNotYes' == 1 | `local_notDrop' == 1 | `local_multiCorr' == 1  {

					*Error multiple input
					if `local_multiInp' == 1 {
						noi {
							display as error "{phang}The following observations have more than one correction. Only one correction (correct, drop or newID) per row is allowed{p_end}"
							list `varlist' dupListID correct drop newID if `tmpvar_multiInp' > 1
							di ""
						}
					}

					*Error multiple correct
					if `local_multiCorr' == 1 {
						noi {
							display as error "{phang}The following observations are in a duplicate group where more than one observation is listed as correct. Only one observation per duplicate group can be correct{p_end}"
							list `varlist' dupListID correct drop newID if `tempvar_numcorrect' > 1
							di ""
						}

					}

					*Error in incorrect string
					if `local_inputNotYes' == 1 {
						noi {
							display as error "{phang}The following observations have an answer in either correct or drop that is neither yes nor y{p_end}"
							list `varlist' dupListID correct drop if `tmpvar_inputNotYes' == 1
							di ""
						}
					}

					*Error is not specfied as drop
					if `local_notDrop' == 1 {
						noi {
							display as error "{phang}The following observations are not explicitly indicated as drop while other duplicates in the same duplicate group are corrected. Either manually indicate as drop or see option droprest{p_end}"
							list `varlist' dupListID correct drop newID if `tmpvar_notDrop' == 1
							di ""
						}
					}

				*Same error for any incorrect input
				error 119
				exit
				}

				/******************
					Section 3.5
					Save the prepared report to be used later
				******************/

				*Keep only the variables needed for matching and variables used for input in the Excel file
				keep 	`idvar' `uniquevars' `excelvars' `groupAnyCorrection'

				*Save imported data set with all corrections
				save	`preppedReport'
			}

			/***********************************************************************
			************************************************************************

				Section 4

				Merge corrections with original data

			************************************************************************
			***********************************************************************/

			*Re-load original
			use `restart', clear

			* Merge original data with imported Excel file (if Excel file exists)
			if `fileExists'  {

				*Create a tempvar for merging results
				tempvar iedup_merge

				*Merge the corrections with the data set
				merge 1:1 `uniquevars' using `preppedReport', generate(`iedup_merge')

				*Make sure that obsrevations listed in the duplicate report is still in the data set
				cap assert `iedup_merge' != 2

				*Display error message if assertion is not true and some duplicates in the excle file are no longer in the data set
				if _rc {

					display as error "{phang}One or several observations in the Excel report are no longer found in the data set. Always run ieduplicates on the raw data set that include all the duplicates, both new duplicates and those you have already identified. After removing duplicates, save the data set using a different name. You might also recieve this error if you are using an old ieduplicates Excel report on a new data set.{p_end}"
					error 9
					exit
				}

				*Explicitly drop temporary variable. Temporary variables might
				*be exported to excel so delete explicitly before that. Only
				*using tempvar here to create a name with no conflicts
				drop `iedup_merge'

			}

			*save the data set to be used when correcting the data
			save `datawithreportmerged'

			/***********************************************************************
			************************************************************************

			/***********************************************************************
			************************************************************************

				Section 6

				Test if there are duplicates in ID var. If any duplicates exist,
				tehn update the Excel file with new and unaddressed cases

			************************************************************************
			***********************************************************************/

			/******************
				Section 6.1
				Test if there are any duplicates in ID var
			******************/

			* Generate variable that is not 0
			* if observation is a duplicate
			tempvar dup
			duplicates tag `varlist', gen(`dup')

			*Test if there are any duplicates
			cap assert `dup'==0
			if _rc {

				/******************
				/******************
					Section 6.3
					Keep only duplicates for the report
				******************/

				*Keep if observation is part of duplicate group
				keep if `dup' != 0

				if `fileExists'  {
					* If Excel file exists keep excel vars and
					* variables passed as arguments in the
					* command
					keep 	`agrumentVars' `excelvars'
				}
				else {
					* Keep only variables passed as arguments in
					* the command and the string ID var as no Excel file exists
					keep 	`agrumentVars'

					*Generate the excel variables used for indicating correction
					foreach excelvar of local excelvars {

						*Create all variables apart from dupListID as string vars
						if "`excelvar'" == "dupListID" {
							gen `excelvar' = .
						}
						else {
							gen `excelvar' = ""
						}
					}

				}

				/******************
					Section 6.4
					Update the excel vars that are not updated manually
				******************/

				* Generate a local that is 1 if there are new duplicates
				local unaddressedNewExcel 0
				count if dateFixed == ""
				if `r(N)' > 0 local unaddressedNewExcel 1

				/******************
					Section 6.4.1 Date variables
				******************/

				* Add date first time duplicvate was identified
				replace dateListed 	= "`date'" if dateListed == ""

				** Add today's date to variable dateFixed if dateFixed
				*  is empty and at least one correction is added
				replace dateFixed 	= "`date'" if dateFixed == "" & (correct != "" | drop != "" | newID != "")

				/******************
					Section 6.4.2 Duplicate report list ID
				******************/

				** Sort after dupListID and after ID var for
				*  duplicates currently without dupListID
				sort dupListID `varlist'

				** Assign dupListID 1 to the top row if no duplicate
				*  list IDs have been generated so far.
				replace dupListID = 1 if _n == 1 & dupListID == .

				** Generate new IDs based on the row above instead of directly
				*  from the row number. That prevents duplicates in the list in
				*  case an observation is deleted. The first observation with
				*  missing value will have an ID that is one digit higher than
				*  the highest ID already in the list
				replace dupListID = dupListID[_n - 1] + 1 if dupListID == .

				/******************
					Section 6.5
					Keep and order the variables and output the Excel files
				******************/

				* If cases unaddressed then update the Excel file
				if `unaddressedNewExcel'  {

					keep 	`agrumentVars' `excelvars'
					order	`varlist' `excelvars' `uniquevars' `keepvars'

					if "`daily'" == "" {

						*Returns 0 if folder does not exist, 1 if it does
						mata : st_numscalar("r(dirExist)", direxists("`folder'/Daily"))

						** If the daily folder is not created, just create it
						if `r(dirExist)' == 0  {

							*Create the folder since it does not exist
							mkdir "`folder'/Daily"
						}

						*Export the daily file
						cap export excel using "`folder'/Daily/iedupreport`suffix'_`date'.xlsx"	, firstrow(variables) replace nolabel

						*Print error if daily report cannot be saved
						if _rc {

							display as error "{phang}There the Daily copy could not be saved to the `folder'/Daily folder. Make sure to close any old daily copy or see the option nodaily{p_end}"
							error 603
							exit

						}

						*Prepare local for output
						local daily_output "and a daily copy have been saved to the Daily folder"
					}

					*Export main report
					export excel using "`folder'/iedupreport`suffix'.xlsx"	, firstrow(variables) replace  nolabel

					*Produce output
					noi di `"{phang}Excel file created at: {browse "`folder'/iedupreport`suffix'.xlsx":`folder'/iedupreport`suffix'.xlsx} `daily_output'{p_end}"'
					noi di ""
				}
			}

		/***********************************************************************
		************************************************************************

			Section 7

			Update the data set and with the new corrections.

		************************************************************************
		***********************************************************************/

		* Load the original data set merged with correction. Duplicates
		* in all variables are already dropped in this data set
		use 	`restart', clear

		* If excel file exists, apply any corrections indicated (if any)
		if `fileExists' {

			/******************
				Section 7.1
				Drop duplicates listed for drop
			******************/

			drop if drop == "yes"

			/******************
				Section 7.2
				Update new ID. ID var can be either numeric or
				string. All numbers can be made strings but not
				all strings can be numeric. Therefore this
				section is complicated.
			******************/

			/******************
				Section 7.2.1
				ID var in original file is string. Either
				newID was imported as string or the variable
				is made string. Easy.
			******************/

			*Test if there are any corrections by new ID
			cap assert missing(newID)
			if _rc {

				local idtype 	: type `idvar'
				local idtypeNew : type newID

				*If ID var is string but newID is not, then just make it string
				if substr("`idtype'",1,3) == "str" & substr("`idtypeNew'",1,3) != "str" {

					tostring newID , replace
					replace  newID = "" if newID == "."
				}

				*If ID var is numeric but the newID is loaded as string
				else if substr("`idtype'",1,3) != "str" & substr("`idtypeNew'",1,3) == "str" {

					* Check if -tostringok- is specificed:
					if "`tostringok'" != "" {

						* Make original ID var string
						tostring `idvar' , replace
						replace  `idvar' = "" if `idvar' == "."

					}

					* Error, IDvar can not be updated
					else {

						* Create a local with all non-numeric values
						levelsof newID if missing(real(newID)), local(NaN_values) clean

						* Output error message
						di as error "{phang}The ID varaible `idvar' is numeric but newID has thes non-numeric values: `NaN_values'. Update newID to only contain numeric values or see option tostringok.{p_end}"
						error 109
						exit
					}
				}

				*After making sure that type is ok, update the IDs
				replace `idvar' = newID if !missing(newID)

				/******************
				******************/
					exit
				}
			}

			/******************
				Section 7.4
				Drop Excel vars
			******************/

			drop `excelvars'
		}

		/***********************************************************************
		************************************************************************

			Section 8

			Return the data set without duplicates and
			output information regarding unresolved duplicates.

		************************************************************************
		***********************************************************************/

		* Generate a variable that is 1 if the observation is a duplicate in varlist
		tempvar dropDup
		duplicates tag `varlist',  gen(`dropDup')
		* Generate a list of the IDs that are still duplicates
		levelsof `varlist' 			if `dropDup' != 0 , local(dup_ids) clean
		* Drop the duplicates (they are exported in Excel)
		drop 						if `dropDup' != 0

		* Test if varlist is now uniquely and fully identifying the data set
		cap isid `varlist'
		if _rc {

			di as error "{phang}The data set is not returned with `varlist' uniquely and fully identifying the data set. Please report this bug to kbjarkefur@worldbank.org{p_end}"
			error 119
			exit

		}

		if `:list sizeof dup_ids' == 0 {

			noi di	"{phang}There are no unresolved duplicates in this data set. The data set is returned with `varlist' uniquely and fully identifying the data set.{p_end}"
		}
		else {
			noi di	"{phang}There are `:list sizeof dup_ids' duplicates unresolved. IDs still contining duplicates: `dup_ids'. The unresolved duplicate observations were exported in the Excel file. The data set is returned without those duplicates and with `varlist' uniquely and fully identifying the data set.{p_end}"
		}

		return scalar numDup	= `:list sizeof dup_ids'

		/***********************************************************************
		************************************************************************

			Section 9

			Save data set to be returned outside preserve/restore.
			Preserve/restore is used so that original data is returned
			in case an error is thrown.

		************************************************************************
		***********************************************************************/

		tempfile returndata
		save 	`returndata'

		restore

		** Using restore above to return the data to
		*  the orignal data set in case of error.
		use `returndata', clear

	}
	end

*! version 5.5 26APR2018 DIME Analytics lcardosodeandrad@worldbank.org

	capture program drop ieduplicates
	program ieduplicates , rclass

	qui {

		syntax varname ,  FOLder(string) UNIQUEvars(varlist) [ KEEPvars(varlist) MINprecision(numlist >0) tostringok droprest nodaily SUFfix(string)]

		version 11.0

		*Add version of Stata fix
		//Make sure that keepvars are still saved if saved if the duplicates file
		*	is generated on a subset of the data. For example, duplicates from
		*	version 1, 2 . If the command is run on only version 1. Then values
		*	for keeepvars in version 2 and 3 are dropped and not reloaded as those
		*	obs are not in current memory

		**Test that observations have not been deleted from the report before readind
		* it. Deleted in a way that the report does not make sense. Provide an error
		* message to this that is more informative.

		preserve

			/***********************************************************************
			************************************************************************

				Section 1 - Set up locals needed in data

				Saving a version of the data to be used before merging corrections
				back to the original data set and before correcting duplicates

			************************************************************************
			***********************************************************************/

			*Tempfiles to be uses
			tempfile originalData preppedReport datawithreportmerged dataToReturn

			** Save a version of original data that can be brought
			*  back withing the preserve/restore section
			save 	`originalData'

			* Create a local with todays date to date the reports
			local  date = subinstr(c(current_date)," ","",.)

			*Put idvar in a local with a more descriptive name
			local idvar `varlist'

			** Making one macro with all variables that will be
			*  imported and exported from the Excel file
			local argumentVars `idvar' `uniquevars' `keepvars'

			* Create a list of the varaibles created by this command to put in the report
			local excelvars dupListID dateListed dateFixed correct drop newID initials notes

			/***********************************************************************
			************************************************************************

				Section 2

				Saving a version of the data to be used before mergin and
				before correcting duplicates

			************************************************************************
			***********************************************************************/

			tempfile restart
			save 	`restart'

			/***********************************************************************
			************************************************************************

				Section 3

				Import and prepare Corrections (if file exists)

			************************************************************************
			***********************************************************************/

			/******************
				Section 3.1
				Check if earlier report exists.
			******************/

			cap confirm file "`folder'/iedupreport`suffix'.xlsx"

			if !_rc {
				local fileExists 1
			}
			else {
				local fileExists 0
			}

			/******************
				Section 3.2
				If report exist, load file and check input
			******************/

			if `fileExists' {

				*Load excel file. Load all vars as string and use meta data from Section 1
				import excel "`folder'/iedupreport`suffix'.xlsx"	, clear firstrow

				** All excelvars but dupListID and newID should be string. dupListID
				*  should be numeric and the type of newID should be based on the user input
				foreach excelvar of local excelvars {
					if !inlist("`excelvar'", "dupListID", "newID") {

						* Make original ID var string
						tostring `excelvar' , replace
						replace  `excelvar' = "" if `excelvar' == "."
					}
				}

				/******************
					Section 3.2.1
					Make sure that the
					ID variable and
					the uniquevars are
					not changed since
					last report.
				******************/

				ds
				local existingexcelvars  `r(varlist)'

				if `:list varlist in existingexcelvars' == 0 {

					noi display as error "{phang}ID variable [`varlist'] does not exist in the previously exported Excle file. If you renamed or changed the ID variable, you need to start over with a new file. Rename or move the already existing file. Create a new file and carefully copy any corrections from the old file to the new.{p_end}"
					noi di ""
					error 111
					exit

				}
				if `:list uniquevars in existingexcelvars' == 0 {

					noi display as error "{phang}One or more unique variables in [`uniquevars'] do not exist in the previously exported Excel file. If you renamed or changed any variable used in uniquevars(), you need to start over with a new file. Rename or move the already existing file. Create a new file and carefully copy any corrections from the old file to the new.{p_end}"
					noi di ""
					error 111
					exit
				}

				/******************
					Section 3.3
					Make sure input is correct
				******************/

				*Temporary variables needed for checking input
				tempvar tmpvar_multiInp tmpvar_inputNotYes tmpvar_maxMultiInp tmpvar_notDrop tempvar_yescorrect tempvar_numcorrect

				*Locals indicating in which ways input is incorrect (if any)
				local local_multiInp 		0
				local local_multiCorr		0
				local local_inputNotYes		0
				local local_notDrop			0

				/******************
					Section 3.3.1
					Make sure there are not too many corrections
				******************/

				* Count the number of corrections (correct drop newID) per
				* observation. Only one correction per observation is allowed.
				egen `tmpvar_multiInp' = rownonmiss(correct drop newID), strok

				*Check that all rows have utmost one correction
				cap assert `tmpvar_multiInp' == 0 | `tmpvar_multiInp' == 1

				if _rc {

					*Error will be outputted below
					local local_multiInp 	1
				}

				/******************
					Section 3.3.2
					Make sure string input is yes or y
				******************/

				* Make string input lower case and change "y" to "yes"
				replace correct = lower(correct)
				replace drop 	= lower(drop)
				replace correct = "yes" if correct 	== "y"
				replace drop 	= "yes" if drop 	== "y"

				*Check that varaibles are wither empty or "yes"
				gen `tmpvar_inputNotYes' = !((correct  == "yes" | correct == "") & (drop  == "yes" | drop == ""))

				cap assert `tmpvar_inputNotYes' == 0
				if _rc {

					*Error will be outputted below
					local local_inputNotYes 	1
				}

				/******************
					Section 3.3.3
					Make sure that either option droprest is specified, or that
					drop was correctly indicated for all observations. i.e.; if
					correct or newID was indicated for at least one duplicate in
					a duplicate group, then all other observations should be
					indicated as drop (unless droprest is specified)

				******************/

				*Check if any other duplicate in duplicate group has at least one correction
				gen `tempvar_yescorrect' = (correct == "yes")
				bys `varlist' : egen `tempvar_numcorrect' =  total(`tempvar_yescorrect')
				count if `tempvar_numcorrect' > 1
				if `r(N)' != 0 local local_multiCorr 1

				*Check if any other duplicate in duplicate group has at least one correction
				bys `varlist' : egen `tmpvar_maxMultiInp' = max(`tmpvar_multiInp')

				*Check that drops are explicitly indicated
				gen `tmpvar_notDrop' = (`tmpvar_multiInp' == 0 & `tmpvar_maxMultiInp' > 0)

				* Check if option droprest is specified
				if "`droprest'" != "" {
					cap assert `tmpvar_notDrop' == 0
					if _rc {

						*Error will be outputted below
						local local_notDrop 	1
					}
				}
				else {

					** Option -droprest- specified. Drop will be changed to yes
					*  for any observations without drop or any other correction
					*  explicitly specified if the observation is in a duplicate
					*  group with at least one observation has a correction
					replace drop 	= "yes" if `tmpvar_notDrop' 	== 1

				}

				/******************
					Section 3.4
					Throw errors if any of the tests were not passed
				******************/

				*Was any error detected
				if `local_multiInp' == 1 | `local_inputNotYes' == 1 | `local_notDrop' == 1 | `local_multiCorr' == 1  {

					*Error multiple input
					if `local_multiInp' == 1 {
						noi {
							display as error "{phang}The following observations have more than one correction. Only one correction (correct, drop or newID) per row is allowed{p_end}"
							list `varlist' dupListID correct drop newID if `tmpvar_multiInp' > 1
							di ""
						}
					}

					*Error multiple correct
					if `local_multiCorr' == 1 {
						noi {
							display as error "{phang}The following observations are in a duplicate group where more than one observation is listed as correct. Only one observation per duplicate group can be correct{p_end}"
							list `varlist' dupListID correct drop newID if `tempvar_numcorrect' > 1
							di ""
						}

					}

					*Error in incorrect string
					if `local_inputNotYes' == 1 {
						noi {
							display as error "{phang}The following observations have an answer in either correct or drop that is neither yes nor y{p_end}"
							list `varlist' dupListID correct drop if `tmpvar_inputNotYes' == 1
							di ""
						}
					}

					*Error is not specfied as drop
					if `local_notDrop' == 1 {
						noi {
							display as error "{phang}The following observations are not explicitly indicated as drop while other duplicates in the same duplicate group are corrected. Either manually indicate as drop or see option droprest{p_end}"
							list `varlist' dupListID correct drop newID if `tmpvar_notDrop' == 1
							di ""
						}
					}

				*Same error for any incorrect input
				error 119
				exit
				}

				*Keep only the variables needed for matching and variables used for input in the Excel file
				keep 	`varlist' `uniquevars' `excelvars'

				*Save imported data set with all corrections
				tempfile imputfile_merge
				save	`imputfile_merge'
			}

			/***********************************************************************
			************************************************************************

				Section 4

				Merge corrections with original data

			************************************************************************
			***********************************************************************/

			*Re-load original
			use `restart', clear

			* Merge original data with imported Excel file (if Excel file exists)
			if `fileExists'  {

				*Create a tempvar for merging results
				tempvar iedup_merge

				*Merge the corrections with the data set
				merge 1:1 `varlist' `uniquevars' using `imputfile_merge', generate(`iedup_merge')

				*Make sure that obsrevations listed in the duplicate report is still in the data set
				cap assert `iedup_merge' != 2

				*Display error message if assertion is not true and some duplicates in the excle file are no longer in the data set
				if _rc {

					display as error "{phang}One or several observations in the Excel report are no longer found in the data set. Always run ieduplicates on the raw data set that include all the duplicates, both new duplicates and those you have already identified. After removing duplicates, save the data set using a different name. You might also recieve this error if you are using an old ieduplicates Excel report on a new data set.{p_end}"
					error 9
					exit

				}

				*Explicitly drop temporary variable. Temporary variables might
				*be exported to excel so delete explicitly before that. Only
				*using tempvar here to create a name with no conflicts
				drop `iedup_merge'

			}

			/***********************************************************************
			************************************************************************

			/***********************************************************************
			************************************************************************

				Section 6

				Test if there are duplicates in ID var. If any duplicates exist,
				tehn update the Excel file with new and unaddressed cases

			************************************************************************
			***********************************************************************/

			/******************
				Section 6.1
				Test if there are any duplicates in ID var
			******************/

			* Generate variable that is not 0
			* if observation is a duplicate
			tempvar dup
			duplicates tag `varlist', gen(`dup')

			*Test if there are any duplicates
			cap assert `dup'==0
			if _rc {

				/******************
					Section 6.2
					Test if the variables passed as ID var and unique var
					uniquely and fully identify the data set. It should be
					possible to merge corrections back to the main file.

				******************/

				cap isid 	`varlist' `uniquevars'
				if _rc {

					display as error "{phang}ID variable `varlist' does not uniquely identify the observations in the data set together with the uniquevars: `uniquevars'. This is needed for exactly merging the right correction to the right observation{p_end}"
					error 119
					exit
				}

				/******************
					Section 6.3
					Keep only duplicates for the report
				******************/

				*Keep if observation is part of duplicate group
				keep if `dup' != 0

				if `fileExists'  {
					* If Excel file exists keep excel vars and
					* variables passed as arguments in the
					* command
					keep 	`agrumentVars' `excelvars'
				}
				else {
					* Keep only variables passed as arguments in
					* the command and the string ID var as no Excel file exists
					keep 	`agrumentVars'

					*Generate the excel variables used for indicating correction
					foreach excelvar of local excelvars {

						*Create all variables apart from dupListID as string vars
						if "`excelvar'" == "dupListID" {
							gen `excelvar' = .
						}
						else {
							gen `excelvar' = ""
						}
					}

				}

				/******************
					Section 6.4
					Update the excel vars that are not updated manually
				******************/

				* Generate a local that is 1 if there are new duplicates
				local unaddressedNewExcel 0
				count if dateFixed == ""
				if `r(N)' > 0 local unaddressedNewExcel 1

				/******************
					Section 6.4.1 Date variables
				******************/

				* Add date first time duplicvate was identified
				replace dateListed 	= "`date'" if dateListed == ""

				** Add today's date to variable dateFixed if dateFixed
				*  is empty and at least one correction is added
				replace dateFixed 	= "`date'" if dateFixed == "" & (correct != "" | drop != "" | newID != "")

				/******************
					Section 6.4.2 Duplicate report list ID
				******************/

				** Sort after dupListID and after ID var for
				*  duplicates currently without dupListID
				sort dupListID `varlist'

				** Assign dupListID 1 to the top row if no duplicate
				*  list IDs have been generated so far.
				replace dupListID = 1 if _n == 1 & dupListID == .

				** Generate new IDs based on the row above instead of directly
				*  from the row number. That prevents duplicates in the list in
				*  case an observation is deleted. The first observation with
				*  missing value will have an ID that is one digit higher than
				*  the highest ID already in the list
				replace dupListID = dupListID[_n - 1] + 1 if dupListID == .

				/******************
					Section 6.5
					Keep and order the variables and output the Excel files
				******************/

				* If cases unaddressed then update the Excel file
				if `unaddressedNewExcel'  {

					keep 	`agrumentVars' `excelvars'
					order	`varlist' `excelvars' `uniquevars' `keepvars'

					if "`daily'" == "" {

						*Returns 0 if folder does not exist, 1 if it does
						mata : st_numscalar("r(dirExist)", direxists("`folder'/Daily"))

						** If the daily folder is not created, just create it
						if `r(dirExist)' == 0  {

							*Create the folder since it does not exist
							mkdir "`folder'/Daily"
						}

						*Export the daily file
						cap export excel using "`folder'/Daily/iedupreport`suffix'_`date'.xlsx"	, firstrow(variables) replace nolabel

						*Print error if daily report cannot be saved
						if _rc {

							display as error "{phang}There the Daily copy could not be saved to the `folder'/Daily folder. Make sure to close any old daily copy or see the option nodaily{p_end}"
							error 603
							exit

						}

						*Prepare local for output
						local daily_output "and a daily copy have been saved to the Daily folder"
					}

					*Export main report
					export excel using "`folder'/iedupreport`suffix'.xlsx"	, firstrow(variables) replace  nolabel

					*Produce output
					noi di `"{phang}Excel file created at: {browse "`folder'/iedupreport`suffix'.xlsx":`folder'/iedupreport`suffix'.xlsx} `daily_output'{p_end}"'
					noi di ""
				}
			}

		/***********************************************************************
		************************************************************************

			Section 7

			Update the data set and with the new corrections.

		************************************************************************
		***********************************************************************/

		* Load the original data set merged with correction. Duplicates
		* in all variables are already dropped in this data set
		use 	`restart', clear

		* If excel file exists, apply any corrections indicated (if any)
		if `fileExists' {

			/******************
				Section 7.1
				Drop duplicates listed for drop
			******************/

			drop if drop == "yes"

			/******************
				Section 7.2
				Update new ID. ID var can be either numeric or
				string. All numbers can be made strings but not
				all strings can be numeric. Therefore this
				section is complicated.
			******************/

			/******************
				Section 7.2.1
				ID var in original file is string. Either
				newID was imported as string or the variable
				is made string. Easy.
			******************/

			*Test if there are any corrections by new ID
			cap assert missing(newID)
			if _rc {

				local idtype 	: type `idvar'
				local idtypeNew : type newID

				*If ID var is string but newID is not, then just make it string
				if substr("`idtype'",1,3) == "str" & substr("`idtypeNew'",1,3) != "str" {

					tostring newID , replace
					replace  newID = "" if newID == "."
				}

				*If ID var is numeric but the newID is loaded as string
				else if substr("`idtype'",1,3) != "str" & substr("`idtypeNew'",1,3) == "str" {

					* Check if -tostringok- is specificed:
					if "`tostringok'" != "" {

						* Make original ID var string
						tostring `idvar' , replace
						replace  `idvar' = "" if `idvar' == "."

					}

					* Error, IDvar can not be updated
					else {

						* Create a local with all non-numeric values
						levelsof newID if missing(real(newID)), local(NaN_values) clean

						* Output error message
						di as error "{phang}The ID varaible `idvar' is numeric but newID has thes non-numeric values: `NaN_values'. Update newID to only contain numeric values or see option tostringok.{p_end}"
						error 109
						exit
					}
				}

				*After making sure that type is ok, update the IDs
				replace `idvar' = newID if !missing(newID)

				/******************
				******************/
					exit
				}
			}

			/******************
				Section 7.4
				Drop Excel vars
			******************/

			drop `excelvars'
		}

		/***********************************************************************
		************************************************************************

			Section 8

			Return the data set without duplicates and
			output information regarding unresolved duplicates.

		************************************************************************
		***********************************************************************/

		* Generate a variable that is 1 if the observation is a duplicate in varlist
		tempvar dropDup
		duplicates tag `varlist',  gen(`dropDup')
		* Generate a list of the IDs that are still duplicates
		levelsof `varlist' 			if `dropDup' != 0 , local(dup_ids) clean
		* Drop the duplicates (they are exported in Excel)
		drop 						if `dropDup' != 0

		* Test if varlist is now uniquely and fully identifying the data set
		cap isid `varlist'
		if _rc {

			di as error "{phang}The data set is not returned with `varlist' uniquely and fully identifying the data set. Please report this bug to kbjarkefur@worldbank.org{p_end}"
			error 119
			exit

		}

		if `:list sizeof dup_ids' == 0 {

			noi di	"{phang}There are no unresolved duplicates in this data set. The data set is returned with `varlist' uniquely and fully identifying the data set.{p_end}"
		}
		else {
			noi di	"{phang}There are `:list sizeof dup_ids' duplicates unresolved. IDs still contining duplicates: `dup_ids'. The unresolved duplicate observations were exported in the Excel file. The data set is returned without those duplicates and with `varlist' uniquely and fully identifying the data set.{p_end}"
		}

		return scalar numDup	= `:list sizeof dup_ids'

		/***********************************************************************
		************************************************************************

			Section 9

			Save data set to be returned outside preserve/restore.
			Preserve/restore is used so that original data is returned
			in case an error is thrown.

		************************************************************************
		***********************************************************************/

		tempfile returndata
		save 	`returndata'

		restore

		** Using restore above to return the data to
		*  the orignal data set in case of error.
		use `returndata', clear

	}
	end


Fix bug where reghdfe fails with -summ- option and factor variables

*! version 4.4.12 10nov2017

program reghdfe, eclass
	* Intercept old+version
	cap syntax, version old
	if !c(rc) {
		reghdfe_old, version
		exit
	}

	* Intercept old
	cap syntax anything(everything) [fw aw pw/], [*] old
	if !c(rc) {
		di as error "(running historical version of reghdfe)"
		if ("`weight'"!="") local weightexp [`weight'=`exp']
		reghdfe_old `anything' `weightexp', `options'
		exit
	}

	* Aux. subcommands
	cap syntax, [*]
	if inlist("`options'", "check", "compile", "reload", "update", "version", "requirements") {
		if ("`options'"=="compile") loc args force
		if ("`options'"=="check") loc options compile
		if ("`options'"=="update") {
			loc args 1
			loc options reload
		}
		loc subcmd = proper("`options'")
		`subcmd' `args'
	}
	else if replay() {
		Replay `0'
	}
	else {
		Cleanup 0
		Compile // takes 0.01s to run this useful check (ensures .mlib exists)
		cap noi Estimate `0'
		Cleanup `c(rc)'
	}
end

program Compile
	args force
	ftools, check // in case lftools.mlib does not exist or is outdated
	ms_get_version reghdfe // save local package_version

	loc list_objects "FixedEffects() fixed_effects() BipartiteGraph()"
	loc list_functions "reghdfe_*() transform_*() accelerate_*() panelmean() panelsolve_*() lsmr()"
	loc list_misc "weighted_quadcolsum() safe_divide() check_convergence() precompute_inv_xx()"
	// TODO: prefix everything with reghdfe_*

	ms_compile_mata, ///
		package(reghdfe) ///
		version(`package_version') ///
		fun("`list_objects' `list_functions' `list_misc'") ///
		verbose ///
		`force'
end

program Reload
	* Internal debugging tool.
	* Updates dependencies and reghdfe from local path or from github
	* Usage:
	* 	reghdfe, update // from c:\git\..
	* 	reghdfe, reload // from github

	args online
	if ("`online'" == "") loc online 0

	di as text _n "{bf:reghdfe: updating required packages}"
	di as text "{hline 64}"

	* -ftools- https://github.com/sergiocorreia/ftools/
	cap ado uninstall ftools
	if (`online') net install ftools, from("https://github.com/sergiocorreia/ftools/raw/master/src/")
	if (!`online') net install ftools, from("c:\git\ftools\src")
	di as text "{hline 64}"
	ftools, compile
	di as text "{hline 64}"

	* Update -reghdfe-
	di as text _n  _n "{bf:reghdfe: updating self}"
	di as text "{hline 64}"
	qui ado uninstall reghdfe
	if (`online') net install reghdfe, from("https://github.com/sergiocorreia/reghdfe/raw/master/src/")
	if (!`online') net install reghdfe, from("c:\git\reghdfe\src")
	qui which reghdfe
	di as text "{hline 64}"
	reghdfe, compile
	di as text "{hline 64}"

	* Cleaning up
	di as text _n "{bf:Note:} You need to run {stata program drop _all} now."
end

program Version
	which reghdfe
	Requirements
end

program Requirements
	di as text _n "Required packages installed?"
	loc reqs ftools
	// ivreg2 avar tuples group3hdfe
	if (c(stata_version)<13) loc reqs `reqs' boottest

	loc ftools_github "https://github.com/sergiocorreia/ftools/raw/master/src/"

	loc error 0

	foreach req of local reqs {
		loc fn `req'.ado
		cap findfile `fn'
		if (_rc) {
			loc error 1
			di as text "{lalign 20:- `req'}" as error "not" _c
			di as text "    {stata ssc install `req':install from SSC}" _c
			if inlist("`req'", "ftools") {
				loc github ``req'_github'
				di as text `"    {stata `"net install `req', from(`"`github'"')"':install from github}"'
			}
			else {
				di as text // newline
			}
		}
		else {
			di as text "{lalign 20:- `req'}" as text "yes"
		}
	}

	if (`error') exit 601
end

program Cleanup
	args rc
	cap mata: mata drop HDFE
	cap mata: mata drop hdfe_*
	cap drop __temp_reghdfe_resid__
	cap matrix drop reghdfe_statsmatrix
	if (`rc') exit `rc'
end

program Parse
	* Trim whitespace (caused by "///" line continuations; aesthetic only)
	mata: st_local("0", stritrim(st_local("0")))

	* Main syntax
	#d;
	syntax anything(id=varlist equalok) [if] [in] [aw pw fw/] , [

		/* Model */
		Absorb(string) NOAbsorb
		SUmmarize SUmmarize2(string asis) /* simulate implicit options */

		/* Standard Errors */
		VCE(string) CLuster(string)

		/* Diagnostic */
		Verbose(numlist min=1 max=1 >=-1 <=5 integer)
		TIMEit

		/* Speedup and memory Tricks */
		NOSAMPle /* do not save e(sample) */

		/* Undocumented */
		KEEPSINgletons
		OLD /* use latest v3 */
		NOTES(string) /* NOTES(key=value ...), will be stored on e() */

		] [*] /* capture optimization options, display options, etc. */
		;
	#d cr

	* Unused
	* SAVEcache
	* USEcache
	* CLEARcache
	* COMPACT /* use as little memory as possible but is slower */

	* Convert options to boolean
	if ("`verbose'" == "") loc verbose 0
	loc timeit = ("`timeit'"!="")
	loc drop_singletons = ("`keepsingletons'"=="")

	if (`timeit') timer on 29

	* Sanity checks
	if (`verbose'>-1 & "`keepsingletons'"!="") {
		loc url "http://scorreia.com/reghdfe/nested_within_cluster.pdf"
		loc msg "WARNING: Singleton observations not dropped; statistical significance is biased"
		di as error `"`msg' {browse "`url'":(link)}"'
	}
	if ("`cluster'"!="") {
		_assert ("`vce'"==""), msg("cannot specify both cluster() and vce()")
		loc vce cluster `cluster'
		loc cluster // clear it to avoid bugs in subsequent lines
	}

	* Parse Varlist
	ms_parse_varlist `anything'
	loc base_varlist "`r(basevars)'"
	foreach cat in depvar indepvars endogvars instruments {
		loc original_`cat' "`r(`cat')'"
	}
	loc model = cond("`r(instruments)'" == "", "ols", "iv")
	loc original_varlist = "`r(varlist)'" // no parens or equal sign

	* Parse Weights
	if ("`weight'"!="") {
		unab exp : `exp', min(1) max(1) // simple weights only
	}

	* Parse VCE
	ms_parse_vce, vce(`vce') weighttype(`weight')
	loc vcetype = "`s(vcetype)'"
	loc num_clusters = `s(num_clusters)'
	loc clustervars = "`s(clustervars)'"
	loc base_clustervars = "`s(base_clustervars)'"
	loc vceextra = "`s(vceextra)'"

	* Select sample (except for absvars)
	loc varlist `original_varlist' `base_clustervars'
	tempvar touse
	marksample touse, strok // based on varlist + cluster + if + in + weight

	* Parse noabsorb
	_assert  ("`absorb'`noabsorb'" != ""), msg("option {bf:absorb()} or {bf:noabsorb} required")
	if ("`noabsorb'" != "") {
		_assert ("`absorb'" == ""), msg("{bf:absorb()} and {bf:noabsorb} are mutually exclusive")
		tempvar c
		gen byte `c' = 1
		loc absorb `c'
	}

	if (`timeit') timer off 29

	* Construct HDFE object
	// SYNTAX: fixed_effects(absvars | , touse, wtype, wtvar, dropsing, verbose)
	mata: st_local("comma", strpos(`"`absorb'"', ",") ? "" : ",")
	if (`timeit') timer on 20
	mata: HDFE = fixed_effects(`"`absorb' `comma' `options'"', "`touse'", "`weight'", "`exp'", `drop_singletons', `verbose')
	if (`timeit') timer off 20
	mata: HDFE.cmdline = "reghdfe " + st_local("0")
	loc options `s(options)'

	mata: st_local("N", strofreal(HDFE.N))
	if (`N' == 0) error 2000

	* Fill out HDFE object
	mata: HDFE.varlist = "`base_varlist'"
	mata: HDFE.original_depvar = "`original_depvar'"
	mata: HDFE.original_indepvars = "`original_indepvars'"
	mata: HDFE.original_endogvars = "`original_endogvars'"
	mata: HDFE.original_instruments = "`original_instruments'"
	mata: HDFE.original_varlist = "`original_varlist'"
	mata: HDFE.model = "`model'"

	mata: HDFE.vcetype = "`vcetype'"
	mata: HDFE.num_clusters = `num_clusters'
	mata: HDFE.clustervars = tokens("`clustervars'")
	mata: HDFE.base_clustervars = tokens("`base_clustervars'")
	mata: HDFE.vceextra = "`vceextra'"

	* Parse summarize
	if ("`summarize'" != "") {
		_assert ("`summarize2'" == ""), msg("summarize() syntax error")
		loc summarize2 mean min max  // default values
	}
	ParseSummarize `summarize2'
	mata: HDFE.summarize_stats = "`s(stats)'"
	mata: HDFE.summarize_quietly = `s(quietly)'

	* Parse misc options
	mata: HDFE.notes = `"`notes'"'
	mata: HDFE.store_sample = ("`nosample'"=="")
	mata: HDFE.timeit = `timeit'

	* Parse Coef Table Options (do this last!)
	_get_diopts diopts options, `options' // store in `diopts', and the rest back to `options'
	_assert (`"`options'"'==""), msg(`"invalid options: `options'"')
	if ("`hascons'"!="") di in ye "(option ignored: `hascons')"
	if ("`tsscons'"!="") di in ye "(option ignored: `tsscons')"
	mata: HDFE.diopts = `"`diopts'"'
end

program ParseSummarize, sclass
	sreturn clear
	syntax [namelist(name=stats)] , [QUIetly]
	local quietly = ("`quietly'"!="")
	sreturn loc stats "`stats'"
	sreturn loc quietly = `quietly'
end

// --------------------------------------------------------------------------

program Estimate, eclass
	ereturn clear

	* Parse and fill out HDFE object
	Parse `0'
	mata: st_local("timeit", strofreal(HDFE.timeit))

	* Compute degrees-of-freedom
	if (`timeit') timer on 21
	mata: HDFE.estimate_dof()
	if (`timeit') timer off 21

	* Save updated e(sample) (singletons reduce sample);
	* required to parse factor variables to partial out
	if (`timeit') timer on 29
	tempvar touse
	mata: HDFE.save_touse("`touse'")
	if (`timeit') timer off 29

	* Expand varlists
	if (`timeit') timer on 22
	foreach cat in /*varlist*/ depvar indepvars endogvars instruments {
		mata: st_local("vars", HDFE.original_`cat')
		if ("`vars'" == "") continue

		loc varlist
		// Expand (need to do it after the dataset has been reduced in size)
		loc 0 `vars' if `touse'
		syntax varlist(ts fv numeric) if
		//di as error "VAR1=<`vars'>"
		//di as error "VAR2=<`varlist'>"

		// HACK: addbn replaces 0.foreign with 0bn.foreign , to prevent st_data() from loading a bunch of zeros
		ms_fvstrip `varlist' if `touse', expand dropomit addbn  // onebyone
		// If we don't use onebyone, then 1.x 2.x ends up as 2.x
		loc varlist "`r(varlist)'"
		//di as error "VAR3=<`varlist'>"
		loc allvars `allvars' `varlist'
		mata: HDFE.`cat' = "`varlist'"

		loc varlist // (optional) clear it up to prevent bugs
	}
	mata: HDFE.varlist = "`allvars'"
	if (`timeit') timer off 22

	* Stats
	mata: st_local("stats", HDFE.summarize_stats)
	if ("`stats'" != "") Stats `touse'

	* Condition number
	mata: HDFE.estimate_cond()

	* Partial out; save TSS of depvar
	if (`timeit') timer on 23
	mata: hdfe_variables = HDFE.partial_out(HDFE.varlist, 1) // 1=Save TSS of first var if HDFE.tss is missing
	if (`timeit') timer off 23

	* Regress
	mata: assert_msg(HDFE.model=="ols", "You can only run OLS with reghdfe; for IV see https://github.com/sergiocorreia/ivreg2_demo")
	if (`timeit') timer on 24
	RegressOLS `touse'
	if (`timeit') timer off 24

	* (optional) Store FEs
	if (`timeit') timer on 29
	reghdfe_store_alphas
	if (`timeit') timer off 29

	* View estimation tables
	mata: st_local("diopts", HDFE.diopts)
	Replay, `diopts'

	// ~~ Preserve relevant dataset ~~-
	//
	// if (`compact') {
	// 	preserve
	// 	keep `varlist' `absorb'
	// 	loc N = c(N)
	// }
	// else {
	// 	qui cou if `touse'
	// 	loc N = r(N)
	// }

	if (`timeit') {
		di as text _n "{bf: Timer results:}"
		timer list
		di as text "Legend: 20: Create HDFE object; 21: Estimate DoF; 22: expand varlists; 23: partial out; 24: regress; 29: rest"
		di
	}
end

program RegressOLS, eclass
	args touse

	tempname b V N rank df_r
	mata: reghdfe_post_ols(HDFE, hdfe_variables, "`b'", "`V'", "`N'", "`rank'", "`df_r'")
	mata: st_local("indepvars", HDFE.indepvars)
	mata: hdfe_variables = .

	loc esample
	mata: st_local("store_sample", strofreal(HDFE.store_sample))
	if (`store_sample') loc esample "esample(`touse')"

	if ("`indepvars'" != "") {
		matrix colnames `b' = `indepvars'
		matrix colnames `V' = `indepvars'
		matrix rownames `V' = `indepvars'
		_ms_findomitted `b' `V'
		ereturn post `b' `V', `esample' buildfvinfo depname(`depvar')
	}
	else {
		ereturn post, `esample' buildfvinfo depname(`depvar')
	}

	ereturn scalar N       = `N'
	ereturn scalar rank    = `rank'
	ereturn scalar df_r    = `df_r'
	ereturn local  cmd     "reghdfe"
	mata: HDFE.post()

	* Post stats
	cap conf matrix reghdfe_statsmatrix
	if (!c(rc)) {
		ereturn matrix summarize = reghdfe_statsmatrix
		mata: st_local("summarize_quietly", strofreal(HDFE.summarize_quietly))
		ereturn scalar summarize_quietly = `summarize_quietly'
	}
end

program Replay, rclass
	syntax [, *]

	if `"`e(cmd)'"' != "reghdfe"  {
	        error 301
	}

	_get_diopts options, `options'
	reghdfe_header // _coef_table_header
	di ""
	_coef_table, `options' // ereturn display, `options'
	return add // adds r(level), r(table), etc. to ereturn (before the footnote deletes them)
	reghdfe_footnote

	* Replay stats
	if (e(summarize_quietly)==0) {
		di as text _n "{sf:Regression Summary Statistics:}" _c
		matlist e(summarize)', border(top bottom) rowtitle(Variable) // twidth(18)
	}
end

program Stats
	args touse
	* Optional weights
	mata: st_local("weight", sprintf("[%s=%s]", HDFE.weight_type, HDFE.weight_var))
	assert "`weight'" != ""
	if ("`weight'" == "[=]") loc weight
	loc weight : subinstr local weight "[pweight" "[aweight"

	mata: st_local("stats", HDFE.summarize_stats)
	mata: st_local("varlist", HDFE.varlist)
	mata: st_local("cvars", invtokens(HDFE.cvars))
	loc full_varlist `varlist' `cvars'

	* quick workaround b/c -tabstat- does not support factor variables
	fvrevar `full_varlist', list
	loc full_varlist `r(varlist)'

	qui tabstat `full_varlist' if `touse' `weight' , stat(`stats') col(stat) save
	matrix reghdfe_statsmatrix = r(StatTotal)
end

*! version 4.4.11 26oct2017

program reghdfe, eclass
	* Intercept old+version
	cap syntax, version old
	if !c(rc) {
		reghdfe_old, version
		exit
	}

	* Intercept old
	cap syntax anything(everything) [fw aw pw/], [*] old
	if !c(rc) {
		di as error "(running historical version of reghdfe)"
		if ("`weight'"!="") local weightexp [`weight'=`exp']
		reghdfe_old `anything' `weightexp', `options'
		exit
	}

	* Aux. subcommands
	cap syntax, [*]
	if inlist("`options'", "check", "compile", "reload", "update", "version", "requirements") {
		if ("`options'"=="compile") loc args force
		if ("`options'"=="check") loc options compile
		if ("`options'"=="update") {
			loc args 1
			loc options reload
		}
		loc subcmd = proper("`options'")
		`subcmd' `args'
	}
	else if replay() {
		Replay `0'
	}
	else {
		Cleanup 0
		Compile // takes 0.01s to run this useful check (ensures .mlib exists)
		cap noi Estimate `0'
		Cleanup `c(rc)'
	}
end

program Compile
	args force
	ftools, check // in case lftools.mlib does not exist or is outdated
	ms_get_version reghdfe // save local package_version

	loc list_objects "FixedEffects() fixed_effects() BipartiteGraph()"
	loc list_functions "reghdfe_*() transform_*() accelerate_*() panelmean() panelsolve_*() lsmr()"
	loc list_misc "weighted_quadcolsum() safe_divide() check_convergence() precompute_inv_xx()"
	// TODO: prefix everything with reghdfe_*

	ms_compile_mata, ///
		package(reghdfe) ///
		version(`package_version') ///
		fun("`list_objects' `list_functions' `list_misc'") ///
		verbose ///
		`force'
end

program Reload
	* Internal debugging tool.
	* Updates dependencies and reghdfe from local path or from github
	* Usage:
	* 	reghdfe, update // from c:\git\..
	* 	reghdfe, reload // from github

	args online
	if ("`online'" == "") loc online 0

	di as text _n "{bf:reghdfe: updating required packages}"
	di as text "{hline 64}"

	* -ftools- https://github.com/sergiocorreia/ftools/
	cap ado uninstall ftools
	if (`online') net install ftools, from("https://github.com/sergiocorreia/ftools/raw/master/src/")
	if (!`online') net install ftools, from("c:\git\ftools\src")
	di as text "{hline 64}"
	ftools, compile
	di as text "{hline 64}"

	* Update -reghdfe-
	di as text _n  _n "{bf:reghdfe: updating self}"
	di as text "{hline 64}"
	qui ado uninstall reghdfe
	if (`online') net install reghdfe, from("https://github.com/sergiocorreia/reghdfe/raw/master/src/")
	if (!`online') net install reghdfe, from("c:\git\reghdfe\src")
	qui which reghdfe
	di as text "{hline 64}"
	reghdfe, compile
	di as text "{hline 64}"

	* Cleaning up
	di as text _n "{bf:Note:} You need to run {stata program drop _all} now."
end

program Version
	which reghdfe
	Requirements
end

program Requirements
	di as text _n "Required packages installed?"
	loc reqs ftools
	// ivreg2 avar tuples group3hdfe
	if (c(stata_version)<13) loc reqs `reqs' boottest

	loc ftools_github "https://github.com/sergiocorreia/ftools/raw/master/src/"

	loc error 0

	foreach req of local reqs {
		loc fn `req'.ado
		cap findfile `fn'
		if (_rc) {
			loc error 1
			di as text "{lalign 20:- `req'}" as error "not" _c
			di as text "    {stata ssc install `req':install from SSC}" _c
			if inlist("`req'", "ftools") {
				loc github ``req'_github'
				di as text `"    {stata `"net install `req', from(`"`github'"')"':install from github}"'
			}
			else {
				di as text // newline
			}
		}
		else {
			di as text "{lalign 20:- `req'}" as text "yes"
		}
	}

	if (`error') exit 601
end

program Cleanup
	args rc
	cap mata: mata drop HDFE
	cap mata: mata drop hdfe_*
	cap drop __temp_reghdfe_resid__
	cap matrix drop reghdfe_statsmatrix
	if (`rc') exit `rc'
end

program Parse
	* Trim whitespace (caused by "///" line continuations; aesthetic only)
	mata: st_local("0", stritrim(st_local("0")))

	* Main syntax
	#d;
	syntax anything(id=varlist equalok) [if] [in] [aw pw fw/] , [

		/* Model */
		Absorb(string) NOAbsorb
		SUmmarize SUmmarize2(string asis) /* simulate implicit options */

		/* Standard Errors */
		VCE(string) CLuster(string)

		/* Diagnostic */
		Verbose(numlist min=1 max=1 >=-1 <=5 integer)
		TIMEit

		/* Speedup and memory Tricks */
		NOSAMPle /* do not save e(sample) */

		/* Undocumented */
		KEEPSINgletons
		OLD /* use latest v3 */
		NOTES(string) /* NOTES(key=value ...), will be stored on e() */

		] [*] /* capture optimization options, display options, etc. */
		;
	#d cr

	* Unused
	* SAVEcache
	* USEcache
	* CLEARcache
	* COMPACT /* use as little memory as possible but is slower */

	* Convert options to boolean
	if ("`verbose'" == "") loc verbose 0
	loc timeit = ("`timeit'"!="")
	loc drop_singletons = ("`keepsingletons'"=="")

	if (`timeit') timer on 29

	* Sanity checks
	if (`verbose'>-1 & "`keepsingletons'"!="") {
		loc url "http://scorreia.com/reghdfe/nested_within_cluster.pdf"
		loc msg "WARNING: Singleton observations not dropped; statistical significance is biased"
		di as error `"`msg' {browse "`url'":(link)}"'
	}
	if ("`cluster'"!="") {
		_assert ("`vce'"==""), msg("cannot specify both cluster() and vce()")
		loc vce cluster `cluster'
		loc cluster // clear it to avoid bugs in subsequent lines
	}

	* Parse Varlist
	ms_parse_varlist `anything'
	loc base_varlist "`r(basevars)'"
	foreach cat in depvar indepvars endogvars instruments {
		loc original_`cat' "`r(`cat')'"
	}
	loc model = cond("`r(instruments)'" == "", "ols", "iv")
	loc original_varlist = "`r(varlist)'" // no parens or equal sign

	* Parse Weights
	if ("`weight'"!="") {
		unab exp : `exp', min(1) max(1) // simple weights only
	}

	* Parse VCE
	ms_parse_vce, vce(`vce') weighttype(`weight')
	loc vcetype = "`s(vcetype)'"
	loc num_clusters = `s(num_clusters)'
	loc clustervars = "`s(clustervars)'"
	loc base_clustervars = "`s(base_clustervars)'"
	loc vceextra = "`s(vceextra)'"

	* Select sample (except for absvars)
	loc varlist `original_varlist' `base_clustervars'
	tempvar touse
	marksample touse, strok // based on varlist + cluster + if + in + weight

	* Parse noabsorb
	_assert  ("`absorb'`noabsorb'" != ""), msg("option {bf:absorb()} or {bf:noabsorb} required")
	if ("`noabsorb'" != "") {
		_assert ("`absorb'" == ""), msg("{bf:absorb()} and {bf:noabsorb} are mutually exclusive")
		tempvar c
		gen byte `c' = 1
		loc absorb `c'
	}

	if (`timeit') timer off 29

	* Construct HDFE object
	// SYNTAX: fixed_effects(absvars | , touse, wtype, wtvar, dropsing, verbose)
	mata: st_local("comma", strpos(`"`absorb'"', ",") ? "" : ",")
	if (`timeit') timer on 20
	mata: HDFE = fixed_effects(`"`absorb' `comma' `options'"', "`touse'", "`weight'", "`exp'", `drop_singletons', `verbose')
	if (`timeit') timer off 20
	mata: HDFE.cmdline = "reghdfe " + st_local("0")
	loc options `s(options)'

	mata: st_local("N", strofreal(HDFE.N))
	if (`N' == 0) error 2000

	* Fill out HDFE object
	mata: HDFE.varlist = "`base_varlist'"
	mata: HDFE.original_depvar = "`original_depvar'"
	mata: HDFE.original_indepvars = "`original_indepvars'"
	mata: HDFE.original_endogvars = "`original_endogvars'"
	mata: HDFE.original_instruments = "`original_instruments'"
	mata: HDFE.original_varlist = "`original_varlist'"
	mata: HDFE.model = "`model'"

	mata: HDFE.vcetype = "`vcetype'"
	mata: HDFE.num_clusters = `num_clusters'
	mata: HDFE.clustervars = tokens("`clustervars'")
	mata: HDFE.base_clustervars = tokens("`base_clustervars'")
	mata: HDFE.vceextra = "`vceextra'"

	* Parse summarize
	if ("`summarize'" != "") {
		_assert ("`summarize2'" == ""), msg("summarize() syntax error")
		loc summarize2 mean min max  // default values
	}
	ParseSummarize `summarize2'
	mata: HDFE.summarize_stats = "`s(stats)'"
	mata: HDFE.summarize_quietly = `s(quietly)'

	* Parse misc options
	mata: HDFE.notes = `"`notes'"'
	mata: HDFE.store_sample = ("`nosample'"=="")
	mata: HDFE.timeit = `timeit'

	* Parse Coef Table Options (do this last!)
	_get_diopts diopts options, `options' // store in `diopts', and the rest back to `options'
	_assert (`"`options'"'==""), msg(`"invalid options: `options'"')
	if ("`hascons'"!="") di in ye "(option ignored: `hascons')"
	if ("`tsscons'"!="") di in ye "(option ignored: `tsscons')"
	mata: HDFE.diopts = `"`diopts'"'
end

program ParseSummarize, sclass
	sreturn clear
	syntax [namelist(name=stats)] , [QUIetly]
	local quietly = ("`quietly'"!="")
	sreturn loc stats "`stats'"
	sreturn loc quietly = `quietly'
end

// --------------------------------------------------------------------------

program Estimate, eclass
	ereturn clear

	* Parse and fill out HDFE object
	Parse `0'
	mata: st_local("timeit", strofreal(HDFE.timeit))

	* Compute degrees-of-freedom
	if (`timeit') timer on 21
	mata: HDFE.estimate_dof()
	if (`timeit') timer off 21

	* Save updated e(sample) (singletons reduce sample);
	* required to parse factor variables to partial out
	if (`timeit') timer on 29
	tempvar touse
	mata: HDFE.save_touse("`touse'")
	if (`timeit') timer off 29

	* Expand varlists
	if (`timeit') timer on 22
	foreach cat in /*varlist*/ depvar indepvars endogvars instruments {
		mata: st_local("vars", HDFE.original_`cat')
		if ("`vars'" == "") continue

		loc varlist
		// Expand (need to do it after the dataset has been reduced in size)
		loc 0 `vars' if `touse'
		syntax varlist(ts fv numeric) if
		//di as error "VAR1=<`vars'>"
		//di as error "VAR2=<`varlist'>"

		// HACK: addbn replaces 0.foreign with 0bn.foreign , to prevent st_data() from loading a bunch of zeros
		ms_fvstrip `varlist' if `touse', expand dropomit addbn  // onebyone
		// If we don't use onebyone, then 1.x 2.x ends up as 2.x
		loc varlist "`r(varlist)'"
		//di as error "VAR3=<`varlist'>"
		loc allvars `allvars' `varlist'
		mata: HDFE.`cat' = "`varlist'"

		loc varlist // (optional) clear it up to prevent bugs
	}
	mata: HDFE.varlist = "`allvars'"
	if (`timeit') timer off 22

	* Stats
	mata: st_local("stats", HDFE.summarize_stats)
	if ("`stats'" != "") Stats `touse'

	* Condition number
	mata: HDFE.estimate_cond()

	* Partial out; save TSS of depvar
	if (`timeit') timer on 23
	mata: hdfe_variables = HDFE.partial_out(HDFE.varlist, 1) // 1=Save TSS of first var if HDFE.tss is missing
	if (`timeit') timer off 23

	* Regress
	mata: assert_msg(HDFE.model=="ols", "You can only run OLS with reghdfe; for IV see https://github.com/sergiocorreia/ivreg2_demo")
	if (`timeit') timer on 24
	RegressOLS `touse'
	if (`timeit') timer off 24

	* (optional) Store FEs
	if (`timeit') timer on 29
	reghdfe_store_alphas
	if (`timeit') timer off 29

	* View estimation tables
	mata: st_local("diopts", HDFE.diopts)
	Replay, `diopts'

	// ~~ Preserve relevant dataset ~~-
	//
	// if (`compact') {
	// 	preserve
	// 	keep `varlist' `absorb'
	// 	loc N = c(N)
	// }
	// else {
	// 	qui cou if `touse'
	// 	loc N = r(N)
	// }

	if (`timeit') {
		di as text _n "{bf: Timer results:}"
		timer list
		di as text "Legend: 20: Create HDFE object; 21: Estimate DoF; 22: expand varlists; 23: partial out; 24: regress; 29: rest"
		di
	}
end

program RegressOLS, eclass
	args touse

	tempname b V N rank df_r
	mata: reghdfe_post_ols(HDFE, hdfe_variables, "`b'", "`V'", "`N'", "`rank'", "`df_r'")
	mata: st_local("indepvars", HDFE.indepvars)
	mata: hdfe_variables = .

	loc esample
	mata: st_local("store_sample", strofreal(HDFE.store_sample))
	if (`store_sample') loc esample "esample(`touse')"

	if ("`indepvars'" != "") {
		matrix colnames `b' = `indepvars'
		matrix colnames `V' = `indepvars'
		matrix rownames `V' = `indepvars'
		_ms_findomitted `b' `V'
		ereturn post `b' `V', `esample' buildfvinfo depname(`depvar')
	}
	else {
		ereturn post, `esample' buildfvinfo depname(`depvar')
	}

	ereturn scalar N       = `N'
	ereturn scalar rank    = `rank'
	ereturn scalar df_r    = `df_r'
	ereturn local  cmd     "reghdfe"
	mata: HDFE.post()

	* Post stats
	cap conf matrix reghdfe_statsmatrix
	if (!c(rc)) {
		ereturn matrix summarize = reghdfe_statsmatrix
		mata: st_local("summarize_quietly", strofreal(HDFE.summarize_quietly))
		ereturn scalar summarize_quietly = `summarize_quietly'
	}
end

program Replay, rclass
	syntax [, *]

	if `"`e(cmd)'"' != "reghdfe"  {
	        error 301
	}

	_get_diopts options, `options'
	reghdfe_header // _coef_table_header
	di ""
	_coef_table, `options' // ereturn display, `options'
	return add // adds r(level), r(table), etc. to ereturn (before the footnote deletes them)
	reghdfe_footnote

	* Replay stats
	if (e(summarize_quietly)==0) {
		di as text _n "{sf:Regression Summary Statistics:}" _c
		matlist e(summarize)', border(top bottom) rowtitle(Variable) // twidth(18)
	}
end

program Stats
	args touse
	* Optional weights
	mata: st_local("weight", sprintf("[%s=%s]", HDFE.weight_type, HDFE.weight_var))
	assert "`weight'" != ""
	if ("`weight'" == "[=]") loc weight
	loc weight : subinstr local weight "[pweight" "[aweight"

	mata: st_local("stats", HDFE.summarize_stats)
	mata: st_local("varlist", HDFE.varlist)
	mata: st_local("cvars", invtokens(HDFE.cvars))
	loc full_varlist `varlist' `cvars'

	qui tabstat `full_varlist' if `touse' `weight' , stat(`stats') col(stat) save
	matrix reghdfe_statsmatrix = r(StatTotal)
end


Add tests for closed bug reports

Add tests for closed bug reports

noi cscript "reghdfe: complex factor variables that fvstrip handles" adofile reghdfe

cap pr drop TrimBoth
pr TrimBoth, eclass
	matrix trim_b = e(b)
	matrix trim_V = e(V)
	loc size = colsof(trim_b) - 1

	mata: idx = 2, 3, 5
	mata: b = st_matrix("trim_b")[idx]
	mata: st_matrix("trim_b", b)
	mata: V = st_matrix("trim_V")[idx, idx]
	mata: st_matrix("trim_V", V)

	ereturn matrix trim_b = trim_b
	ereturn matrix trim_V = trim_V
end

cap pr drop TrimBoth2
pr TrimBoth2, eclass
	matrix trim_b = e(b)
	matrix trim_V = e(V)
	loc size = colsof(trim_b) - 1

	// mata: idx = 2..13 , 15..25 , 27 , 30
	mata: idx = 2..13 , 15..25 , 27 , 29, 30
	mata: b = st_matrix("trim_b")[idx]
	mata: st_matrix("trim_b", b)
	mata: V = st_matrix("trim_V")[idx, idx]
	mata: st_matrix("trim_V", V)

	ereturn matrix trim_b = trim_b
	ereturn matrix trim_V = trim_V
end

local included_e ///
	scalar: N rmse tss rss mss r2 r2_a F df_r df_m ll ll_0 ///
	matrix: trim_b trim_V ///
	macros: wexp wtype

* [TEST] https://github.com/sergiocorreia/reghdfe/issues/86

	sysuse auto, clear
	gen byte turn43 = (turn==43)

	* 1) Benchmark
	reghdfe price weight, absorb(rep78) vce(cl foreign)
	storedresults save benchmark e()

	* 2) Reghdfe
	reghdfe price weight, absorb(rep78) vce(cl i.foreign)
	storedresults compare benchmark e(), exclude(macro: depvar cmdline)

	* 3) Cleanup
	storedresults drop benchmark

****

	sysuse auto, clear
	gen byte turn43 = (turn==43)

	* 1) Benchmark
	reghdfe price weight, absorb(rep78) vce(cl foreign#turn)
	storedresults save benchmark e()

	* 2) Reghdfe
	reghdfe price weight, absorb(rep78) vce(cl i.foreign#i.turn)
	storedresults compare benchmark e(), exclude(macro: depvar cmdline)

	* 3) Cleanup
	storedresults drop benchmark

* [TEST] https://github.com/sergiocorreia/reghdfe/issues/88

	* Setup
	sysuse auto, clear
	bys turn: gen t = _n
	xtset turn t

	* 1) Benchmark
	areg price L.(weight gear), a(turn)
	trim_cons
	storedresults save benchmark e()
	local bench_df_a = e(df_a)

	* 2) Compare w/reghdfe
	reghdfe price L.(weight gear), a(turn) keepsing
	notrim
	matrix list e(trim_V)
	storedresults compare benchmark e(), tol(1e-10) include(`included_e')
	assert `bench_df_a'==e(df_a)-1

	* 3) Cleanup
	storedresults drop benchmark

* [TEST] https://github.com/sergiocorreia/reghdfe/issues/28

	sysuse auto, clear
	gen byte turn43 = (turn==43)

	* 1) Benchmark
	reghdfe turn43 price weight, a(trunk)
	storedresults save benchmark e()

	* 2) Reghdfe
	reghdfe 43.turn price weight, a(trunk)
	storedresults compare benchmark e(), exclude(macro: depvar cmdline)

	* 3) Cleanup
	storedresults drop benchmark

* [TEST] https://github.com/sergiocorreia/reghdfe/issues/22

	sysuse auto, clear
	gen x = foreign

	* 1) Benchmark
	areg price i.x##c.gear, a(turn)
	matrix list e(V)
	TrimBoth
	matrix list e(trim_V)
	local bench_df_a = e(df_a)
	storedresults save benchmark e()

	* 2) Reghdfe
	reghdfe price i.x##c.gear, a(turn) keepsing v(-1)
	notrim
	matrix list e(trim_V)
	storedresults compare benchmark e(), tol(1e-10) include(`included_e')
	assert `bench_df_a'==e(df_a)-1

	* 3) Cleanup
	storedresults drop benchmark

* [TEST] https://github.com/sergiocorreia/reghdfe/issues/35

	sysuse auto, clear
	fvset base 1 foreign

	* 1) Benchmark
	areg price weight, a(foreign)
	trim_cons
	local bench_df_a = e(df_a)
	storedresults save benchmark e()

	* 2) Reghdfe
	reghdfe price weight, a(foreign) keepsing v(-1)
	notrim
	storedresults compare benchmark e(), tol(1e-10) include(`included_e')
	assert `bench_df_a'==e(df_a)-1

	* 3) Cleanup
	storedresults drop benchmark

* [TEST] https://github.com/sergiocorreia/reghdfe/issues/43

	sysuse nlsw88.dta, clear

	* 1) Benchmark
	areg wage i.industry##c.age i.union##c.age, a(race)
	matrix list e(b)
	TrimBoth2
	matrix list e(trim_b)
	local bench_df_a = e(df_a)
	storedresults save benchmark e()

	* 2) Reghdfe
	reghdfe wage i.industry##c.age i.union##c.age, a(race) keepsing  v(-1)
	notrim
	storedresults compare benchmark e(), tol(1e-9) include(`included_e')
	assert `bench_df_a'==e(df_a)-1

	* 3) Cleanup
	storedresults drop benchmark

cd "C:/Git/reghdfe/test"
exit

noi cscript "reghdfe: complex factor variables that fvstrip handles" adofile reghdfe

cap pr drop TrimBoth
pr TrimBoth, eclass
	matrix trim_b = e(b)
	matrix trim_V = e(V)
	loc size = colsof(trim_b) - 1

	mata: idx = 2, 3, 5
	mata: b = st_matrix("trim_b")[idx]
	mata: st_matrix("trim_b", b)
	mata: V = st_matrix("trim_V")[idx, idx]
	mata: st_matrix("trim_V", V)

	ereturn matrix trim_b = trim_b
	ereturn matrix trim_V = trim_V
end

cap pr drop TrimBoth2
pr TrimBoth2, eclass
	matrix trim_b = e(b)
	matrix trim_V = e(V)
	loc size = colsof(trim_b) - 1

	// mata: idx = 2..13 , 15..25 , 27 , 30
	mata: idx = 2..13 , 15..25 , 27 , 29, 30
	mata: b = st_matrix("trim_b")[idx]
	mata: st_matrix("trim_b", b)
	mata: V = st_matrix("trim_V")[idx, idx]
	mata: st_matrix("trim_V", V)

	ereturn matrix trim_b = trim_b
	ereturn matrix trim_V = trim_V
end

local included_e ///
	scalar: N rmse tss rss mss r2 r2_a F df_r df_m ll ll_0 ///
	matrix: trim_b trim_V ///
	macros: wexp wtype

* [TEST] https://github.com/sergiocorreia/reghdfe/issues/88

	* Setup
	sysuse auto, clear
	bys turn: gen t = _n
	xtset turn t

	* 1) Benchmark
	areg price L.(weight gear), a(turn)
	trim_cons
	storedresults save benchmark e()
	local bench_df_a = e(df_a)

	* 2) Compare w/reghdfe
	reghdfe price L.(weight gear), a(turn) keepsing
	notrim
	matrix list e(trim_V)
	storedresults compare benchmark e(), tol(1e-10) include(`included_e')
	assert `bench_df_a'==e(df_a)-1

	* 3) Cleanup
	storedresults drop benchmark

* [TEST] https://github.com/sergiocorreia/reghdfe/issues/28

	sysuse auto, clear
	gen byte turn43 = (turn==43)

	* 1) Benchmark
	reghdfe turn43 price weight, a(trunk)
	storedresults save benchmark e()

	* 2) Reghdfe
	reghdfe 43.turn price weight, a(trunk)
	storedresults compare benchmark e(), exclude(macro: depvar cmdline)

	* 3) Cleanup
	storedresults drop benchmark

* [TEST] https://github.com/sergiocorreia/reghdfe/issues/22

	sysuse auto, clear
	gen x = foreign

	* 1) Benchmark
	areg price i.x##c.gear, a(turn)
	matrix list e(V)
	TrimBoth
	matrix list e(trim_V)
	local bench_df_a = e(df_a)
	storedresults save benchmark e()

	* 2) Reghdfe
	reghdfe price i.x##c.gear, a(turn) keepsing v(-1)
	notrim
	matrix list e(trim_V)
	storedresults compare benchmark e(), tol(1e-10) include(`included_e')
	assert `bench_df_a'==e(df_a)-1

	* 3) Cleanup
	storedresults drop benchmark

* [TEST] https://github.com/sergiocorreia/reghdfe/issues/35

	sysuse auto, clear
	fvset base 1 foreign

	* 1) Benchmark
	areg price weight, a(foreign)
	trim_cons
	local bench_df_a = e(df_a)
	storedresults save benchmark e()

	* 2) Reghdfe
	reghdfe price weight, a(foreign) keepsing v(-1)
	notrim
	storedresults compare benchmark e(), tol(1e-10) include(`included_e')
	assert `bench_df_a'==e(df_a)-1

	* 3) Cleanup
	storedresults drop benchmark

* [TEST] https://github.com/sergiocorreia/reghdfe/issues/43

	sysuse nlsw88.dta, clear

	* 1) Benchmark
	areg wage i.industry##c.age i.union##c.age, a(race)
	matrix list e(b)
	TrimBoth2
	matrix list e(trim_b)
	local bench_df_a = e(df_a)
	storedresults save benchmark e()

	* 2) Reghdfe
	reghdfe wage i.industry##c.age i.union##c.age, a(race) keepsing  v(-1)
	notrim
	storedresults compare benchmark e(), tol(1e-9) include(`included_e')
	assert `bench_df_a'==e(df_a)-1

	* 3) Cleanup
	storedresults drop benchmark

cd "C:/Git/reghdfe/test"
exit


Update repo URL

Update repo URL

{smcl}
{* *! version 1.0.0 Matthew White 30dec2014}{...}
{title:Title}

{phang}
{cmd:stgit} {hline 2} Retrieve information about a Git repository

{marker syntax}{...}
{title:Syntax}

{p 8 10 2}
{cmd:stgit} [{cmd:status}]{cmd:,} [{it:options}]

{* Using -help odbc- as a template.}{...}
{* 26 is the position of the last character in the first column + 3.}{...}
{synoptset 26 tabbed}{...}
{synopthdr}
{synoptline}
{synopt:{opt git_dir(directory_name)}}GIT_DIR, the directory containing
the repository metadata{p_end}
{synoptline}
{p2colreset}{...}

{marker description}{...}
{title:Description}

{pstd}
{cmd:stgit} retrieves information about a Git repository,
including the current branch and the SHA-1 hash of the current commit.
In Stata 13 and above, it uses
the Java library {browse "https://eclipse.org/jgit/":JGit} to return
the status of the working tree.

{marker remarks}{...}
{title:Remarks}

{pstd}
In Stata 13 and above, {cmd:stgit} requires
the {help SSC} package {cmd:statajava}.
You must also {browse "https://eclipse.org/jgit/download/":download}
the JGit {cmd:.jar} file. (On the download page, it may be called
the "raw API library.") Place the {cmd:.jar} file on your {help ado-path}.
For instance, you may save it in your {help PERSONAL} system directory.
On your computer, this is {cmd}{ccl sysdir_personal}{txt}.

{pstd}
{cmd:stgit} respects {help version:version control} when deciding
which values to retrieve. Even in Stata 13 and above,
if the command interpreter is set to before version 13,
{cmd:stgit} does not use JGit, and returns limited information.

{pstd}
One use of {cmd:stgit} is adding commit hashes to dataset {help notes} and
to exported results such as tables. This facilitates reproducible research,
pinpointing the code that produced the files.

{pstd}
The GitHub repository for {cmd:stgit} is
{browse "https://github.com/matthew-white/stgit":here}.

{marker options}{...}
{title:Options}

{phang}
{opt git_dir()} specifies the path of GIT_DIR,
the repository metadata directory. This is typically named {cmd:.git}.
If {cmd:git_dir()} is not specified, {cmd:stgit} attempts to find GIT_DIR;
it assumes that the current working directory is within the Git repository.
Whether or not {cmd:git_dir()} is specified, {cmd:stgit} stores
the absolute path of GIT_DIR in {cmd:r(git_dir)}.

{marker examples}{...}
{title:Examples}

{pstd}Retrieve information about the repository within which
the current working directory is located{p_end}
{phang2}{cmd:stgit}

{pstd}Same as the previous {cmd:stgit} command{p_end}
{phang2}{cmd:stgit status}

{pstd}Retrieve information about the repository whose GIT_DIR is
{cmd:GitHub/cfout/.git}{p_end}
{phang2}{cmd:stgit, git_dir("GitHub/cfout/.git")}{p_end}

{marker results}{...}
{title:Stored results}

{pstd}
In Stata 9 and above, {cmd:stgit} stores the following in {cmd:r()}:

{synoptset 23 tabbed}{...}
{p2col 5 23 27 2: Scalars}{p_end}
{synopt:{cmd:r(has_detached_head)}}{cmd:1} if HEAD is detached,
{cmd:0} if not{p_end}

{p2col 5 23 27 2: Macros}{p_end}
{synopt:{cmd:r(git_dir)}}absolute path of GIT_DIR{p_end}
{synopt:{cmd:r(sha)}}SHA-1 hash of current commit{p_end}
{synopt:{cmd:r(branch)}}name of current branch or
{cmd:r(sha)} if HEAD is detached{p_end}
{p2colreset}{...}

{pstd}
In Stata 13 and above, {cmd:stgit} also stores the following in {cmd:r()}.
{cmd:stgit} retrieves these values using
the JGit class {cmd:org.eclipse.jgit.api.Status}, and much of
the language below is copied from
that class's {browse "https://eclipse.org/jgit/documentation/":API}
(version 3.5.3).

{synoptset 29 tabbed}{...}
{p2col 5 29 33 2: Scalars}{p_end}
{synopt:{cmd:r(is_clean)}}{cmd:1} if no differences exist between
the working tree, the index, and the current HEAD,
{cmd:0} if differences do exist{p_end}
{synopt:{cmd:r(has_uncommitted_changes)}}{cmd:1} if any tracked file is changed,
{cmd:0} if not{p_end}

{synoptset 26 tabbed}{...}
{p2col 5 26 30 2: Macros}{p_end}
{synopt:{cmd:r(untracked)}}list of files that are not ignored and not in
the index (e.g., what you get if you create a new file without adding it to
the index){p_end}
{synopt:{cmd:r(untracked_folders)}}list of directories that are not ignored and
not in the index{p_end}
{synopt:{cmd:r(uncommitted_changes)}}list of files and folders that are known to
the repository and changed either in the index or in the working tree{p_end}
{synopt:{cmd:r(added)}}list of files added to the index, not in HEAD (e.g.,
what you get if you call {cmd:git add ...} on a newly created file){p_end}
{synopt:{cmd:r(modified)}}list of files modified on disk relative to
the index (e.g., what you get if you modify an existing file without adding
it to the index){p_end}
{synopt:{cmd:r(changed)}}list of files changed from HEAD to index (e.g.,
what you get if you modify an existing file and call {cmd:git add ...} on
it){p_end}
{synopt:{cmd:r(missing)}}list of files in index but not filesystem (e.g.,
what you get if you call {cmd:rm ...} on an existing file){p_end}
{synopt:{cmd:r(removed)}}list of files removed from index, but in HEAD (e.g.,
what you get if you call {cmd:git rm ...} on an existing file){p_end}
{synopt:{cmd:r(conflicting)}}list of files that are in conflict (e.g.,
what you get if you modify a file that was modified by someone else in
the meantime){p_end}
{synopt:{cmd:r(ignored_not_in_index)}}list of files and folders that are
ignored and not in the index{p_end}
{p2colreset}{...}

{marker author}{...}
{title:Author}

{pstd}Matthew White, Innovations for Poverty Action{p_end}
{pstd}mwhite@poverty-action.org{p_end}

{title:Also see}

{psee}
User-written:  {helpb git}
{p_end}

{smcl}
{* *! version 1.0.0 Matthew White 30dec2014}{...}
{title:Title}

{phang}
{cmd:stgit} {hline 2} Retrieve information about a Git repository

{marker syntax}{...}
{title:Syntax}

{p 8 10 2}
{cmd:stgit} [{cmd:status}]{cmd:,} [{it:options}]

{* Using -help odbc- as a template.}{...}
{* 26 is the position of the last character in the first column + 3.}{...}
{synoptset 26 tabbed}{...}
{synopthdr}
{synoptline}
{synopt:{opt git_dir(directory_name)}}GIT_DIR, the directory containing
the repository metadata{p_end}
{synoptline}
{p2colreset}{...}

{marker description}{...}
{title:Description}

{pstd}
{cmd:stgit} retrieves information about a Git repository,
including the current branch and the SHA-1 hash of the current commit.
In Stata 13 and above, it uses
the Java library {browse "https://eclipse.org/jgit/":JGit} to return
the status of the working tree.

{marker remarks}{...}
{title:Remarks}

{pstd}
In Stata 13 and above, {cmd:stgit} requires
the {help SSC} package {cmd:statajava}.
You must also {browse "https://eclipse.org/jgit/download/":download}
the JGit {cmd:.jar} file. (On the download page, it may be called
the "raw API library.") Place the {cmd:.jar} file on your {help ado-path}.
For instance, you may save it in your {help PERSONAL} system directory.
On your computer, this is {cmd}{ccl sysdir_personal}{txt}.

{pstd}
{cmd:stgit} respects {help version:version control} when deciding
which values to retrieve. Even in Stata 13 and above,
if the command interpreter is set to before version 13,
{cmd:stgit} does not use JGit, and returns limited information.

{pstd}
One use of {cmd:stgit} is adding commit hashes to dataset {help notes} and
to exported results such as tables. This facilitates reproducible research,
pinpointing the code that produced the files.

{pstd}
The GitHub repository for {cmd:stgit} is
{browse "https://github.com/matthew-white/stata-git-tools":here}.

{marker options}{...}
{title:Options}

{phang}
{opt git_dir()} specifies the path of GIT_DIR,
the repository metadata directory. This is typically named {cmd:.git}.
If {cmd:git_dir()} is not specified, {cmd:stgit} attempts to find GIT_DIR;
it assumes that the current working directory is within the Git repository.
Whether or not {cmd:git_dir()} is specified, {cmd:stgit} stores
the absolute path of GIT_DIR in {cmd:r(git_dir)}.

{marker examples}{...}
{title:Examples}

{pstd}Retrieve information about the repository within which
the current working directory is located{p_end}
{phang2}{cmd:stgit}

{pstd}Same as the previous {cmd:stgit} command{p_end}
{phang2}{cmd:stgit status}

{pstd}Retrieve information about the repository whose GIT_DIR is
{cmd:GitHub/cfout/.git}{p_end}
{phang2}{cmd:stgit, git_dir("GitHub/cfout/.git")}{p_end}

{marker results}{...}
{title:Stored results}

{pstd}
In Stata 9 and above, {cmd:stgit} stores the following in {cmd:r()}:

{synoptset 23 tabbed}{...}
{p2col 5 23 27 2: Scalars}{p_end}
{synopt:{cmd:r(has_detached_head)}}{cmd:1} if HEAD is detached,
{cmd:0} if not{p_end}

{p2col 5 23 27 2: Macros}{p_end}
{synopt:{cmd:r(git_dir)}}absolute path of GIT_DIR{p_end}
{synopt:{cmd:r(sha)}}SHA-1 hash of current commit{p_end}
{synopt:{cmd:r(branch)}}name of current branch or
{cmd:r(sha)} if HEAD is detached{p_end}
{p2colreset}{...}

{pstd}
In Stata 13 and above, {cmd:stgit} also stores the following in {cmd:r()}.
{cmd:stgit} retrieves these values using
the JGit class {cmd:org.eclipse.jgit.api.Status}, and much of
the language below is copied from
that class's {browse "https://eclipse.org/jgit/documentation/":API}
(version 3.5.3).

{synoptset 29 tabbed}{...}
{p2col 5 29 33 2: Scalars}{p_end}
{synopt:{cmd:r(is_clean)}}{cmd:1} if no differences exist between
the working tree, the index, and the current HEAD,
{cmd:0} if differences do exist{p_end}
{synopt:{cmd:r(has_uncommitted_changes)}}{cmd:1} if any tracked file is changed,
{cmd:0} if not{p_end}

{synoptset 26 tabbed}{...}
{p2col 5 26 30 2: Macros}{p_end}
{synopt:{cmd:r(untracked)}}list of files that are not ignored and not in
the index (e.g., what you get if you create a new file without adding it to
the index){p_end}
{synopt:{cmd:r(untracked_folders)}}list of directories that are not ignored and
not in the index{p_end}
{synopt:{cmd:r(uncommitted_changes)}}list of files and folders that are known to
the repository and changed either in the index or in the working tree{p_end}
{synopt:{cmd:r(added)}}list of files added to the index, not in HEAD (e.g.,
what you get if you call {cmd:git add ...} on a newly created file){p_end}
{synopt:{cmd:r(modified)}}list of files modified on disk relative to
the index (e.g., what you get if you modify an existing file without adding
it to the index){p_end}
{synopt:{cmd:r(changed)}}list of files changed from HEAD to index (e.g.,
what you get if you modify an existing file and call {cmd:git add ...} on
it){p_end}
{synopt:{cmd:r(missing)}}list of files in index but not filesystem (e.g.,
what you get if you call {cmd:rm ...} on an existing file){p_end}
{synopt:{cmd:r(removed)}}list of files removed from index, but in HEAD (e.g.,
what you get if you call {cmd:git rm ...} on an existing file){p_end}
{synopt:{cmd:r(conflicting)}}list of files that are in conflict (e.g.,
what you get if you modify a file that was modified by someone else in
the meantime){p_end}
{synopt:{cmd:r(ignored_not_in_index)}}list of files and folders that are
ignored and not in the index{p_end}
{p2colreset}{...}

{marker author}{...}
{title:Author}

{pstd}Matthew White, Innovations for Poverty Action{p_end}
{pstd}mwhite@poverty-action.org{p_end}

{title:Also see}

{psee}
User-written:  {helpb git}
{p_end}