ieduplicates : addition to todo

ieduplicates : addition to todo

Add a todo for testing that observations in the report has not been deleted in a way that the report do no longer make sense. And provide an informative error message

	*! version 2.1 1Nov2016  Kristoffer Bjarkefur kbjarkefur@worldbank.org

	capture program drop ieduplicates
	program ieduplicates , rclass

	qui {

		syntax varname ,  FOLder(string) UNIQUEvars(varlist) [ KEEPvars(varlist) MINprecision(numlist >0) tostringok droprest nodaily SUFfix(string)]

		version 11.0

		*Add version of Stata fix
		//Make sure that keepvars are still saved if saved if the duplicates file
		*	is generated on a subset of the data. For example, duplicates from
		*	version 1, 2 . If the command is run on only version 1. Then values
		*	for keeepvars in version 2 and 3 are dropped and not reloaded as those
		*	obs are not in current memory

		**Test that observations have not been deleted from the report before readind
		* it. Deleted in a way that the report does not make sense. Provide an error
		* message to this that is more informative.

		preserve

			noi di ""

			** Making one macro with all variables that will be
			*  imported and exported from the Excel file
			local agrumentVars `varlist' `uniquevars' `keepvars'

			** When migrating data from Stata to Excel format, time variables loose precision.
			*  If time varaibles are used for uniquely identifying observations, then this loss
			*  of precision might make a time variable unfit to merge the observations back
			*  to Stata after they have been exported to Excel.
			if "`minprecision'" != "" local milliprecision = `minprecision' * 1000 * 60

			local excelvars dupListID dateListed dateFixed correct drop newID initials notes

			local  date = subinstr(c(current_date)," ","",.)

			/***********************************************************************
			************************************************************************

				Section 1

				Storing format and type of the variables that will be imported and
				exported from the Excel file. The export and import function may
				sometimes alter type and format. Merging with different types results
				in an error and format is required for date and time variabels.
				By using the information stored in this loop, the code can enusre
				that the variables have correct type and format before merging.
				For more information, see below.

			************************************************************************
			***********************************************************************/

			local i 0
			foreach var in `agrumentVars' {

				local type_`i'		: type		`var'
				local format_`i'	: format 	`var'

				if substr("`format_`i''",1,2) == "%t" & substr("`format_`i''",1,3) != "%td" {

					if "`minprecision'" != "" replace `var' =  (floor(`var' / `milliprecision')*`milliprecision')
				}

				local ++i
			}

			/***********************************************************************
			************************************************************************

				Section 2

				Saving a version of the data to be used before mergin and
				before correcting duplicates

			************************************************************************
			***********************************************************************/

			tempfile restart
			save 	`restart'

			/***********************************************************************
			************************************************************************

				Section 3

				Import and prepare Corrections (if file exists)

			************************************************************************
			***********************************************************************/

			local suffix _`suffix'

			/******************
				Section 3.1
				Check if earlier report exists.
			******************/

			cap confirm file "`folder'/iedupreport`suffix'.xlsx"

			if !_rc {
				local fileExists 1
			}
			else {
				local fileExists 0
			}

			/******************
				Section 3.2
				If report exist, load file and check input
			******************/

			if `fileExists' {

				*Load excel file. Load all vars as string and use meta data from Section 1
				import excel "`folder'/iedupreport`suffix'.xlsx"	, clear firstrow allstring

				*dupListID is always numeric
				destring dupListID, replace

				/******************
					Section 3.2.1
					Make sure that the
					ID variable and
					the uniquevars are
					not changed since
					last report.
				******************/

				ds
				local existingexcelvars  `r(varlist)'

				if `:list varlist in existingexcelvars' == 0 {

					noi display as error "{phang}ID variable [`varlist'] does not exist in the previously exported Excle file. If you renamed or changed the ID variable, you need to start over with a new file. Rename or move the already existing file. Create a new file and carefully copy any corrections from the old file to the new.{p_end}"
					noi di ""
					error 111
					exit

				}
				if `:list uniquevars in existingexcelvars' == 0 {

					noi display as error "{phang}One or more unique variables in [`uniquevars'] do not exist in the previously exported Excel file. If you renamed or changed any variable used in uniquevars(), you need to start over with a new file. Rename or move the already existing file. Create a new file and carefully copy any corrections from the old file to the new.{p_end}"
					noi di ""
					error 111
					exit
				}

				/******************
					Section 3.2.2
					Make sure that all variables are same
					type and format as in original Stata
					file regardless of how they were
					imported from Excel.
				******************/

				local i 0
				foreach argVar in `agrumentVars' {

					cap confirm variable `argVar' //In case the variable was added since last export
					if !_rc {

						if substr("`type_`i''",1,3) == "str" {

							*No need for any action since all varaibles are loaded as string

						}
						else if substr("`format_`i''",1,2) == "%t" {

							** All variables are loaded as strings. The letters
							*  in date/time varibles makes -destring- not
							*  applicable. The code generates a new variable
							*  that read the date/time string using the date()
							*  and clock() functions. Then the format from
							*  Section 1 is applied to the new variable. The old
							*  string variable is dropped and the newly
							*  generated variable takes its place.
							if substr("`format_`i''",1,3) == "%td" {

								*Read date var from string
								gen double	`argVar'_tmp = date(`argVar', "MDY")
							}
							else {

								*Read time var from string
								gen double	`argVar'_tmp = clock( `argVar', "MDY hm")
								*Manually applying lower precision. Read more in Section 0 for details

								if "`minprecision'" != "" replace `argVar'_tmp =  (floor(`argVar'_tmp / `milliprecision')*`milliprecision')

							}

							** Order the newly generated var after the imported
							*  string var, and then drop the string var
							order 	`argVar'_tmp, after(`argVar')
							drop 	`argVar'

							** Format the new variable to match its format in
							*  the original Stata file and then take away the
							*  _tmp suffix
							format  `argVar'_tmp `format_`i''
							rename 	`argVar'_tmp `argVar'

						}
						else {

							*Destring numeric variables
							destring `argVar' , replace
						}
					}
					local ++i
				}

				/******************
					Section 3.3
					Make sure input is correct
				******************/

				*Temporary variables needed for checking input
				tempvar tmpvar_multiInp tmpvar_inputNotYes tmpvar_maxMultiInp tmpvar_notDrop tempvar_yescorrect tempvar_numcorrect

				*Locals indicating in which ways input is incorrect (if any)
				local local_multiInp 		0
				local local_multiCorr		0
				local local_inputNotYes		0
				local local_notDrop			0

				/******************
					Section 3.3.1
					Make sure there are not too many corrections
				******************/

				* Count the number of corrections (correct drop newID) per
				* observation. Only one correction per observation is allowed.
				egen `tmpvar_multiInp' = rownonmiss(correct drop newID), strok

				*Check that all rows have utmost one correction
				cap assert `tmpvar_multiInp' == 0 | `tmpvar_multiInp' == 1

				if _rc {

					*Error will be outputted below
					local local_multiInp 	1
				}

				/******************
					Section 3.3.2
					Make sure string input is yes or y
				******************/

				* Make string input lower case and change "y" to "yes"
				replace correct = lower(correct)
				replace drop 	= lower(drop)
				replace correct = "yes" if correct 	== "y"
				replace drop 	= "yes" if drop 	== "y"

				*Check that varaibles are wither empty or "yes"
				gen `tmpvar_inputNotYes' = !((correct  == "yes" | correct == "") & (drop  == "yes" | drop == ""))

				cap assert `tmpvar_inputNotYes' == 0
				if _rc {

					*Error will be outputted below
					local local_inputNotYes 	1
				}

				/******************
					Section 3.3.3
					Make sure that either option droprest is specified, or that
					drop was correctly indicated for all observations. i.e.; if
					correct or newID was indicated for at least one duplicate in
					a duplicate group, then all other observations should be
					indicated as drop (unless droprest is specified)

				******************/

				*Check if any other duplicate in duplicate group has at least one correction
				gen `tempvar_yescorrect' = (correct == "yes")
				bys `varlist' : egen `tempvar_numcorrect' =  total(`tempvar_yescorrect')
				count if `tempvar_numcorrect' > 1
				if `r(N)' != 0 local local_multiCorr 1

				*Check if any other duplicate in duplicate group has at least one correction
				bys `varlist' : egen `tmpvar_maxMultiInp' = max(`tmpvar_multiInp')

				*Check that drops are explicitly indicated
				gen `tmpvar_notDrop' = (`tmpvar_multiInp' == 0 & `tmpvar_maxMultiInp' > 0)

				* Check if option droprest is specified
				if "`droprest'" != "" {
					cap assert `tmpvar_notDrop' == 0
					if _rc {

						*Error will be outputted below
						local local_notDrop 	1
					}
				}
				else {

					** Option -droprest- specified. Drop will be changed to yes
					*  for any observations without drop or any other correction
					*  explicitly specified if the observation is in a duplicate
					*  group with at least one observation has a correction
					replace drop 	= "yes" if `tmpvar_notDrop' 	== 1

				}

				/******************
					Section 3.4
					Throw errors if any of the tests were not passed
				******************/

				*Was any error detected
				if `local_multiInp' == 1 | `local_inputNotYes' == 1 | `local_notDrop' == 1 | `local_multiCorr' == 1  {

					*Error multiple input
					if `local_multiInp' == 1 {
						noi {
							display as error "{phang}The following observations have more than one correction. Only one correction (correct, drop or newID) per row is allowed{p_end}"
							list `varlist' dupListID correct drop newID if `tmpvar_multiInp' > 1
							di ""
						}
					}

					*Error multiple correct
					if `local_multiCorr' == 1 {
						noi {
							display as error "{phang}The following observations are in a duplicate group where more than one observation is listed as correct. Only one observation per duplicate group can be correct{p_end}"
							list `varlist' dupListID correct drop newID if `tempvar_numcorrect' > 1
							di ""
						}

					}

					*Error in incorrect string
					if `local_inputNotYes' == 1 {
						noi {
							display as error "{phang}The following observations have an answer in either correct or drop that is neither yes nor y{p_end}"
							list `varlist' dupListID correct drop if `tmpvar_inputNotYes' == 1
							di ""
						}
					}

					*Error is not specfied as drop
					if `local_notDrop' == 1 {
						noi {
							display as error "{phang}The following observations are not explicitly indicated as drop while other duplicates in the same duplicate group are corrected. Either manually indicate as drop or see option droprest{p_end}"
							list `varlist' dupListID correct drop newID if `tmpvar_notDrop' == 1
							di ""
						}
					}

				*Same error for any incorrect input
				error 119
				exit
				}

				*Keep only the variables needed for matching and variables used for input in the Excel file
				keep 	`varlist' `uniquevars' `excelvars'

				*Save imported data set with all corrections
				tempfile imputfile_merge
				save	`imputfile_merge'
			}

			/***********************************************************************
			************************************************************************

				Section 4

				Merge corrections with original data

			************************************************************************
			***********************************************************************/

			*Re-load original
			use `restart', clear

			* Merge original data with imported Excel file (if Excel file exists)
			if `fileExists'  {

				*Create a tempvar for merging results
				tempvar iedup_merge

				*Merge the corrections with the data set
				merge 1:1 `varlist' `uniquevars' using `imputfile_merge', generate(`iedup_merge')

				*Make sure that obsrevations listed in the duplicate report is still in the data set
				cap assert `iedup_merge' != 2

				*Display error message if assertion is not true and some duplicates in the excle file are no longer in the data set
				if _rc {

					display as error "{phang}One or several observations in the Excel report are no longer found in the data set. Do not manually delete duplicates identified by this command. Use the Excel file generated by this command to delete duplicates. And do not use the same Excel report for a new data set. If you create a new data set you need a new location for your Excel file.{p_end}"
					error 9
					exit

				}

				*Explicitly drop temporary variable. Temporary variables might
				*be exported to excel so delete explicitly before that. Only
				*using tempvar here to create a name with no conflicts
				drop `iedup_merge'

			}

			/***********************************************************************
			************************************************************************

				Section 5

				Drop all but one observations that are duplicates in all variables

			************************************************************************
			***********************************************************************/

			tempvar id_string allDup

			/******************
				Section 5.1
				Next section 5.2 needs the ID var in the same type for the
				if statement in duplicates drop. And since all numeric variables
				can be expressed as string, we generate an temporary variable
				that is always string.
			******************/

			*Test if ID var is already string
			cap confirm string variable `varlist'

			*if ID var not string:
			if _rc {
				*Generate string copy of ID var
				tostring `varlist' , generate(`id_string') force
			}

			*if ID var is string:
			else {
				*Simply copy the ID var to the temporary variable
				gen `id_string' = `varlist'
			}

			/******************
				Section 5.2
				Throw errors if any of the tests was not passed
			******************/

			** Generate variables that are not 0 if any observations are
			*  duplicates in all variables
			duplicates tag , gen(`allDup')

			*Test if any observations is duplicates in all variables
			count if `allDup' != 0
			if `r(N)' != 0 {

				* Output message indicating that some observations
				* were dropped automatically as they were duplicates
				* in all variables.
				noi di "{phang}The following IDs are duplicates in all variable so only one version is kept. The other observations in the same duplicate group are automatically dropped:{p_end}"

				*Create a local of all IDs that are to be deleted.
				levelsof `id_string' if `allDup' != 0
				foreach alldupID in `r(levels)' {

					*Output the ID
					noi di "{phang2}ID: `alldupID'{p_end}"

					*Drop all but one duplicates in duplicate
					*groups that are duplicated in all variables
					duplicates drop 	if `id_string' == "`alldupID'",  force
				}

				*Add an empty row after the output
				noi di ""
			}

			** Save data set including dropping duplicates in all variables.
			*  The command returns the data set without these observations.
			save `restart', replace

			/***********************************************************************
			************************************************************************

				Section 6

				Test if there are duplicates in ID var. If any duplicates exist,
				tehn update the Excel file with new and unaddressed cases

			************************************************************************
			***********************************************************************/

			/******************
				Section 6.1
				Test if there are any duplicates in ID var
			******************/

			* Generate variable that is not 0
			* if observation is a duplicate
			tempvar dup
			duplicates tag `varlist', gen(`dup')

			*Test if there are any duplicates
			cap assert `dup'==0
			if _rc {

				/******************
					Section 6.2
					Test if the variables passed as ID var and unique var
					uniquely and fully identify the data set. It should be
					possible to merge corrections back to the main file.

				******************/

				cap isid 	`varlist' `uniquevars'
				if _rc {

					display as error "{phang}ID variable `varlist' does not uniquely identify the observations in the data set together with the uniquevars: `uniquevars'. This is needed for exactly merging the right correction to the right observation{p_end}"
					error 119
					exit
				}

				/******************
					Section 6.3
					Keep only duplicates for the report
				******************/

				*Keep if observation is part of duplicate group
				keep if `dup' != 0

				if `fileExists'  {
					* If Excel file exists keep excel vars and
					* variables passed as arguments in the
					* command
					keep 	`agrumentVars' `excelvars'
				}
				else {
					* Keep only variables passed as arguments in
					* the command and the string ID var as no Excel file exists
					keep 	`agrumentVars'

					*Generate the excel variables used for indicating correction
					foreach excelvar of local excelvars {

						*Create all variables apart from dupListID as string vars
						if "`excelvar'" == "dupListID" {
							gen `excelvar' = .
						}
						else {
							gen `excelvar' = ""
						}
					}

				}

				/******************
					Section 6.4
					Update the excel vars that are not updated manually
				******************/

				* Generate a local that is 1 if there are new duplicates
				local unaddressedNewExcel 0
				count if dateFixed == ""
				if `r(N)' > 0 local unaddressedNewExcel 1

				/******************
					Section 6.4.1 Date variables
				******************/

				* Add date first time duplicvate was identified
				replace dateListed 	= "`date'" if dateListed == ""

				** Add today's date to variable dateFixed if dateFixed
				*  is empty and at least one correction is added
				replace dateFixed 	= "`date'" if dateFixed == "" & (correct != "" | drop != "" | newID != "")

				/******************
					Section 6.4.2 Duplicate report list ID
				******************/

				** Sort after dupListID and after ID var for
				*  duplicates currently without dupListID
				sort dupListID `varlist'

				** Assign dupListID 1 to the top row if no duplicate
				*  list IDs have been generated so far.
				replace dupListID = 1 if _n == 1 & dupListID == .

				** Generate new IDs based on the row above instead of directly
				*  from the row number. That prevents duplicates in the list in
				*  case an observation is deleted. The first observation with
				*  missing value will have an ID that is one digit higher than
				*  the highest ID already in the list
				replace dupListID = dupListID[_n - 1] + 1 if dupListID == .

				/******************
					Section 6.5
					Keep and order the variables and output the Excel files
				******************/

				* If cases unaddressed then update the Excel file
				if `unaddressedNewExcel'  {

					keep 	`agrumentVars' `excelvars'
					order	`varlist' `excelvars' `uniquevars' `keepvars'

					if "`daily'" == "" {

						cap export excel using "`folder'/Daily/iedupreport`suffix'_`date'.xlsx"	, firstrow(variables) replace nolabel
						if _rc {

							display as error "{phang}There is no folder named Daily in `folder'. Either create that folder or see option nodaily{p_end}"
							error 603
							exit

						}

						local daily_output "and a daily copy have been saved to the Daily folder"
					}
					export excel using "`folder'/iedupreport`suffix'.xlsx"					, firstrow(variables) replace  nolabel

					noi di `"{phang}Excel file created at: {browse "`folder'/iedupreport`suffix'.xlsx":`folder'/iedupreport`suffix'.xlsx} `daily_output'{p_end}"'
					noi di ""
				}
			}

		/***********************************************************************
		************************************************************************

			Section 7

			Update the data set and with the new corrections.

		************************************************************************
		***********************************************************************/

		* Load the original data set merged with correction. Duplicates
		* in all variables are already dropped in this data set
		use 	`restart', clear

		* If excel file exists, apply any corrections indicated (if any)
		if `fileExists' {

			/******************
				Section 7.1
				Drop duplicates listed for drop
			******************/

			drop if drop == "yes"

			/******************
				Section 7.2
				Update new ID. ID var can be either numeric or
				string. All numbers can be made strings but not
				all strings can be numeric. Therefore this
				section is complicated.
			******************/

			/******************
				Section 7.2.1
				ID var in original file is string. Either
				newID was imported as string or the variable
				is made string. Easy.
			******************/

			* If ID variable is string in original Stata file
			if substr("`type_0'",1,3) == "str" {

				* Tostring the newID
				tostring newID , replace
				* Replace missing value with the empty string
				replace  newID = "" if newID == "."
				* Update ID
				replace `varlist' = newID if newID != ""
			}

			/******************
				Section 7.2.2
				ID var in original file is numeric. Test first
				if newID is numeric or can be made numeric.
			******************/

			* ID var is numeric
			else {

				** Trying to convert newID. If newID is already numeric, nothing
				*  happens. If it is not possible to make it numeric (having
				*  non-numeric characters), then it will remain as string.
				destring newID, replace
				* Test if newID now is numeric
				cap confirm numeric variable newID

				/******************
					Section 7.2.2.1
					newID is numeric and original ID can easily be updated.
				******************/

				if !_rc {
					replace `varlist' = newID if newID != .
				}

				/******************
					Section 7.2.2.2
					newID cannot be made numeric but origianl ID var is numeric.
					To update original ID var, it has to be made a string, but
					that will be allowed only if option tostringok is specified.
				******************/

				else {

					* Check if -tostringok- is specificed:
					if "`tostringok'" != "" {

						* Make original ID var string
						tostring `varlist' , replace
						* Replace any missing values with empty string
						replace  `varlist' = "" if `varlist' == "."
						* Update the original ID var with value in newID
						replace  `varlist' = newID if newID != ""

					}

					* Error, IDvar can not be updated
					else {

						* Create a local with all non-numeric values
						levelsof newID if missing(real(newID)), local(NaN_values)

						* Output error message
						di as error "{phang}`varlist' is numeric but newID has thes non-numeric values `NaN_values'. Update newID to only contain numeric values or see option tostringok{p_end}"
						error 109
						exit
					}
				}
			}

			/******************
				Section 7.3
				Test that values in newID
				were neither used twice
				nor already existed
			******************/

			* Loop over all values in newID
			levelsof newID
			foreach  newID in `r(levels)' {

				/******************
					Section 7.1
					Different test depending on ID var being string
					or numeric. Count number of observations with each
					of the values used in newID and test that that
					the number is exactly one for each value.
				******************/

				if substr("`type_0'",1,3) == "str" {
					count if `varlist' 	== "`newID'"
				}
				else {
					count if `varlist' 	== `newID'
				}

				if `r(N)'	== 1 {

					*Do nothing, each value in newID should be used exactly once.
				}
				else if `r(N)' 	== 0 {

					di as error "{phang}New ID value `newID' used as a correction in the Excel file is after corrections was never used. Please ensure that `newID' is a valid input. If problem remains, please report this bug to kbjarkefur@worldbank.org{p_end}"
					error 119
					exit
				}
				else {

					di as error "{phang}New ID value `newID' used as a correction in the Excel file is after corrections used in `r(N)' obsevations. New ID value `newID' is already used in original data or is used more than once in the Excel file to correct duplicates{p_end}"
					error 119
					exit
				}
			}

			/******************
				Section 7.4
				Drop Excel vars
			******************/

			drop `excelvars'
		}

		/***********************************************************************
		************************************************************************

			Section 8

			Return the data set without duplicates and
			output information regarding unresolved duplicates.

		************************************************************************
		***********************************************************************/

		* Generate a variable that is 1 if the observation is a duplicate in varlist
		tempvar dropDup
		duplicates tag `varlist',  gen(`dropDup')
		* Generate a list of the IDs that are still duplicates
		levelsof `varlist' 			if `dropDup' != 0 , local(dup_ids) clean
		* Drop the duplicates (they are exported in Excel)
		drop 						if `dropDup' != 0

		* Test if varlist is now uniqely and fully identifying the data set
		cap isid `varlist'
		if _rc {

			di as error "{phang}The data set is not returned with `varlist' uniqely and fully identifying the data set. Please report this bug to kbjarkefur@worldbank.org{p_end}"
			error 119
			exit

		}

		if `:list sizeof dup_ids' == 0 {

			noi di	"{phang}There are no unresolved duplicates in this data set. The data set is returned with `varlist' uniqely and fully identifying the data set.{p_end}"
		}
		else {
			noi di	"{phang}There are `:list sizeof dup_ids' duplicates unresolved. IDs still contining duplicates: `dup_ids'. The unresolved duplicate observations were exported in the Excel file. The data set is returned without those duplicates and with `varlist' uniquely and fully identifying the data set.{p_end}"
		}

		return scalar numDup	= `:list sizeof dup_ids'

		/***********************************************************************
		************************************************************************

			Section 9

			Save data set to be returned outside preserve/restore.
			Preserve/restore is used so that original data is returned
			in case an error is thrown.

		************************************************************************
		***********************************************************************/

		tempfile returndata
		save 	`returndata'

		restore

		** Using restore above to return the data to
		*  the orignal data set in case of error.
		use `returndata', clear

	}
	end

	*! version 2.1 1Nov2016  Kristoffer Bjarkefur kbjarkefur@worldbank.org

	capture program drop ieduplicates
	program ieduplicates , rclass

	qui {

		syntax varname ,  FOLder(string) UNIQUEvars(varlist) [ KEEPvars(varlist) MINprecision(numlist >0) tostringok droprest nodaily SUFfix(string)]

		version 11.0

		*Add version of Stata fix
		//Make sure that keepvars are still saved if saved if the duplicates file
		*	is generated on a subset of the data. For example, duplicates from
		*	version 1, 2 . If the command is run on only version 1. Then values
		*	for keeepvars in version 2 and 3 are dropped and not reloaded as those
		*	obs are not in current memory

		preserve

			noi di ""

			** Making one macro with all variables that will be
			*  imported and exported from the Excel file
			local agrumentVars `varlist' `uniquevars' `keepvars'

			** When migrating data from Stata to Excel format, time variables loose precision.
			*  If time varaibles are used for uniquely identifying observations, then this loss
			*  of precision might make a time variable unfit to merge the observations back
			*  to Stata after they have been exported to Excel.
			if "`minprecision'" != "" local milliprecision = `minprecision' * 1000 * 60

			local excelvars dupListID dateListed dateFixed correct drop newID initials notes

			local  date = subinstr(c(current_date)," ","",.)

			/***********************************************************************
			************************************************************************

				Section 1

				Storing format and type of the variables that will be imported and
				exported from the Excel file. The export and import function may
				sometimes alter type and format. Merging with different types results
				in an error and format is required for date and time variabels.
				By using the information stored in this loop, the code can enusre
				that the variables have correct type and format before merging.
				For more information, see below.

			************************************************************************
			***********************************************************************/

			local i 0
			foreach var in `agrumentVars' {

				local type_`i'		: type		`var'
				local format_`i'	: format 	`var'

				if substr("`format_`i''",1,2) == "%t" & substr("`format_`i''",1,3) != "%td" {

					if "`minprecision'" != "" replace `var' =  (floor(`var' / `milliprecision')*`milliprecision')
				}

				local ++i
			}

			/***********************************************************************
			************************************************************************

				Section 2

				Saving a version of the data to be used before mergin and
				before correcting duplicates

			************************************************************************
			***********************************************************************/

			tempfile restart
			save 	`restart'

			/***********************************************************************
			************************************************************************

				Section 3

				Import and prepare Corrections (if file exists)

			************************************************************************
			***********************************************************************/

			local suffix _`suffix'

			/******************
				Section 3.1
				Check if earlier report exists.
			******************/

			cap confirm file "`folder'/iedupreport`suffix'.xlsx"

			if !_rc {
				local fileExists 1
			}
			else {
				local fileExists 0
			}

			/******************
				Section 3.2
				If report exist, load file and check input
			******************/

			if `fileExists' {

				*Load excel file. Load all vars as string and use meta data from Section 1
				import excel "`folder'/iedupreport`suffix'.xlsx"	, clear firstrow allstring

				*dupListID is always numeric
				destring dupListID, replace

				/******************
					Section 3.2.1
					Make sure that the
					ID variable and
					the uniquevars are
					not changed since
					last report.
				******************/

				ds
				local existingexcelvars  `r(varlist)'

				if `:list varlist in existingexcelvars' == 0 {

					noi display as error "{phang}ID variable [`varlist'] does not exist in the previously exported Excle file. If you renamed or changed the ID variable, you need to start over with a new file. Rename or move the already existing file. Create a new file and carefully copy any corrections from the old file to the new.{p_end}"
					noi di ""
					error 111
					exit

				}
				if `:list uniquevars in existingexcelvars' == 0 {

					noi display as error "{phang}One or more unique variables in [`uniquevars'] do not exist in the previously exported Excel file. If you renamed or changed any variable used in uniquevars(), you need to start over with a new file. Rename or move the already existing file. Create a new file and carefully copy any corrections from the old file to the new.{p_end}"
					noi di ""
					error 111
					exit
				}

				/******************
					Section 3.2.2
					Make sure that all variables are same
					type and format as in original Stata
					file regardless of how they were
					imported from Excel.
				******************/

				local i 0
				foreach argVar in `agrumentVars' {

					cap confirm variable `argVar' //In case the variable was added since last export
					if !_rc {

						if substr("`type_`i''",1,3) == "str" {

							*No need for any action since all varaibles are loaded as string

						}
						else if substr("`format_`i''",1,2) == "%t" {

							** All variables are loaded as strings. The letters
							*  in date/time varibles makes -destring- not
							*  applicable. The code generates a new variable
							*  that read the date/time string using the date()
							*  and clock() functions. Then the format from
							*  Section 1 is applied to the new variable. The old
							*  string variable is dropped and the newly
							*  generated variable takes its place.
							if substr("`format_`i''",1,3) == "%td" {

								*Read date var from string
								gen double	`argVar'_tmp = date(`argVar', "MDY")
							}
							else {

								*Read time var from string
								gen double	`argVar'_tmp = clock( `argVar', "MDY hm")
								*Manually applying lower precision. Read more in Section 0 for details

								if "`minprecision'" != "" replace `argVar'_tmp =  (floor(`argVar'_tmp / `milliprecision')*`milliprecision')

							}

							** Order the newly generated var after the imported
							*  string var, and then drop the string var
							order 	`argVar'_tmp, after(`argVar')
							drop 	`argVar'

							** Format the new variable to match its format in
							*  the original Stata file and then take away the
							*  _tmp suffix
							format  `argVar'_tmp `format_`i''
							rename 	`argVar'_tmp `argVar'

						}
						else {

							*Destring numeric variables
							destring `argVar' , replace
						}
					}
					local ++i
				}

				/******************
					Section 3.3
					Make sure input is correct
				******************/

				*Temporary variables needed for checking input
				tempvar tmpvar_multiInp tmpvar_inputNotYes tmpvar_maxMultiInp tmpvar_notDrop tempvar_yescorrect tempvar_numcorrect

				*Locals indicating in which ways input is incorrect (if any)
				local local_multiInp 		0
				local local_multiCorr		0
				local local_inputNotYes		0
				local local_notDrop			0

				/******************
					Section 3.3.1
					Make sure there are not too many corrections
				******************/

				* Count the number of corrections (correct drop newID) per
				* observation. Only one correction per observation is allowed.
				egen `tmpvar_multiInp' = rownonmiss(correct drop newID), strok

				*Check that all rows have utmost one correction
				cap assert `tmpvar_multiInp' == 0 | `tmpvar_multiInp' == 1

				if _rc {

					*Error will be outputted below
					local local_multiInp 	1
				}

				/******************
					Section 3.3.2
					Make sure string input is yes or y
				******************/

				* Make string input lower case and change "y" to "yes"
				replace correct = lower(correct)
				replace drop 	= lower(drop)
				replace correct = "yes" if correct 	== "y"
				replace drop 	= "yes" if drop 	== "y"

				*Check that varaibles are wither empty or "yes"
				gen `tmpvar_inputNotYes' = !((correct  == "yes" | correct == "") & (drop  == "yes" | drop == ""))

				cap assert `tmpvar_inputNotYes' == 0
				if _rc {

					*Error will be outputted below
					local local_inputNotYes 	1
				}

				/******************
					Section 3.3.3
					Make sure that either option droprest is specified, or that
					drop was correctly indicated for all observations. i.e.; if
					correct or newID was indicated for at least one duplicate in
					a duplicate group, then all other observations should be
					indicated as drop (unless droprest is specified)

				******************/

				*Check if any other duplicate in duplicate group has at least one correction
				gen `tempvar_yescorrect' = (correct == "yes")
				bys `varlist' : egen `tempvar_numcorrect' =  total(`tempvar_yescorrect')
				count if `tempvar_numcorrect' > 1
				if `r(N)' != 0 local local_multiCorr 1

				*Check if any other duplicate in duplicate group has at least one correction
				bys `varlist' : egen `tmpvar_maxMultiInp' = max(`tmpvar_multiInp')

				*Check that drops are explicitly indicated
				gen `tmpvar_notDrop' = (`tmpvar_multiInp' == 0 & `tmpvar_maxMultiInp' > 0)

				* Check if option droprest is specified
				if "`droprest'" != "" {
					cap assert `tmpvar_notDrop' == 0
					if _rc {

						*Error will be outputted below
						local local_notDrop 	1
					}
				}
				else {

					** Option -droprest- specified. Drop will be changed to yes
					*  for any observations without drop or any other correction
					*  explicitly specified if the observation is in a duplicate
					*  group with at least one observation has a correction
					replace drop 	= "yes" if `tmpvar_notDrop' 	== 1

				}

				/******************
					Section 3.4
					Throw errors if any of the tests were not passed
				******************/

				*Was any error detected
				if `local_multiInp' == 1 | `local_inputNotYes' == 1 | `local_notDrop' == 1 | `local_multiCorr' == 1  {

					*Error multiple input
					if `local_multiInp' == 1 {
						noi {
							display as error "{phang}The following observations have more than one correction. Only one correction (correct, drop or newID) per row is allowed{p_end}"
							list `varlist' dupListID correct drop newID if `tmpvar_multiInp' > 1
							di ""
						}
					}

					*Error multiple correct
					if `local_multiCorr' == 1 {
						noi {
							display as error "{phang}The following observations are in a duplicate group where more than one observation is listed as correct. Only one observation per duplicate group can be correct{p_end}"
							list `varlist' dupListID correct drop newID if `tempvar_numcorrect' > 1
							di ""
						}

					}

					*Error in incorrect string
					if `local_inputNotYes' == 1 {
						noi {
							display as error "{phang}The following observations have an answer in either correct or drop that is neither yes nor y{p_end}"
							list `varlist' dupListID correct drop if `tmpvar_inputNotYes' == 1
							di ""
						}
					}

					*Error is not specfied as drop
					if `local_notDrop' == 1 {
						noi {
							display as error "{phang}The following observations are not explicitly indicated as drop while other duplicates in the same duplicate group are corrected. Either manually indicate as drop or see option droprest{p_end}"
							list `varlist' dupListID correct drop newID if `tmpvar_notDrop' == 1
							di ""
						}
					}

				*Same error for any incorrect input
				error 119
				exit
				}

				*Keep only the variables needed for matching and variables used for input in the Excel file
				keep 	`varlist' `uniquevars' `excelvars'

				*Save imported data set with all corrections
				tempfile imputfile_merge
				save	`imputfile_merge'
			}

			/***********************************************************************
			************************************************************************

				Section 4

				Merge corrections with original data

			************************************************************************
			***********************************************************************/

			*Re-load original
			use `restart', clear

			* Merge original data with imported Excel file (if Excel file exists)
			if `fileExists'  {

				*Create a tempvar for merging results
				tempvar iedup_merge

				*Merge the corrections with the data set
				merge 1:1 `varlist' `uniquevars' using `imputfile_merge', generate(`iedup_merge')

				*Make sure that obsrevations listed in the duplicate report is still in the data set
				cap assert `iedup_merge' != 2

				*Display error message if assertion is not true and some duplicates in the excle file are no longer in the data set
				if _rc {

					display as error "{phang}One or several observations in the Excel report are no longer found in the data set. Do not manually delete duplicates identified by this command. Use the Excel file generated by this command to delete duplicates. And do not use the same Excel report for a new data set. If you create a new data set you need a new location for your Excel file.{p_end}"
					error 9
					exit

				}

				*Explicitly drop temporary variable. Temporary variables might
				*be exported to excel so delete explicitly before that. Only
				*using tempvar here to create a name with no conflicts
				drop `iedup_merge'

			}

			/***********************************************************************
			************************************************************************

				Section 5

				Drop all but one observations that are duplicates in all variables

			************************************************************************
			***********************************************************************/

			tempvar id_string allDup

			/******************
				Section 5.1
				Next section 5.2 needs the ID var in the same type for the
				if statement in duplicates drop. And since all numeric variables
				can be expressed as string, we generate an temporary variable
				that is always string.
			******************/

			*Test if ID var is already string
			cap confirm string variable `varlist'

			*if ID var not string:
			if _rc {
				*Generate string copy of ID var
				tostring `varlist' , generate(`id_string') force
			}

			*if ID var is string:
			else {
				*Simply copy the ID var to the temporary variable
				gen `id_string' = `varlist'
			}

			/******************
				Section 5.2
				Throw errors if any of the tests was not passed
			******************/

			** Generate variables that are not 0 if any observations are
			*  duplicates in all variables
			duplicates tag , gen(`allDup')

			*Test if any observations is duplicates in all variables
			count if `allDup' != 0
			if `r(N)' != 0 {

				* Output message indicating that some observations
				* were dropped automatically as they were duplicates
				* in all variables.
				noi di "{phang}The following IDs are duplicates in all variable so only one version is kept. The other observations in the same duplicate group are automatically dropped:{p_end}"

				*Create a local of all IDs that are to be deleted.
				levelsof `id_string' if `allDup' != 0
				foreach alldupID in `r(levels)' {

					*Output the ID
					noi di "{phang2}ID: `alldupID'{p_end}"

					*Drop all but one duplicates in duplicate
					*groups that are duplicated in all variables
					duplicates drop 	if `id_string' == "`alldupID'",  force
				}

				*Add an empty row after the output
				noi di ""
			}

			** Save data set including dropping duplicates in all variables.
			*  The command returns the data set without these observations.
			save `restart', replace

			/***********************************************************************
			************************************************************************

				Section 6

				Test if there are duplicates in ID var. If any duplicates exist,
				tehn update the Excel file with new and unaddressed cases

			************************************************************************
			***********************************************************************/

			/******************
				Section 6.1
				Test if there are any duplicates in ID var
			******************/

			* Generate variable that is not 0
			* if observation is a duplicate
			tempvar dup
			duplicates tag `varlist', gen(`dup')

			*Test if there are any duplicates
			cap assert `dup'==0
			if _rc {

				/******************
					Section 6.2
					Test if the variables passed as ID var and unique var
					uniquely and fully identify the data set. It should be
					possible to merge corrections back to the main file.

				******************/

				cap isid 	`varlist' `uniquevars'
				if _rc {

					display as error "{phang}ID variable `varlist' does not uniquely identify the observations in the data set together with the uniquevars: `uniquevars'. This is needed for exactly merging the right correction to the right observation{p_end}"
					error 119
					exit
				}

				/******************
					Section 6.3
					Keep only duplicates for the report
				******************/

				*Keep if observation is part of duplicate group
				keep if `dup' != 0

				if `fileExists'  {
					* If Excel file exists keep excel vars and
					* variables passed as arguments in the
					* command
					keep 	`agrumentVars' `excelvars'
				}
				else {
					* Keep only variables passed as arguments in
					* the command and the string ID var as no Excel file exists
					keep 	`agrumentVars'

					*Generate the excel variables used for indicating correction
					foreach excelvar of local excelvars {

						*Create all variables apart from dupListID as string vars
						if "`excelvar'" == "dupListID" {
							gen `excelvar' = .
						}
						else {
							gen `excelvar' = ""
						}
					}

				}

				/******************
					Section 6.4
					Update the excel vars that are not updated manually
				******************/

				* Generate a local that is 1 if there are new duplicates
				local unaddressedNewExcel 0
				count if dateFixed == ""
				if `r(N)' > 0 local unaddressedNewExcel 1

				/******************
					Section 6.4.1 Date variables
				******************/

				* Add date first time duplicvate was identified
				replace dateListed 	= "`date'" if dateListed == ""

				** Add today's date to variable dateFixed if dateFixed
				*  is empty and at least one correction is added
				replace dateFixed 	= "`date'" if dateFixed == "" & (correct != "" | drop != "" | newID != "")

				/******************
					Section 6.4.2 Duplicate report list ID
				******************/

				** Sort after dupListID and after ID var for
				*  duplicates currently without dupListID
				sort dupListID `varlist'

				** Assign dupListID 1 to the top row if no duplicate
				*  list IDs have been generated so far.
				replace dupListID = 1 if _n == 1 & dupListID == .

				** Generate new IDs based on the row above instead of directly
				*  from the row number. That prevents duplicates in the list in
				*  case an observation is deleted. The first observation with
				*  missing value will have an ID that is one digit higher than
				*  the highest ID already in the list
				replace dupListID = dupListID[_n - 1] + 1 if dupListID == .

				/******************
					Section 6.5
					Keep and order the variables and output the Excel files
				******************/

				* If cases unaddressed then update the Excel file
				if `unaddressedNewExcel'  {

					keep 	`agrumentVars' `excelvars'
					order	`varlist' `excelvars' `uniquevars' `keepvars'

					if "`daily'" == "" {

						cap export excel using "`folder'/Daily/iedupreport`suffix'_`date'.xlsx"	, firstrow(variables) replace nolabel
						if _rc {

							display as error "{phang}There is no folder named Daily in `folder'. Either create that folder or see option nodaily{p_end}"
							error 603
							exit

						}

						local daily_output "and a daily copy have been saved to the Daily folder"
					}
					export excel using "`folder'/iedupreport`suffix'.xlsx"					, firstrow(variables) replace  nolabel

					noi di `"{phang}Excel file created at: {browse "`folder'/iedupreport`suffix'.xlsx":`folder'/iedupreport`suffix'.xlsx} `daily_output'{p_end}"'
					noi di ""
				}
			}

		/***********************************************************************
		************************************************************************

			Section 7

			Update the data set and with the new corrections.

		************************************************************************
		***********************************************************************/

		* Load the original data set merged with correction. Duplicates
		* in all variables are already dropped in this data set
		use 	`restart', clear

		* If excel file exists, apply any corrections indicated (if any)
		if `fileExists' {

			/******************
				Section 7.1
				Drop duplicates listed for drop
			******************/

			drop if drop == "yes"

			/******************
				Section 7.2
				Update new ID. ID var can be either numeric or
				string. All numbers can be made strings but not
				all strings can be numeric. Therefore this
				section is complicated.
			******************/

			/******************
				Section 7.2.1
				ID var in original file is string. Either
				newID was imported as string or the variable
				is made string. Easy.
			******************/

			* If ID variable is string in original Stata file
			if substr("`type_0'",1,3) == "str" {

				* Tostring the newID
				tostring newID , replace
				* Replace missing value with the empty string
				replace  newID = "" if newID == "."
				* Update ID
				replace `varlist' = newID if newID != ""
			}

			/******************
				Section 7.2.2
				ID var in original file is numeric. Test first
				if newID is numeric or can be made numeric.
			******************/

			* ID var is numeric
			else {

				** Trying to convert newID. If newID is already numeric, nothing
				*  happens. If it is not possible to make it numeric (having
				*  non-numeric characters), then it will remain as string.
				destring newID, replace
				* Test if newID now is numeric
				cap confirm numeric variable newID

				/******************
					Section 7.2.2.1
					newID is numeric and original ID can easily be updated.
				******************/

				if !_rc {
					replace `varlist' = newID if newID != .
				}

				/******************
					Section 7.2.2.2
					newID cannot be made numeric but origianl ID var is numeric.
					To update original ID var, it has to be made a string, but
					that will be allowed only if option tostringok is specified.
				******************/

				else {

					* Check if -tostringok- is specificed:
					if "`tostringok'" != "" {

						* Make original ID var string
						tostring `varlist' , replace
						* Replace any missing values with empty string
						replace  `varlist' = "" if `varlist' == "."
						* Update the original ID var with value in newID
						replace  `varlist' = newID if newID != ""

					}

					* Error, IDvar can not be updated
					else {

						* Create a local with all non-numeric values
						levelsof newID if missing(real(newID)), local(NaN_values)

						* Output error message
						di as error "{phang}`varlist' is numeric but newID has thes non-numeric values `NaN_values'. Update newID to only contain numeric values or see option tostringok{p_end}"
						error 109
						exit
					}
				}
			}

			/******************
				Section 7.3
				Test that values in newID
				were neither used twice
				nor already existed
			******************/

			* Loop over all values in newID
			levelsof newID
			foreach  newID in `r(levels)' {

				/******************
					Section 7.1
					Different test depending on ID var being string
					or numeric. Count number of observations with each
					of the values used in newID and test that that
					the number is exactly one for each value.
				******************/

				if substr("`type_0'",1,3) == "str" {
					count if `varlist' 	== "`newID'"
				}
				else {
					count if `varlist' 	== `newID'
				}

				if `r(N)'	== 1 {

					*Do nothing, each value in newID should be used exactly once.
				}
				else if `r(N)' 	== 0 {

					di as error "{phang}New ID value `newID' used as a correction in the Excel file is after corrections was never used. Please ensure that `newID' is a valid input. If problem remains, please report this bug to kbjarkefur@worldbank.org{p_end}"
					error 119
					exit
				}
				else {

					di as error "{phang}New ID value `newID' used as a correction in the Excel file is after corrections used in `r(N)' obsevations. New ID value `newID' is already used in original data or is used more than once in the Excel file to correct duplicates{p_end}"
					error 119
					exit
				}
			}

			/******************
				Section 7.4
				Drop Excel vars
			******************/

			drop `excelvars'
		}

		/***********************************************************************
		************************************************************************

			Section 8

			Return the data set without duplicates and
			output information regarding unresolved duplicates.

		************************************************************************
		***********************************************************************/

		* Generate a variable that is 1 if the observation is a duplicate in varlist
		tempvar dropDup
		duplicates tag `varlist',  gen(`dropDup')
		* Generate a list of the IDs that are still duplicates
		levelsof `varlist' 			if `dropDup' != 0 , local(dup_ids) clean
		* Drop the duplicates (they are exported in Excel)
		drop 						if `dropDup' != 0

		* Test if varlist is now uniqely and fully identifying the data set
		cap isid `varlist'
		if _rc {

			di as error "{phang}The data set is not returned with `varlist' uniqely and fully identifying the data set. Please report this bug to kbjarkefur@worldbank.org{p_end}"
			error 119
			exit

		}

		if `:list sizeof dup_ids' == 0 {

			noi di	"{phang}There are no unresolved duplicates in this data set. The data set is returned with `varlist' uniqely and fully identifying the data set.{p_end}"
		}
		else {
			noi di	"{phang}There are `:list sizeof dup_ids' duplicates unresolved. IDs still contining duplicates: `dup_ids'. The unresolved duplicate observations were exported in the Excel file. The data set is returned without those duplicates and with `varlist' uniquely and fully identifying the data set.{p_end}"
		}

		return scalar numDup	= `:list sizeof dup_ids'

		/***********************************************************************
		************************************************************************

			Section 9

			Save data set to be returned outside preserve/restore.
			Preserve/restore is used so that original data is returned
			in case an error is thrown.

		************************************************************************
		***********************************************************************/

		tempfile returndata
		save 	`returndata'

		restore

		** Using restore above to return the data to
		*  the orignal data set in case of error.
		use `returndata', clear

	}
	end


Some manual indentation fixes.

Some manual indentation fixes.

Yes, again.
My mix of editors (mg, vim, Notepad++, and Visual Studio) don't get along.

* The variables created will be 'y' and 'x%d' for %d=[1 through max(feature_id)].
* Feature IDs are always positive integers, in svmlight format, according to its source code.

* TODO: rename to svm_use and figure out how to support the dual 'svm use filename' and 'svm use varlist using filename' that the built-in use does
*        it will be possible, just maybe ugly

program _svm, plugin /*load the C extension if not already loaded*/

program define svm_load
  version 13
  syntax using/, [clip]

   quietly {

    * Do the pre-loading, to count how much space we need
    plugin call _svm, "_load" "pre" "`using'"

    * HACK: Stata's various versions all have a hard upper limit on the number of variables; for example StataIC has 2048 (2^11) and StataMP has 2^15
    * ADDITIONALLY, Stata has an off-by-one bug: the max you can actually pass to a C plugin is one less [citation needed]
    * We simply clamp the number of variables to get around this,  leaving room for 1 for the Y variable and 1 to avoid the off-by-one bug
    * This needs to be handled better. Perhaps we should let the user give varlist (but if they don't give it, default to all in the file??)
    if(`=_svm_load_M+1' > `c(max_k_theory)'-1-1) {
      di as error "Warning: your version of Stata will not allow `=_svm_load_M+1' variables nor be able to use the C plugin with that many."
      if("`clip'"!="") {
        di as error "Clamping to `=c(max_k_theory)-1-1'."
        scalar _svm_load_M = `=c(max_k_theory)-1-1-1' /*remember: the extra -1 is to account for the Y column, and the extra extra -1 is the leave room for a prediction column*/
      }
      else {
        exit 1
      }
    }

    * handle error cases; I do this explicitly  so
    if(`=_svm_load_M'<1) {
      * because Stata programming is all with macros, if this is a bad variable it doesn't cause a sensible crash,
      * instead of causes either "invalid syntax" or some sort of mysterious "invalid operation" error
      * (in particular "newlist x1-x0" is invalid)
      * checking this doesn't cover all the ways M can be bad (e.g. it could be a string)
      di as error "Need at least one feature to load"
      exit 1
    }
    if(`=_svm_load_N'<1) {
      * this one
      di as error "Need at least one observation to load"
      exit 1
    }

    * make a new, empty, dataset of exactly the size we need
    clear

    * Make variables y x1 x2 x3 ... x`=_svm_load_M'
    generate double y = .

    * this weird newlist syntax is the official suggestion for making a set of new variables in "help foreach"
    foreach j of newlist x1-x`=_svm_load_M'  {
      * make a new variable named "xj" where j is an integer
      * specify "double" because libsvm uses doubles and the C interface uses doubles, yet the default is floats
      generate double `j' = .
    }

    * Make observations 1 .. `=_svm_load_N'
    * Stata will fill in the missing value for each at this point
    set obs `=_svm_load_N'

    * Delete the "local variables"
    * Do this here in case the next step crashes
    * I am programming in BASIC.
    scalar drop _svm_load_N _svm_load_M

    * Do the actual loading
    * "*" means "all variables". We need to pass this in because in addition to C plugins only being able to read and write to variables that already exist,
    * they can only read and write to variables specified in varlist
    * (mata does not have this sort of restriction.)
    capture plugin call _svm *, "_load" "`using'"

  }
end

* load the given svmlight-format file into memory
* the outcome variable (the first one on each line) is loaded in y, the rest are loaded into x<label>, where <label> is the label listed in the file before each value
* note! this *will* clear your current dataset
* NB: it is not clear to me if it is easier or hard to do this in pure-Stata than to try to jerry-rig C into the mix (the main trouble with C is that extensions cannot create new variables, and we need to create new variables as we discover them)
* it is *definitely* *SLOWER* to do this in pure Stata. svm-train loads the same test dataset in a fraction of a second where this takes 90s (on an SSD and i7).
program define svm_load_purestata

  * this makes macro `using' contain a filename
  syntax using/

  * .svmlight is meant to be a sparse format where variables go missing all the time
  * so we do the possibly-quadratic-runtime thing: add one row at a time to the dataset
  * using the deep magic of "set obs `=_N+1`; replace var = value in l". I think 'in l' means 'in last' but 'in last' doesn't work.
  * tip originally from Nick Cox: http://statalist.1588530.n2.nabble.com/Adding-rows-to-datasheet-td4784525.html
  * I suspect this inefficency is intrinsic.
  *  (libsvm's svm-train.c handles this problem by doing two passes over the data: once to count what it has to load, and twice to actually allocate memory and load it; we should profile for which method is faster in Stata)
  tempname fd
  file open `fd' using "`using'", read text

  * get rid of the old data
  clear

  * we know svmlight files always have exactly one y vector
  generate double y = .

  file read `fd' line
  while r(eof)==0 {
    *display "read `line'" /*DEBUG*/

  	quiet set obs `=_N+1'

  	gettoken Y T : line
  	quiet replace y = `Y' in l
  	*di "T=`T'" /*DEBUG*/

  	* this does [(name, value = X.split(":")) for X in line.split()]
  	* and it puts the results into the table.
  	local j = 1
  	while("`T'" != "") {
          *if(`j' > 10) continue, break /*DEBUG*/
  	  gettoken X T : T
  	  gettoken name X : X, parse(":")
  	  gettoken X value : X, parse(":")
  	  *di "@ `=_N' `name' = `value'" /*DEBUG*/

          capture quiet generate double x`name' = . /*UNCONDITIONALLY make a new variable*/
          capture quiet replace x`name' = `value' in l
  	  if(`=_rc' != 0) continue, break /*something went wrong, probably that we couldn't make a new variable (due to memory or built-in Stata constraints). Just try the next observation*/

			local j = `j' + 1
  	}
  	*list /*DEBUG: see the state after after new observation*/

  	file read `fd' line
  }
  file close `fd'
end

* The variables created will be 'y' and 'x%d' for %d=[1 through max(feature_id)].
* Feature IDs are always positive integers, in svmlight format, according to its source code.

* TODO: rename to svm_use and figure out how to support the dual 'svm use filename' and 'svm use varlist using filename' that the built-in use does
*        it will be possible, just maybe ugly

program _svm, plugin /*load the C extension if not already loaded*/

program define svm_load
  version 13
  syntax using/, [clip]

   quietly {

    * Do the pre-loading, to count how much space we need
    plugin call _svm, "_load" "pre" "`using'"

    * HACK: Stata's various versions all have a hard upper limit on the number of variables; for example StataIC has 2048 (2^11) and StataMP has 2^15
    * ADDITIONALLY, Stata has an off-by-one bug: the max you can actually pass to a C plugin is one less [citation needed]
    * We simply clamp the number of variables to get around this,  leaving room for 1 for the Y variable and 1 to avoid the off-by-one bug
    * This needs to be handled better. Perhaps we should let the user give varlist (but if they don't give it, default to all in the file??)
    if(`=_svm_load_M+1' > `c(max_k_theory)'-1-1) {
      di as error "Warning: your version of Stata will not allow `=_svm_load_M+1' variables nor be able to use the C plugin with that many."
	  di as error "clip=`clip'"
	  if("`clip'"!="") {
        di as error "Clamping to `=c(max_k_theory)-1-1'."
        scalar _svm_load_M = `=c(max_k_theory)-1-1-1' /*remember: the extra -1 is to account for the Y column, and the extra extra -1 is the leave room for a prediction column*/
      }
	  else {
	    exit 1
	  }
    }

    * handle error cases; I do this explicitly  so
    if(`=_svm_load_M'<1) {
      * because Stata programming is all with macros, if this is a bad variable it doesn't cause a sensible crash,
      * instead of causes either "invalid syntax" or some sort of mysterious "invalid operation" error
      * (in particular "newlist x1-x0" is invalid)
      * checking this doesn't cover all the ways M can be bad (e.g. it could be a string)
      di as error "Need at least one feature to load"
      exit 1
    }
    if(`=_svm_load_N'<1) {
      * this one
      di as error "Need at least one observation to load"
      exit 1
    }

    * make a new, empty, dataset of exactly the size we need
    clear

    * Make variables y x1 x2 x3 ... x`=_svm_load_M'
    generate double y = .

    * this weird newlist syntax is the official suggestion for making a set of new variables in "help foreach"
    foreach j of newlist x1-x`=_svm_load_M'  {
      * make a new variable named "xj" where j is an integer
      * specify "double" because libsvm uses doubles and the C interface uses doubles, yet the default is floats
      generate double `j' = .
    }

    * Make observations 1 .. `=_svm_load_N'
    * Stata will fill in the missing value for each at this point
    set obs `=_svm_load_N'

    * Delete the "local variables"
    * Do this here in case the next step crashes
    * I am programming in BASIC.
    scalar drop _svm_load_N _svm_load_M

    * Do the actual loading
    * "*" means "all variables". We need to pass this in because in addition to C plugins only being able to read and write to variables that already exist,
    * they can only read and write to variables specified in varlist
    * (mata does not have this sort of restriction.)
    capture plugin call _svm *, "_load" "`using'"

  }
end

* load the given svmlight-format file into memory
* the outcome variable (the first one on each line) is loaded in y, the rest are loaded into x<label>, where <label> is the label listed in the file before each value
* note! this *will* clear your current dataset
* NB: it is not clear to me if it is easier or hard to do this in pure-Stata than to try to jerry-rig C into the mix (the main trouble with C is that extensions cannot create new variables, and we need to create new variables as we discover them)
* it is *definitely* *SLOWER* to do this in pure Stata. svm-train loads the same test dataset in a fraction of a second where this takes 90s (on an SSD and i7).
program define svm_load_purestata

  * this makes macro `using' contain a filename
  syntax using/

  * .svmlight is meant to be a sparse format where variables go missing all the time
  * so we do the possibly-quadratic-runtime thing: add one row at a time to the dataset
  * using the deep magic of "set obs `=_N+1`; replace var = value in l". I think 'in l' means 'in last' but 'in last' doesn't work.
  * tip originally from Nick Cox: http://statalist.1588530.n2.nabble.com/Adding-rows-to-datasheet-td4784525.html
  * I suspect this inefficency is intrinsic.
  *  (libsvm's svm-train.c handles this problem by doing two passes over the data: once to count what it has to load, and twice to actually allocate memory and load it; we should profile for which method is faster in Stata)
  tempname fd
  file open `fd' using "`using'", read text

  * get rid of the old data
  clear

  * we know svmlight files always have exactly one y vector
  generate double y = .

  file read `fd' line
  while r(eof)==0 {
  	*display "read `line'" /*DEBUG*/

  	quiet set obs `=_N+1'

  	gettoken Y T : line
  	quiet replace y = `Y' in l
  	*di "T=`T'" /*DEBUG*/

  	* this does [(name, value = X.split(":")) for X in line.split()]
  	* and it puts the results into the table.
  	local j = 1
  	while("`T'" != "") {
      *if(`j' > 10) continue, break /*DEBUG*/
  	  gettoken X T : T
  	  gettoken name X : X, parse(":")
  	  gettoken X value : X, parse(":")
  	  *di "@ `=_N' `name' = `value'" /*DEBUG*/

  	  capture quiet generate double x`name' = . /*UNCONDITIONALLY make a new variable*/
    	capture quiet replace x`name' = `value' in l
  	  if(`=_rc' != 0) continue, break /*something went wrong, probably that we couldn't make a new variable (due to memory or built-in Stata constraints). Just try the next observation*/

			local j = `j' + 1
  	}
  	*list /*DEBUG: see the state after after new observation*/

  	file read `fd' line
  }
  file close `fd'
end


Bump version

Bump version

*! version 5.5.0 26jan2018

program reghdfe, eclass
	* Intercept old+version
	cap syntax, version old
	if !c(rc) {
		reghdfe_old, version
		exit
	}

	* Intercept old
	cap syntax anything(everything) [fw aw pw/], [*] old
	if !c(rc) {
		di as error "(running historical version of reghdfe)"
		if ("`weight'"!="") local weightexp [`weight'=`exp']
		reghdfe_old `anything' `weightexp', `options'
		exit
	}

	* Aux. subcommands
	cap syntax, [*]
	if inlist("`options'", "check", "compile", "reload", "update", "version", "requirements") {
		if ("`options'"=="compile") loc args force
		if ("`options'"=="check") loc options compile
		if ("`options'"=="update") {
			loc args 1
			loc options reload
		}
		loc subcmd = proper("`options'")
		`subcmd' `args'
	}
	else if replay() {
		Replay `0'
	}
	else {
		Cleanup 0
		ms_get_version ftools, min_version("2.34.0") // Compile // takes 0.01s to run this useful check (ensures .mlib exists)
		cap noi Estimate `0'
		Cleanup `c(rc)'
	}
end

program Compile
	args force

	* Check dependencies
	ftools, check // in case lftools.mlib does not exist or is outdated
	ms_get_version ftools, min_version("2.34.0")
	ms_get_version reghdfe // save local package_version
	loc list_objects "FixedEffects() fixed_effects() BipartiteGraph()"
	loc list_functions "reghdfe_*() transform_*() accelerate_*() panelmean() panelsolve_*() lsmr()"
	loc list_misc "weighted_quadcolsum() safe_divide() check_convergence() precompute_inv_xx() _st_data_wrapper()"
	// TODO: prefix everything with reghdfe_*

	ms_compile_mata, ///
		package(reghdfe) ///
		version(`package_version') ///
		fun("`list_objects' `list_functions' `list_misc'") ///
		verbose ///
		`force'
end

program Reload
	* Internal debugging tool.
	* Updates dependencies and reghdfe from local path or from github
	* Usage:
	* 	reghdfe, update // from c:\git\..
	* 	reghdfe, reload // from github

	args online
	if ("`online'" == "") loc online 0

	di as text _n "{bf:reghdfe: updating required packages}"
	di as text "{hline 64}"

	* -ftools- https://github.com/sergiocorreia/ftools/
	cap ado uninstall ftools
	if (`online') net install ftools, from("https://github.com/sergiocorreia/ftools/raw/master/src/")
	if (!`online') net install ftools, from("c:\git\ftools\src")
	di as text "{hline 64}"
	ftools, compile
	di as text "{hline 64}"

	* Update -reghdfe-
	di as text _n  _n "{bf:reghdfe: updating self}"
	di as text "{hline 64}"
	qui ado uninstall reghdfe
	if (`online') net install reghdfe, from("https://github.com/sergiocorreia/reghdfe/raw/master/src/")
	if (!`online') net install reghdfe, from("c:\git\reghdfe\src")
	qui which reghdfe
	di as text "{hline 64}"
	reghdfe, compile
	di as text "{hline 64}"

	* Cleaning up
	di as text _n "{bf:Note:} You need to run {stata program drop _all} now."
end

program Version
	which reghdfe
	Requirements
end

program Requirements
	di as text _n "Required packages installed?"
	loc reqs ftools
	// ivreg2 avar tuples group3hdfe
	if (c(stata_version)<13) loc reqs `reqs' boottest

	loc ftools_github "https://github.com/sergiocorreia/ftools/raw/master/src/"

	loc error 0

	foreach req of local reqs {
		loc fn `req'.ado
		cap findfile `fn'
		if (_rc) {
			loc error 1
			di as text "{lalign 20:- `req'}" as error "not" _c
			di as text "    {stata ssc install `req':install from SSC}" _c
			if inlist("`req'", "ftools") {
				loc github ``req'_github'
				di as text `"    {stata `"net install `req', from(`"`github'"')"':install from github}"'
			}
			else {
				di as text // newline
			}
		}
		else {
			di as text "{lalign 20:- `req'}" as text "yes"
		}
	}

	if (`error') exit 601
end

program Cleanup
	args rc
	cap mata: mata drop HDFE
	cap mata: mata drop hdfe_*
	cap drop __temp_reghdfe_resid__
	cap matrix drop reghdfe_statsmatrix
	if (`rc' == 132) {
		di as text "- If you got the {it:parentheses unbalanced} error, note that IV/2SLS was moved to {help ivreghdfe}"
		di as smcl `"- Latest version: {browse "https://github.com/sergiocorreia/ivreghdfe":https://github.com/sergiocorreia/ivreghdfe}"'
		di as smcl `"- SSC version: {stata "net describe ivreghdfe, from(http://fmwww.bc.edu/RePEc/bocode/i)"}"'
		di as smcl `"- Note: the older functionality can still be accessed through the {it:old} option"'
	}
	if (`rc') exit `rc'
end

program Parse
	* Trim whitespace (caused by "///" line continuations; aesthetic only)
	mata: st_local("0", stritrim(st_local("0")))

	* Main syntax
	#d;
	syntax varlist(fv ts numeric) [if] [in] [aw pw fw/] , [

		/* Model */
		Absorb(string) NOAbsorb
		SUmmarize SUmmarize2(string asis) /* simulate implicit options */

		/* Standard Errors */
		VCE(string) CLuster(string)

		/* Diagnostic */
		Verbose(numlist min=1 max=1 >=-1 <=5 integer)
		TIMEit

		/* Speedup and memory Tricks */
		NOSAMPle /* do not save e(sample) */
		COMPACT /* use as little memory as possible but is slower */

		/* Undocumented */
		KEEPSINgletons
		OLD /* use latest v3 */
		NOTES(string) /* NOTES(key=value ...), will be stored on e() */

		] [*] /* capture optimization options, display options, etc. */
		;
	#d cr

	* Unused
	* SAVEcache
	* USEcache
	* CLEARcache

	* Convert options to boolean
	if ("`verbose'" == "") loc verbose 0
	loc timeit = ("`timeit'"!="")
	loc drop_singletons = ("`keepsingletons'" == "")
	loc compact = ("`compact'" != "")

	if (`timeit') timer on 29

	* Sanity checks
	if (`verbose'>-1 & "`keepsingletons'"!="") {
		loc url "http://scorreia.com/reghdfe/nested_within_cluster.pdf"
		loc msg "WARNING: Singleton observations not dropped; statistical significance is biased"
		di as error `"`msg' {browse "`url'":(link)}"'
	}
	if ("`cluster'"!="") {
		_assert ("`vce'"==""), msg("cannot specify both cluster() and vce()")
		loc vce cluster `cluster'
		loc cluster // clear it to avoid bugs in subsequent lines
	}

	* Split varlist into <depvar> and <indepvars>
	ms_parse_varlist `varlist'
	if (`verbose' > 0) {
		di as text _n "## Parsing varlist: {res}`varlist'"
		return list
	}
	loc depvar `r(depvar)'
	loc indepvars `r(indepvars)'
	loc fe_format "`r(fe_format)'"
	loc basevars `r(basevars)'

	* Parse Weights
	if ("`weight'"!="") {
		unab exp : `exp', min(1) max(1) // simple weights only
	}

	* Parse VCE
	ms_parse_vce, vce(`vce') weighttype(`weight')
	if (`verbose' > 0) {
		di as text _n "## Parsing vce({res}`vce'{txt})"
		sreturn list
	}
	loc vcetype = "`s(vcetype)'"
	loc num_clusters = `s(num_clusters)'
	loc clustervars = "`s(clustervars)'"
	loc base_clustervars = "`s(base_clustervars)'"
	loc vceextra = "`s(vceextra)'"

	* Select sample (except for absvars)
	loc varlist `depvar' `indepvars' `base_clustervars'
	tempvar touse
	marksample touse, strok // based on varlist + cluster + if + in + weight

	* Parse noabsorb
	_assert  ("`absorb'`noabsorb'" != ""), msg("option {bf:absorb()} or {bf:noabsorb} required")
	if ("`noabsorb'" != "") {
		_assert ("`absorb'" == ""), msg("{bf:absorb()} and {bf:noabsorb} are mutually exclusive")
	}

	if (`timeit') timer off 29

	* Construct HDFE object
	// SYNTAX: fixed_effects(absvars | , touse, wtype, wtvar, dropsing, verbose)
	ms_add_comma, loc(absorb) cmd(`"`absorb'"') opt(`"`options'"')
	if (`timeit') timer on 20
	mata: HDFE = fixed_effects(`"`absorb'"', "`touse'", "`weight'", "`exp'", `drop_singletons', `verbose')
	if (`timeit') timer off 20
	mata: HDFE.cmdline = "reghdfe " + st_local("0")
	loc options `s(options)'

	mata: st_local("N", strofreal(HDFE.N))
	if (`N' == 0) error 2000

	* Fill out HDFE object
	* mata: HDFE.varlist = "`base_varlist'"
	mata: HDFE.depvar = "`depvar'"
	mata: HDFE.indepvars = "`indepvars'"
	mata: HDFE.vcetype = "`vcetype'"
	mata: HDFE.num_clusters = `num_clusters'
	mata: HDFE.clustervars = tokens("`clustervars'")
	mata: HDFE.base_clustervars = tokens("`base_clustervars'")
	mata: HDFE.vceextra = "`vceextra'"

	* Preserve memory
	mata: HDFE.compact = `compact'
	if (`compact') {
		loc panelvar "`_dta[_TSpanel]'"
		loc timevar "`_dta[_TStvar]'"
		mata: HDFE.panelvar = "`panelvar'"
		mata: HDFE.timevar = "`timevar'"
		c_local keepvars `basevars' `base_clustervars' `panelvar' `timevar' // `exp'
	}

	* Parse summarize
	if ("`summarize'" != "") {
		_assert ("`summarize2'" == ""), msg("summarize() syntax error")
		loc summarize2 mean min max  // default values
	}
	ParseSummarize `summarize2'
	mata: HDFE.summarize_stats = "`s(stats)'"
	mata: HDFE.summarize_quietly = `s(quietly)'

	* Parse misc options
	mata: HDFE.notes = `"`notes'"'
	mata: HDFE.store_sample = ("`nosample'"=="")
	mata: HDFE.timeit = `timeit'

	* Parse Coef Table Options (do this last!)
	_get_diopts diopts options, `options' // store in `diopts', and the rest back to `options'
	_assert (`"`options'"'==""), msg(`"invalid options: `options'"')
	if ("`hascons'"!="") di in ye "(option ignored: `hascons')"
	if ("`tsscons'"!="") di in ye "(option ignored: `tsscons')"
	mata: HDFE.diopts = `"`diopts'"'
end

program ParseSummarize, sclass
	sreturn clear
	syntax [namelist(name=stats)] , [QUIetly]
	local quietly = ("`quietly'"!="")
	sreturn loc stats "`stats'"
	sreturn loc quietly = `quietly'
end

// --------------------------------------------------------------------------

program Estimate, eclass
	ereturn clear

	* Parse and fill out HDFE object
	Parse `0'
	mata: st_local("timeit", strofreal(HDFE.timeit))
	mata: st_local("compact", strofreal(HDFE.compact))
	mata: st_local("verbose", strofreal(HDFE.verbose))

	* Compute degrees-of-freedom
	if (`timeit') timer on 21
	mata: HDFE.estimate_dof()
	if (`timeit') timer off 21

	* Save updated e(sample) (singletons reduce sample);
	* required to parse factor variables to partial out
	if (`timeit') timer on 29
	tempvar touse
	mata: HDFE.save_touse("`touse'")
	if (`timeit') timer off 29

	* Expand varlists
	if (`timeit') timer on 22
	mata: st_local("depvar", HDFE.depvar)
	mata: st_local("indepvars", HDFE.indepvars)
	if (`verbose' > 0) di as text _n "## Parsing and expanding indepvars: {res}`indepvars'"
	ms_expand_varlist `indepvars' if `touse'
	if (`verbose' > 0) return list
	mata: HDFE.fullindepvars = "`r(fullvarlist)'"
	mata: HDFE.indepvars = "`r(varlist)'"
	mata: HDFE.not_basevar = strtoreal(tokens("`r(not_omitted)'"))
	mata: HDFE.varlist = "`depvar' `r(varlist)'"
	if (`timeit') timer off 22

	* Stats
	mata: st_local("stats", HDFE.summarize_stats)
	if ("`stats'" != "") Stats `touse'

	* Condition number
	mata: HDFE.estimate_cond()

	* Preserve
	if (`compact') {
		if (`verbose' > 0) di as text "## Preserving dataset"
		preserve
		keep `keepvars'
	}

	* Partial out; save TSS of depvar
	if (`timeit') timer on 23
	// partial_out(Varlist/Matrix | , Save TSS if HDFE.tss is missing? [0], Standardize data? [1], First col is depvar? [1])
	mata: hdfe_variables = HDFE.partial_out(HDFE.varlist, 1, ., .)
	if (`timeit') timer off 23

	* Regress
	if (`timeit') timer on 24
	tempname b V N rank df_r
	mata: reghdfe_post_ols(HDFE, hdfe_variables, "`b'", "`V'", "`N'", "`rank'", "`df_r'")
	mata: hdfe_variables = .
	* Restore
	if (`compact') {
		if (`verbose' > 0) di as text "## Restoring dataset"
		restore
		mata: st_local("residuals", HDFE.residuals)
		if ("`residuals'" != "") mata: HDFE.save_variable(HDFE.residuals, HDFE.residuals_vector, "Residuals")
	}
	RegressOLS `touse' `b' `V' `N' `rank' `df_r'
	if (`timeit') timer off 24

	* (optional) Store FEs
	if (`timeit') timer on 29
	reghdfe_store_alphas
	if (`timeit') timer off 29

	* View estimation tables
	mata: st_local("diopts", HDFE.diopts)
	Replay, `diopts'

	if (`timeit') {
		di as text _n "{bf: Timer results:}"
		timer list
		di as text "Legend: 20: Create HDFE object; 21: Estimate DoF; 22: expand varlists; 23: partial out; 24: regress; 29: rest"
		di
	}
end

program RegressOLS, eclass
	args touse b V N rank df_r

	mata: st_local("store_sample", strofreal(HDFE.store_sample))
	if (`store_sample') loc esample "esample(`touse')"

	mata: st_local("indepvars", HDFE.fullindepvars)
	if ("`indepvars'" != "") {
		matrix colnames `b' = `indepvars'
		matrix colnames `V' = `indepvars'
		matrix rownames `V' = `indepvars'
		_ms_findomitted `b' `V'
		ereturn post `b' `V', `esample' buildfvinfo depname(`depvar')
	}
	else {
		ereturn post, `esample' buildfvinfo depname(`depvar')
	}

	ereturn scalar N       = `N'
	ereturn scalar rank    = `rank'
	ereturn scalar df_r    = `df_r'
	ereturn local  cmd     "reghdfe"
	mata: HDFE.post()

	* Post stats
	cap conf matrix reghdfe_statsmatrix
	if (!c(rc)) {
		ereturn matrix summarize = reghdfe_statsmatrix
		mata: st_local("summarize_quietly", strofreal(HDFE.summarize_quietly))
		ereturn scalar summarize_quietly = `summarize_quietly'
	}
end

program Replay, rclass
	syntax [, *]

	if `"`e(cmd)'"' != "reghdfe"  {
	        error 301
	}

	_get_diopts options, `options'
	reghdfe_header // _coef_table_header
	di ""
	_coef_table, `options' // ereturn display, `options'
	return add // adds r(level), r(table), etc. to ereturn (before the footnote deletes them)
	reghdfe_footnote

	* Replay stats
	if (e(summarize_quietly)==0) {
		di as text _n "{sf:Regression Summary Statistics:}" _c
		matlist e(summarize)', border(top bottom) rowtitle(Variable) // twidth(18)
	}
end

program Stats
	args touse
	* Optional weights
	mata: st_local("weight", sprintf("[%s=%s]", HDFE.weight_type, HDFE.weight_var))
	assert "`weight'" != ""
	if ("`weight'" == "[=]") loc weight
	loc weight : subinstr local weight "[pweight" "[aweight"

	mata: st_local("stats", HDFE.summarize_stats)
	mata: st_local("varlist", HDFE.varlist)
	mata: st_local("cvars", invtokens(HDFE.cvars))
	loc full_varlist `varlist' `cvars'

	* quick workaround b/c -tabstat- does not support factor variables
	fvrevar `full_varlist', list
	loc full_varlist `r(varlist)'

	qui tabstat `full_varlist' if `touse' `weight' , stat(`stats') col(stat) save
	matrix reghdfe_statsmatrix = r(StatTotal)
end

findfile "reghdfe.mata"
include "`r(fn)'"

exit

*! version 5.4.0 26jan2018

program reghdfe, eclass
	* Intercept old+version
	cap syntax, version old
	if !c(rc) {
		reghdfe_old, version
		exit
	}

	* Intercept old
	cap syntax anything(everything) [fw aw pw/], [*] old
	if !c(rc) {
		di as error "(running historical version of reghdfe)"
		if ("`weight'"!="") local weightexp [`weight'=`exp']
		reghdfe_old `anything' `weightexp', `options'
		exit
	}

	* Aux. subcommands
	cap syntax, [*]
	if inlist("`options'", "check", "compile", "reload", "update", "version", "requirements") {
		if ("`options'"=="compile") loc args force
		if ("`options'"=="check") loc options compile
		if ("`options'"=="update") {
			loc args 1
			loc options reload
		}
		loc subcmd = proper("`options'")
		`subcmd' `args'
	}
	else if replay() {
		Replay `0'
	}
	else {
		Cleanup 0
		//Compile // takes 0.01s to run this useful check (ensures .mlib exists)
		cap noi Estimate `0'
		Cleanup `c(rc)'
	}
end

program Compile
	args force

	* Check dependencies
	ftools, check // in case lftools.mlib does not exist or is outdated
	ms_get_version ftools, min_version("2.33.0")
	ms_get_version reghdfe // save local package_version
	loc list_objects "FixedEffects() fixed_effects() BipartiteGraph()"
	loc list_functions "reghdfe_*() transform_*() accelerate_*() panelmean() panelsolve_*() lsmr()"
	loc list_misc "weighted_quadcolsum() safe_divide() check_convergence() precompute_inv_xx() _st_data_wrapper()"
	// TODO: prefix everything with reghdfe_*

	ms_compile_mata, ///
		package(reghdfe) ///
		version(`package_version') ///
		fun("`list_objects' `list_functions' `list_misc'") ///
		verbose ///
		`force'
end

program Reload
	* Internal debugging tool.
	* Updates dependencies and reghdfe from local path or from github
	* Usage:
	* 	reghdfe, update // from c:\git\..
	* 	reghdfe, reload // from github

	args online
	if ("`online'" == "") loc online 0

	di as text _n "{bf:reghdfe: updating required packages}"
	di as text "{hline 64}"

	* -ftools- https://github.com/sergiocorreia/ftools/
	cap ado uninstall ftools
	if (`online') net install ftools, from("https://github.com/sergiocorreia/ftools/raw/master/src/")
	if (!`online') net install ftools, from("c:\git\ftools\src")
	di as text "{hline 64}"
	ftools, compile
	di as text "{hline 64}"

	* Update -reghdfe-
	di as text _n  _n "{bf:reghdfe: updating self}"
	di as text "{hline 64}"
	qui ado uninstall reghdfe
	if (`online') net install reghdfe, from("https://github.com/sergiocorreia/reghdfe/raw/master/src/")
	if (!`online') net install reghdfe, from("c:\git\reghdfe\src")
	qui which reghdfe
	di as text "{hline 64}"
	reghdfe, compile
	di as text "{hline 64}"

	* Cleaning up
	di as text _n "{bf:Note:} You need to run {stata program drop _all} now."
end

program Version
	which reghdfe
	Requirements
end

program Requirements
	di as text _n "Required packages installed?"
	loc reqs ftools
	// ivreg2 avar tuples group3hdfe
	if (c(stata_version)<13) loc reqs `reqs' boottest

	loc ftools_github "https://github.com/sergiocorreia/ftools/raw/master/src/"

	loc error 0

	foreach req of local reqs {
		loc fn `req'.ado
		cap findfile `fn'
		if (_rc) {
			loc error 1
			di as text "{lalign 20:- `req'}" as error "not" _c
			di as text "    {stata ssc install `req':install from SSC}" _c
			if inlist("`req'", "ftools") {
				loc github ``req'_github'
				di as text `"    {stata `"net install `req', from(`"`github'"')"':install from github}"'
			}
			else {
				di as text // newline
			}
		}
		else {
			di as text "{lalign 20:- `req'}" as text "yes"
		}
	}

	if (`error') exit 601
end

program Cleanup
	args rc
	cap mata: mata drop HDFE
	cap mata: mata drop hdfe_*
	cap drop __temp_reghdfe_resid__
	cap matrix drop reghdfe_statsmatrix
	if (`rc' == 132) {
		di as text "- If you got the {it:parentheses unbalanced} error, note that IV/2SLS was moved to {help ivreghdfe}"
		di as smcl `"- Latest version: {browse "https://github.com/sergiocorreia/ivreghdfe":https://github.com/sergiocorreia/ivreghdfe}"'
		di as smcl `"- SSC version: {stata "net describe ivreghdfe, from(http://fmwww.bc.edu/RePEc/bocode/i)"}"'
		di as smcl `"- Note: the older functionality can still be accessed through the {it:old} option"'
	}
	if (`rc') exit `rc'
end

program Parse
	* Trim whitespace (caused by "///" line continuations; aesthetic only)
	mata: st_local("0", stritrim(st_local("0")))

	* Main syntax
	#d;
	syntax varlist(fv ts numeric) [if] [in] [aw pw fw/] , [

		/* Model */
		Absorb(string) NOAbsorb
		SUmmarize SUmmarize2(string asis) /* simulate implicit options */

		/* Standard Errors */
		VCE(string) CLuster(string)

		/* Diagnostic */
		Verbose(numlist min=1 max=1 >=-1 <=5 integer)
		TIMEit

		/* Speedup and memory Tricks */
		NOSAMPle /* do not save e(sample) */
		COMPACT /* use as little memory as possible but is slower */

		/* Undocumented */
		KEEPSINgletons
		OLD /* use latest v3 */
		NOTES(string) /* NOTES(key=value ...), will be stored on e() */

		] [*] /* capture optimization options, display options, etc. */
		;
	#d cr

	* Unused
	* SAVEcache
	* USEcache
	* CLEARcache

	* Convert options to boolean
	if ("`verbose'" == "") loc verbose 0
	loc timeit = ("`timeit'"!="")
	loc drop_singletons = ("`keepsingletons'" == "")
	loc compact = ("`compact'" != "")

	if (`timeit') timer on 29

	* Sanity checks
	if (`verbose'>-1 & "`keepsingletons'"!="") {
		loc url "http://scorreia.com/reghdfe/nested_within_cluster.pdf"
		loc msg "WARNING: Singleton observations not dropped; statistical significance is biased"
		di as error `"`msg' {browse "`url'":(link)}"'
	}
	if ("`cluster'"!="") {
		_assert ("`vce'"==""), msg("cannot specify both cluster() and vce()")
		loc vce cluster `cluster'
		loc cluster // clear it to avoid bugs in subsequent lines
	}

	* Split varlist into <depvar> and <indepvars>
	ms_parse_varlist `varlist'
	if (`verbose' > 0) {
		di as text _n "## Parsing varlist: {res}`varlist'"
		return list
	}
	loc depvar `r(depvar)'
	loc indepvars `r(indepvars)'
	loc fe_format "`r(fe_format)'"
	loc basevars `r(basevars)'

	* Parse Weights
	if ("`weight'"!="") {
		unab exp : `exp', min(1) max(1) // simple weights only
	}

	* Parse VCE
	ms_parse_vce, vce(`vce') weighttype(`weight')
	if (`verbose' > 0) {
		di as text _n "## Parsing vce({res}`vce'{txt})"
		sreturn list
	}
	loc vcetype = "`s(vcetype)'"
	loc num_clusters = `s(num_clusters)'
	loc clustervars = "`s(clustervars)'"
	loc base_clustervars = "`s(base_clustervars)'"
	loc vceextra = "`s(vceextra)'"

	* Select sample (except for absvars)
	loc varlist `depvar' `indepvars' `base_clustervars'
	tempvar touse
	marksample touse, strok // based on varlist + cluster + if + in + weight

	* Parse noabsorb
	_assert  ("`absorb'`noabsorb'" != ""), msg("option {bf:absorb()} or {bf:noabsorb} required")
	if ("`noabsorb'" != "") {
		_assert ("`absorb'" == ""), msg("{bf:absorb()} and {bf:noabsorb} are mutually exclusive")
	}

	if (`timeit') timer off 29

	* Construct HDFE object
	// SYNTAX: fixed_effects(absvars | , touse, wtype, wtvar, dropsing, verbose)
	ms_add_comma, loc(absorb) cmd(`"`absorb'"') opt(`"`options'"')
	if (`timeit') timer on 20
	mata: HDFE = fixed_effects(`"`absorb'"', "`touse'", "`weight'", "`exp'", `drop_singletons', `verbose')
	if (`timeit') timer off 20
	mata: HDFE.cmdline = "reghdfe " + st_local("0")
	loc options `s(options)'

	mata: st_local("N", strofreal(HDFE.N))
	if (`N' == 0) error 2000

	* Fill out HDFE object
	* mata: HDFE.varlist = "`base_varlist'"
	mata: HDFE.depvar = "`depvar'"
	mata: HDFE.indepvars = "`indepvars'"
	mata: HDFE.vcetype = "`vcetype'"
	mata: HDFE.num_clusters = `num_clusters'
	mata: HDFE.clustervars = tokens("`clustervars'")
	mata: HDFE.base_clustervars = tokens("`base_clustervars'")
	mata: HDFE.vceextra = "`vceextra'"

	* Preserve memory
	mata: HDFE.compact = `compact'
	if (`compact') {
		loc panelvar "`_dta[_TSpanel]'"
		loc timevar "`_dta[_TStvar]'"
		mata: HDFE.panelvar = "`panelvar'"
		mata: HDFE.timevar = "`timevar'"
		c_local keepvars `basevars' `base_clustervars' `panelvar' `timevar' // `exp'
	}

	* Parse summarize
	if ("`summarize'" != "") {
		_assert ("`summarize2'" == ""), msg("summarize() syntax error")
		loc summarize2 mean min max  // default values
	}
	ParseSummarize `summarize2'
	mata: HDFE.summarize_stats = "`s(stats)'"
	mata: HDFE.summarize_quietly = `s(quietly)'

	* Parse misc options
	mata: HDFE.notes = `"`notes'"'
	mata: HDFE.store_sample = ("`nosample'"=="")
	mata: HDFE.timeit = `timeit'

	* Parse Coef Table Options (do this last!)
	_get_diopts diopts options, `options' // store in `diopts', and the rest back to `options'
	_assert (`"`options'"'==""), msg(`"invalid options: `options'"')
	if ("`hascons'"!="") di in ye "(option ignored: `hascons')"
	if ("`tsscons'"!="") di in ye "(option ignored: `tsscons')"
	mata: HDFE.diopts = `"`diopts'"'
end

program ParseSummarize, sclass
	sreturn clear
	syntax [namelist(name=stats)] , [QUIetly]
	local quietly = ("`quietly'"!="")
	sreturn loc stats "`stats'"
	sreturn loc quietly = `quietly'
end

// --------------------------------------------------------------------------

program Estimate, eclass
	ereturn clear

	* Parse and fill out HDFE object
	Parse `0'
	mata: st_local("timeit", strofreal(HDFE.timeit))
	mata: st_local("compact", strofreal(HDFE.compact))
	mata: st_local("verbose", strofreal(HDFE.verbose))

	* Compute degrees-of-freedom
	if (`timeit') timer on 21
	mata: HDFE.estimate_dof()
	if (`timeit') timer off 21

	* Save updated e(sample) (singletons reduce sample);
	* required to parse factor variables to partial out
	if (`timeit') timer on 29
	tempvar touse
	mata: HDFE.save_touse("`touse'")
	if (`timeit') timer off 29

	* Expand varlists
	if (`timeit') timer on 22
	mata: st_local("depvar", HDFE.depvar)
	mata: st_local("indepvars", HDFE.indepvars)
	if (`verbose' > 0) di as text _n "## Parsing and expanding indepvars: {res}`indepvars'"
	ms_expand_varlist `indepvars' if `touse'
	if (`verbose' > 0) return list
	mata: HDFE.fullindepvars = "`r(fullvarlist)'"
	mata: HDFE.indepvars = "`r(varlist)'"
	mata: HDFE.not_basevar = strtoreal(tokens("`r(not_omitted)'"))
	mata: HDFE.varlist = "`depvar' `r(varlist)'"
	if (`timeit') timer off 22

	* Stats
	mata: st_local("stats", HDFE.summarize_stats)
	if ("`stats'" != "") Stats `touse'

	* Condition number
	mata: HDFE.estimate_cond()

	* Preserve
	if (`compact') {
		if (`verbose' > 0) di as text "## Preserving dataset"
		preserve
		keep `keepvars'
	}

	* Partial out; save TSS of depvar
	if (`timeit') timer on 23
	// partial_out(Varlist/Matrix | , Save TSS if HDFE.tss is missing? [0], Standardize data? [1], First col is depvar? [1])
	mata: hdfe_variables = HDFE.partial_out(HDFE.varlist, 1, ., .)
	if (`timeit') timer off 23

	* Regress
	if (`timeit') timer on 24
	tempname b V N rank df_r
	mata: reghdfe_post_ols(HDFE, hdfe_variables, "`b'", "`V'", "`N'", "`rank'", "`df_r'")
	mata: hdfe_variables = .
	* Restore
	if (`compact') {
		if (`verbose' > 0) di as text "## Restoring dataset"
		restore
		mata: st_local("residuals", HDFE.residuals)
		if ("`residuals'" != "") mata: HDFE.save_variable(HDFE.residuals, HDFE.residuals_vector, "Residuals")
	}
	RegressOLS `touse' `b' `V' `N' `rank' `df_r'
	if (`timeit') timer off 24

	* (optional) Store FEs
	if (`timeit') timer on 29
	reghdfe_store_alphas
	if (`timeit') timer off 29

	* View estimation tables
	mata: st_local("diopts", HDFE.diopts)
	Replay, `diopts'

	if (`timeit') {
		di as text _n "{bf: Timer results:}"
		timer list
		di as text "Legend: 20: Create HDFE object; 21: Estimate DoF; 22: expand varlists; 23: partial out; 24: regress; 29: rest"
		di
	}
end

program RegressOLS, eclass
	args touse b V N rank df_r

	mata: st_local("store_sample", strofreal(HDFE.store_sample))
	if (`store_sample') loc esample "esample(`touse')"

	mata: st_local("indepvars", HDFE.fullindepvars)
	if ("`indepvars'" != "") {
		matrix colnames `b' = `indepvars'
		matrix colnames `V' = `indepvars'
		matrix rownames `V' = `indepvars'
		_ms_findomitted `b' `V'
		ereturn post `b' `V', `esample' buildfvinfo depname(`depvar')
	}
	else {
		ereturn post, `esample' buildfvinfo depname(`depvar')
	}

	ereturn scalar N       = `N'
	ereturn scalar rank    = `rank'
	ereturn scalar df_r    = `df_r'
	ereturn local  cmd     "reghdfe"
	mata: HDFE.post()

	* Post stats
	cap conf matrix reghdfe_statsmatrix
	if (!c(rc)) {
		ereturn matrix summarize = reghdfe_statsmatrix
		mata: st_local("summarize_quietly", strofreal(HDFE.summarize_quietly))
		ereturn scalar summarize_quietly = `summarize_quietly'
	}
end

program Replay, rclass
	syntax [, *]

	if `"`e(cmd)'"' != "reghdfe"  {
	        error 301
	}

	_get_diopts options, `options'
	reghdfe_header // _coef_table_header
	di ""
	_coef_table, `options' // ereturn display, `options'
	return add // adds r(level), r(table), etc. to ereturn (before the footnote deletes them)
	reghdfe_footnote

	* Replay stats
	if (e(summarize_quietly)==0) {
		di as text _n "{sf:Regression Summary Statistics:}" _c
		matlist e(summarize)', border(top bottom) rowtitle(Variable) // twidth(18)
	}
end

program Stats
	args touse
	* Optional weights
	mata: st_local("weight", sprintf("[%s=%s]", HDFE.weight_type, HDFE.weight_var))
	assert "`weight'" != ""
	if ("`weight'" == "[=]") loc weight
	loc weight : subinstr local weight "[pweight" "[aweight"

	mata: st_local("stats", HDFE.summarize_stats)
	mata: st_local("varlist", HDFE.varlist)
	mata: st_local("cvars", invtokens(HDFE.cvars))
	loc full_varlist `varlist' `cvars'

	* quick workaround b/c -tabstat- does not support factor variables
	fvrevar `full_varlist', list
	loc full_varlist `r(varlist)'

	qui tabstat `full_varlist' if `touse' `weight' , stat(`stats') col(stat) save
	matrix reghdfe_statsmatrix = r(StatTotal)
end

findfile "reghdfe.mata"
include "`r(fn)'"

exit