[iegitaddmd] pssing skipfolders to recursive call

*! version 6.3 5NOV2019 DIME Analytics dimeanalytics@worldbank.org

cap program drop   iegitaddmd
	program define iegitaddmd

qui {
	syntax , folder(string) [comparefolder(string) customfile(string) all skip replace AUTOmatic DRYrun skipfolders(string)]

	/******************************
	*******************************
		Test user input
	*******************************
	******************************/

	******************************
	*	Test input: folder

	*Test that folder exist
	mata : st_numscalar("r(dirExist)", direxists("`folder'"))

	if (`r(dirExist)' == 0) | (length("`folder'")<=10) {

		noi di as error `"{phang}The folder used in [folder(`folder')] does not exist or you have not entered the full path. For example, full paths on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}. Important: Specify the whole file path to the repository folder, not just {it:C:/} or {it:/user/} as that would create the placeholder file in every empty folder on your computer!{p_end}"'
		error 693
		exit
	}

	* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
	local folderStd			= subinstr(`"`folder'"',"\","/",.)

	******************************
	*	Test input: comparefolder

	if !missing("`comparefolder'") {
		mata : st_numscalar("r(dirExist)", direxists("`comparefolder'"))

		if (`r(dirExist)' == 0) | (length("`comparefolder'")<=10) {

			noi di as error `"{phang}The folder used in [comaparefolder(`comparefolder')] does not exist or you have not entered the full path. For example, full paths on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}. Important: Specify the whole file path to the repository folder, not just {it:C:/} or {it:/user/} as that would create the placeholder file in every empty folder on your computer!{p_end}"'
			error 693
			exit
		}

		* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
		local comparefolderStd	= subinstr(`"`comparefolder'"',"\","/",.)

		*Get the name of the last folder in both folder() and comparefolder()
		local thisLastSlash = strpos(strreverse(`"`folderStd'"'),"/")
		local thisFolder 	= substr(`"`folderStd'"', (-1 * `thisLastSlash')+1 ,.)
		local compLastSlash = strpos(strreverse(`"`comparefolderStd'"'),"/")
		local compFolder 	= substr(`"`comparefolderStd'"', (-1 * `compLastSlash')+1 ,.)

		*The last folder name should always be the same for folder() and
		* comparefolder() otherwise there it is likely that the to paths
		* point to differnet starting points in the two fodler trees that
		* are to be compared.
		if ("`compFolder'" != "`thisFolder'") {
			noi di as error `"{phang}The last folder [`thisFolder'] in [folder(`folder')] is not identical to the last folder [`compFolder'] in [comparefolder(`comparefolder')]. This is an indication that the the two fodler trees to be compared are not two versions of the same folder tree.{p_end}"'
			error 693
			exit
		}
	}

	******************************
	*	Test input: customfile

	*Test that file exist if not using default file
	if "`customfile'" != "" {
		*File option used test if file exists
		capture confirm file "`customfile'"
		if _rc {
			*File does not exist, throw error
			noi di as error `"{phang}File "`customfile'" was not found. Remember that you must enter the full path. For example, on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}.{p_end}"'
			error 693
			exit
		}

		* Get the name of file (include path to keep )
		* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
		local customFileStd	= subinstr("`customfile'","\","/",.)
		*Find the last slash in the file name
		local lastSlash = strpos(strreverse(`"`customFileStd'"'),"/")
		*Use that position to get the file name. Multiply with minus one as we count from the back
		local customFileName = substr(`"`customFileStd'"', (-1 * `lastSlash')+1 ,.)
		*Used for the recursive call of the command when there is a subfolder
		local customFileRecurse `"customfile(`customFileStd')"'

		*Path and name of thile regardless if customfile were used
		local newPlaceholderFile `"`customFileName'"'
	}
	else {
		*Path and name of thile regardless if customfile were not used
		local newPlaceholderFile "README.md"
	}

	******************************
	*	Test input: skip & replace

	*Options skip and replace cannot be used at the same time
	if !missing("`skip'") & !missing("`replace'") {
		noi di as error `"{phang}The options skip and replace may not be used at the same time.{p_end}"'
		error 198
		exit
	}

	*Options all, skip and replace cannot be used together with comparefolder()
	if !missing("`comparefolder'") & (!missing("`all'") | !missing("`skip'") | !missing("`replace'")) {
		noi di as error `"{phang}The options all, skip and replace may not be used when option comparefolder() is used.{p_end}"'
		error 198
		exit
	}

	*Add .git folder to folders to be skipped
	local skipfolders `skipfolders' ".git"

	/******************************
	*******************************
		List Files and folders
	*******************************
	******************************/

	******************************
	*	List all files and folders

	*List files, directories and other files
	local flist : dir `"`folderStd'"' files "*"	, respectcase
	local dlist : dir `"`folderStd'"' dirs  "*" , respectcase
	local olist : dir `"`folderStd'"' other "*"	, respectcase

	/******************************
	*******************************
		Exectute command without compare option
	*******************************
	******************************/

	if missing("`comparefolder'") {

		******************************
		*	Call command recursively on all sub-folders

		*Use the command on each subfolder to this folder (if any)
		foreach dir of local dlist {
			*Test if directory is in
			if `:list dir in skipfolders' {
				noi di as result "{pstd}SKIPPED: Folder [`folder'/`dir'] is skipped. See option skipvars() in {help iegitaddmd}.{p_end}"
			}
			else {
				*Recursive call on each subfolder
				noi iegitaddmd , folder(`"`folderStd'/`dir'"') `all' `customFileRecurse' `skip' `replace' `automatic' `dryrun' skipfolders(`skipfolders')
			}
		}

		******************************
		*	Evaluate if file will be crated in this folder

		*Test if all of those lists are empty, meaning there are
		* no folders or files in this folder
		if `"`flist'`dlist'`olist'"' == "" noi writePlaceholder, folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `automatic' `dryrun'

		*If the folder is not empty, test if option all were used.
		else if ("`all'" == "all") {

			*Test if a file with exactly that name is already used
			cap confirm file `"`folderStd'/`newPlaceholderFile'"'

			* No file with this name exists in this folder, create file without further ado
			if _rc != 0 noi writePlaceholder, folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `automatic' `dryrun'

			* File exist, but no sintructions on how to deal with this has been given, throw error
			else if missing("`skip'") & missing("`replace'")  {
				noi di as error `"{phang}The file  `newPlaceholderFile' already exists. Either remove option {it:all} or use either option {it:skip} or {it:replace}. See help file before using either of these options unless you are already familiar with them.{p_end}"'
				error 602
			}

			* File exist, and instruction is to replace, create files and overwrite as needed
			else if !missing("`replace'") noi writePlaceholder,  folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `automatic' `dryrun'

			* File exist, and instruction is to skip, do nothing
			else if !missing("`skip'") {
					*Nothing is done, this if-block is only included for readability and completion
			}

			* The code should nevere get to this point
			else  {
				noi di as error `"{phang}Coding error in if-chain in "All" block.Please report this error to dimeanalytics@worldbank.org or here: https://github.com/worldbank/ietoolkit/issues/new{p_end}"'
				error 602
			}
		}
		else {
			* Folder is not empty and option all is not used, so do nothing in this folder
		}
	}
	/******************************
	*******************************
		Exectute command with compare option
	*******************************
	******************************/

	else if !missing("`comparefolder'") {

		*List folders that are in comparefolder() but not in in folder()
		local comp_dlist : dir `"`comparefolderStd'"' dirs  "*" , respectcase
		local only_in_cdlist : list comp_dlist - dlist

		*Loop over all folders only in comparefolder()
		foreach dir of local only_in_cdlist {
			*Find all leaf folders. A leaf folder is a folder with no
			*sub-folder. Think of the folder structure as a tree, and
			*each folder with no sub-folders is an end of a branch where
			*you find leaves.
			noi findLeafFolders, folder(`"`comparefolderStd'/`dir'"')

			*Loop over all leaves and create accordingly
			foreach leaf in `r(leaves)' {
				*Leaf exist in compare folder, create the path to the leaf
				* to be created in the corresponding folder
				local folderleaf = subinstr(`"`leaf'"',`"`comparefolderStd'"', `"`folder'"', 1)

				*Create placeholder file in this new leaf
				noi writePlaceholder, folder(`"`folderleaf'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `automatic' `dryrun'
			}
		}

		*When using the comparfolder() option, only recurse over files that exist in
		*both folder() and comparefolder(). Folders only in folder() are not relevant
		*when comapring. And folders only in comaprefolder() have already been addressed.
		local in_both_dlists : list comp_dlist & dlist
		foreach dir of local in_both_dlists {
			*Recursive call on each subfolder
			noi iegitaddmd , folder(`"`folderStd'/`dir'"') comparefolder(`"`comparefolder'/`dir'"') `customFileRecurse' `automatic' `dryrun' skipfolders(`skipfolders')
		}
	}
}

end

*Write a README.md file when iegitaddmd finds an empty folder
cap program drop writePlaceholder
program define   writePlaceholder

qui {
	syntax , folder(string) newfilename(string) [customfiletocopy(string) automatic dryrun]

	*Reset locals to be used in this command
	local dryrun_prompt ""
	local createfile ""

	*Create message to show in dryrun mode
	if !missing("`dryrun'") local dryrun_prompt " NO FILE WILL BE CREATED AS OPTION {bf:dryrun} IS USED!"

	*If manual was used, get manual confirmation for each file
	if missing("`automatic'") {
		noi di ""
		global confirmation "" //Reset global

		*Keep aslking for input until the input is either Y, y, N, n or BREAK
		while (upper("${confirmation}") != "Y" & upper("${confirmation}") != "N" & "${confirmation}" != "BREAK") {
		  noi di as txt "{pstd}You are about to create a file [`newfilename'] in the folder [`folder']. If the folder does not exist it will be created. Do you want to do that? To confirm type {bf:Y} and hit enter, to abort type {bf:N} and hit enter. Type {bf:BREAK} and hit enter to stop the code. See option {help iegitaddmd:automatic} to not be prompted before creating files.`dryrun_prompt'{p_end}", _request(confirmation)
		}
		*Copy user input to local
		local createfile = upper("${confirmation}")
		* If user wrote "BREAK" then exit the code
		if ("`createfile'" == "BREAK") error 1
	}
	*Automtic is used, always create the file
	else local createfile "Y"

	*Manual was used and input was N, no file were creaetd
	if 	("`createfile'" == "N") noi di as result "{pstd}No file or folder created.{p_end}"

	*Dryrun, list where file would have been created
	else if !missing("`dryrun'") noi di as result "{pstd}DRY RUN! Without option {bf:dryrun} file [`folder'/`newfilename'] would have been created.{p_end}"

	*If "manual" were used and input was Y or if manual was not used, create the file
	else if ("`createfile'" == "Y") {

		*Recursively create folder if needed
		noi rmkdir, folder(`"`folder'"')

		*Copy custom file if custom file is used
		if !missing(`"`customfile'"') copy "`customfile'" `"`folder'/`newfilename'"', replace
		*Custom file is not used, so cretae defeault file
		else {
			*Create file
			tempname 	newHandle
			cap file close 	`newHandle'
			file open  	`newHandle' using "`folder'/`newfilename'", text write replace
			*Write the content to the file
			file write  `newHandle' ///
				"# Placeholder file" _n _n ///
				"This file has been created automatically by the command **iegitaddmd** from the Stata package [**ietoolkit**](https://worldbank.github.io/ietoolkit) to make GitHub sync this folder. GitHub does not sync empty folders or folders that only contain ignored files, but in research projects it is often important to share the full standardized folder structure along with the actual files. This command is intended to be used with **iefolder**, but it can be used in any folder structure intended to be shared on GitHub." _n _n ///
				"In recently started projects, it is typical to create data folders, script folders (do-files, r-files) and output folders, among others. The output folders are initially empty, but the script files might include a file path to them for later use. If an output folder is empty and another collaborator clones the repository, then the output folder expected by the scripts will not be included by GitHub in the cloned repository, and the script will not run properly." _n _n ///
				"## You should replace the content of this placeholder file" _n _n ///
				"The text in this file should be replaced with text that describes the intended use of this folder. This file is written in markdown (.md) format, which is suitable for GitHub syncing (unlike .doc/.docx). If the file is named *README.md*, GitHub automatically displays its contents when someone navigates to the containing folder on GitHub.com using a web browser." _n _n ///
				"If you are new to markup languages (markdown, html etc.) then this [Markdown Tutorial](https://www.markdowntutorial.com/) is a great place to start. If you have some experience with markup languages, then this [Markdown Cheat Sheet](https://guides.github.com/pdfs/markdown-cheatsheet-online.pdf) is a great resource." _n _n ///
				"## Add similar files in other folders" _n _n ///
				"The Stata command **iegitaddmd** does not add anything to folders that already have content unless this is explicitly requested using the option `all`. It is best practice to create a *README.md* file in any folders that have content, for example, in folders which purpose might not obvious to someone using the repository for the first time. Again, if the file is named *README.md*, then the content of the file will be shown in the browser when someone explores the repository on GitHub.com. This is a very good way to document your code and your data work." _n _n ///
				"Another great use of a *README.md* file is to use it as a documentation on how the folder it sits in and its subfolders are organized,  where its content can be found, and where new content is meant to be saved. For example, if you have a folder called `/Baseline/`, then you can give a short description of the activities conducted during the baseline and where data, scripts and outputs related to it can be found." _n _n ///
				"## Removing this file" _n _n ///
				"Our recommendation is to not remove this file, as GitHub may stop syncing the parent folder unless the folder now has other content. We recommend to not even remove this file when content that is committed to this repository is added to this folder and the file can be removed without breaking the GitHub functionality, as it is better practice to replace the content of this file with content describing this specific folder rather than deleting it." _n _n

			*Closing the file
			file close 		`newHandle'
		}
		*Output that the file was created
		noi di as result "{pstd}File [`folder'/`newfilename'] created.{p_end}"
	}
}
end

*Write a README.md file when iegitaddmd finds an empty folder
cap program drop findLeafFolders
program define   findLeafFolders, rclass

qui {
	syntax, folder(string)

	local dlist : dir `"`folder'"' dirs  "*" , respectcase

	if missing(`"`dlist'"') {
		*This is a leaf, pass it back
		return local leaves `""`folder'""'
	}
	else {
		local return_leaves
		foreach dir of local dlist {
			*Recursive call on each subfolder
			findLeafFolders, folder(`"`folder'/`dir'"')
			local return_leaves `"`return_leaves' `r(leaves)'"'
		}
		return local leaves `"`return_leaves'"'
	}
}
end

*Recursively call parent folders in folderpath needed to be created until
* folder found that already exist, then creaet all subfolders.
cap program drop rmkdir
program define   rmkdir, rclass

qui {
	syntax, folder(string) [folderstocreate(string)]

	*Test if this folder exists
	mata : st_numscalar("r(dirExist)", direxists(`"`folder'"'))

	*Folder does not exist, find parent folder and make recursive call
	if (`r(dirExist)' == 0) {
		*Get the parent folder of folder
		local lastSlash = strpos(strreverse(`"`folder'"'),"/")
		local parentFolder = substr(`"`folder'"',1,strlen("`folder'")-`lastSlash')
		local thisFolder = substr(`"`folder'"', (-1 * `lastSlash')+1 ,.)

		*Make the coll recursivly on the parent folder and add this folder to folderstocreate()
		noi rmkdir , folder(`"`parentFolder'"') folderstocreate(`""`thisFolder'" `folderstocreate'"')
	}
	*Folder exist, create sub-folders to create if any
	else {
		*Create the folders needed
		foreach dir of local folderstocreate {
			mkdir "`folder'/`dir'"
			local folder "`folder'/`dir'"
		}
	}
}
end

*! version 6.3 5NOV2019 DIME Analytics dimeanalytics@worldbank.org

cap program drop   iegitaddmd
	program define iegitaddmd

qui {
	syntax , folder(string) [comparefolder(string) customfile(string) all skip replace AUTOmatic DRYrun skipfolders(string)]

	/******************************
	*******************************
		Test user input
	*******************************
	******************************/

	******************************
	*	Test input: folder

	*Test that folder exist
	mata : st_numscalar("r(dirExist)", direxists("`folder'"))

	if (`r(dirExist)' == 0) | (length("`folder'")<=10) {

		noi di as error `"{phang}The folder used in [folder(`folder')] does not exist or you have not entered the full path. For example, full paths on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}. Important: Specify the whole file path to the repository folder, not just {it:C:/} or {it:/user/} as that would create the placeholder file in every empty folder on your computer!{p_end}"'
		error 693
		exit
	}

	* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
	local folderStd			= subinstr(`"`folder'"',"\","/",.)

	******************************
	*	Test input: comparefolder

	if !missing("`comparefolder'") {
		mata : st_numscalar("r(dirExist)", direxists("`comparefolder'"))

		if (`r(dirExist)' == 0) | (length("`comparefolder'")<=10) {

			noi di as error `"{phang}The folder used in [comaparefolder(`comparefolder')] does not exist or you have not entered the full path. For example, full paths on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}. Important: Specify the whole file path to the repository folder, not just {it:C:/} or {it:/user/} as that would create the placeholder file in every empty folder on your computer!{p_end}"'
			error 693
			exit
		}

		* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
		local comparefolderStd	= subinstr(`"`comparefolder'"',"\","/",.)

		*Get the name of the last folder in both folder() and comparefolder()
		local thisLastSlash = strpos(strreverse(`"`folderStd'"'),"/")
		local thisFolder 	= substr(`"`folderStd'"', (-1 * `thisLastSlash')+1 ,.)
		local compLastSlash = strpos(strreverse(`"`comparefolderStd'"'),"/")
		local compFolder 	= substr(`"`comparefolderStd'"', (-1 * `compLastSlash')+1 ,.)

		*The last folder name should always be the same for folder() and
		* comparefolder() otherwise there it is likely that the to paths
		* point to differnet starting points in the two fodler trees that
		* are to be compared.
		if ("`compFolder'" != "`thisFolder'") {
			noi di as error `"{phang}The last folder [`thisFolder'] in [folder(`folder')] is not identical to the last folder [`compFolder'] in [comparefolder(`comparefolder')]. This is an indication that the the two fodler trees to be compared are not two versions of the same folder tree.{p_end}"'
			error 693
			exit
		}
	}

	******************************
	*	Test input: customfile

	*Test that file exist if not using default file
	if "`customfile'" != "" {
		*File option used test if file exists
		capture confirm file "`customfile'"
		if _rc {
			*File does not exist, throw error
			noi di as error `"{phang}File "`customfile'" was not found. Remember that you must enter the full path. For example, on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}.{p_end}"'
			error 693
			exit
		}

		* Get the name of file (include path to keep )
		* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
		local customFileStd	= subinstr("`customfile'","\","/",.)
		*Find the last slash in the file name
		local lastSlash = strpos(strreverse(`"`customFileStd'"'),"/")
		*Use that position to get the file name. Multiply with minus one as we count from the back
		local customFileName = substr(`"`customFileStd'"', (-1 * `lastSlash')+1 ,.)
		*Used for the recursive call of the command when there is a subfolder
		local customFileRecurse `"customfile(`customFileStd')"'

		*Path and name of thile regardless if customfile were used
		local newPlaceholderFile `"`customFileName'"'
	}
	else {
		*Path and name of thile regardless if customfile were not used
		local newPlaceholderFile "README.md"
	}

	******************************
	*	Test input: skip & replace

	*Options skip and replace cannot be used at the same time
	if !missing("`skip'") & !missing("`replace'") {
		noi di as error `"{phang}The options skip and replace may not be used at the same time.{p_end}"'
		error 198
		exit
	}

	*Options all, skip and replace cannot be used together with comparefolder()
	if !missing("`comparefolder'") & (!missing("`all'") | !missing("`skip'") | !missing("`replace'")) {
		noi di as error `"{phang}The options all, skip and replace may not be used when option comparefolder() is used.{p_end}"'
		error 198
		exit
	}

	*Add .git folder to folders to be skipped
	local skipfolders `skipfolders' ".git"

	/******************************
	*******************************
		List Files and folders
	*******************************
	******************************/

	******************************
	*	List all files and folders

	*List files, directories and other files
	local flist : dir `"`folderStd'"' files "*"	, respectcase
	local dlist : dir `"`folderStd'"' dirs  "*" , respectcase
	local olist : dir `"`folderStd'"' other "*"	, respectcase

	/******************************
	*******************************
		Exectute command without compare option
	*******************************
	******************************/

	if missing("`comparefolder'") {

		******************************
		*	Call command recursively on all sub-folders

		*Use the command on each subfolder to this folder (if any)
		foreach dir of local dlist {
			*Test if directory is in
			if `:list dir in skipfolders' {
				noi di as result "{pstd}SKIPPED: Folder [`folder'/`dir'] is skipped. See option skipvars() in {help iegitaddmd}.{p_end}"
			}
			else {
				*Recursive call on each subfolder
				noi iegitaddmd , folder(`"`folderStd'/`dir'"') `all' `customFileRecurse' `skip' `replace' `automatic' `dryrun'
			}
		}

		******************************
		*	Evaluate if file will be crated in this folder

		*Test if all of those lists are empty, meaning there are
		* no folders or files in this folder
		if `"`flist'`dlist'`olist'"' == "" noi writePlaceholder, folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `automatic' `dryrun'

		*If the folder is not empty, test if option all were used.
		else if ("`all'" == "all") {

			*Test if a file with exactly that name is already used
			cap confirm file `"`folderStd'/`newPlaceholderFile'"'

			* No file with this name exists in this folder, create file without further ado
			if _rc != 0 noi writePlaceholder, folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `automatic' `dryrun'

			* File exist, but no sintructions on how to deal with this has been given, throw error
			else if missing("`skip'") & missing("`replace'")  {
				noi di as error `"{phang}The file  `newPlaceholderFile' already exists. Either remove option {it:all} or use either option {it:skip} or {it:replace}. See help file before using either of these options unless you are already familiar with them.{p_end}"'
				error 602
			}

			* File exist, and instruction is to replace, create files and overwrite as needed
			else if !missing("`replace'") noi writePlaceholder,  folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `automatic' `dryrun'

			* File exist, and instruction is to skip, do nothing
			else if !missing("`skip'") {
					*Nothing is done, this if-block is only included for readability and completion
			}

			* The code should nevere get to this point
			else  {
				noi di as error `"{phang}Coding error in if-chain in "All" block.Please report this error to dimeanalytics@worldbank.org or here: https://github.com/worldbank/ietoolkit/issues/new{p_end}"'
				error 602
			}
		}
		else {
			* Folder is not empty and option all is not used, so do nothing in this folder
		}
	}
	/******************************
	*******************************
		Exectute command with compare option
	*******************************
	******************************/

	else if !missing("`comparefolder'") {

		*List folders that are in comparefolder() but not in in folder()
		local comp_dlist : dir `"`comparefolderStd'"' dirs  "*" , respectcase
		local only_in_cdlist : list comp_dlist - dlist

		*Loop over all folders only in comparefolder()
		foreach dir of local only_in_cdlist {
			*Find all leaf folders. A leaf folder is a folder with no
			*sub-folder. Think of the folder structure as a tree, and
			*each folder with no sub-folders is an end of a branch where
			*you find leaves.
			noi findLeafFolders, folder(`"`comparefolderStd'/`dir'"')

			*Loop over all leaves and create accordingly
			foreach leaf in `r(leaves)' {
				*Leaf exist in compare folder, create the path to the leaf
				* to be created in the corresponding folder
				local folderleaf = subinstr(`"`leaf'"',`"`comparefolderStd'"', `"`folder'"', 1)

				*Create placeholder file in this new leaf
				noi writePlaceholder, folder(`"`folderleaf'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `automatic' `dryrun'
			}
		}

		*When using the comparfolder() option, only recurse over files that exist in
		*both folder() and comparefolder(). Folders only in folder() are not relevant
		*when comapring. And folders only in comaprefolder() have already been addressed.
		local in_both_dlists : list comp_dlist & dlist
		foreach dir of local in_both_dlists {
			*Recursive call on each subfolder
			noi iegitaddmd , folder(`"`folderStd'/`dir'"') comparefolder(`"`comparefolder'/`dir'"') `customFileRecurse' `automatic' `dryrun'
		}
	}
}

end

*Write a README.md file when iegitaddmd finds an empty folder
cap program drop writePlaceholder
program define   writePlaceholder

qui {
	syntax , folder(string) newfilename(string) [customfiletocopy(string) automatic dryrun]

	*Reset locals to be used in this command
	local dryrun_prompt ""
	local createfile ""

	*Create message to show in dryrun mode
	if !missing("`dryrun'") local dryrun_prompt " NO FILE WILL BE CREATED AS OPTION {bf:dryrun} IS USED!"

	*If manual was used, get manual confirmation for each file
	if missing("`automatic'") {
		noi di ""
		global confirmation "" //Reset global

		*Keep aslking for input until the input is either Y, y, N, n or BREAK
		while (upper("${confirmation}") != "Y" & upper("${confirmation}") != "N" & "${confirmation}" != "BREAK") {
		  noi di as txt "{pstd}You are about to create a file [`newfilename'] in the folder [`folder']. If the folder does not exist it will be created. Do you want to do that? To confirm type {bf:Y} and hit enter, to abort type {bf:N} and hit enter. Type {bf:BREAK} and hit enter to stop the code. See option {help iegitaddmd:automatic} to not be prompted before creating files.`dryrun_prompt'{p_end}", _request(confirmation)
		}
		*Copy user input to local
		local createfile = upper("${confirmation}")
		* If user wrote "BREAK" then exit the code
		if ("`createfile'" == "BREAK") error 1
	}
	*Automtic is used, always create the file
	else local createfile "Y"

	*Manual was used and input was N, no file were creaetd
	if 	("`createfile'" == "N") noi di as result "{pstd}No file or folder created.{p_end}"

	*Dryrun, list where file would have been created
	else if !missing("`dryrun'") noi di as result "{pstd}DRY RUN! Without option {bf:dryrun} file [`folder'/`newfilename'] would have been created.{p_end}"

	*If "manual" were used and input was Y or if manual was not used, create the file
	else if ("`createfile'" == "Y") {

		*Recursively create folder if needed
		noi rmkdir, folder(`"`folder'"')

		*Copy custom file if custom file is used
		if !missing(`"`customfile'"') copy "`customfile'" `"`folder'/`newfilename'"', replace
		*Custom file is not used, so cretae defeault file
		else {
			*Create file
			tempname 	newHandle
			cap file close 	`newHandle'
			file open  	`newHandle' using "`folder'/`newfilename'", text write replace
			*Write the content to the file
			file write  `newHandle' ///
				"# Placeholder file" _n _n ///
				"This file has been created automatically by the command **iegitaddmd** from the Stata package [**ietoolkit**](https://worldbank.github.io/ietoolkit) to make GitHub sync this folder. GitHub does not sync empty folders or folders that only contain ignored files, but in research projects it is often important to share the full standardized folder structure along with the actual files. This command is intended to be used with **iefolder**, but it can be used in any folder structure intended to be shared on GitHub." _n _n ///
				"In recently started projects, it is typical to create data folders, script folders (do-files, r-files) and output folders, among others. The output folders are initially empty, but the script files might include a file path to them for later use. If an output folder is empty and another collaborator clones the repository, then the output folder expected by the scripts will not be included by GitHub in the cloned repository, and the script will not run properly." _n _n ///
				"## You should replace the content of this placeholder file" _n _n ///
				"The text in this file should be replaced with text that describes the intended use of this folder. This file is written in markdown (.md) format, which is suitable for GitHub syncing (unlike .doc/.docx). If the file is named *README.md*, GitHub automatically displays its contents when someone navigates to the containing folder on GitHub.com using a web browser." _n _n ///
				"If you are new to markup languages (markdown, html etc.) then this [Markdown Tutorial](https://www.markdowntutorial.com/) is a great place to start. If you have some experience with markup languages, then this [Markdown Cheat Sheet](https://guides.github.com/pdfs/markdown-cheatsheet-online.pdf) is a great resource." _n _n ///
				"## Add similar files in other folders" _n _n ///
				"The Stata command **iegitaddmd** does not add anything to folders that already have content unless this is explicitly requested using the option `all`. It is best practice to create a *README.md* file in any folders that have content, for example, in folders which purpose might not obvious to someone using the repository for the first time. Again, if the file is named *README.md*, then the content of the file will be shown in the browser when someone explores the repository on GitHub.com. This is a very good way to document your code and your data work." _n _n ///
				"Another great use of a *README.md* file is to use it as a documentation on how the folder it sits in and its subfolders are organized,  where its content can be found, and where new content is meant to be saved. For example, if you have a folder called `/Baseline/`, then you can give a short description of the activities conducted during the baseline and where data, scripts and outputs related to it can be found." _n _n ///
				"## Removing this file" _n _n ///
				"Our recommendation is to not remove this file, as GitHub may stop syncing the parent folder unless the folder now has other content. We recommend to not even remove this file when content that is committed to this repository is added to this folder and the file can be removed without breaking the GitHub functionality, as it is better practice to replace the content of this file with content describing this specific folder rather than deleting it." _n _n

			*Closing the file
			file close 		`newHandle'
		}
		*Output that the file was created
		noi di as result "{pstd}File [`folder'/`newfilename'] created.{p_end}"
	}
}
end

*Write a README.md file when iegitaddmd finds an empty folder
cap program drop findLeafFolders
program define   findLeafFolders, rclass

qui {
	syntax, folder(string)

	local dlist : dir `"`folder'"' dirs  "*" , respectcase

	if missing(`"`dlist'"') {
		*This is a leaf, pass it back
		return local leaves `""`folder'""'
	}
	else {
		local return_leaves
		foreach dir of local dlist {
			*Recursive call on each subfolder
			findLeafFolders, folder(`"`folder'/`dir'"')
			local return_leaves `"`return_leaves' `r(leaves)'"'
		}
		return local leaves `"`return_leaves'"'
	}
}
end

*Recursively call parent folders in folderpath needed to be created until
* folder found that already exist, then creaet all subfolders.
cap program drop rmkdir
program define   rmkdir, rclass

qui {
	syntax, folder(string) [folderstocreate(string)]

	*Test if this folder exists
	mata : st_numscalar("r(dirExist)", direxists(`"`folder'"'))

	*Folder does not exist, find parent folder and make recursive call
	if (`r(dirExist)' == 0) {
		*Get the parent folder of folder
		local lastSlash = strpos(strreverse(`"`folder'"'),"/")
		local parentFolder = substr(`"`folder'"',1,strlen("`folder'")-`lastSlash')
		local thisFolder = substr(`"`folder'"', (-1 * `lastSlash')+1 ,.)

		*Make the coll recursivly on the parent folder and add this folder to folderstocreate()
		noi rmkdir , folder(`"`parentFolder'"') folderstocreate(`""`thisFolder'" `folderstocreate'"')
	}
	*Folder exist, create sub-folders to create if any
	else {
		*Create the folders needed
		foreach dir of local folderstocreate {
			mkdir "`folder'/`dir'"
			local folder "`folder'/`dir'"
		}
	}
}
end


Document import_svmlight, clip

Document import_svmlight, clip

{smcl}
{* *! version 0.0.1  28may2015}{...}
{vieweralsosee "[D] export" "mansection D export"}{...}
{vieweralsosee "[D] import" "mansection D import"}{...}
{viewerjumpto "Syntax" "svm##syntax"}{...}
{viewerjumpto "Description" "svm##description"}{...}
{* {viewerjumpto "Options" "svm##options"}}{...}
{viewerjumpto "Examples" "svm##examples"}{...}
{viewerjumpto "Stored results" "svm##results"}{...}
{viewerjumpto "Gotchas" "svm##gotchas"}{...}
{viewerjumpto "Copyright" "svm##copyright"}{...}
{viewerjumpto "Exeunt" "svm##exeunt"}{...}
{viewerjumpto "References" "svm##references"}{...}
{...}{* NB: these hide the newlines }
{...}
{...}
{title:Title}

{p2colset 5 18 20 2}{...}
{p2col :{cmd:svm} {hline 2}}SVM^Light data format{p_end}
{p2colreset}{...}

{marker syntax}{...}
{title:Syntax}

{p 8 16 2}
export_svmlight {varlist} using {filename}

{p 8 16 2}
import_svmlight using {filename}[, clip]

{synoptset 20 tabbed}{...}
{synopthdr}
{synoptline}
{synopt :{cmd: using}}The filename of the svmlight dataset.{p_end}

{marker description}{...}
{title:Description}

{pstd}
{help svmlight##svmlight} is a SVM software which created a simple de-facto plaintext standard for medium-sized numeric datasets, which was adopted by {help svmlight##libsvm}.
The kind libsvm authors even provide a {browse "http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/":repository} of datasets in compressed svmlight format
(note: this module does not handle compression; you will need to use an external program for that).

{pstd}
{cmd:export_svmlight} saves the current Stata dataset to disk in svmlight format. The first variable is used as the outcome variable, which gets written without an attached column index.

{pstd}
{cmd:import_svmlight} loads the a dataset svmlight format into Stata (clearing the current dataset). The outcome variable will become the first column.
{opt clip} tells the importer to silently drop trailing columns if there are more than {help matsize} allows.

{pstd}
The svmlight format is designed to hold a single outcome variable and a sparse set of predictor variables.
Exporting and reimporting a dataset through svmlight will lose your data labels, description, and all other metadata.
Missing data in your dataset will be skipped during export, and missing "features" will become missing data when imported.

{marker examples}{...}
{title:Examples:  Loading a dataset}

{phang2}{cmd:. !wget http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/duke.bz2}{p_end}
{phang2}{cmd:. !bunzip2 duke.bz2 -c > duke.svmlight}{p_end}
{phang2}{cmd:. import_svmlight using "duke.svmlight"}{p_end}
{phang2}{cmd:. list}{p_end}

{title:Examples:  Saving a dataset}

{phang2}{cmd:. webuse highschool}{p_end}
{phang2}{cmd:. export_svmlight * using "highschool.svmlight"}{p_end}
{phang2}{cmd:. type "highschool.svmlight"}{p_end}

{smcl}
{* *! version 0.0.1  28may2015}{...}
{vieweralsosee "[D] export" "mansection D export"}{...}
{vieweralsosee "[D] import" "mansection D import"}{...}
{viewerjumpto "Syntax" "svm##syntax"}{...}
{viewerjumpto "Description" "svm##description"}{...}
{* {viewerjumpto "Options" "svm##options"}}{...}
{viewerjumpto "Examples" "svm##examples"}{...}
{viewerjumpto "Stored results" "svm##results"}{...}
{viewerjumpto "Gotchas" "svm##gotchas"}{...}
{viewerjumpto "Copyright" "svm##copyright"}{...}
{viewerjumpto "Exeunt" "svm##exeunt"}{...}
{viewerjumpto "References" "svm##references"}{...}
{...}{* NB: these hide the newlines }
{...}
{...}
{title:Title}

{p2colset 5 18 20 2}{...}
{p2col :{cmd:svm} {hline 2}}SVM^Light data format{p_end}
{p2colreset}{...}

{marker syntax}{...}
{title:Syntax}

{p 8 16 2}
export_svmlight {varlist} using {filename}

{p 8 16 2}
import_svmlight {using} {filename}

{synoptset 20 tabbed}{...}
{synopthdr}
{synoptline}
{synopt :{cmd: using}}The filename of the svmlight dataset.{p_end}

{marker description}{...}
{title:Description}

{pstd}
{help svmlight##svmlight} is a SVM software which created a simple de-facto plaintext standard for medium-sized numeric datasets, which was adopted by {help svmlight##libsvm}.
The kind libsvm authors even provide a {browse "http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/":repository} of datasets in compressed svmlight format
(note: this module does not handle compression; you will need to use an external program for that).

{pstd}
{cmd:export_svmlight} saves the current Stata dataset to disk in svmlight format. The first variable is used as the outcome variable, which gets written without an attached column index.

{pstd}
{cmd:import_svmlight} loads the a dataset svmlight format into Stata (clearing the current dataset). The outcome variable will become the first column.

{pstd}
The svmlight format is designed to hold a single outcome variable and a sparse set of predictor variables.
Exporting and reimporting a dataset through svmlight will lose your data labels, description, and all other metadata.
Missing data in your dataset will be skipped during export, and missing "features" will become missing data when imported.

{marker examples}{...}
{title:Examples:  Loading a dataset}

{phang2}{cmd:. !wget http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/duke.bz2}{p_end}
{phang2}{cmd:. !bunzip2 duke.bz2 -c > duke.svmlight}{p_end}
{phang2}{cmd:. import_svmlight using "duke.svmlight"}{p_end}
{phang2}{cmd:. list}{p_end}

{title:Examples:  Saving a dataset}

{phang2}{cmd:. webuse highschool}{p_end}
{phang2}{cmd:. export_svmlight * using "highschool.svmlight"}{p_end}
{phang2}{cmd:. type "highschool.svmlight"}{p_end}


minor tweak to save_touse()

minor tweak to save_touse()

can replace instead of just generate a new var

// --------------------------------------------------------------------------
// FixedEffects main class
// --------------------------------------------------------------------------

mata:

class FixedEffects
{
    // Factors
    `Integer'               G // number of FE levels (2 for two-way-fe, etc.)
    `Integer'               N // number of obs
    `Boolean'               has_intercept
    `Boolean'               save_any_fe
    `Boolean'               save_all_fe
    `Varlist'               absvars
    `Varlist'               ivars
    `Varlist'               cvars
    `Varlist'               targets
    `RowVector'             intercepts
    `RowVector'             num_slopes
    `Factors'               factors
    `Vector'                sample
    `Integer'               num_singletons

    // Weight-specific
    `Boolean'               has_weights
    `String'                weighttype
    `Varname'               weightvar
    `Variabe'               weight // unsorted weight

    // Misc
    `Integer'               verbose
    `Boolean'               timeit

    // Optimization options
    `Real'                  tolerance
    `Integer'               maxiter
    `String'                transform           // Kaczmarz Cimmino Symmetric_kaczmarz (k c s)
    `String'                acceleration        // Acceleration method. None/No/Empty is none\
    `Integer'               accel_start         // Iteration where we start to accelerate // set it at 6? 2?3?
    `string'                slope_method
    `Boolean'               prune               // Whether to recursively prune degree-1 edges
    `Boolean'               abort               // Raise error if convergence failed?

    // Specific to Aitken's acceleration
    `Integer'               accel_freq

    // Used when pruning 1-core vertices
    `BipartiteGraph'        bg

    `Vector'                pruned_weight       // temp. weight for the factors that were pruned
    `Integer'               prune_g1            // Factor 1/2 in the bipartite subgraph that gets pruned
    `Integer'               prune_g2            // Factor 2/2 in the bipartite subgraph that gets pruned

    // Additional options not used here
    `Options'               options

    // Results object
    `Output'                output

    // Methods
    `Void'                  new()
    `Matrix'                partial_out()
    `Void'                  _partial_out()
    `Variables'             project_one_fe()
    `Void'                  prune_1core()
    `Void'                  _expand_1core()
    `Void'                  estimate_dof()
    `Void'                  save_touse()
    `Void'                  save_variable()
}

// Set default value of properties
`Void' FixedEffects::new()
{
    num_singletons = .
    sample = J(0, 1, .)
    weight = 1 // set to 1 so cross(x, S.weight, y)==cross(x, y)

    verbose = 0

    // Optimization defaults
    slope_method = "invsym"
    maxiter = 1e4
    tolerance = 1e-8
    transform = "symmetric_kaczmarz"
    acceleration = "conjugate_gradient"
    accel_start = 6
    output.converged = 0
    prune = 1
    abort = 1

    // Specific to Aitken:
    accel_freq = 3
}

`Variables' FixedEffects::partial_out(`Anything' data, | `Boolean' save_tss)
{
    // -data- is either a varlist or a matrix
    `Variables'             y
    `Varlist'               vars
    `RowVector'             idx
    `Integer'               i

    if (eltype(data) == "string") {
        vars = tokens(invtokens(data))
        if (verbose > 0) printf("\n{txt} ## Partialling out %g variables: {res}%s{txt}\n\n", cols(vars), invtokens(vars))
        if (verbose > 0) printf("{txt}    - Loading variables into Mata\n")
        y = st_data(sample, invtokens(vars))

        if (cols(y) < cols(vars)) {
            printf("{err}(some columns were dropped due to a bug/quirk in st_data() (%g cols created instead of %g for %s); running slower workaround)\n", cols(y), cols(vars), invtokens(vars))
            assert(0) // when does this happen?
        }
        else if (cols(y) > cols(vars)) {
            printf("{err}(some empty columns were added due to a bug/quirk in {bf:st_data()}; %g cols created instead of %g for {it:%s}; running slower workaround)\n", cols(y), cols(vars), invtokens(vars))
            y = J(rows(y), 0, .)
            for (i=1; i<=cols(vars); i++) {
                y = y, st_data(sample, vars[i])
            }
        }

        assert_msg(cols(y)==cols(vars), "st_data() constructed more columns than expected")

        idx = selectindex(colmax(abs(colminmax(y))) :== 0)
        if (cols(idx) & verbose>-1) {
            printf("{err}WARNING: after demeaning, some variables are just zeros: %s\n", invtokens(vars[idx]))
        }

        _partial_out(y, save_tss)
    }
    else {
        if (verbose > 0) printf("\n{txt} ## Partialling out %g variables\n\n", cols(data))
        _partial_out(y=data, save_tss)
    }
    //if (verbose==0) printf("\n")
    if (verbose==0) printf("{txt}(converged in %s iterations)\n", strofreal(output.iteration_count))
    return(y)
}

`Void' FixedEffects::_partial_out(`Variables' y, | `Boolean' save_tss)
{
    `RowVector'             stdevs, needs_zeroing
    `FunctionP'             funct_transform, func_accel // transform
    `Real'                y_mean

    if (args()<2 | save_tss==.) save_tss = 0

    // Solver Warnings
    if (transform=="kaczmarz" & acceleration=="conjugate_gradient") {
        printf("{err}(WARNING: convergence is {bf:unlikely} with transform=kaczmarz and accel=CG)\n")
    }

    // Load transform pointer
    if (transform=="cimmino") funct_transform = &transform_cimmino()
    if (transform=="kaczmarz") funct_transform = &transform_kaczmarz()
    if (transform=="symmetric_kaczmarz") funct_transform = &transform_sym_kaczmarz()
    if (transform=="random_kaczmarz") funct_transform = &transform_rand_kaczmarz()

    // Pointer to acceleration routine
    if (acceleration=="test") func_accel = &accelerate_test()
    if (acceleration=="none") func_accel = &accelerate_none()
    if (acceleration=="conjugate_gradient") func_accel = &accelerate_cg()
    if (acceleration=="steepest_descent") func_accel = &accelerate_sd()
    if (acceleration=="aitken") func_accel = &accelerate_aitken()
    if (acceleration=="hybrid") func_accel = &accelerate_hybrid()

    // Shortcut for trivial case (1 FE)
    if (G==1) func_accel = &accelerate_none()

    // Compute TSS of depvar
    if (save_tss & output.tss==.) {
        if (has_intercept) {
            y_mean = mean(y[., 1], weight)
            output.tss = crossdev(y[., 1], y_mean, weight, y[., 1], y_mean) // Sum of w[i] * (y[i]-y_mean) ^ 2
        }
        else {
            output.tss = cross(y[., 1], weight, y[., 1]) // Sum of w[i] * y[i] ^ 2
        }
        if (weighttype=="aweight" | weighttype=="pweight") output.tss = output.tss * rows(y) / sum(weight)
    }

    // Standardize variables
    if (verbose > 0) printf("{txt}    - Standardizing variables\n")
    stdevs = reghdfe_standardize(y)

    // Solve
    if (verbose>0) printf("{txt}    - Running solver (acceleration={res}%s{txt}, transform={res}%s{txt} tol={res}%-1.0e{txt})\n", acceleration, transform, tolerance)
    if (verbose==1) printf("{txt}    - Iterating:")
    if (verbose>1) printf("{txt}      ")
    output.converged = 0
    output.iteration_count = 0
    y = (*func_accel)(this, y, funct_transform) :* stdevs
    // output.converged gets updated by check_convergence()

    if (prune) {
        _expand_1core(y)
    }

    // Standardizing makes it hard to detect values that are perfectly collinear with the absvars
    // in which case they should be 0.00 but they end up as 1e-16
    // EG: reghdfe price ibn.foreign , absorb(foreign)

    // This will edit to zero entire columns where *ALL* values are very close to zero
    needs_zeroing = colmax(abs(colminmax(y))) :< 1e-8 // chose something relatively close to epsilon() ~ 1e-16
    needs_zeroing = selectindex(needs_zeroing)
    if (cols(needs_zeroing)) {
        y[., needs_zeroing] = J(rows(y), cols(needs_zeroing), 0)
    }

    if (weight==1) weight = J(rows(y), 1, .) // bugbug remove this!??!
}

`Variables' FixedEffects::project_one_fe(`Variables' y, `Integer' g)
{
    `Factor'                f

    // Cons+K+W, Cons+K, K+W, K, Cons+W, Cons = 6 variants

    f = factors[g]

    if (num_slopes[g]==0) {
        // bugbug see if we can remove the .]
        return(panelmean(f.sort(y), f)[f.levels, .])
    }
    else if (intercepts[g]) {
        return(panelsolve_invsym(f.sort(y), f, intercepts[g]))
    }
    else {
        return(panelsolve_invsym(f.sort(y), f, intercepts[g]))
    }
}

`Void' FixedEffects::estimate_dof()
{
    `Boolean'               has_int
    `Integer'               SuperG                      // Number of "extended absvars" (intercepts and slopes in the G absvars)
    `Integer'               g, h                        // index FEs (1..G)
    `Integer'               num_intercepts              // Number of absvars with an intercept term
    `Integer'               i_cluster, i_intercept, j_intercept
    `Integer'               i                           // index 1..SuperG
    `Integer'               j
    `Integer'               bg_verbose                  // verbose level when calling BipartiteGraph()
    `Integer'               m                           // Mobility groups between a specific pair of FEs
    `Integer'               M_due_to_nested             // Used for r2_a r2_a_within rmse
    `RowVector'             SubGs, M, M_is_exact, M_is_nested
    `RowVector'             offsets, idx, zeros, results
    `Matrix'                tmp
    `Variables'             data
    `DataCol'               cluster_data
    `String'                absvar, clustervar
    `Factor'                F
    `BipartiteGraph'        BG

    // `Boolean'               adj_clusters

    if (verbose > 0) printf("\n{txt} ## Estimating degrees-of-freedom absorbed by the fixed effects\n\n")

    // Options
    // adj_clusters = 1
    // adj_first_pair = 1
    // adj_pairwise = 1

    // Count all FE intercepts and slopes
    SubGs = intercepts + num_slopes
    SuperG = sum(SubGs)
    num_intercepts = sum(intercepts)
    offsets = runningsum(SubGs) - SubGs :+ 1 // start of each FE within the extended list
    idx = selectindex(intercepts) // Select all FEs with intercepts
    if (verbose > 0) printf("{txt}    - there are %f fixed intercepts and slopes in the %f absvars\n", SuperG, G)

    // Initialize result vectors and scalars
    M_is_exact = J(1, SuperG, 0)
    M_is_nested = J(1, SuperG, 0)
    M_due_to_nested = 0

    // (1) M will hold the redundant coefs for each extended absvar (G_extended, not G)
    M = J(1, SuperG, 0)

    // (2) (Intercept-Only) Look for absvars that are clustervars
    assert(0 <= options.num_clusters & options.num_clusters <= 10)
    if (options.num_clusters > 0) {
        for (i_intercept=1; i_intercept<=length(idx); i_intercept++) {
            g = idx[i_intercept]
            i = offsets[g]
            absvar = invtokens(tokens(ivars[g]), "#")
            if (anyof(options.clustervars, absvar)) {
                M[i] = factors[g].num_levels
                M_due_to_nested = M_due_to_nested + M[i]
                M_is_exact[i] = M_is_nested[i] = 1
                idx[i_intercept] = 0
                if (verbose > 0) printf("{txt} - categorical variable {res}%s{txt} is also a cluster variable, so it doesn't reduce DoF\n", absvar)
            }
        }
    }
    idx = select(idx, idx)

    // (3) (Intercept-Only) Look for absvars that are nested within a clustervar
    for (i_cluster=1; i_cluster<= options.num_clusters; i_cluster++) {
        cluster_data = .
        if (!length(idx)) break // no more absvars to process
        for (i_intercept=1; i_intercept<=length(idx); i_intercept++) {

            g = idx[i_intercept]
            i = offsets[g]
            absvar = invtokens(tokens(ivars[g]), "#")
            clustervar = options.clustervars[i_cluster]
            if (M_is_exact[i]) continue // nothing to do

            if (cluster_data == .) {
                if (strpos(clustervar, "#")) {
                    clustervar = options.base_clustervars[i_cluster]
                    F = factor(clustervar, sample, ., "", 0, 0, ., 0)
                    cluster_data = F.levels
                    F = Factor() // clear
                }
                else {
                    cluster_data = __fload_data(clustervar, sample)
                }
            }

            if (factors[g].nested_within(cluster_data)) {
                M[i] = factors[g].num_levels
                M_is_exact[i] = M_is_nested[i] = 1
                M_due_to_nested = M_due_to_nested + M[i]
                idx[i_intercept] = 0
                if (verbose > 0) printf("{txt} - categorical variable {res}%s{txt} is nested within a cluster variable, so it doesn't reduce DoF\n", absvar)
            }
        }
        idx = select(idx, idx)
    }
    cluster_data = . // save memory

    // (4) (Intercept-Only) Every intercept but the first has at least one redundant coef.
    if (length(idx) > 1) {
        if (verbose > 0) printf("{txt}    - there is at least one redundant coef. for every set of FE intercepts after the first one\n")
        M[offsets[idx[2..num_intercepts]]] = J(1, num_intercepts-1, 1) // Set DoF loss of all intercepts but the first one to 1
    }

    // (5) (Intercept-only) Mobility group algorithm
    // Excluding those already solved, the first absvar is exact

    if (length(idx)) {
        i = idx[1]
        M_is_exact[i] = 1
    }

    // Compute number of dijsoint subgraphs / mobility groups for each pair of remaining FEs
    BG = BipartiteGraph()
    bg_verbose = max(( verbose - 1 , 0 ))

    for (i_intercept=1; i_intercept<=length(idx)-1; i_intercept++) {
        for (j_intercept=i_intercept+1; j_intercept<=length(idx); j_intercept++) {
            g = idx[i_intercept]
            h = idx[j_intercept]
            i = offsets[h]
            BG.init(&factors[g], &factors[h], bg_verbose)
            m = BG.init_zigzag()
            if (verbose > 0) printf("{txt}    - mobility groups between FE intercepts #%f and #%f: {res}%f\n", g, h, m)
            M[i] = max(( M[i] , m ))
            if (j_intercept==2) M_is_exact[i] = 1
        }
    }
    BG = BipartiteGraph() // clear
    // TODO: add group3hdfe
    // TODO: add option to only do the first pair

    // (6) See if cvars are zero (w/out intercept) or just constant (w/intercept)
    i = 1
    for (g=1; g<=G; g++) {
        // If model has intercept, redundant cvars are those that are CONSTANT
        // Without intercept, a cvar has to be zero within a FE for it to be redundant
        // Since S.fes[g].x are already demeaned IF they have intercept, we don't have to worry about the two cases
        has_int = intercepts[g]
        if (has_int) i++

        if (num_slopes[g]) {
            data = asarray(factors[g].extra, "x")
            assert(num_slopes[g]==cols(data))
            results = J(1, cols(data), 0)
            // float(1.1) - 1 == 2.384e-08 , so let's pick something bigger, 1e-6
            zeros = J(1, cols(data), 1e-6)
            // This can be speed up by moving the -if- outside the -for-
            for (j = 1; j <= factors[g].num_levels; j++) {
                tmp = colminmax(panelsubmatrix(data, j, factors[g].info))
                if (has_int) {
                    results = results + ((tmp[2, .] - tmp[1, .]) :<= zeros)
                }
                else {
                    results = results + (colsum(abs(tmp)) :<= zeros)
                }
            }
            data = .
            if (sum(results)) {
                if (has_int  & verbose) printf("{txt}    - the slopes in the FE #%f are constant for {res}%f{txt} levels, which don't reduce DoF\n", g, sum(results))
                if (!has_int & verbose) printf("{txt}    - the slopes in the FE #%f are zero for {res}%f{txt} levels, which don't reduce DoF\n", g, sum(results))
                M[i..i+num_slopes[g]-1] = results
            }
            i = i + num_slopes[g]
        }
    }

    // Store results
    output.G_extended = SuperG
    output.doflist_M = M
    output.doflist_M_is_exact = M_is_exact
    output.doflist_M_is_nested = M_is_nested
    output.doflist_K = J(1, SuperG, .)
    for (g=1; g<=G; g++) {
        i = offsets[g]
        j = g==G ? SuperG : offsets[g+1]
        output.doflist_K[i..j] = J(1, j-i+1, factors[g].num_levels)
    }

    output.dof_M = sum(M)
    output.df_a = sum(output.doflist_K) - output.dof_M
    output.M_due_to_nested = M_due_to_nested
}

`Void' FixedEffects::prune_1core()
{
    // Note that we can't prune degree-2 nodes, to keep the graph bipartite
    `Integer'               i, j, g
    `Vector'                subgraph_id

    `Vector'                idx
    `RowVector'             i_prune

    assert(G==2) // bugbug remove?

    // Abort if the user set HDFE.prune = 0
    if (!prune) return

    // Pick two factors, and check if we really benefit from pruning
    prune = 0
    i_prune = J(1, 2, 0)
    for (g=i=1; g<=2; g++) {
        //if (intercepts[g] & !num_slopes[g] & factors[g].num_levels>100) {
        if (intercepts[g] & !num_slopes[g]) {
            i_prune[i++] = g // increments at the end
            if (i > 2) { // success!
                prune = 1
                break
            }
        }
    }

    if (!prune) return

    // for speed, the factor with more levels goes first
    i = i_prune[1]
    j = i_prune[2]
    //if (factors[i].num_levels < factors[j].num_levels) swap(i, j) // bugbug uncomment it!
    prune_g1 = i
    prune_g2 = j

    bg = BipartiteGraph()
    bg.init(&factors[prune_g1], &factors[prune_g2], verbose)
    bg.init_zigzag(1) // 1 => save subgraphs into bg.subgraph_id
    bg.compute_cores()
    bg.prune_1core(weight)
}

// bugbug fix or remove this fn altogether
`Void' FixedEffects::_expand_1core(`Variables' y)
{
    y = bg.expand_1core(y)
}

`Void' FixedEffects::save_touse(`Varname' touse, | `Boolean' replace)
{
    `Integer'               idx
    `Vector'                mask

    if (verbose > 0) printf("\n{txt} ## Saving e(sample)\n")
    mask = J(st_nobs(), 1, 0)
    mask[sample] = J(rows(sample), 1, 1)

    if (args()<2 | replace==.) {
        // Generate
        idx = st_addvar("byte", touse)
        st_store(., idx, mask)
    }
    else {
        // Replace
        st_store(., touse, mask)
    }
}

`Void' FixedEffects::save_variable(`Varname' varname,
                                   `Variable' data,
                                 | `String' varlabel)
{
    `Integer'               idx
    `Vector'                mask
    idx = st_addvar("double", varname)
    st_store(sample, idx, data)
    if (args()>=3 & varlabel!="") st_varlabel(idx, varlabel)

}

end

// --------------------------------------------------------------------------
// FixedEffects main class
// --------------------------------------------------------------------------

mata:

class FixedEffects
{
    // Factors
    `Integer'               G // number of FE levels (2 for two-way-fe, etc.)
    `Integer'               N // number of obs
    `Boolean'               has_intercept
    `Boolean'               save_any_fe
    `Boolean'               save_all_fe
    `Varlist'               absvars
    `Varlist'               ivars
    `Varlist'               cvars
    `Varlist'               targets
    `RowVector'             intercepts
    `RowVector'             num_slopes
    `Factors'               factors
    `Vector'                sample
    `Integer'               num_singletons

    // Weight-specific
    `Boolean'               has_weights
    `String'                weighttype
    `Varname'               weightvar
    `Variabe'               weight // unsorted weight

    // Misc
    `Integer'               verbose
    `Boolean'               timeit

    // Optimization options
    `Real'                  tolerance
    `Integer'               maxiter
    `String'                transform           // Kaczmarz Cimmino Symmetric_kaczmarz (k c s)
    `String'                acceleration        // Acceleration method. None/No/Empty is none\
    `Integer'               accel_start         // Iteration where we start to accelerate // set it at 6? 2?3?
    `string'                slope_method
    `Boolean'               prune               // Whether to recursively prune degree-1 edges
    `Boolean'               abort               // Raise error if convergence failed?

    // Specific to Aitken's acceleration
    `Integer'               accel_freq

    // Used when pruning 1-core vertices
    `BipartiteGraph'        bg

    `Vector'                pruned_weight       // temp. weight for the factors that were pruned
    `Integer'               prune_g1            // Factor 1/2 in the bipartite subgraph that gets pruned
    `Integer'               prune_g2            // Factor 2/2 in the bipartite subgraph that gets pruned

    // Additional options not used here
    `Options'               options

    // Results object
    `Output'                output

    // Methods
    `Void'                  new()
    `Matrix'                partial_out()
    `Void'                  _partial_out()
    `Variables'             project_one_fe()
    `Void'                  prune_1core()
    `Void'                  _expand_1core()
    `Void'                  estimate_dof()
    `Void'                  save_touse()
    `Void'                  save_variable()
}

// Set default value of properties
`Void' FixedEffects::new()
{
    num_singletons = .
    sample = J(0, 1, .)
    weight = 1 // set to 1 so cross(x, S.weight, y)==cross(x, y)

    verbose = 0

    // Optimization defaults
    slope_method = "invsym"
    maxiter = 1e4
    tolerance = 1e-8
    transform = "symmetric_kaczmarz"
    acceleration = "conjugate_gradient"
    accel_start = 6
    output.converged = 0
    prune = 1
    abort = 1

    // Specific to Aitken:
    accel_freq = 3
}

`Variables' FixedEffects::partial_out(`Anything' data, | `Boolean' save_tss)
{
    // -data- is either a varlist or a matrix
    `Variables'             y
    `Varlist'               vars
    `RowVector'             idx
    `Integer'               i

    if (eltype(data) == "string") {
        vars = tokens(invtokens(data))
        if (verbose > 0) printf("\n{txt} ## Partialling out %g variables: {res}%s{txt}\n\n", cols(vars), invtokens(vars))
        if (verbose > 0) printf("{txt}    - Loading variables into Mata\n")
        y = st_data(sample, invtokens(vars))

        if (cols(y) < cols(vars)) {
            printf("{err}(some columns were dropped due to a bug/quirk in st_data() (%g cols created instead of %g for %s); running slower workaround)\n", cols(y), cols(vars), invtokens(vars))
            assert(0) // when does this happen?
        }
        else if (cols(y) > cols(vars)) {
            printf("{err}(some empty columns were added due to a bug/quirk in {bf:st_data()}; %g cols created instead of %g for {it:%s}; running slower workaround)\n", cols(y), cols(vars), invtokens(vars))
            y = J(rows(y), 0, .)
            for (i=1; i<=cols(vars); i++) {
                y = y, st_data(sample, vars[i])
            }
        }

        assert_msg(cols(y)==cols(vars), "st_data() constructed more columns than expected")

        idx = selectindex(colmax(abs(colminmax(y))) :== 0)
        if (cols(idx) & verbose>-1) {
            printf("{err}WARNING: after demeaning, some variables are just zeros: %s\n", invtokens(vars[idx]))
        }

        _partial_out(y, save_tss)
    }
    else {
        if (verbose > 0) printf("\n{txt} ## Partialling out %g variables\n\n", cols(data))
        _partial_out(y=data, save_tss)
    }
    //if (verbose==0) printf("\n")
    if (verbose==0) printf("{txt}(converged in %s iterations)\n", strofreal(output.iteration_count))
    return(y)
}

`Void' FixedEffects::_partial_out(`Variables' y, | `Boolean' save_tss)
{
    `RowVector'             stdevs, needs_zeroing
    `FunctionP'             funct_transform, func_accel // transform
    `Real'                y_mean

    if (args()<2 | save_tss==.) save_tss = 0

    // Solver Warnings
    if (transform=="kaczmarz" & acceleration=="conjugate_gradient") {
        printf("{err}(WARNING: convergence is {bf:unlikely} with transform=kaczmarz and accel=CG)\n")
    }

    // Load transform pointer
    if (transform=="cimmino") funct_transform = &transform_cimmino()
    if (transform=="kaczmarz") funct_transform = &transform_kaczmarz()
    if (transform=="symmetric_kaczmarz") funct_transform = &transform_sym_kaczmarz()
    if (transform=="random_kaczmarz") funct_transform = &transform_rand_kaczmarz()

    // Pointer to acceleration routine
    if (acceleration=="test") func_accel = &accelerate_test()
    if (acceleration=="none") func_accel = &accelerate_none()
    if (acceleration=="conjugate_gradient") func_accel = &accelerate_cg()
    if (acceleration=="steepest_descent") func_accel = &accelerate_sd()
    if (acceleration=="aitken") func_accel = &accelerate_aitken()
    if (acceleration=="hybrid") func_accel = &accelerate_hybrid()

    // Shortcut for trivial case (1 FE)
    if (G==1) func_accel = &accelerate_none()

    // Compute TSS of depvar
    if (save_tss & output.tss==.) {
        if (has_intercept) {
            y_mean = mean(y[., 1], weight)
            output.tss = crossdev(y[., 1], y_mean, weight, y[., 1], y_mean) // Sum of w[i] * (y[i]-y_mean) ^ 2
        }
        else {
            output.tss = cross(y[., 1], weight, y[., 1]) // Sum of w[i] * y[i] ^ 2
        }
        if (weighttype=="aweight" | weighttype=="pweight") output.tss = output.tss * rows(y) / sum(weight)
    }

    // Standardize variables
    if (verbose > 0) printf("{txt}    - Standardizing variables\n")
    stdevs = reghdfe_standardize(y)

    // Solve
    if (verbose>0) printf("{txt}    - Running solver (acceleration={res}%s{txt}, transform={res}%s{txt} tol={res}%-1.0e{txt})\n", acceleration, transform, tolerance)
    if (verbose==1) printf("{txt}    - Iterating:")
    if (verbose>1) printf("{txt}      ")
    output.converged = 0
    output.iteration_count = 0
    y = (*func_accel)(this, y, funct_transform) :* stdevs
    // output.converged gets updated by check_convergence()

    if (prune) {
        _expand_1core(y)
    }

    // Standardizing makes it hard to detect values that are perfectly collinear with the absvars
    // in which case they should be 0.00 but they end up as 1e-16
    // EG: reghdfe price ibn.foreign , absorb(foreign)

    // This will edit to zero entire columns where *ALL* values are very close to zero
    needs_zeroing = colmax(abs(colminmax(y))) :< 1e-8 // chose something relatively close to epsilon() ~ 1e-16
    needs_zeroing = selectindex(needs_zeroing)
    if (cols(needs_zeroing)) {
        y[., needs_zeroing] = J(rows(y), cols(needs_zeroing), 0)
    }

    if (weight==1) weight = J(rows(y), 1, .) // bugbug remove this!??!
}

`Variables' FixedEffects::project_one_fe(`Variables' y, `Integer' g)
{
    `Factor'                f

    // Cons+K+W, Cons+K, K+W, K, Cons+W, Cons = 6 variants

    f = factors[g]

    if (num_slopes[g]==0) {
        // bugbug see if we can remove the .]
        return(panelmean(f.sort(y), f)[f.levels, .])
    }
    else if (intercepts[g]) {
        return(panelsolve_invsym(f.sort(y), f, intercepts[g]))
    }
    else {
        return(panelsolve_invsym(f.sort(y), f, intercepts[g]))
    }
}

`Void' FixedEffects::estimate_dof()
{
    `Boolean'               has_int
    `Integer'               SuperG                      // Number of "extended absvars" (intercepts and slopes in the G absvars)
    `Integer'               g, h                        // index FEs (1..G)
    `Integer'               num_intercepts              // Number of absvars with an intercept term
    `Integer'               i_cluster, i_intercept, j_intercept
    `Integer'               i                           // index 1..SuperG
    `Integer'               j
    `Integer'               bg_verbose                  // verbose level when calling BipartiteGraph()
    `Integer'               m                           // Mobility groups between a specific pair of FEs
    `Integer'               M_due_to_nested             // Used for r2_a r2_a_within rmse
    `RowVector'             SubGs, M, M_is_exact, M_is_nested
    `RowVector'             offsets, idx, zeros, results
    `Matrix'                tmp
    `Variables'             data
    `DataCol'               cluster_data
    `String'                absvar, clustervar
    `Factor'                F
    `BipartiteGraph'        BG

    // `Boolean'               adj_clusters

    if (verbose > 0) printf("\n{txt} ## Estimating degrees-of-freedom absorbed by the fixed effects\n\n")

    // Options
    // adj_clusters = 1
    // adj_first_pair = 1
    // adj_pairwise = 1

    // Count all FE intercepts and slopes
    SubGs = intercepts + num_slopes
    SuperG = sum(SubGs)
    num_intercepts = sum(intercepts)
    offsets = runningsum(SubGs) - SubGs :+ 1 // start of each FE within the extended list
    idx = selectindex(intercepts) // Select all FEs with intercepts
    if (verbose > 0) printf("{txt}    - there are %f fixed intercepts and slopes in the %f absvars\n", SuperG, G)

    // Initialize result vectors and scalars
    M_is_exact = J(1, SuperG, 0)
    M_is_nested = J(1, SuperG, 0)
    M_due_to_nested = 0

    // (1) M will hold the redundant coefs for each extended absvar (G_extended, not G)
    M = J(1, SuperG, 0)

    // (2) (Intercept-Only) Look for absvars that are clustervars
    assert(0 <= options.num_clusters & options.num_clusters <= 10)
    if (options.num_clusters > 0) {
        for (i_intercept=1; i_intercept<=length(idx); i_intercept++) {
            g = idx[i_intercept]
            i = offsets[g]
            absvar = invtokens(tokens(ivars[g]), "#")
            if (anyof(options.clustervars, absvar)) {
                M[i] = factors[g].num_levels
                M_due_to_nested = M_due_to_nested + M[i]
                M_is_exact[i] = M_is_nested[i] = 1
                idx[i_intercept] = 0
                if (verbose > 0) printf("{txt} - categorical variable {res}%s{txt} is also a cluster variable, so it doesn't reduce DoF\n", absvar)
            }
        }
    }
    idx = select(idx, idx)

    // (3) (Intercept-Only) Look for absvars that are nested within a clustervar
    for (i_cluster=1; i_cluster<= options.num_clusters; i_cluster++) {
        cluster_data = .
        if (!length(idx)) break // no more absvars to process
        for (i_intercept=1; i_intercept<=length(idx); i_intercept++) {

            g = idx[i_intercept]
            i = offsets[g]
            absvar = invtokens(tokens(ivars[g]), "#")
            clustervar = options.clustervars[i_cluster]
            if (M_is_exact[i]) continue // nothing to do

            if (cluster_data == .) {
                if (strpos(clustervar, "#")) {
                    clustervar = options.base_clustervars[i_cluster]
                    F = factor(clustervar, sample, ., "", 0, 0, ., 0)
                    cluster_data = F.levels
                    F = Factor() // clear
                }
                else {
                    cluster_data = __fload_data(clustervar, sample)
                }
            }

            if (factors[g].nested_within(cluster_data)) {
                M[i] = factors[g].num_levels
                M_is_exact[i] = M_is_nested[i] = 1
                M_due_to_nested = M_due_to_nested + M[i]
                idx[i_intercept] = 0
                if (verbose > 0) printf("{txt} - categorical variable {res}%s{txt} is nested within a cluster variable, so it doesn't reduce DoF\n", absvar)
            }
        }
        idx = select(idx, idx)
    }
    cluster_data = . // save memory

    // (4) (Intercept-Only) Every intercept but the first has at least one redundant coef.
    if (length(idx) > 1) {
        if (verbose > 0) printf("{txt}    - there is at least one redundant coef. for every set of FE intercepts after the first one\n")
        M[offsets[idx[2..num_intercepts]]] = J(1, num_intercepts-1, 1) // Set DoF loss of all intercepts but the first one to 1
    }

    // (5) (Intercept-only) Mobility group algorithm
    // Excluding those already solved, the first absvar is exact

    if (length(idx)) {
        i = idx[1]
        M_is_exact[i] = 1
    }

    // Compute number of dijsoint subgraphs / mobility groups for each pair of remaining FEs
    BG = BipartiteGraph()
    bg_verbose = max(( verbose - 1 , 0 ))

    for (i_intercept=1; i_intercept<=length(idx)-1; i_intercept++) {
        for (j_intercept=i_intercept+1; j_intercept<=length(idx); j_intercept++) {
            g = idx[i_intercept]
            h = idx[j_intercept]
            i = offsets[h]
            BG.init(&factors[g], &factors[h], bg_verbose)
            m = BG.init_zigzag()
            if (verbose > 0) printf("{txt}    - mobility groups between FE intercepts #%f and #%f: {res}%f\n", g, h, m)
            M[i] = max(( M[i] , m ))
            if (j_intercept==2) M_is_exact[i] = 1
        }
    }
    BG = BipartiteGraph() // clear
    // TODO: add group3hdfe
    // TODO: add option to only do the first pair

    // (6) See if cvars are zero (w/out intercept) or just constant (w/intercept)
    i = 1
    for (g=1; g<=G; g++) {
        // If model has intercept, redundant cvars are those that are CONSTANT
        // Without intercept, a cvar has to be zero within a FE for it to be redundant
        // Since S.fes[g].x are already demeaned IF they have intercept, we don't have to worry about the two cases
        has_int = intercepts[g]
        if (has_int) i++

        if (num_slopes[g]) {
            data = asarray(factors[g].extra, "x")
            assert(num_slopes[g]==cols(data))
            results = J(1, cols(data), 0)
            // float(1.1) - 1 == 2.384e-08 , so let's pick something bigger, 1e-6
            zeros = J(1, cols(data), 1e-6)
            // This can be speed up by moving the -if- outside the -for-
            for (j = 1; j <= factors[g].num_levels; j++) {
                tmp = colminmax(panelsubmatrix(data, j, factors[g].info))
                if (has_int) {
                    results = results + ((tmp[2, .] - tmp[1, .]) :<= zeros)
                }
                else {
                    results = results + (colsum(abs(tmp)) :<= zeros)
                }
            }
            data = .
            if (sum(results)) {
                if (has_int  & verbose) printf("{txt}    - the slopes in the FE #%f are constant for {res}%f{txt} levels, which don't reduce DoF\n", g, sum(results))
                if (!has_int & verbose) printf("{txt}    - the slopes in the FE #%f are zero for {res}%f{txt} levels, which don't reduce DoF\n", g, sum(results))
                M[i..i+num_slopes[g]-1] = results
            }
            i = i + num_slopes[g]
        }
    }

    // Store results
    output.G_extended = SuperG
    output.doflist_M = M
    output.doflist_M_is_exact = M_is_exact
    output.doflist_M_is_nested = M_is_nested
    output.doflist_K = J(1, SuperG, .)
    for (g=1; g<=G; g++) {
        i = offsets[g]
        j = g==G ? SuperG : offsets[g+1]
        output.doflist_K[i..j] = J(1, j-i+1, factors[g].num_levels)
    }

    output.dof_M = sum(M)
    output.df_a = sum(output.doflist_K) - output.dof_M
    output.M_due_to_nested = M_due_to_nested
}

`Void' FixedEffects::prune_1core()
{
    // Note that we can't prune degree-2 nodes, to keep the graph bipartite
    `Integer'               i, j, g
    `Vector'                subgraph_id

    `Vector'                idx
    `RowVector'             i_prune

    assert(G==2) // bugbug remove?

    // Abort if the user set HDFE.prune = 0
    if (!prune) return

    // Pick two factors, and check if we really benefit from pruning
    prune = 0
    i_prune = J(1, 2, 0)
    for (g=i=1; g<=2; g++) {
        //if (intercepts[g] & !num_slopes[g] & factors[g].num_levels>100) {
        if (intercepts[g] & !num_slopes[g]) {
            i_prune[i++] = g // increments at the end
            if (i > 2) { // success!
                prune = 1
                break
            }
        }
    }

    if (!prune) return

    // for speed, the factor with more levels goes first
    i = i_prune[1]
    j = i_prune[2]
    //if (factors[i].num_levels < factors[j].num_levels) swap(i, j) // bugbug uncomment it!
    prune_g1 = i
    prune_g2 = j

    bg = BipartiteGraph()
    bg.init(&factors[prune_g1], &factors[prune_g2], verbose)
    bg.init_zigzag(1) // 1 => save subgraphs into bg.subgraph_id
    bg.compute_cores()
    bg.prune_1core(weight)
}

// bugbug fix or remove this fn altogether
`Void' FixedEffects::_expand_1core(`Variables' y)
{
    y = bg.expand_1core(y)
}

`Void' FixedEffects::save_touse(`Varname' touse)
{
    `Integer'               idx
    `Vector'                mask
    if (verbose > 0) printf("\n{txt} ## Saving e(sample)\n")
    idx = st_addvar("byte", touse)
    mask = J(st_nobs(), 1, 0)
    mask[sample] = J(rows(sample), 1, 1)
    st_store(., idx, mask)

}

`Void' FixedEffects::save_variable(`Varname' varname,
                                   `Variable' data,
                                 | `String' varlabel)
{
    `Integer'               idx
    `Vector'                mask
    idx = st_addvar("double", varname)
    st_store(sample, idx, data)
    if (args()>=3 & varlabel!="") st_varlabel(idx, varlabel)

}

end


Adding timing to bs subcommand

Adding timing to bs subcommand

*! parallel_bs vers 0.14.6.24 24jun2014
*! auth George G. Vega

program def parallel_bs, eclass

	vers 11.0

	if !replay() { // If it is a replay,
		parallel_bootstrap `0'
	}
	else if ("`e(prefix)'" != "bootstrap" | !inlist("`e(pll)'","1") ) { // If the last command runned wasn't nnmatch2
		di as error "Last estimation was not {bf:parallel bs}, was -{bf:`=e(prefix)'}-"
		exit 301
	}
	else { // if the last command was pll bootstrap, it replays the results
		bs, title(parallel bootstrapping)
	}
end

program def parallel_bootstrap, rclass

	vers 11.0

	#delimit ;
	syntax anything(name=model equalok everything) [,
		EXPression(string asis)
		PROGrams(string)
		Mata
		NOGlobals
		KEEPTiming
		Seeds(passthru)
		Randtype(passthru)
		Timeout(integer 60)
		PROCessors(integer 0)
		argopt(string)
		SAVing(string) Reps(integer 50) Keep KEEPLast *];
	#delimit cr

	/* Checking whereas parallel has been config */
	if length("$PLL_CLUSTERS") == 0 {
		di "{error:You haven't set the number of clusters}" _n "{error:Please set it with: {cmd:parallel setclusters} {it:#}}"
		exit 198
	}

	if ("`keeptiming'" == "") {
		timer clear 97
		timer clear 98
		timer clear 99
	}

	timer on 98

	/* Setting sizes */
	//BS needs normally reps at least 2 (per cluster)
	if `reps'<2*$PLL_CLUSTERS {
		_assert `reps'>1, msg("reps() must be an integer greater than 1") rc(198)

		local orig_PLL_CLUSTERS = ${PLL_CLUSTERS}
		global PLL_CLUSTERS = floor(`reps'/2)
		di "Small workload. Temporarily setting number of clusters to ${PLL_CLUSTERS}"
	}
	local csize = floor(`reps'/$PLL_CLUSTERS)
	local lsize = `csize' + (`reps' - `csize'*$PLL_CLUSTERS)

	/* Reserving a pll_id. This will be stored in the -parallelid- local
	macro */
	mata: parallel_sandbox(5)

	/* Saving the tmpfile */
	local tmpdta = "__pll`parallelid'_bs_dta.dta"
	if (`"`saving'"' == "") {
		if (c(os)=="Windows") local saving = `"`c(tmpdir)'__pll`parallelid'_bs_outdta.dta"'
		else local saving = `"`c(tmpdir)'/__pll`parallelid'_bs_outdta.dta"'
		local save = 0
	}
	else local save = 1
	local simul = `"__pll`parallelid'_bs_simul.do"'

	qui save `tmpdta'

	/* Parsing saving */
	_prefix_saving `saving'
	local saving    `"`s(filename)'"'
	if "`double'" == "" {
			local double    `"`s(double)'"'
	}
	local every     `"`s(every)'"'
	local replace   `"`s(replace)'"'

	cap confirm file `saving'
	if (!_rc & "`replace'" == "") {
		di "{error:File -`saving'- already exists, use the -replace- option}"
		if "`orig_PLL_CLUSTERS'"!="" global PLL_CLUSTERS=`orig_PLL_CLUSTERS'
		exit 602
	}

	/* Parsing a program */
	if (regexm(`"`model'"', "^([a-zA-Z0-9_]+)")) local cmd = regexs(1)
	cap findfile `cmd'.ado
	if (_rc) local programs `programs' `cmd'

	/* Creating a tmp program */
	tempname fh
	cap file open `fh' using `"`simul'"', w replace
	file write `fh' `"use `tmpdta', clear"' _n
	file write `fh' "if (\`pll_instance'==\$PLL_CLUSTERS) local reps = `lsize'" _n
	file write `fh' "else local reps = `csize'" _n
	file write `fh' `"local pll_instance : di %04.0f \`pll_instance'"' _n
	file write `fh' `"bs `expression', sav(__pll\`pll_id'_bs_eststore\`pll_instance', replace `double' `every') `options' rep(\`reps'): `model' `argopt'"' _n
	file close `fh'

	timer off 98
	cap timer list
	if (r(t98) == .) local pll_t_setu = 0
	else local pll_t_setu = r(t98)

	/* Running parallel */
	cap noi parallel do `simul', nodata programs(`programs') `mata' `noglobals' `seeds' ///
		`randtype' timeout(`timeout') processors(`processors') setparallelid(`parallelid') `keep' `keeplast'
	local pllseeds = r(pll_seeds)
	local nerrors  = r(pll_errs)
	local pll_dir r(pll_dir)

	local pll_t_setu = r(pll_t_setu) + `pll_t_setu'
	local pll_t_calc = r(pll_t_calc)
	local pll_t_fini = r(pll_t_fini)

	timer on 97

	if (_rc) {
		if ("`keep'"=="" & "`keeplast'"=="") qui parallel clean, e(${LAST_PLL_ID}) force nologs
		mata: parallel_sandbox(2, "`parallelid'")
		if "`orig_PLL_CLUSTERS'"!="" global PLL_CLUSTERS=`orig_PLL_CLUSTERS'
		exit _rc
	}

	preserve

	/* Appending datasets */
	forval i=1/$PLL_CLUSTERS {
		quietly {
			local pll_instance : di %04.0f `i'
			use `"__pll$LAST_PLL_ID`'_bs_eststore`pll_instance'"', clear

			if ((`=`i'-1')) append using `saving'
			save `saving', replace
		}
	}

	/* Storing macros */
	local macros : r(macros)
	local scalars : r(scalars)
	foreach m of local macros {
		local `m' = r(`m')
	}
	foreach s of local scalars {
		local `s' = r(`s')
	}

	restore

	/* Returning bs data */
	bstat using `saving', title(parallel bootstrapping)

	/* Cleaning up */
	if ("`keep'"=="" & "`keeplast'"=="") parallel clean, e(${LAST_PLL_ID}) nologs
	mata: parallel_sandbox(2, "`parallelid'")

	parallel_bs_ereturn

	/* Getting macros back */
	if "`orig_PLL_CLUSTERS'"!="" global PLL_CLUSTERS=`orig_PLL_CLUSTERS'
	foreach m of local macros {
		return local `m'  `"``m''"'
	}
	foreach s of local scalars {
		return scalar `s' = ``s''
	}

	timer off 97
	cap timer list
	if (r(t97) == .) local pll_t_fini = 0 + `pll_t_fini'
	else local pll_t_fini = r(t97) + `pll_t_fini'

	return local  pll_seeds="`pllseeds'"
	return scalar pll_errs = `nerrors'
	return local  pll_dir "`pll_dir'"
	// return scalar pll_t_reps = `pll_t_reps'
	return scalar pll_t_setu = `pll_t_setu'
	return scalar pll_t_calc = `pll_t_calc'
	return scalar pll_t_fini = `pll_t_fini'
	return local pll_id = "`parallelid'"

	return local pll_seeds = "`pllseeds'"

end

program def parallel_bs_ereturn, eclass
	vers 11.0
	ereturn local pll 1
end

*! parallel_bs vers 0.14.6.24 24jun2014
*! auth George G. Vega

program def parallel_bs, eclass

	vers 11.0

	if !replay() { // If it is a replay,
		parallel_bootstrap `0'
	}
	else if ("`e(prefix)'" != "bootstrap" | !inlist("`e(pll)'","1") ) { // If the last command runned wasn't nnmatch2
		di as error "Last estimation was not {bf:parallel bs}, was -{bf:`=e(prefix)'}-"
		exit 301
	}
	else { // if the last command was pll bootstrap, it replays the results
		bs, title(parallel bootstrapping)
	}
end

program def parallel_bootstrap, rclass

	vers 11.0

	#delimit ;
	syntax anything(name=model equalok everything) [,
		EXPression(string asis)
		PROGrams(string)
		Mata
		NOGlobals
		Seeds(passthru)
		Randtype(passthru)
		Timeout(integer 60)
		PROCessors(integer 0)
		argopt(string)
		SAVing(string) Reps(integer 50) Keep KEEPLast *];
	#delimit cr

	/* Checking whereas parallel has been config */
	if length("$PLL_CLUSTERS") == 0 {
		di "{error:You haven't set the number of clusters}" _n "{error:Please set it with: {cmd:parallel setclusters} {it:#}}"
		exit 198
	}

	/* Setting sizes */
	//BS needs normally reps at least 2 (per cluster)
	if `reps'<2*$PLL_CLUSTERS {
		_assert `reps'>1, msg("reps() must be an integer greater than 1") rc(198)

		local orig_PLL_CLUSTERS = ${PLL_CLUSTERS}
		global PLL_CLUSTERS = floor(`reps'/2)
		di "Small workload. Temporarily setting number of clusters to ${PLL_CLUSTERS}"
	}
	local csize = floor(`reps'/$PLL_CLUSTERS)
	local lsize = `csize' + (`reps' - `csize'*$PLL_CLUSTERS)

	/* Reserving a pll_id. This will be stored in the -parallelid- local
	macro */
	mata: parallel_sandbox(5)

	/* Saving the tmpfile */
	local tmpdta = "__pll`parallelid'_bs_dta.dta"
	if (`"`saving'"' == "") {
		if (c(os)=="Windows") local saving = `"`c(tmpdir)'__pll`parallelid'_bs_outdta.dta"'
		else local saving = `"`c(tmpdir)'/__pll`parallelid'_bs_outdta.dta"'
		local save = 0
	}
	else local save = 1
	local simul = `"__pll`parallelid'_bs_simul.do"'

	qui save `tmpdta'

	/* Parsing saving */
	_prefix_saving `saving'
	local saving    `"`s(filename)'"'
	if "`double'" == "" {
			local double    `"`s(double)'"'
	}
	local every     `"`s(every)'"'
	local replace   `"`s(replace)'"'

	cap confirm file `saving'
	if (!_rc & "`replace'" == "") {
		di "{error:File -`saving'- already exists, use the -replace- option}"
		if "`orig_PLL_CLUSTERS'"!="" global PLL_CLUSTERS=`orig_PLL_CLUSTERS'
		exit 602
	}

	/* Parsing a program */
	if (regexm(`"`model'"', "^([a-zA-Z0-9_]+)")) local cmd = regexs(1)
	cap findfile `cmd'.ado
	if (_rc) local programs `programs' `cmd'

	/* Creating a tmp program */
	tempname fh
	cap file open `fh' using `"`simul'"', w replace
	file write `fh' `"use `tmpdta', clear"' _n
	file write `fh' "if (\`pll_instance'==\$PLL_CLUSTERS) local reps = `lsize'" _n
	file write `fh' "else local reps = `csize'" _n
	file write `fh' `"local pll_instance : di %04.0f \`pll_instance'"' _n
	file write `fh' `"bs `expression', sav(__pll\`pll_id'_bs_eststore\`pll_instance', replace `double' `every') `options' rep(\`reps'): `model' `argopt'"' _n
	file close `fh'

	/* Running parallel */
	cap noi parallel do `simul', nodata programs(`programs') `mata' `noglobals' `seeds' ///
		`randtype' timeout(`timeout') processors(`processors') setparallelid(`parallelid') `keep' `keeplast'
	local pllseeds = r(pll_seeds)

	if (_rc) {
		if ("`keep'"=="" & "`keeplast'"=="") qui parallel clean, e(${LAST_PLL_ID}) force nologs
		mata: parallel_sandbox(2, "`parallelid'")
		if "`orig_PLL_CLUSTERS'"!="" global PLL_CLUSTERS=`orig_PLL_CLUSTERS'
		exit _rc
	}

	preserve

	/* Appending datasets */
	forval i=1/$PLL_CLUSTERS {
		quietly {
			local pll_instance : di %04.0f `i'
			use `"__pll$LAST_PLL_ID`'_bs_eststore`pll_instance'"', clear

			if ((`=`i'-1')) append using `saving'
			save `saving', replace
		}
	}

	/* Storing macros */
	local macros : r(macros)
	local scalars : r(scalars)
	foreach m of local macros {
		local `m' = r(`m')
	}
	foreach s of local scalars {
		local `s' = r(`s')
	}

	restore

	/* Returning bs data */
	bstat using `saving', title(parallel bootstrapping)

	/* Cleaning up */
	if ("`keep'"=="" & "`keeplast'"=="") parallel clean, e(${LAST_PLL_ID}) nologs
	mata: parallel_sandbox(2, "`parallelid'")

	parallel_bs_ereturn

	/* Getting macros back */
	if "`orig_PLL_CLUSTERS'"!="" global PLL_CLUSTERS=`orig_PLL_CLUSTERS'
	foreach m of local macros {
		return local `m'  `"``m''"'
	}
	foreach s of local scalars {
		return scalar `s' = ``s''
	}
	return local pll_seeds = "`pllseeds'"

end

program def parallel_bs_ereturn, eclass
	vers 11.0
	ereturn local pll 1
end