[iegitaddmd] small edits and comments

[iegitaddmd] small edits and comments

*! version 6.2 31JAN2019 DIME Analytics dimeanalytics@worldbank.org

cap program drop   iegitaddmd
	program define iegitaddmd

qui {

		syntax , folder(string) [customfile(string) all skip replace manual]

		/******************************
		*******************************
			Test user input
		*******************************
		******************************/

		******************************
		*	Test input: folder

		*Test that folder exist
		mata : st_numscalar("r(dirExist)", direxists("`folder'"))

		if (`r(dirExist)' == 0) | (length("`folder'")<=10) {

			noi di as error `"{phang}The "`folder'" folder does not exist or you have not entered the full path. For example, full paths on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}. Important: Specify the whole file path to the repository folder, not just {it:C:/} or {it:/user/} as that would create the placeholder file in every empty folder on your computer!{p_end}"'
			error 693
			exit
		}

		* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
		local folderStd	= subinstr("`folder'","\","/",.)

		******************************
		*	Test input: customfile

		*Test that file exist if not using default file
		if "`customfile'" != "" {
			*File option used test if file exists
			capture confirm file "`customfile'"
			if _rc {
				*File does not exist, throw error
				noi di as error `"{phang}File "`customfile'" was not found. Remember that you must enter the full path. For example, on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}.{p_end}"'
				error 693
				exit
			}

			* Get the name of file (include path to keep )
			* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
			local customFileStd	= subinstr("`customfile'","\","/",.)
			*Find the last slash in the file name
			local lastSlash = strpos(strreverse(`"`customFileStd'"'),"/")
			*Use that position to get the file name. Multiply with minus one as we count from the back
			local customFileName = substr(`"`customFileStd'"', (-1 * `lastSlash') ,.)
			*Used for the recursive call of the command when there is a subfolder
			local customFileRecurse `"customfile(`customFileStd')"'

			*Path and name of thile regardless if customfile were used
			local newPlaceholderFile `"`folderStd'`customFileName'"'
		}
		else {
			*Path and name of thile regardless if customfile were not used
			local newPlaceholderFile `"`folderStd'/README.md"'
		}

		******************************
		*	Test input: skip & replace

		*Options skip and replace cannot be used together
		if "`skip'" != "" & "`replace'" != "" {
			noi di as error `"{phang}The options skip and replace may not be used together.{p_end}"'
			error 198
			exit
		}

		/******************************
		*******************************
			Execute command
		*******************************
		******************************/

		******************************
		*	List all files and folders

		*List files, directories and other files
		local flist : dir `"`folderStd'"' files "*"	, respectcase
		local dlist : dir `"`folderStd'"' dirs  "*" , respectcase
		local olist : dir `"`folderStd'"' other "*"	, respectcase

		******************************
		*	Call command recursively on all sub-folders

		*Use the command on each subfolder to this folder (if any)
		foreach dir of local dlist {
			*Recursive call on each subfolder
			noi iegitaddmd , folder(`"`folderStd'/`dir'"') `all' `customFileRecurse' `skip' `replace' `manual'
		}

		******************************
		*	Evaluate if file will be crated in this folder

		*Test if all of those lists are empty, meaning there are
		* no folders or files in this folder
		if `"`flist'`dlist'`olist'"' == "" noi writeDefaultPlaceholder, newfileinclone(`"`newPlaceholderFile'"') customfile(`"`customFileStd'"') `manual'

		*If the folder is not empty, test if option all were used.
		else if ("`all'" == "all") {

				*Test if a file with exactly that name is already used
				cap confirm file `"`newPlaceholderFile'"'

				* No file with this name exists in this folder, create file without further ado
				if _rc != 0 noi writeDefaultPlaceholder, newfileinclone(`"`newPlaceholderFile'"') customfile(`"`customFileStd'"') `manual'

				* File exist, but no sintructions on how to deal with this has been given, throw error
				else if missing("`skip'") & missing("`replace'")  {
					noi di as error `"{phang}The file  `newPlaceholderFile' already exists. Either remove option {it:all} or use either option {it:skip} or {it:replace}. See help file before using either of these options unless you are already familiar with them.{p_end}"'
					error 602
				}

				* File exist, and instruction is to replace, create files and overwrite as needed
				else if !missing("`replace'") noi writeDefaultPlaceholder, newfileinclone(`"`newPlaceholderFile'"') customfile(`"`customFileStd'"') `manual'

				* File exist, and instruction is to skip, do nothing
				else if !missing("`skip'") {
						*Nothing is done, this if-block is only included for readability and completion
				}

				* The code should nevere get to this point
				else  {
					noi di as error `"{phang}Coding error in if-chain in "All" block.Please report this error to dimeanalytics@worldbank.org or here: https://github.com/worldbank/ietoolkit/issues/new{p_end}"'
					error 602
				}
		}
		else {
			* Folder is not empty and option all is not used, so do nothing in this folder
		}
}

end

	*Write a README.md file when iegitaddmd finds an empty folder
cap program drop writeDefaultPlaceholder
program define   writeDefaultPlaceholder

qui {

	syntax , newfileinclone(string) [customfile(string) manual]

	*If manual was used, get manual confirmation for each file
	if !missing("`manual'") {
		noi di ""
		global confirmation "" //Reset global

		*Get the current folder name from newfileinclone
		local lastSlash = strpos(strreverse(`"`newfileinclone'"'),"/")
		local folder = substr(`"`newfileinclone'"',1,strlen("`newfileinclone'")-`lastSlash')
		local fileName = substr(`"`newfileinclone'"', (-1 * `lastSlash')+1 ,.)

		*Keep aslking for input until the input is either Y, y, N, n or BREAK
		while (upper("${confirmation}") != "Y" & upper("${confirmation}") != "N" & "${confirmation}" != "BREAK") {
		  noi di as txt "{pstd}You are about to create a file [`fileName'] in the folder [`folder']. Do you want to do that? To confirm type {bf:Y} and hit enter, to abort type {bf:N} and hit enter. Type {bf:BREAK} and hit enter to stop the code.{p_end}", _request(confirmation)
		}
		*Copy user input to local
		local createfile = upper("${confirmation}")
		* If user wrote "BREAK" then exit the code
		if ("`createfile'" == "BREAK") error 1
	}

	*If "manual" were used and input was Y or if manual was not used, create the file
	if ("`createfile'" == "Y") | (missing("`manual'")) {

		*Copy custom file if custom file is used
		if !missing(`"`customfile'"') copy "`customfile'" `"`newfileinclone'"', replace
		*Custom file is not used, so cretae defeault file
		else {
			*Create file
			tempname 	newHandle
			cap file close 	`newHandle'
			file open  	`newHandle' using "`newfileinclone'", text write replace
			*Write the content to the file
			file write  `newHandle' ///
				"# Placeholder file" _n _n ///
				"This file has been created automatically by the command **iegitaddmd** from the Stata package [**ietoolkit**](https://worldbank.github.io/ietoolkit) to make GitHub sync this folder. GitHub does not sync empty folders or folders that only contain ignored files, but in research projects it is often important to share the full standardized folder structure along with the actual files. This command is intended to be used with **iefolder**, but it can be used in any folder structure intended to be shared on GitHub." _n _n ///
				"In recently started projects, it is typical to create data folders, script folders (do-files, r-files) and output folders, among others. The output folders are initially empty, but the script files might include a file path to them for later use. If an output folder is empty and another collaborator clones the repository, then the output folder expected by the scripts will not be included by GitHub in the cloned repository, and the script will not run properly." _n _n ///
				"## You should replace the content of this placeholder file" _n _n ///
				"The text in this file should be replaced with text that describes the intended use of this folder. This file is written in markdown (.md) format, which is suitable for GitHub syncing (unlike .doc/.docx). If the file is named *README.md*, GitHub automatically displays its contents when someone opens the containing folder on GitHub.com using a web browser." _n _n ///
				"If you are new to markup languages (markdown, html etc.) then this [Markdown Tutorial](https://www.markdowntutorial.com/) is a great place to start. If you have some experience with markup languages, then this [Markdown Cheat Sheet](https://guides.github.com/pdfs/markdown-cheatsheet-online.pdf) is a great place to start." _n _n ///
				"## Add similar files in other folders" _n _n ///
				"The Stata command **iegitaddmd** does not add anything to folders that already have content unless this is explicitly requested using the option `all`. It is best practice to create a *README.md* file in any folders that have content, for example, in folders whose purpose might not obvious to someone using the repository for the first time. Again, if the file is named *README.md*, then the content of the file will be shown in the browser when someone explores the repository on GitHub.com." _n _n ///
				"Another great use of a *README.md* file is to use it as a documentation on how the folder it sits in and its subfolders are organized,  where its content can be found, and where new content is meant to be saved. For example, if you have a folder called `/Baseline/`, then you can give a short description of the activities conducted during the baseline and where data, scripts and outputs related to it can be found." _n _n ///
				"## Removing this file" _n _n ///
				"Removing this file is not recommended, as GitHub may stop syncing the parent folder unless the folder now has other content. However, even when content is added and the file can be removed without breaking the GitHub functionality, our recommendation is to replace the content with more relevant content rather than deleting it." _n _n

			*Closing the file
			file close 		`newHandle'

		}
		*Uoutput that the file was created
		noi di as result "{pstd}File [`newfileinclone'] created.{p_end}"
	}
	*Manual was used and input was N, no file were creaetd
	else noi di as result "{pstd}No file created.{p_end}"
}
end

*! version 6.2 31JAN2019 DIME Analytics dimeanalytics@worldbank.org

cap program drop   iegitaddmd
	program define iegitaddmd

qui {

		syntax , folder(string) [customfile(string) all skip replace manual]

		/******************************
		*******************************
			Test user input
		*******************************
		******************************/

		******************************
		*	Test input: folder

		*Test that folder exist
		mata : st_numscalar("r(dirExist)", direxists("`folder'"))

		if (`r(dirExist)' == 0) | (length("`folder'")<=10) {

			noi di as error `"{phang}The "`folder'" folder does not exist or you have not entered the full path. For example, full paths on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}. Important: Specify the whole file path to the repository folder, not just {it:C:/} or {it:/user/} as that would create the placeholder file in every empty folder on your computer!{p_end}"'
			error 693
			exit
		}

		* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
		local folderStd	= subinstr("`folder'","\","/",.)

		******************************
		*	Test input: customfile

		*Test that file exist if not using default file
		if "`customfile'" != "" {
			*File option used test if file exists
			capture confirm file "`customfile'"
			if _rc {
				*File does not exist, throw error
				noi di as error `"{phang}File "`customfile'" was not found. Remember that you must enter the full path. For example, on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}.{p_end}"'
				error 693
				exit
			}

			* Get the name of file (include path to keep )
			* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
			local customFileStd	= subinstr("`customfile'","\","/",.)
			*Find the last slash in the file name
			local lastSlash = strpos(strreverse(`"`customFileStd'"'),"/")
			*Use that position to get the file name. Multiply with minus one as we count from the back
			local customFileName = substr(`"`customFileStd'"', (-1 * `lastSlash') ,.)
			*Used for the recursive call of the command when there is a subfolder
			local customFileRecurse `"customfile(`customFileStd')"'

			*Path and name of thile regardless if customfile were used
			local newPlaceholderFile `"`folderStd'`customFileName'"'
		}
		else {
			*Path and name of thile regardless if customfile were not used
			local newPlaceholderFile `"`folderStd'/README.md"'
		}

		******************************
		*	Test input: skip & replace

		*Options skip and replace cannot be used together
		if "`skip'" != "" & "`replace'" != "" {
			noi di as error `"{phang}The options skip and replace may not be used together.{p_end}"'
			error 198
			exit
		}

		/******************************
		*******************************
			Execute command
		*******************************
		******************************/

		******************************
		*	List all files and folders

		*List files, directories and other files
		local flist : dir `"`folderStd'"' files "*"	, respectcase
		local dlist : dir `"`folderStd'"' dirs  "*" , respectcase
		local olist : dir `"`folderStd'"' other "*"	, respectcase

		******************************
		*	Call command recursively on all sub-folders

		*Use the command on each subfolder to this folder (if any)
		foreach dir of local dlist {
			*Recursive call on each subfolder
			noi iegitaddmd , folder(`"`folderStd'/`dir'"') `all' `customFileRecurse' `skip' `replace' `manual'
		}

		******************************
		*	Evaluate if file will be crated in this folder

		*Test if all of those lists are empty, meaning there are
		* no folders or files in this folder
		if `"`flist'`dlist'`olist'"' == "" noi writeDefaultPlaceholder, newfileinclone(`"`newPlaceholderFile'"') customfile(`"`customFileStd'"') `manual'

		*If the folder is not empty, test if option all were used.
		else if ("`all'" == "all") {

				cap confirm file `"`newPlaceholderFile'"'

				* No file with this name exists in this folder, create file without further ado
				if _rc != 0 noi writeDefaultPlaceholder, newfileinclone(`"`newPlaceholderFile'"') customfile(`"`customFileStd'"') `manual'

				* File exist, but no sintructions on how to deal with this has been given, throw error
				else if missing("`skip'") & missing("`replace'")  {
					noi di as error `"{phang}The file  `newPlaceholderFile' already exists. Either remove option {it:all} or use either option {it:skip} or {it:replace}. See help file before using either of these options unless you are already familiar with them.{p_end}"'
					error 602
				}

				* File exist, and instruction is to replace, create files and overwrite as needed
				else if !missing("`replace'") noi writeDefaultPlaceholder, newfileinclone(`"`newPlaceholderFile'"') customfile(`"`customFileStd'"') `manual'

				* File exist, and instruction is to skip, do nothing
				else if !missing("`skip'") {
						*Nothing is done, this if-block is only included for readability and completion
				}

				* The code should nevere get to this point
				else  {
					noi di as error `"{phang}Coding error in if-chain in "All" block.Please report this error to dimeanalytics@worldbank.org or here: https://github.com/worldbank/ietoolkit/issues/new{p_end}"'
					error 602
				}

		}
}

end

	*Write a README.md file when iegitaddmd finds an empty folder
cap program drop writeDefaultPlaceholder
program define   writeDefaultPlaceholder

qui {

	syntax , newfileinclone(string) [customfile(string) manual]

	*Get the current folder name from newfileinclone
	local lastSlash = strpos(strreverse(`"`newfileinclone'"'),"/")
	local folder = substr(`"`newfileinclone'"',1,strlen("`newfileinclone'")-`lastSlash')
	local fileName = substr(`"`newfileinclone'"', (-1 * `lastSlash')+1 ,.)

	*If manual was used, get manual confirmation for each file
	if !missing("`manual'") {
		noi di ""
		global confirmation "" //Reset global

		*Keep aslking for input until the input is either Y, y, N, n or BREAK
		while (upper("${confirmation}") != "Y" & upper("${confirmation}") != "N" & "${confirmation}" != "BREAK") {
		  noi di as txt "{pstd}You are about to create a file [`fileName'] in the folder [`folder']. Do you want to do that? To confirm type {bf:Y} and hit enter, to abort type {bf:N} and hit enter. Type {bf:BREAK} and hit enter to stop the code.{p_end}", _request(confirmation)
		}
		*Copy user input to local
		local createfile = upper("${confirmation}")
		* If user wrote "BREAK" then exit the code
		if ("`createfile'" == "BREAK") error 1
	}

	*If "manual" were used and input was Y or if manual was not used, create the file
	if ("`createfile'" == "Y") | (missing("`manual'")) {

		*Copy custom file if custom file is used
		if !missing(`"`customfile'"') copy "`customfile'" `"`newfileinclone'"', replace
		*Custom file is not used, so cretae defeault file
		else {
			*Create file
			tempname 	newHandle
			cap file close 	`newHandle'
			file open  	`newHandle' using "`newfileinclone'", text write replace
			*Write the content to the file
			file write  `newHandle' ///
				"# Placeholder file" _n _n ///
				"This file has been created automatically by the command **iegitaddmd** from the Stata package [**ietoolkit**](https://worldbank.github.io/ietoolkit) to make GitHub sync this folder. GitHub does not sync empty folders or folders that only contain ignored files, but in research projects it is often important to share the full standardized folder structure along with the actual files. This command is intended to be used with **iefolder**, but it can be used in any folder structure intended to be shared on GitHub." _n _n ///
				"In recently started projects, it is typical to create data folders, script folders (do-files, r-files) and output folders, among others. The output folders are initially empty, but the script files might include a file path to them for later use. If an output folder is empty and another collaborator clones the repository, then the output folder expected by the scripts will not be included by GitHub in the cloned repository, and the script will not run properly." _n _n ///
				"## You should replace the content of this placeholder file" _n _n ///
				"The text in this file should be replaced with text that describes the intended use of this folder. This file is written in markdown (.md) format, which is suitable for GitHub syncing (unlike .doc/.docx). If the file is named *README.md*, GitHub automatically displays its contents when someone opens the containing folder on GitHub.com using a web browser." _n _n ///
				"If you are new to markup languages (markdown, html etc.) then this [Markdown Tutorial](https://www.markdowntutorial.com/) is a great place to start. If you have some experience with markup languages, then this [Markdown Cheat Sheet](https://guides.github.com/pdfs/markdown-cheatsheet-online.pdf) is a great place to start." _n _n ///
				"## Add similar files in other folders" _n _n ///
				"The Stata command **iegitaddmd** does not add anything to folders that already have content unless this is explicitly requested using the option `all`. It is best practice to create a *README.md* file in any folders that have content, for example, in folders whose purpose might not obvious to someone using the repository for the first time. Again, if the file is named *README.md*, then the content of the file will be shown in the browser when someone explores the repository on GitHub.com." _n _n ///
				"Another great use of a *README.md* file is to use it as a documentation on how the folder it sits in and its subfolders are organized,  where its content can be found, and where new content is meant to be saved. For example, if you have a folder called `/Baseline/`, then you can give a short description of the activities conducted during the baseline and where data, scripts and outputs related to it can be found." _n _n ///
				"## Removing this file" _n _n ///
				"Removing this file is not recommended, as GitHub may stop syncing the parent folder unless the folder now has other content. However, even when content is added and the file can be removed without breaking the GitHub functionality, our recommendation is to replace the content with more relevant content rather than deleting it." _n _n

			*Closing the file
			file close 		`newHandle'

		}
		*Uoutput that the file was created
		noi di as result "{pstd}File [`newfileinclone'] created.{p_end}"
	}
	*Manual was used and input was N, no file were creaetd
	else noi di as result "{pstd}No file created.{p_end}"
}
end


missing value for year before 2007

missing value for year before 2007

program make_traf

insheet using ${stash}traf.csv, clear

rename v13 wikihandle
rename v14 year

egen traf = rowtotal(v1-v12)
replace traf = traf/12 if year > 2007

// traffic data not available
replace traf = . if year < 2007

drop v*

save ${stash}traf, replace

end

program make_traf

insheet using ${stash}traf.csv, clear

egen traf = rowtotal(v1-v12)
replace traf = traf/12

rename v13 wikihandle
rename v14 year

drop v*

save ${stash}traf, replace

end


[iegitaddmd ?] remove names

[iegitaddmd ?] remove names

{smcl}
{* 5 Nov 2019}{...}
{hline}
help for {hi:iegitaddmd}
{hline}

{title:Title}

{cmd:iegitaddmd} {hline 2} Creates a placeholder file in subfolders of a GitHub repository folder, which allows committing folder structures with empty folders.

{phang2}For a more descriptive discussion on the intended usage and work flow of this
command please see the {browse "https://dimewiki.worldbank.org/wiki/Iegitaddmd":DIME Wiki}.

{title:Syntax}

{phang} {cmdab:iegitaddmd} , {opt folder(full_file_path)} [{opt comparefolder(full_file_path)} {cmd:customfile(}{it:{help filename}}{cmd:)} {opt all} {opt skip} {opt replace} {opt skipfolders(folder_name)}  {opt auto:matic} {opt dry:run}]

{marker opts}{...}
{synoptset 28}{...}
{synopthdr:options}
{synoptline}
{synopt :{opt folder(full_file_path)}}Specifies the folder path to the project folder clone where placeholder files will be created{p_end}
{synopt :{opt comparefolder(full_file_path)}}Specifies the top folder of a folder tree to be compared with {opt folder()}{p_end}
{synopt :{cmd:customfile(}{it:{help filename}}{cmd:)}}Specifies a file saved on disk that is used instead of the default file as placeholder{p_end}
{synopt :{cmd:all}}Creates the placeholder file in every subfolder of {opt folder()}, whether empty or not{p_end}
{synopt :{opt skip}}If option {opt all} is used and a folder has a file with same name as placeholder file, then nothing is done{p_end}
{synopt :{opt replace}}If option {opt all} is used and a folder has a file with same name as placeholder file, then the file is overwritten{p_end}
{synopt :{opt skipfolders(folder_name)}}List of folders to be skipped. The folder {it:.git} is always added to this list, even when option is not used{p_end}
{synopt :{opt auto:matic}}Makes the command create placeholder files without prompting the user for each file{p_end}
{synopt :{opt dry:run}}Makes the command list all the files that would have been created without this option.{p_end}
{synoptline}

{marker desc}
{title:Description}

{pstd}GitHub does not sync empty folders, or folder that only contain ignored files. However, it is common in research projects that these types of folders needs to be shared anyways. {cmd:iegitaddmd} provides a solution to this by creating placeholder files in in these folders. This placeholder file can then be shared to GitHub and that way the folder is also shared. {cmd:iegitaddmd} have two ways to identify which those folders are that are developed with the two use cases described below in mind.{p_end}

{pstd}{bf:Use case 1.} It is common in the beginning of a research project that a standardized folder structure is set up (for example with {help iefolder}) where some folders are created that are not yet needed. Examples of folders like that could be the output folders. If these folders are not shared at the time they are created, there is a risk that different team members name them differently which will be confusing and could lead to errors. {cmd:iegitaddmd} can solve this by scanning a folder and all its sub-folders for completely empty folders, and create a placeholder file in that folder.{p_end}

{pstd}The solution to use case 1 in {cmd:iegitaddmd} is a Stata adaptation of {it:Solution B} in {browse "http://bytefreaks.net/gnulinux/bash/how-to-add-automatically-all-empty-folders-in-git-repository" :this post}.{p_end}

{pstd}{bf:Use case 2.} A common method when transferring an already ongoing research project to GitHub is to copy all of the data work folder from wherever the files and folders where shared before (for example DropBox) to a clone of the repository. Then only the files that are meant to be shared on GitHub are uploaded to the cloud using a {it:.gitignore file}. Empty folders or folders where all files are ignored is then not shared through GitHub. {cmd:iegitaddmd} can solve this by scanning the DropBox folder (or wherever the project folder was shared before) for sub-folders that exist in that folder but not in the clone and crate that folder in the clone and put a placeholder in it.{p_end}

{pstd}The way you differentiate between these two use cases when using {cmd:iegitaddmd} is whether the option {opt comparefolder()} is used or not. If that option is used, then {cmd:iegitaddmd} will identify all folders that are in {opt comparefolder()} that is not in {opt folder()}, and create a placeholder file there. If {opt comparefolder()} is not used, then {cmd:iegitaddmd} will identify all completely empty folders (no files or subfolders) and create a placeholder file there.{p_end}

{pstd}The default placeholder file used if no file is specified using option {opt customfile()} is called README.md, which is a name and format recognized by GitHub.com so that the content of the file is displayed on GitHub.com when navigating to that folder using a browser. The placeholder file may be removed as	soon as files that are not ignored have been added to the folder, or even better is to keep the file but edit the content of it so it documents the content of that folder for other and future team members.{p_end}

{marker optslong}
{title:Options}

{phang}{opt folder(file_path)} is the folder path to the project folder clone where the placeholder files will be created. Unless option {opt comparefolder()} or {opt all} is used, then placeholder files will be created in all empty folders. See option {opt manual} for a way to manually confirm each folder before a placeholder file is created in it.{p_end}

{phang}{opt comparefolder(file_path)} is used to compare a file tree in a different location and create the folders with placeholder files in all folders that exist in {opt comparefolder()} but not in {opt folder()}. If a folder that does no exist in {opt folder()} has a subfolder, then both the folder and the subfolder will be created and the placeholder file is only crated in the sub-folder. This is enough to share both folder and the subfolder in GitHub. If the {opt comparefolder()} and {opt folder()} ends with folders with different names, for example {opt folder(${project}/datawork/baseline)} and {opt comparefolder(${project}/datawork)}, then the command will throw an error as it is likely that the paths do not point to corresponding folders in the two folder trees. See option {opt manual} for a way to manually confirm each folder before a placeholder file is created in it.{p_end}

{phang}{cmd:file(}{it:{help filename}}{cmd:)} allows the user to specify a file saved on
	the computer to be used as the placeholder file	instead of the default README.md file. This
	allows people and organizations to write their own template placeholder files according to their own
	preferences. We recommend that a file of type .md (markdown) and name README.md is used as
	GitHub.com recognizes this name and format and will display the content of the file when someone
	browse to that folder on GitHub.com. But this is not a technical requirement, any file type
	and name can be used as placeholder file.{p_end}

{phang}{opt all} creates the placeholder file in {opt folder()} and in every subfolder,
	regardless if they are empty or not. This allows the user to create a placeholder file in every
	folder that can be edited with documentation and instructions to the purpose and usage of
	each subfolder. This option can also be important when .{it:gitignore} is used, as {cmd:iegitaddmd} will not
	create files to subfolders that only has ignored file -- in which case the folder will not
	be synced by GitHub.{p_end}

{phang}{opt skip} and {opt replace} tells {cmd:iegitaddmd} what to do if the option {opt all} is used
	and if any of the folders contain a file with the same name as the placeholder file the command
	is trying to create. If a file with the same name as the placeholder file exist and neither of these options
	are used, then the command will throw an error. If {opt skip} is used, then nothing is done in the
	folder where the file with the same name exists and the command proceeds to the next folder.
	If {opt replace} is used then the file with the same name is overwritten with the new placeholder
	file before the command proceeds to the next folder.{p_end}

{phang}{opt skipfolders(folder_name)} can be used to tell {cmd:iegitaddmd} which folders
in which a placeholder file should never be created. The best example of this is the {it:.git}
folder in which placeholders never should be created. That name is always skipped regardless of
this option being used or not. Use this option to list additional folders to be skipped. You
should not list the full folder path in this option, just the folder name. All folders with
that name will be skipped regardless of their location in the project folder. The folder names may
be listed with or without quotation marks as long as there are no spaces in the names. If any of
the folder names has spaces, then quotation marks must be used for all folder names.

{phang}{opt auto:matic} can be used to speed up the creation of placeholders by telling the
command to not prompt the users for confirmation for each file before it is created. The
default is that the command is asking the user before creating each place holder file.
This option should only be used when you are confident you have specified the correct folder
paths. We recommend that you use the {opt dryrun} with this option to make sure that the
folder paths are correct.{p_end}

{phang}{opt dry:run} can be used to safely test that the folder paths are specified correctly
before any placeholder files are created. When this option it used the command simply lists
the file that would have been created if this option were not used. Once you are confident
that list is correct, you can remove this option and re-run the command and the files will
be created.{p_end}

{title:Example}

{pstd}{inp:global github_folder "C:\Users\JohnSmith\Documents\GitHub\ProjectA"}{break}{inp:iegitaddmd , folder({it:"$github_folder"})}{p_end}

{pstd}In the example above, there is a GitHub repository in the folder ProjectA. This
	repository has a folder structure where some folders are still empty but will later
	be populated with files. In order to have all folders, even the empty ones, synced on all
	collaborators' cloned local copies of the repository, the folders need to contain at least
	one file, which is being created by the command.{p_end}

{title:Acknowledgements}

{pstd}I would like to acknowledge the help in testing and proofreading I received in relation to this command and help file from (in alphabetic order):{p_end}
{pmore}Guadalupe Bedoya{break}Luiza Cardoso de Andrade{break}Mrijan Rimal{break}Benjamin Daniels{break}

{title:Author}

{phang}All commands in ietoolkit is developed by DIME Analytics at DECIE, The World Bank's unit for Development Impact Evaluations.

{phang}Author: DIME Analytics, The World Bank Group

{phang}Please send bug-reports, suggestions and requests for clarifications
		 writing "ietoolkit iegitaddmd" in the subject line to:{break}
		 dimeanalytics@worldbank.org

{phang}You can also see the code, make comments to the code, see the version
		 history of the code, and submit additions or edits to the code through {browse "https://github.com/worldbank/ietoolkit":the GitHub repository of ietoolkit}.{p_end}

{smcl}
{* 5 Nov 2019}{...}
{hline}
help for {hi:iegitaddmd}
{hline}

{title:Title}

{cmd:iegitaddmd} {hline 2} Creates a placeholder file in subfolders of a GitHub repository folder, which allows committing folder structures with empty folders.

{phang2}For a more descriptive discussion on the intended usage and work flow of this
command please see the {browse "https://dimewiki.worldbank.org/wiki/Iegitaddmd":DIME Wiki}.

{title:Syntax}

{phang} {cmdab:iegitaddmd} , {opt folder(full_file_path)} [{opt comparefolder(full_file_path)} {cmd:customfile(}{it:{help filename}}{cmd:)} {opt all} {opt skip} {opt replace} {opt skipfolders(folder_name)}  {opt auto:matic} {opt dry:run}]

{marker opts}{...}
{synoptset 28}{...}
{synopthdr:options}
{synoptline}
{synopt :{opt folder(full_file_path)}}Specifies the folder path to the project folder clone where placeholder files will be created{p_end}
{synopt :{opt comparefolder(full_file_path)}}Specifies the top folder of a folder tree to be compared with {opt folder()}{p_end}
{synopt :{cmd:customfile(}{it:{help filename}}{cmd:)}}Specifies a file saved on disk that is used instead of the default file as placeholder{p_end}
{synopt :{cmd:all}}Creates the placeholder file in every subfolder of {opt folder()}, whether empty or not{p_end}
{synopt :{opt skip}}If option {opt all} is used and a folder has a file with same name as placeholder file, then nothing is done{p_end}
{synopt :{opt replace}}If option {opt all} is used and a folder has a file with same name as placeholder file, then the file is overwritten{p_end}
{synopt :{opt skipfolders(folder_name)}}List of folders to be skipped. The folder {it:.git} is always added to this list, even when option is not used{p_end}
{synopt :{opt auto:matic}}Makes the command create placeholder files without prompting the user for each file{p_end}
{synopt :{opt dry:run}}Makes the command list all the files that would have been created without this option.{p_end}
{synoptline}

{marker desc}
{title:Description}

{pstd}GitHub does not sync empty folders, or folder that only contain ignored files. However, it is common in research projects that these types of folders needs to be shared anyways. {cmd:iegitaddmd} provides a solution to this by creating placeholder files in in these folders. This placeholder file can then be shared to GitHub and that way the folder is also shared. {cmd:iegitaddmd} have two ways to identify which those folders are that are developed with the two use cases described below in mind.{p_end}

{pstd}{bf:Use case 1.} It is common in the beginning of a research project that a standardized folder structure is set up (for example with {help iefolder}) where some folders are created that are not yet needed. Examples of folders like that could be the output folders. If these folders are not shared at the time they are created, there is a risk that different team members name them differently which will be confusing and could lead to errors. {cmd:iegitaddmd} can solve this by scanning a folder and all its sub-folders for completely empty folders, and create a placeholder file in that folder.{p_end}

{pstd}The solution to use case 1 in {cmd:iegitaddmd} is a Stata adaptation of {it:Solution B} in {browse "http://bytefreaks.net/gnulinux/bash/how-to-add-automatically-all-empty-folders-in-git-repository" :this post}.{p_end}

{pstd}{bf:Use case 2.} A common method when transferring an already ongoing research project to GitHub is to copy all of the data work folder from wherever the files and folders where shared before (for example DropBox) to a clone of the repository. Then only the files that are meant to be shared on GitHub are uploaded to the cloud using a {it:.gitignore file}. Empty folders or folders where all files are ignored is then not shared through GitHub. {cmd:iegitaddmd} can solve this by scanning the DropBox folder (or wherever the project folder was shared before) for sub-folders that exist in that folder but not in the clone and crate that folder in the clone and put a placeholder in it.{p_end}

{pstd}The way you differentiate between these two use cases when using {cmd:iegitaddmd} is whether the option {opt comparefolder()} is used or not. If that option is used, then {cmd:iegitaddmd} will identify all folders that are in {opt comparefolder()} that is not in {opt folder()}, and create a placeholder file there. If {opt comparefolder()} is not used, then {cmd:iegitaddmd} will identify all completely empty folders (no files or subfolders) and create a placeholder file there.{p_end}

{pstd}The default placeholder file used if no file is specified using option {opt customfile()} is called README.md, which is a name and format recognized by GitHub.com so that the content of the file is displayed on GitHub.com when navigating to that folder using a browser. The placeholder file may be removed as	soon as files that are not ignored have been added to the folder, or even better is to keep the file but edit the content of it so it documents the content of that folder for other and future team members.{p_end}

{marker optslong}
{title:Options}

{phang}{opt folder(file_path)} is the folder path to the project folder clone where the placeholder files will be created. Unless option {opt comparefolder()} or {opt all} is used, then placeholder files will be created in all empty folders. See option {opt manual} for a way to manually confirm each folder before a placeholder file is created in it.{p_end}

{phang}{opt comparefolder(file_path)} is used to compare a file tree in a different location and create the folders with placeholder files in all folders that exist in {opt comparefolder()} but not in {opt folder()}. If a folder that does no exist in {opt folder()} has a subfolder, then both the folder and the subfolder will be created and the placeholder file is only crated in the sub-folder. This is enough to share both folder and the subfolder in GitHub. If the {opt comparefolder()} and {opt folder()} ends with folders with different names, for example {opt folder(${project}/datawork/baseline)} and {opt comparefolder(${project}/datawork)}, then the command will throw an error as it is likely that the paths do not point to corresponding folders in the two folder trees. See option {opt manual} for a way to manually confirm each folder before a placeholder file is created in it.{p_end}

{phang}{cmd:file(}{it:{help filename}}{cmd:)} allows the user to specify a file saved on
	the computer to be used as the placeholder file	instead of the default README.md file. This
	allows people and organizations to write their own template placeholder files according to their own
	preferences. We recommend that a file of type .md (markdown) and name README.md is used as
	GitHub.com recognizes this name and format and will display the content of the file when someone
	browse to that folder on GitHub.com. But this is not a technical requirement, any file type
	and name can be used as placeholder file.{p_end}

{phang}{opt all} creates the placeholder file in {opt folder()} and in every subfolder,
	regardless if they are empty or not. This allows the user to create a placeholder file in every
	folder that can be edited with documentation and instructions to the purpose and usage of
	each subfolder. This option can also be important when .{it:gitignore} is used, as {cmd:iegitaddmd} will not
	create files to subfolders that only has ignored file -- in which case the folder will not
	be synced by GitHub.{p_end}

{phang}{opt skip} and {opt replace} tells {cmd:iegitaddmd} what to do if the option {opt all} is used
	and if any of the folders contain a file with the same name as the placeholder file the command
	is trying to create. If a file with the same name as the placeholder file exist and neither of these options
	are used, then the command will throw an error. If {opt skip} is used, then nothing is done in the
	folder where the file with the same name exists and the command proceeds to the next folder.
	If {opt replace} is used then the file with the same name is overwritten with the new placeholder
	file before the command proceeds to the next folder.{p_end}

{phang}{opt skipfolders(folder_name)} can be used to tell {cmd:iegitaddmd} which folders
in which a placeholder file should never be created. The best example of this is the {it:.git}
folder in which placeholders never should be created. That name is always skipped regardless of
this option being used or not. Use this option to list additional folders to be skipped. You
should not list the full folder path in this option, just the folder name. All folders with
that name will be skipped regardless of their location in the project folder. The folder names may
be listed with or without quotation marks as long as there are no spaces in the names. If any of
the folder names has spaces, then quotation marks must be used for all folder names.

{phang}{opt auto:matic} can be used to speed up the creation of placeholders by telling the
command to not prompt the users for confirmation for each file before it is created. The
default is that the command is asking the user before creating each place holder file.
This option should only be used when you are confident you have specified the correct folder
paths. We recommend that you use the {opt dryrun} with this option to make sure that the
folder paths are correct.{p_end}

{phang}{opt dry:run} can be used to safely test that the folder paths are specified correctly
before any placeholder files are created. When this option it used the command simply lists
the file that would have been created if this option were not used. Once you are confident
that list is correct, you can remove this option and re-run the command and the files will
be created.{p_end}

{title:Example}

{pstd}{inp:global github_folder "C:\Users\JohnSmith\Documents\GitHub\ProjectA"}{break}{inp:iegitaddmd , folder({it:"$github_folder"})}{p_end}

{pstd}In the example above, there is a GitHub repository in the folder ProjectA. This
	repository has a folder structure where some folders are still empty but will later
	be populated with files. In order to have all folders, even the empty ones, synced on all
	collaborators' cloned local copies of the repository, the folders need to contain at least
	one file, which is being created by the command.{p_end}

{title:Acknowledgements}

{pstd}I would like to acknowledge the help in testing and proofreading I received in relation to this command and help file from (in alphabetic order):{p_end}
{pmore}Guadalupe Bedoya{break}Luiza Cardoso de Andrade{break}Mrijan Rimal{break}Benjamin Daniels{break}

{title:Author}

{phang}All commands in ietoolkit is developed by DIME Analytics at DECIE, The World Bank's unit for Development Impact Evaluations.

{phang}Main author: Kristoffer Bjarkefur, DIME Analytics, The World Bank Group

{phang}Please send bug-reports, suggestions and requests for clarifications
		 writing "ietoolkit iegitaddmd" in the subject line to:{break}
		 dimeanalytics@worldbank.org

{phang}You can also see the code, make comments to the code, see the version
		 history of the code, and submit additions or edits to the code through {browse "https://github.com/worldbank/ietoolkit":the GitHub repository of ietoolkit}.{p_end}


more outcomes

more outcomes

program make_merge

insheet using ${stash}tmp.csv, clear

renamevar

merge m:1 wikihandle using ${stash}bbk_master, nogen

merge 1:1 wikihandle year using ${stash}traf, nogen

merge 1:1 wikihandle year using ${stash}rev, keep(master match) nogen

egen id = group(wikihandle)
xtset id year

gen post = (year>2008)

gen treat = (debutyear < 1964)

local outcomes "img text bd traf numrev numuser numregusers numnew_wiki numnewreg_wiki numnewreg_page numserious1 numserious3 numserious21 numserious278 numserious1851 avgsize numcountry numregion numtimezone numnewregion numnewregion_wiki"

foreach x in `outcomes'{
    gen ln`x' = ln(`x'+1)
}

drop if debut < 1944 | debut > 1984

genqual

gen decade = round(debut, 10)
egen dy = group(decade year)
egen qy = group(quality year)

save ${stash}master, replace

end

program renamevar

rename v1 wikihandle
rename v2 year
rename v3 user
rename v4 revid
rename v5 size
rename v6 text
rename v7 img
rename v8 bd

end

program genqual

gen quality = .
_pctile numallstar, p(33.333, 66.667 90)

replace qual = 1 if numallstar <= `r(r1)' & isb == 1
replace qual = 2 if numallstar > `r(r1)'  & numall <=`r(r2)' & isb == 1
replace qual = 3 if numallstar > `r(r2)'  & numall <=`r(r3)' & isb == 1
replace qual = 4 if numallstar > `r(r3)' & isb == 1

_pctile minutesrank, p(33.333, 66.667 90)

replace qual = 1 if minutesrank <= `r(r1)' & isb == 0
replace qual = 2 if minutesrank > `r(r1)' & minutesrank <=`r(r2)' & isb == 0
replace qual = 3 if minutesrank > `r(r2)' & minutesrank <=`r(r3)' & isb == 0
replace qual = 4 if minutesrank > `r(r3)' & isb == 0

end

program make_merge

insheet using ${stash}tmp.csv, clear

renamevar

merge m:1 wikihandle using ${stash}bbk_master, nogen

merge 1:1 wikihandle year using ${stash}traf, nogen

merge 1:1 wikihandle year using ${stash}rev, keep(master match) nogen

egen id = group(wikihandle)
xtset id year

gen post = (year>2008)

gen treat = (debutyear < 1964)

local outcomes "img text bd traf numrev numuser numregusers numnew_wiki numnewreg_wiki numnewreg_page numserious1 numserious3 numserious21 numserious278 numserious1851 avgsize numcountry numregion numtimezone numnewregion numnewregion_wiki"

foreach x in `outvars'{
    gen ln`x' = ln(`x'+1)
}

drop if debut < 1944 | debut > 1984

genqual

gen decade = round(debut, 10)
egen dy = group(decade year)
egen qy = group(quality year)

save ${stash}master, replace

end

program renamevar

rename v1 wikihandle
rename v2 year
rename v3 user
rename v4 revid
rename v5 size
rename v6 text
rename v7 img
rename v8 bd

end

program genqual

gen quality = .
_pctile numallstar, p(33.333, 66.667 90)

replace qual = 1 if numallstar <= `r(r1)' & isb == 1
replace qual = 2 if numallstar > `r(r1)'  & numall <=`r(r2)' & isb == 1
replace qual = 3 if numallstar > `r(r2)'  & numall <=`r(r3)' & isb == 1
replace qual = 4 if numallstar > `r(r3)' & isb == 1

_pctile minutesrank, p(33.333, 66.667 90)

replace qual = 1 if minutesrank <= `r(r1)' & isb == 0
replace qual = 2 if minutesrank > `r(r1)' & minutesrank <=`r(r2)' & isb == 0
replace qual = 3 if minutesrank > `r(r2)' & minutesrank <=`r(r3)' & isb == 0
replace qual = 4 if minutesrank > `r(r3)' & isb == 0

end


Update reghdfe_old.sthlp

Update reghdfe_old.sthlp


Fix for default.concepts.json.do

Fix for default.concepts.json.do

#!/usr/bin/env python3

import json
import yaml
import comap
import redo

with redo.ifchange(semantic_types='config/semantic_types.yaml',
                   coding_systems='config/coding_systems.yaml',
                   index=redo.base + '.index.json') as files:
    coding_systems = yaml.load(files['coding_systems'])
    semantic_types = yaml.load(files['semantic_types'])
    index = json.load(files['index'])

cuis = [s['cui'] for s in index['spans']]

client = comap.ComapClient()

concepts = [
    concept
    for concept in client.umls_concepts(cuis, coding_systems)
    if set(concept['semanticTypes']) & set(semantic_types)
]

with redo.output() as f:
    json.dump(concepts, f)

#!/usr/bin/env python3

import json
import yaml
import comap
import redo

with redo.ifchange(comap_config='config/comap.yaml',
                   semantic_types='config/semantic_types.yaml',
                   coding_systems='config/coding_systems.yaml',
                   index=redo.base + '.index.json') as files:
    coding_systems = yaml.load(files['coding_systems'])
    semantic_types = yaml.load(files['semantic_types'])
    comap_api_url = yaml.load(files['comap_config'])['api']['url']
    index = json.load(files['index'])

cuis = [s['cui'] for s in index['spans']]

client = comap.ComapClient(comap_api_url)

concepts = [
    concept
    for concept in client.umls_concepts(cuis, coding_systems)
    if set(concept['semanticTypes']) & set(semantic_types)
]

with redo.output() as f:
    json.dump(concepts, f)


Put find arguments in preferred order

Put find arguments in preferred order

rm -f index.html
find build -mindepth 1 -maxdepth 1 -type f ! -name 'do' ! -name '*.do' -exec rm {} \;

rm -f index.html
find build -type f ! -name 'do' ! -name '*.do' -mindepth 1 -maxdepth 1 -exec rm {} \;


more musings

more musings

= to.do: =

  o bloggish directories should have a "make wc" target that subtracts the word count of
    the template file.  Or, better, use sed to filter out tags.

  o bootstrap should check ~/.ssh to see whether to use savitzky.net or github.

========================================================================================>
Work Log:
========

2015:
=====

1111We
  @ OpenSSH/Cookbook/Multiplexing - Wikibooks
    Makes ssh much faster, and circumvents dreamhost's connection limits.
    > mkdir ~/.ssh/controlmasters; chmod 700 ~/.ssh/controlmasters
  * Add the following to .ssh/config, in the global section at the top:
      ControlPath ~/.ssh/controlmasters/%r@%h:%p
      ControlPersist 10m
      ControlMaster auto
  @ https://github.com/blipvert/ljupdate/commits/master marginally more up to date
    * update emacs/depends.make

2016:
=====

0214Su
  * much hacking in gnome-setup and font setup over the last week or so.

0215Mo
  * added adobe-source-code-pro font, tell emacs to use system font.

... Really ought to copy the config-related stuff out of starport/Admin/admin.to.do

0626
  ~ alltray looked useful - lets you put an icon for any program into the tray.
    It starts the program if it's not already running, but fails in xmonad because it's
    waiting for the program to be iconified.
  * gsimplecal, fbpanel, and probably others have subdirectories in .config
    probably best is to make another directory called dotconfig, parallel to dotfiles

0627Mo
  * made the Makefiles independendent of Tools/Makefile.

0704Mo
  * git subtree add --prefix bin ../bin master
  * The idea is that if there's a pre-existing ~/bin, we can symlink into it.

0705Tu
  * regular dotfiles should start with "_" rather than ".", to make them visible.
    -> move dotfiles/.* -> dotfiles/_*; update dotfiles/Makefile

0716Sa
  . Some dotfiles may need to be modified by a script rather than simply symlinked.
    That's because things like username and email address change from one system to
    another.  In most cases, though, one can use include files generated by make install.

0719Tu
  * When site-dependent parameters can be in separate files (include files/modules).
    the various install targets can initialize them from templates.  A good place to start
    would be .xmonad/Local.hs.  Could also do this for .bashrc; a conditional include
    would make it unnecessary to generate the file if it's not needed. (0716)
    : for gitconfig - note that the file doesn't need to exist
      [include]
	  path = /path/to/file

0725Mo
  * color-coded workspace labels.  ws N makes an xclock with WS=N and the appropriate
    resistor color code.
  * the ability to make ad-hoc workspace labels might help.  Put the list in a file that
    can be read in either xmonad or the bottom-bars script.  Possibly a top dbar with
    vertical menu. (0719) -> add a second argument to ws.
  * Re-running ws N should grep ps for the appropriate color code and kill it.
    An alternative to xclock would be dzen -p.

0726Tu
  x use dzen for workspace labels?  Note that dzen is smaller, but only slightly.
    Problem is that it doesn't seem possible to have it shrinkwrap text.
  * separate widths for first and other top bars; make default 950; adjust ws

0827Sa
  * Allow MakeStuff as well as Tools.  Eventually it will be possible to drop Tools as
    an alternative.

0828Su
  ^ Config -> Honu.  The only real question is whether I want to make Honu a separate
    version, cleaned up and generalized for distribution. -> I don't think so.  It would
    be hard to keep them in sync.  The main me-specific things are the link pages in web,
    and some of the stuff in Archive, but there are also some of the dotfiles (e.g.
    .signature and .Xdefaults-*), stuff in the emacs config, and...  On second thought: I
    think I'd be a very uncomfortable about putting .mailrc, .gnus, and the bookmarks out
    there.  So there's going to have to be a rewrite anyway.  Those can go in -Personal-
  ^ Problematic stuff -- bookmarks, .mailrc, .gnus, etc. belongs in a separate git repo
    that's a subdirectory of local!  Can of course have dotfiles and dotconfig subdirs.
    Can have its own bootstrap, which would get the user-specific out of there, too.  In
    fact, call it local/local-Config or local/My-Honu or some such and derive it from the
    original, with all the old history.  Make a relatively clean start for Honu.

========================================================================================>|
Local Variables:
    fill-column:90
End:

= to.do: =

  o bloggish directories should have a "make wc" target that subtracts the word count of
    the template file.  Or, better, use sed to filter out tags.

  o bootstrap should check ~/.ssh to see whether to use savitzky.net or github.

========================================================================================>
Work Log:
========

2015:
=====

1111We
  @ OpenSSH/Cookbook/Multiplexing - Wikibooks
    Makes ssh much faster, and circumvents dreamhost's connection limits.
    > mkdir ~/.ssh/controlmasters; chmod 700 ~/.ssh/controlmasters
  * Add the following to .ssh/config, in the global section at the top:
      ControlPath ~/.ssh/controlmasters/%r@%h:%p
      ControlPersist 10m
      ControlMaster auto
  @ https://github.com/blipvert/ljupdate/commits/master marginally more up to date
    * update emacs/depends.make

2016:
=====

0214Su
  * much hacking in gnome-setup and font setup over the last week or so.

0215Mo
  * added adobe-source-code-pro font, tell emacs to use system font.

... Really ought to copy the config-related stuff out of starport/Admin/admin.to.do

0626
  ~ alltray looked useful - lets you put an icon for any program into the tray.
    It starts the program if it's not already running, but fails in xmonad because it's
    waiting for the program to be iconified.
  * gsimplecal, fbpanel, and probably others have subdirectories in .config
    probably best is to make another directory called dotconfig, parallel to dotfiles

0627Mo
  * made the Makefiles independendent of Tools/Makefile.

0704Mo
  * git subtree add --prefix bin ../bin master
  * The idea is that if there's a pre-existing ~/bin, we can symlink into it.

0705Tu
  * regular dotfiles should start with "_" rather than ".", to make them visible.
    -> move dotfiles/.* -> dotfiles/_*; update dotfiles/Makefile

0716Sa
  . Some dotfiles may need to be modified by a script rather than simply symlinked.
    That's because things like username and email address change from one system to
    another.  In most cases, though, one can use include files generated by make install.

0719Tu
  * When site-dependent parameters can be in separate files (include files/modules).
    the various install targets can initialize them from templates.  A good place to start
    would be .xmonad/Local.hs.  Could also do this for .bashrc; a conditional include
    would make it unnecessary to generate the file if it's not needed. (0716)
    : for gitconfig - note that the file doesn't need to exist
      [include]
	  path = /path/to/file

0725Mo
  * color-coded workspace labels.  ws N makes an xclock with WS=N and the appropriate
    resistor color code.
  * the ability to make ad-hoc workspace labels might help.  Put the list in a file that
    can be read in either xmonad or the bottom-bars script.  Possibly a top dbar with
    vertical menu. (0719) -> add a second argument to ws.
  * Re-running ws N should grep ps for the appropriate color code and kill it.
    An alternative to xclock would be dzen -p.

0726Tu
  x use dzen for workspace labels?  Note that dzen is smaller, but only slightly.
    Problem is that it doesn't seem possible to have it shrinkwrap text.
  * separate widths for first and other top bars; make default 950; adjust ws

0827Sa
  * Allow MakeStuff as well as Tools.  Eventually it will be possible to drop Tools as
    an alternative.

0828Su
  * Config -> Honu.  The only real question is whether I want to make Honu a separate
    version, cleaned up and generalized for distribution. -> I don't think so.  It would
    be hard to keep them in sync.  The main me-specific things are the link pages in web,
    and some of the stuff in Archive, but there are also some of the dotfiles (e.g.
    .signature and .Xdefaults-*), stuff in the emacs config, and...  On second thought: I
    think I'd be a very uncomfortable about putting .mailrc, .gnus, and the bookmarks out
    there.  So there's going to have to be a rewrite anyway.  Those can go in Personal.

========================================================================================>|
Local Variables:
    fill-column:90
End:


first take result window output

first take result window output


(minor) rewrite ppmlhdfe debug message

(minor) rewrite ppmlhdfe debug message

// --------------------------------------------------------------------------
// FixedEffects constructor (also precomputes factors)
// --------------------------------------------------------------------------

mata:

`FixedEffects' fixed_effects(`Varlist' absvars,
                           | `Varname' touse,
                             `String' weighttype,
                             `Varname' weightvar,
                             `Boolean' drop_singletons,
                             `Boolean' verbose)
{
    `FixedEffects'          S
    `Varname'               absvar, cvars
    `Integer'               i, j, g, gg, remaining
    `Vector'                idx
    `Integer'               spaces
    `Integer'               num_singletons_i
    `Variables'             cvar_data
    `FactorPointer'         pf
    `Matrix'                precond // used for lsmr

    // Set default value of arguments
    if (args()<2) touse = ""
    if (args()<3) weighttype = ""
    if (args()<4) weightvar = ""
    if (args()<5 | drop_singletons==.) drop_singletons = 1
    if (args()<6 | verbose==.) verbose = 0

    S = FixedEffects()
    S.verbose = verbose
    S.drop_singletons = drop_singletons

    // Parse absvars
    if (S.verbose > 0) printf("\n{txt}## Parsing absvars and HDFE options\n")

    if (touse == "") touse = st_tempname()
    st_global("reghdfe_touse", touse)
    stata(`"reghdfe_parse "' + absvars)
    S.sample = `selectindex'(st_data(., touse))
    S.tousevar = touse // useful if later on we want to clone the HDFE object
    st_global("reghdfe_touse", "")

    if (st_global("s(residuals)") != "") S.residuals = st_global("s(residuals)")
    if (st_global("s(verbose)")!="") S.verbose = verbose = strtoreal(st_global("s(verbose)"))
    if (st_global("s(drop_singletons)")!="") S.drop_singletons = drop_singletons = strtoreal(st_global("s(drop_singletons)"))
    assert(S.verbose < .)
    assert(S.drop_singletons==0 | S.drop_singletons==1)

    if (S.verbose > 0) stata("sreturn list")
    S.G = strtoreal(st_global("s(G)"))
    S.absorb = absvars // useful if later on we want to clone the HDFE object
    S.absvars = tokens(st_global("s(absvars)"))
    S.has_intercept = strtoreal(st_global("s(has_intercept)"))
    S.save_any_fe = strtoreal(st_global("s(save_any_fe)"))
    S.save_all_fe = strtoreal(st_global("s(save_all_fe)"))
    S.ivars = tokens(st_global("s(ivars)"))
    S.cvars = tokens(st_global("s(cvars)"))
    S.targets = strtrim(tokens(st_global("s(targets)")))
    S.intercepts = strtoreal(tokens(st_global("s(intercepts)")))
    S.num_slopes = strtoreal(tokens(st_global("s(num_slopes)")))
    S.save_fe = S.targets :!= ""
    S.report_constant = strtoreal(st_global("s(report_constant)"))

    // Ensure that S.report_constant and S.has_intercept are 0/1
    assert(anyof((0,1), S.has_intercept))
    assert(anyof((0,1), S.report_constant))
    S.compute_constant = S.has_intercept & S.report_constant

    if (st_global("s(tolerance)") != "") S.tolerance = strtoreal(st_global("s(tolerance)"))
    if (st_global("s(maxiter)") != "") S.maxiter = strtoreal(st_global("s(maxiter)"))
    if (st_global("s(prune)") != "") S.prune = strtoreal(st_global("s(prune)"))
    if (st_global("s(transform)") != "") S.transform = st_global("s(transform)")
    if (st_global("s(acceleration)") != "") S.acceleration = st_global("s(acceleration)")

    // Override LSMR if G=1
    if (S.G==1 & S.acceleration=="lsmr") S.acceleration = "conjugate_gradient"

    S.dofadjustments = tokens(st_global("s(dofadjustments)"))
    S.groupvar = st_global("s(groupvar)")
    if (st_global("s(finite_condition)")=="1") S.finite_condition = -1 // signal to compute it
    S.compute_rre = (st_global("s(compute_rre)")=="1")
    if (S.compute_rre) S.rre_varname = st_global("s(rre)")

    S.poolsize = strtoreal(st_global("s(poolsize)"))

    if (S.verbose > -1 & !S.has_intercept) printf("{txt}(warning: no intercepts terms in absorb(); regression lacks constant term)\n")

    S.extended_absvars = tokens(st_global("s(extended_absvars)"))
    S.equation_d = st_global("s(equation_d)")
    S.tss = .

    assert(1<=S.G)
    if (S.G>10) printf("{txt}(warning: absorbing %2.0f dimensions of fixed effects; check that you really want that)\n", S.G)
    assert(S.G == cols(S.ivars))
    assert(S.G == cols(S.cvars))
    assert(S.G == cols(S.targets))
    assert(S.G == cols(S.intercepts))
    assert(S.G == cols(S.num_slopes))

    // Fill out object
    S.G = cols(S.absvars)
    S.factors = Factor(S.G)

    assert_msg(anyof(("", "fweight", "pweight", "aweight", "iweight"), weighttype), "wrong weight type")
    S.weight_type = weighttype
    S.weight_var = weightvar

    S.num_singletons = 0
    if (drop_singletons) {
        num_singletons_i = 0
        if (weighttype=="fweight" | weighttype=="iweight") {
            S.weight = st_data(S.sample, weightvar) // just to use it in F.drop_singletons()
        }
    }

    // (1) create the factors and remove singletons
    remaining = S.G
    i = 0
    if (S.verbose > 0) {
        printf("\n{txt}## Initializing Mata object for %g fixed effects\n\n", S.G)
        spaces = max((0, max(strlen(S.absvars))-4))
        printf("{txt}   {c TLC}{hline 4}{c TT}{hline 3}{c TT}{hline 1}%s{hline 6}{c TT}{hline 6}{c TT}{hline 9}{c TT}{hline 11}{c TT}{hline 12}{c TT}{hline 9}{c TT}{hline 14}{c TRC}\n", "{hline 1}" * spaces)
        printf("{txt}   {c |}  i {c |} g {c |} %s Name {c |} Int? {c |} #Slopes {c |}    Obs.   {c |}   Levels   {c |} Sorted? {c |} #Drop Singl. {c |}\n", " " * spaces)
        printf("{txt}   {c LT}{hline 4}{c +}{hline 3}{c +}{hline 1}%s{hline 6}{c +}{hline 6}{c +}{hline 9}{c +}{hline 11}{c +}{hline 12}{c +}{hline 9}{c +}{hline 14}{c RT}\n", "{hline 1}" * spaces)
        displayflush()
    }

    while (remaining) {
        ++i
        g = 1 + mod(i-1, S.G)
        absvar = S.absvars[g]

        if (S.verbose > 0) {
            printf("{txt}   {c |} %2.0f {c |} %1.0f {c |} {res}%s{txt} {c |} ", i, g, (spaces+5-strlen(absvar)) * " " + absvar)
            printf("{txt}{%s}%3s{txt}  {c |}    %1.0f    {c |}", S.intercepts[g] ? "txt" : "err", S.intercepts[g] ? "Yes" : "No", S.num_slopes[g])
            displayflush()
        }

        if (S.verbose > 0) {
            printf("{res}%10.0g{txt} {c |}", rows(S.sample))
            displayflush()
        }

        if (rows(S.sample) < 2) {
            if (S.verbose > 0) printf("\n")
            exit(error(2001))
        }

        if (i<=S.G) {
            // We don't need to save keys (or sort levels but that might change estimates of FEs)
            S.factors[g] = factor(S.ivars[g], S.sample, ., "", ., 1, ., 0)
        }

        if (S.verbose > 0) {
            printf(" {res}%10.0g{txt} {c |} %7s {c |}", S.factors[g].num_levels, S.factors[g].is_sorted ? "Yes" : "No")
            displayflush()
        }

        if (drop_singletons) {

            if (weighttype=="fweight") {
                idx = S.factors[g].drop_singletons(S.weight)
            }
            else if (weighttype=="iweight") {
                idx = S.factors[g].drop_singletons(S.weight, 1) // zero_threshold==1
            }
            else {
                idx = S.factors[g].drop_singletons()
            }

            num_singletons_i = rows(idx)
            S.num_singletons = S.num_singletons + num_singletons_i
            if (S.verbose > 0) {
                printf(" %10.0g   {c |}", num_singletons_i)
                displayflush()
            }

            if (num_singletons_i==0) {
                --remaining
            }
            else {
                remaining = S.G - 1

                // sample[idx] = . // not allowed in Mata; instead, make 0 and then select()
                S.sample[idx] = J(rows(idx), 1, 0)
                S.sample = select(S.sample, S.sample)

                for (j=i-1; j>=max((1, i-remaining)); j--) {
                    gg = 1 + mod(j-1, S.G)
                    S.factors[gg].drop_obs(idx)
                    if (S.verbose > 0) printf("{res} .")
                }
            }
        }
        else {
            if (S.verbose > 0) printf("      n/a     {c |}")
            --remaining
        }
        if (S.verbose > 0) printf("\n")
    }
    if (S.verbose > 0) {
        printf("{txt}   {c BLC}{hline 4}{c BT}{hline 3}{c BT}{hline 1}%s{hline 6}{c BT}{hline 6}{c BT}{hline 9}{c BT}{hline 11}{c BT}{hline 12}{c BT}{hline 9}{c BT}{hline 14}{c BRC}\n", "{hline 1}" * spaces)
    }

    if ( drop_singletons & S.num_singletons>0 & S.verbose>-1 | S.factors[1].num_obs<2) {
        if (weighttype=="iweight") {
            // PPML-specific
            printf(`"{txt}(dropped %s observations that are either singletons or separated by a fixed effect)\n"', strofreal(S.num_singletons))
        }
        else {
            printf(`"{txt}(dropped %s {browse "http://scorreia.com/research/singletons.pdf":singleton observations})\n"', strofreal(S.num_singletons))
        }
    }

    if (S.factors[1].num_obs < 2) {
        exit(error(2001))
    }

    S.N = S.factors[1].num_obs // store number of obs.
    assert(S.N = S.factors[S.G].num_obs)
    assert(S.N > 1)

    // (2) run *.panelsetup() after the sample is defined
    if (S.verbose > 0) printf("\n{txt}## Initializing panelsetup() for each fixed effect\n\n")
    for (g=1; g<=S.G; g++) {
        absvar = S.absvars[g]
        if (S.verbose > 0) printf("{txt}   - panelsetup({res}%s{txt})\n", absvar)
        S.factors[g].panelsetup()
    }

    // (3) load weight
    S.load_weights(weighttype, weightvar, J(0,1,.), 1) // update S.has_weights, S.factors, etc.

    // (4) prune edges of degree-1
    // S.prune = 0 // bugbug
    if (S.prune) S.prune_1core()

    // (5) load cvars
    if (sum(S.num_slopes)) {
        if (S.verbose > 0) printf("\n{txt}## Loading slope variables\n\n")
        for (g=1; g<=S.G; g++) {
            cvars = tokens(S.cvars[g])
            if (S.num_slopes[g]) {
                // Load, standardize, sort by factor, precompute (TODO), and store
                if (S.verbose > 0) printf("{txt}   - cvars({res}%s{txt})\n", invtokens(cvars))
                pf = &(S.factors[g])
                cvar_data = (*pf).sort(st_data(S.sample, cvars))
                asarray((*pf).extra, "x_stdevs", reghdfe_standardize(cvar_data))
                asarray((*pf).extra, "x", cvar_data)
                if (S.intercepts[g]) {
                    asarray((*pf).extra, "xmeans", panelmean(cvar_data, *pf))
                }
                asarray((*pf).extra, "inv_xx", precompute_inv_xx(*pf, S.intercepts[g]))
            }
        }
        cvar_data = .
    }

    // Preconditioners for LSMR
    if (S.acceleration=="lsmr") {
        // Compute M
        S.M = 0
        for (g=1; g<=S.G; g++) {
            S.M = S.M + S.factors[g].num_levels * (S.intercepts[g] + S.num_slopes[g])
        }

        // Preconditioner
        for (g=1; g<=S.G; g++) {
            pf = &(S.factors[g])
            if (S.intercepts[g]) {
                precond = S.has_weights ? asarray((*pf).extra, "weighted_counts") : (*pf).counts
                asarray((*pf).extra, "precond_intercept", sqrt(1 :/ precond))
            }

            if (S.num_slopes[g]) {
                cvars = tokens(S.cvars[g])
                precond = st_data(S.sample, cvars)
                precond = reghdfe_panel_precondition(precond, (*pf))
                asarray((*pf).extra, "precond_slopes", precond)
            }

            precond = .
        }
    }

    // Save "true" residuals for RRE
    if (S.compute_rre) {
        assert_msg(S.rre_varname != "")
        S.rre_true_residual = st_data(S.sample, S.rre_varname)
    }

    return(S)
}

end

// --------------------------------------------------------------------------
// FixedEffects constructor (also precomputes factors)
// --------------------------------------------------------------------------

mata:

`FixedEffects' fixed_effects(`Varlist' absvars,
                           | `Varname' touse,
                             `String' weighttype,
                             `Varname' weightvar,
                             `Boolean' drop_singletons,
                             `Boolean' verbose)
{
    `FixedEffects'          S
    `Varname'               absvar, cvars
    `Integer'               i, j, g, gg, remaining
    `Vector'                idx
    `Integer'               spaces
    `Integer'               num_singletons_i
    `Variables'             cvar_data
    `FactorPointer'         pf
    `Matrix'                precond // used for lsmr

    // Set default value of arguments
    if (args()<2) touse = ""
    if (args()<3) weighttype = ""
    if (args()<4) weightvar = ""
    if (args()<5 | drop_singletons==.) drop_singletons = 1
    if (args()<6 | verbose==.) verbose = 0

    S = FixedEffects()
    S.verbose = verbose
    S.drop_singletons = drop_singletons

    // Parse absvars
    if (S.verbose > 0) printf("\n{txt}## Parsing absvars and HDFE options\n")

    if (touse == "") touse = st_tempname()
    st_global("reghdfe_touse", touse)
    stata(`"reghdfe_parse "' + absvars)
    S.sample = `selectindex'(st_data(., touse))
    S.tousevar = touse // useful if later on we want to clone the HDFE object
    st_global("reghdfe_touse", "")

    if (st_global("s(residuals)") != "") S.residuals = st_global("s(residuals)")
    if (st_global("s(verbose)")!="") S.verbose = verbose = strtoreal(st_global("s(verbose)"))
    if (st_global("s(drop_singletons)")!="") S.drop_singletons = drop_singletons = strtoreal(st_global("s(drop_singletons)"))
    assert(S.verbose < .)
    assert(S.drop_singletons==0 | S.drop_singletons==1)

    if (S.verbose > 0) stata("sreturn list")
    S.G = strtoreal(st_global("s(G)"))
    S.absorb = absvars // useful if later on we want to clone the HDFE object
    S.absvars = tokens(st_global("s(absvars)"))
    S.has_intercept = strtoreal(st_global("s(has_intercept)"))
    S.save_any_fe = strtoreal(st_global("s(save_any_fe)"))
    S.save_all_fe = strtoreal(st_global("s(save_all_fe)"))
    S.ivars = tokens(st_global("s(ivars)"))
    S.cvars = tokens(st_global("s(cvars)"))
    S.targets = strtrim(tokens(st_global("s(targets)")))
    S.intercepts = strtoreal(tokens(st_global("s(intercepts)")))
    S.num_slopes = strtoreal(tokens(st_global("s(num_slopes)")))
    S.save_fe = S.targets :!= ""
    S.report_constant = strtoreal(st_global("s(report_constant)"))

    // Ensure that S.report_constant and S.has_intercept are 0/1
    assert(anyof((0,1), S.has_intercept))
    assert(anyof((0,1), S.report_constant))
    S.compute_constant = S.has_intercept & S.report_constant

    if (st_global("s(tolerance)") != "") S.tolerance = strtoreal(st_global("s(tolerance)"))
    if (st_global("s(maxiter)") != "") S.maxiter = strtoreal(st_global("s(maxiter)"))
    if (st_global("s(prune)") != "") S.prune = strtoreal(st_global("s(prune)"))
    if (st_global("s(transform)") != "") S.transform = st_global("s(transform)")
    if (st_global("s(acceleration)") != "") S.acceleration = st_global("s(acceleration)")

    // Override LSMR if G=1
    if (S.G==1 & S.acceleration=="lsmr") S.acceleration = "conjugate_gradient"

    S.dofadjustments = tokens(st_global("s(dofadjustments)"))
    S.groupvar = st_global("s(groupvar)")
    if (st_global("s(finite_condition)")=="1") S.finite_condition = -1 // signal to compute it
    S.compute_rre = (st_global("s(compute_rre)")=="1")
    if (S.compute_rre) S.rre_varname = st_global("s(rre)")

    S.poolsize = strtoreal(st_global("s(poolsize)"))

    if (S.verbose > -1 & !S.has_intercept) printf("{txt}(warning: no intercepts terms in absorb(); regression lacks constant term)\n")

    S.extended_absvars = tokens(st_global("s(extended_absvars)"))
    S.equation_d = st_global("s(equation_d)")
    S.tss = .

    assert(1<=S.G)
    if (S.G>10) printf("{txt}(warning: absorbing %2.0f dimensions of fixed effects; check that you really want that)\n", S.G)
    assert(S.G == cols(S.ivars))
    assert(S.G == cols(S.cvars))
    assert(S.G == cols(S.targets))
    assert(S.G == cols(S.intercepts))
    assert(S.G == cols(S.num_slopes))

    // Fill out object
    S.G = cols(S.absvars)
    S.factors = Factor(S.G)

    assert_msg(anyof(("", "fweight", "pweight", "aweight", "iweight"), weighttype), "wrong weight type")
    S.weight_type = weighttype
    S.weight_var = weightvar

    S.num_singletons = 0
    if (drop_singletons) {
        num_singletons_i = 0
        if (weighttype=="fweight" | weighttype=="iweight") {
            S.weight = st_data(S.sample, weightvar) // just to use it in F.drop_singletons()
        }
    }

    // (1) create the factors and remove singletons
    remaining = S.G
    i = 0
    if (S.verbose > 0) {
        printf("\n{txt}## Initializing Mata object for %g fixed effects\n\n", S.G)
        spaces = max((0, max(strlen(S.absvars))-4))
        printf("{txt}   {c TLC}{hline 4}{c TT}{hline 3}{c TT}{hline 1}%s{hline 6}{c TT}{hline 6}{c TT}{hline 9}{c TT}{hline 11}{c TT}{hline 12}{c TT}{hline 9}{c TT}{hline 14}{c TRC}\n", "{hline 1}" * spaces)
        printf("{txt}   {c |}  i {c |} g {c |} %s Name {c |} Int? {c |} #Slopes {c |}    Obs.   {c |}   Levels   {c |} Sorted? {c |} #Drop Singl. {c |}\n", " " * spaces)
        printf("{txt}   {c LT}{hline 4}{c +}{hline 3}{c +}{hline 1}%s{hline 6}{c +}{hline 6}{c +}{hline 9}{c +}{hline 11}{c +}{hline 12}{c +}{hline 9}{c +}{hline 14}{c RT}\n", "{hline 1}" * spaces)
        displayflush()
    }

    while (remaining) {
        ++i
        g = 1 + mod(i-1, S.G)
        absvar = S.absvars[g]

        if (S.verbose > 0) {
            printf("{txt}   {c |} %2.0f {c |} %1.0f {c |} {res}%s{txt} {c |} ", i, g, (spaces+5-strlen(absvar)) * " " + absvar)
            printf("{txt}{%s}%3s{txt}  {c |}    %1.0f    {c |}", S.intercepts[g] ? "txt" : "err", S.intercepts[g] ? "Yes" : "No", S.num_slopes[g])
            displayflush()
        }

        if (S.verbose > 0) {
            printf("{res}%10.0g{txt} {c |}", rows(S.sample))
            displayflush()
        }

        if (rows(S.sample) < 2) {
            if (S.verbose > 0) printf("\n")
            exit(error(2001))
        }

        if (i<=S.G) {
            // We don't need to save keys (or sort levels but that might change estimates of FEs)
            S.factors[g] = factor(S.ivars[g], S.sample, ., "", ., 1, ., 0)
        }

        if (S.verbose > 0) {
            printf(" {res}%10.0g{txt} {c |} %7s {c |}", S.factors[g].num_levels, S.factors[g].is_sorted ? "Yes" : "No")
            displayflush()
        }

        if (drop_singletons) {

            if (weighttype=="fweight") {
                idx = S.factors[g].drop_singletons(S.weight)
            }
            else if (weighttype=="iweight") {
                idx = S.factors[g].drop_singletons(S.weight, 1) // zero_threshold==1
            }
            else {
                idx = S.factors[g].drop_singletons()
            }

            num_singletons_i = rows(idx)
            S.num_singletons = S.num_singletons + num_singletons_i
            if (S.verbose > 0) {
                printf(" %10.0g   {c |}", num_singletons_i)
                displayflush()
            }

            if (num_singletons_i==0) {
                --remaining
            }
            else {
                remaining = S.G - 1

                // sample[idx] = . // not allowed in Mata; instead, make 0 and then select()
                S.sample[idx] = J(rows(idx), 1, 0)
                S.sample = select(S.sample, S.sample)

                for (j=i-1; j>=max((1, i-remaining)); j--) {
                    gg = 1 + mod(j-1, S.G)
                    S.factors[gg].drop_obs(idx)
                    if (S.verbose > 0) printf("{res} .")
                }
            }
        }
        else {
            if (S.verbose > 0) printf("      n/a     {c |}")
            --remaining
        }
        if (S.verbose > 0) printf("\n")
    }
    if (S.verbose > 0) {
        printf("{txt}   {c BLC}{hline 4}{c BT}{hline 3}{c BT}{hline 1}%s{hline 6}{c BT}{hline 6}{c BT}{hline 9}{c BT}{hline 11}{c BT}{hline 12}{c BT}{hline 9}{c BT}{hline 14}{c BRC}\n", "{hline 1}" * spaces)
    }

    if ( drop_singletons & S.num_singletons>0 & S.verbose>-1 | S.factors[1].num_obs<2) {
        if (weighttype=="iweight") {
            // PPML-specific
            printf(`"{txt}(dropped %s singleton or separating observations)\n\n"', strofreal(S.num_singletons))
        }
        else {
            printf(`"{txt}(dropped %s {browse "http://scorreia.com/research/singletons.pdf":singleton observations})\n"', strofreal(S.num_singletons))
        }
    }

    if (S.factors[1].num_obs < 2) {
        exit(error(2001))
    }

    S.N = S.factors[1].num_obs // store number of obs.
    assert(S.N = S.factors[S.G].num_obs)
    assert(S.N > 1)

    // (2) run *.panelsetup() after the sample is defined
    if (S.verbose > 0) printf("\n{txt}## Initializing panelsetup() for each fixed effect\n\n")
    for (g=1; g<=S.G; g++) {
        absvar = S.absvars[g]
        if (S.verbose > 0) printf("{txt}   - panelsetup({res}%s{txt})\n", absvar)
        S.factors[g].panelsetup()
    }

    // (3) load weight
    S.load_weights(weighttype, weightvar, J(0,1,.), 1) // update S.has_weights, S.factors, etc.

    // (4) prune edges of degree-1
    // S.prune = 0 // bugbug
    if (S.prune) S.prune_1core()

    // (5) load cvars
    if (sum(S.num_slopes)) {
        if (S.verbose > 0) printf("\n{txt}## Loading slope variables\n\n")
        for (g=1; g<=S.G; g++) {
            cvars = tokens(S.cvars[g])
            if (S.num_slopes[g]) {
                // Load, standardize, sort by factor, precompute (TODO), and store
                if (S.verbose > 0) printf("{txt}   - cvars({res}%s{txt})\n", invtokens(cvars))
                pf = &(S.factors[g])
                cvar_data = (*pf).sort(st_data(S.sample, cvars))
                asarray((*pf).extra, "x_stdevs", reghdfe_standardize(cvar_data))
                asarray((*pf).extra, "x", cvar_data)
                if (S.intercepts[g]) {
                    asarray((*pf).extra, "xmeans", panelmean(cvar_data, *pf))
                }
                asarray((*pf).extra, "inv_xx", precompute_inv_xx(*pf, S.intercepts[g]))
            }
        }
        cvar_data = .
    }

    // Preconditioners for LSMR
    if (S.acceleration=="lsmr") {
        // Compute M
        S.M = 0
        for (g=1; g<=S.G; g++) {
            S.M = S.M + S.factors[g].num_levels * (S.intercepts[g] + S.num_slopes[g])
        }

        // Preconditioner
        for (g=1; g<=S.G; g++) {
            pf = &(S.factors[g])
            if (S.intercepts[g]) {
                precond = S.has_weights ? asarray((*pf).extra, "weighted_counts") : (*pf).counts
                asarray((*pf).extra, "precond_intercept", sqrt(1 :/ precond))
            }

            if (S.num_slopes[g]) {
                cvars = tokens(S.cvars[g])
                precond = st_data(S.sample, cvars)
                precond = reghdfe_panel_precondition(precond, (*pf))
                asarray((*pf).extra, "precond_slopes", precond)
            }

            precond = .
        }
    }

    // Save "true" residuals for RRE
    if (S.compute_rre) {
        assert_msg(S.rre_varname != "")
        S.rre_true_residual = st_data(S.sample, S.rre_varname)
    }

    return(S)
}

end