[iegitaddmd] update content of the default placeholder

*! version 6.2 31JAN2019 DIME Analytics dimeanalytics@worldbank.org

cap program drop   iegitaddmd
	program define iegitaddmd

qui {
	syntax , folder(string) [comparefolder(string) customfile(string) all skip replace manual]

	/******************************
	*******************************
		Test user input
	*******************************
	******************************/

	******************************
	*	Test input: folder

	*Test that folder exist
	mata : st_numscalar("r(dirExist)", direxists("`folder'"))

	if (`r(dirExist)' == 0) | (length("`folder'")<=10) {

		noi di as error `"{phang}The folder used in [folder(`folder')] does not exist or you have not entered the full path. For example, full paths on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}. Important: Specify the whole file path to the repository folder, not just {it:C:/} or {it:/user/} as that would create the placeholder file in every empty folder on your computer!{p_end}"'
		error 693
		exit
	}

	* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
	local folderStd			= subinstr(`"`folder'"',"\","/",.)

	******************************
	*	Test input: comparefolder

	if !missing("`comparefolder'") {
		mata : st_numscalar("r(dirExist)", direxists("`comparefolder'"))

		if (`r(dirExist)' == 0) | (length("`comparefolder'")<=10) {

			noi di as error `"{phang}The folder used in [comaparefolder(`comparefolder')] does not exist or you have not entered the full path. For example, full paths on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}. Important: Specify the whole file path to the repository folder, not just {it:C:/} or {it:/user/} as that would create the placeholder file in every empty folder on your computer!{p_end}"'
			error 693
			exit
		}

		* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
		local comparefolderStd	= subinstr(`"`comparefolder'"',"\","/",.)

		*Get the name of the last folder in both folder() and comparefolder()
		local thisLastSlash = strpos(strreverse(`"`folder'"'),"/")
		local thisFolder 	= substr(`"`folder'"', (-1 * `thisLastSlash')+1 ,.)
		local compLastSlash = strpos(strreverse(`"`comparefolder'"'),"/")
		local compFolder 	= substr(`"`comparefolder'"', (-1 * `compLastSlash')+1 ,.)

		*The last folder name should always be the same for folder() and
		* comparefolder() otherwise there it is likely that the to paths
		* point to differnet starting points in the two fodler trees that
		* are to be compared.
		if ("`compFolder'" != "`thisFolder'") {
			noi di as error `"{phang}The last folder [`thisFolder'] in [folder(`folder')] is not identical to the last folder [`compFolder'] in [comparefolder(`comparefolder')]. This is an indication that the the two fodler trees to be compared are not two versions of the same folder tree.{p_end}"'
			error 693
			exit
		}
	}

	******************************
	*	Test input: customfile

	*Test that file exist if not using default file
	if "`customfile'" != "" {
		*File option used test if file exists
		capture confirm file "`customfile'"
		if _rc {
			*File does not exist, throw error
			noi di as error `"{phang}File "`customfile'" was not found. Remember that you must enter the full path. For example, on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}.{p_end}"'
			error 693
			exit
		}

		* Get the name of file (include path to keep )
		* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
		local customFileStd	= subinstr("`customfile'","\","/",.)
		*Find the last slash in the file name
		local lastSlash = strpos(strreverse(`"`customFileStd'"'),"/")
		*Use that position to get the file name. Multiply with minus one as we count from the back
		local customFileName = substr(`"`customFileStd'"', (-1 * `lastSlash')+1 ,.)
		*Used for the recursive call of the command when there is a subfolder
		local customFileRecurse `"customfile(`customFileStd')"'

		*Path and name of thile regardless if customfile were used
		local newPlaceholderFile `"`customFileName'"'
	}
	else {
		*Path and name of thile regardless if customfile were not used
		local newPlaceholderFile "README.md"
	}

	******************************
	*	Test input: skip & replace

	*Options skip and replace cannot be used at the same time
	if !missing("`skip'") & !missing("`replace'") {
		noi di as error `"{phang}The options skip and replace may not be used at the same time.{p_end}"'
		error 198
		exit
	}

	*Options all, skip and replace cannot be used together with comparefolder()
	if !missing("`comparefolder'") & (!missing("`all'") | !missing("`skip'") | !missing("`replace'")) {
		noi di as error `"{phang}The options all, skip and replace may not be used when option comparefolder() is used.{p_end}"'
		error 198
		exit
	}

	/******************************
	*******************************
		List Files and folders
	*******************************
	******************************/

	******************************
	*	List all files and folders

	*List files, directories and other files
	local flist : dir `"`folderStd'"' files "*"	, respectcase
	local dlist : dir `"`folderStd'"' dirs  "*" , respectcase
	local olist : dir `"`folderStd'"' other "*"	, respectcase

	/******************************
	*******************************
		Exectute command without compare option
	*******************************
	******************************/

	if missing("`comparefolder'") {

		******************************
		*	Call command recursively on all sub-folders

		*Use the command on each subfolder to this folder (if any)
		foreach dir of local dlist {
			*Recursive call on each subfolder
			noi iegitaddmd , folder(`"`folderStd'/`dir'"') `all' `customFileRecurse' `skip' `replace' `manual'
		}

		******************************
		*	Evaluate if file will be crated in this folder

		*Test if all of those lists are empty, meaning there are
		* no folders or files in this folder
		if `"`flist'`dlist'`olist'"' == "" noi writePlaceholder, folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'

		*If the folder is not empty, test if option all were used.
		else if ("`all'" == "all") {

			*Test if a file with exactly that name is already used
			cap confirm file `"`folderStd'/`newPlaceholderFile'"'

			* No file with this name exists in this folder, create file without further ado
			if _rc != 0 noi writePlaceholder, folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'

			* File exist, but no sintructions on how to deal with this has been given, throw error
			else if missing("`skip'") & missing("`replace'")  {
				noi di as error `"{phang}The file  `newPlaceholderFile' already exists. Either remove option {it:all} or use either option {it:skip} or {it:replace}. See help file before using either of these options unless you are already familiar with them.{p_end}"'
				error 602
			}

			* File exist, and instruction is to replace, create files and overwrite as needed
			else if !missing("`replace'") noi writePlaceholder,  folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'

			* File exist, and instruction is to skip, do nothing
			else if !missing("`skip'") {
					*Nothing is done, this if-block is only included for readability and completion
			}

			* The code should nevere get to this point
			else  {
				noi di as error `"{phang}Coding error in if-chain in "All" block.Please report this error to dimeanalytics@worldbank.org or here: https://github.com/worldbank/ietoolkit/issues/new{p_end}"'
				error 602
			}
		}
		else {
			* Folder is not empty and option all is not used, so do nothing in this folder
		}
	}
	/******************************
	*******************************
		Exectute command with compare option
	*******************************
	******************************/

	else if !missing("`comparefolder'") {

		*List folders that are in comparefolder() but not in in folder()
		local comp_dlist : dir `"`comparefolderStd'"' dirs  "*" , respectcase
		local only_in_cdlist : list comp_dlist - dlist

		*Loop over all folders only in comparefolder()
		foreach dir of local only_in_cdlist {
			*Find all leaf folders. A leaf folder is a folder with no
			*sub-folder. Think of the folder structure as a tree, and
			*each folder with no sub-folders is an end of a branch where
			*you find leaves.
			noi findLeafFolders, folder(`"`comparefolderStd'/`dir'"')

			*Loop over all leaves and create accordingly
			foreach leaf in `r(leaves)' {
				*Leaf exist in compare folder, create the path to the leaf
				* to be created in the corresponding folder
				local folderleaf = subinstr(`"`leaf'"',`"`comparefolderStd'"', `"`folder'"', 1)

				*Create placeholder file in this new leaf
				noi writePlaceholder, folder(`"`folderleaf'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'
			}
		}

		*Recursivley do the same call on all sub-folders
		foreach dir of local dlist {
			*Recursive call on each subfolder
			noi iegitaddmd , folder(`"`folderStd'/`dir'"') comparefolder(`"`comparefolder'/`dir'"') `customFileRecurse' `manual'
		}
	}
}

end

*Write a README.md file when iegitaddmd finds an empty folder
cap program drop writePlaceholder
program define   writePlaceholder

qui {
	syntax , folder(string) newfilename(string) [customfiletocopy(string) manual]

	*If manual was used, get manual confirmation for each file
	if !missing("`manual'") {
		noi di ""
		global confirmation "" //Reset global

		*Keep aslking for input until the input is either Y, y, N, n or BREAK
		while (upper("${confirmation}") != "Y" & upper("${confirmation}") != "N" & "${confirmation}" != "BREAK") {
		  noi di as txt "{pstd}You are about to create a file [`newfilename'] in the folder [`folder']. If the folder does not exist it will be created. Do you want to do that? To confirm type {bf:Y} and hit enter, to abort type {bf:N} and hit enter. Type {bf:BREAK} and hit enter to stop the code.{p_end}", _request(confirmation)
		}
		*Copy user input to local
		local createfile = upper("${confirmation}")
		* If user wrote "BREAK" then exit the code
		if ("`createfile'" == "BREAK") error 1
	}

	*If "manual" were used and input was Y or if manual was not used, create the file
	if ("`createfile'" == "Y") | (missing("`manual'")) {

		*Recursively create folder if needed
		noi rmkdir, folder(`"`folder'"')

		*Copy custom file if custom file is used
		if !missing(`"`customfile'"') copy "`customfile'" `"`folder'/`newfilename'"', replace
		*Custom file is not used, so cretae defeault file
		else {
			*Create file
			tempname 	newHandle
			cap file close 	`newHandle'
			file open  	`newHandle' using "`folder'/`newfilename'", text write replace
			*Write the content to the file
			file write  `newHandle' ///
				"# Placeholder file" _n _n ///
				"This file has been created automatically by the command **iegitaddmd** from the Stata package [**ietoolkit**](https://worldbank.github.io/ietoolkit) to make GitHub sync this folder. GitHub does not sync empty folders or folders that only contain ignored files, but in research projects it is often important to share the full standardized folder structure along with the actual files. This command is intended to be used with **iefolder**, but it can be used in any folder structure intended to be shared on GitHub." _n _n ///
				"In recently started projects, it is typical to create data folders, script folders (do-files, r-files) and output folders, among others. The output folders are initially empty, but the script files might include a file path to them for later use. If an output folder is empty and another collaborator clones the repository, then the output folder expected by the scripts will not be included by GitHub in the cloned repository, and the script will not run properly." _n _n ///
				"## You should replace the content of this placeholder file" _n _n ///
				"The text in this file should be replaced with text that describes the intended use of this folder. This file is written in markdown (.md) format, which is suitable for GitHub syncing (unlike .doc/.docx). If the file is named *README.md*, GitHub automatically displays its contents when someone navigates to the containing folder on GitHub.com using a web browser." _n _n ///
				"If you are new to markup languages (markdown, html etc.) then this [Markdown Tutorial](https://www.markdowntutorial.com/) is a great place to start. If you have some experience with markup languages, then this [Markdown Cheat Sheet](https://guides.github.com/pdfs/markdown-cheatsheet-online.pdf) is a great resource." _n _n ///
				"## Add similar files in other folders" _n _n ///
				"The Stata command **iegitaddmd** does not add anything to folders that already have content unless this is explicitly requested using the option `all`. It is best practice to create a *README.md* file in any folders that have content, for example, in folders which purpose might not obvious to someone using the repository for the first time. Again, if the file is named *README.md*, then the content of the file will be shown in the browser when someone explores the repository on GitHub.com. This is a very good way to document your code and your data work." _n _n ///
				"Another great use of a *README.md* file is to use it as a documentation on how the folder it sits in and its subfolders are organized,  where its content can be found, and where new content is meant to be saved. For example, if you have a folder called `/Baseline/`, then you can give a short description of the activities conducted during the baseline and where data, scripts and outputs related to it can be found." _n _n ///
				"## Removing this file" _n _n ///
				"Our recommendation is to not remove this file, as GitHub may stop syncing the parent folder unless the folder now has other content. We recommend to not even remove this file when content that is committed to this repository is added to this folder and the file can be removed without breaking the GitHub functionality, as it is better practice to replace the content of this file with content describing this specific folder rather than deleting it." _n _n

			*Closing the file
			file close 		`newHandle'
		}
		*Output that the file was created
		noi di as result "{pstd}File [`folder'/`newfilename'] created.{p_end}"
	}
	*Manual was used and input was N, no file were creaetd
	else noi di as result "{pstd}No file or folder created.{p_end}"
}
end

*Write a README.md file when iegitaddmd finds an empty folder
cap program drop findLeafFolders
program define   findLeafFolders, rclass

qui {
	syntax, folder(string)

	local dlist : dir `"`folder'"' dirs  "*" , respectcase

	if missing(`"`dlist'"') {
		*This is a leaf, pass it back
		return local leaves `""`folder'""'
	}
	else {
		local return_leaves
		foreach dir of local dlist {
			*Recursive call on each subfolder
			findLeafFolders, folder(`"`folder'/`dir'"')
			local return_leaves `"`return_leaves' `r(leaves)'"'
		}
		return local leaves `"`return_leaves'"'
	}
}
end

*Recursively call parent folders in folderpath needed to be created until
* folder found that already exist, then creaet all subfolders.
cap program drop rmkdir
program define   rmkdir, rclass

qui {
	syntax, folder(string) [folderstocreate(string)]

	*Test if this folder exists
	mata : st_numscalar("r(dirExist)", direxists(`"`folder'"'))

	*Folder does not exist, find parent folder and make recursive call
	if (`r(dirExist)' == 0) {
		*Get the parent folder of folder
		local lastSlash = strpos(strreverse(`"`folder'"'),"/")
		local parentFolder = substr(`"`folder'"',1,strlen("`folder'")-`lastSlash')
		local thisFolder = substr(`"`folder'"', (-1 * `lastSlash')+1 ,.)

		*Make the coll recursivly on the parent folder and add this folder to folderstocreate()
		noi rmkdir , folder(`"`parentFolder'"') folderstocreate(`""`thisFolder'" `folderstocreate'"')
	}
	*Folder exist, create sub-folders to create if any
	else {
		*Create the folders needed
		foreach dir of local folderstocreate {
			mkdir "`folder'/`dir'"
			local folder "`folder'/`dir'"
		}
	}
}
end

*! version 6.2 31JAN2019 DIME Analytics dimeanalytics@worldbank.org

cap program drop   iegitaddmd
	program define iegitaddmd

qui {
	syntax , folder(string) [comparefolder(string) customfile(string) all skip replace manual]

	/******************************
	*******************************
		Test user input
	*******************************
	******************************/

	******************************
	*	Test input: folder

	*Test that folder exist
	mata : st_numscalar("r(dirExist)", direxists("`folder'"))

	if (`r(dirExist)' == 0) | (length("`folder'")<=10) {

		noi di as error `"{phang}The folder used in [folder(`folder')] does not exist or you have not entered the full path. For example, full paths on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}. Important: Specify the whole file path to the repository folder, not just {it:C:/} or {it:/user/} as that would create the placeholder file in every empty folder on your computer!{p_end}"'
		error 693
		exit
	}

	* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
	local folderStd			= subinstr(`"`folder'"',"\","/",.)

	******************************
	*	Test input: comparefolder

	if !missing("`comparefolder'") {
		mata : st_numscalar("r(dirExist)", direxists("`comparefolder'"))

		if (`r(dirExist)' == 0) | (length("`comparefolder'")<=10) {

			noi di as error `"{phang}The folder used in [comaparefolder(`comparefolder')] does not exist or you have not entered the full path. For example, full paths on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}. Important: Specify the whole file path to the repository folder, not just {it:C:/} or {it:/user/} as that would create the placeholder file in every empty folder on your computer!{p_end}"'
			error 693
			exit
		}

		* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
		local comparefolderStd	= subinstr(`"`comparefolder'"',"\","/",.)

		*Get the name of the last folder in both folder() and comparefolder()
		local thisLastSlash = strpos(strreverse(`"`folder'"'),"/")
		local thisFolder 	= substr(`"`folder'"', (-1 * `thisLastSlash')+1 ,.)
		local compLastSlash = strpos(strreverse(`"`comparefolder'"'),"/")
		local compFolder 	= substr(`"`comparefolder'"', (-1 * `compLastSlash')+1 ,.)

		*The last folder name should always be the same for folder() and
		* comparefolder() otherwise there it is likely that the to paths
		* point to differnet starting points in the two fodler trees that
		* are to be compared.
		if ("`compFolder'" != "`thisFolder'") {
			noi di as error `"{phang}The last folder [`thisFolder'] in [folder(`folder')] is not identical to the last folder [`compFolder'] in [comparefolder(`comparefolder')]. This is an indication that the the two fodler trees to be compared are not two versions of the same folder tree.{p_end}"'
			error 693
			exit
		}
	}

	******************************
	*	Test input: customfile

	*Test that file exist if not using default file
	if "`customfile'" != "" {
		*File option used test if file exists
		capture confirm file "`customfile'"
		if _rc {
			*File does not exist, throw error
			noi di as error `"{phang}File "`customfile'" was not found. Remember that you must enter the full path. For example, on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}.{p_end}"'
			error 693
			exit
		}

		* Get the name of file (include path to keep )
		* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
		local customFileStd	= subinstr("`customfile'","\","/",.)
		*Find the last slash in the file name
		local lastSlash = strpos(strreverse(`"`customFileStd'"'),"/")
		*Use that position to get the file name. Multiply with minus one as we count from the back
		local customFileName = substr(`"`customFileStd'"', (-1 * `lastSlash')+1 ,.)
		*Used for the recursive call of the command when there is a subfolder
		local customFileRecurse `"customfile(`customFileStd')"'

		*Path and name of thile regardless if customfile were used
		local newPlaceholderFile `"`customFileName'"'
	}
	else {
		*Path and name of thile regardless if customfile were not used
		local newPlaceholderFile "README.md"
	}

	******************************
	*	Test input: skip & replace

	*Options skip and replace cannot be used at the same time
	if !missing("`skip'") & !missing("`replace'") {
		noi di as error `"{phang}The options skip and replace may not be used at the same time.{p_end}"'
		error 198
		exit
	}

	*Options all, skip and replace cannot be used together with comparefolder()
	if !missing("`comparefolder'") & (!missing("`all'") | !missing("`skip'") | !missing("`replace'")) {
		noi di as error `"{phang}The options all, skip and replace may not be used when option comparefolder() is used.{p_end}"'
		error 198
		exit
	}

	/******************************
	*******************************
		List Files and folders
	*******************************
	******************************/

	******************************
	*	List all files and folders

	*List files, directories and other files
	local flist : dir `"`folderStd'"' files "*"	, respectcase
	local dlist : dir `"`folderStd'"' dirs  "*" , respectcase
	local olist : dir `"`folderStd'"' other "*"	, respectcase

	/******************************
	*******************************
		Exectute command without compare option
	*******************************
	******************************/

	if missing("`comparefolder'") {

		******************************
		*	Call command recursively on all sub-folders

		*Use the command on each subfolder to this folder (if any)
		foreach dir of local dlist {
			*Recursive call on each subfolder
			noi iegitaddmd , folder(`"`folderStd'/`dir'"') `all' `customFileRecurse' `skip' `replace' `manual'
		}

		******************************
		*	Evaluate if file will be crated in this folder

		*Test if all of those lists are empty, meaning there are
		* no folders or files in this folder
		if `"`flist'`dlist'`olist'"' == "" noi writePlaceholder, folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'

		*If the folder is not empty, test if option all were used.
		else if ("`all'" == "all") {

			*Test if a file with exactly that name is already used
			cap confirm file `"`folderStd'/`newPlaceholderFile'"'

			* No file with this name exists in this folder, create file without further ado
			if _rc != 0 noi writePlaceholder, folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'

			* File exist, but no sintructions on how to deal with this has been given, throw error
			else if missing("`skip'") & missing("`replace'")  {
				noi di as error `"{phang}The file  `newPlaceholderFile' already exists. Either remove option {it:all} or use either option {it:skip} or {it:replace}. See help file before using either of these options unless you are already familiar with them.{p_end}"'
				error 602
			}

			* File exist, and instruction is to replace, create files and overwrite as needed
			else if !missing("`replace'") noi writePlaceholder,  folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'

			* File exist, and instruction is to skip, do nothing
			else if !missing("`skip'") {
					*Nothing is done, this if-block is only included for readability and completion
			}

			* The code should nevere get to this point
			else  {
				noi di as error `"{phang}Coding error in if-chain in "All" block.Please report this error to dimeanalytics@worldbank.org or here: https://github.com/worldbank/ietoolkit/issues/new{p_end}"'
				error 602
			}
		}
		else {
			* Folder is not empty and option all is not used, so do nothing in this folder
		}
	}
	/******************************
	*******************************
		Exectute command with compare option
	*******************************
	******************************/

	else if !missing("`comparefolder'") {

		*List folders that are in comparefolder() but not in in folder()
		local comp_dlist : dir `"`comparefolderStd'"' dirs  "*" , respectcase
		local only_in_cdlist : list comp_dlist - dlist

		*Loop over all folders only in comparefolder()
		foreach dir of local only_in_cdlist {
			*Find all leaf folders. A leaf folder is a folder with no
			*sub-folder. Think of the folder structure as a tree, and
			*each folder with no sub-folders is an end of a branch where
			*you find leaves.
			noi findLeafFolders, folder(`"`comparefolderStd'/`dir'"')

			*Loop over all leaves and create accordingly
			foreach leaf in `r(leaves)' {
				*Leaf exist in compare folder, create the path to the leaf
				* to be created in the corresponding folder
				local folderleaf = subinstr(`"`leaf'"',`"`comparefolderStd'"', `"`folder'"', 1)

				*Create placeholder file in this new leaf
				noi writePlaceholder, folder(`"`folderleaf'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'
			}
		}

		*Recursivley do the same call on all sub-folders
		foreach dir of local dlist {
			*Recursive call on each subfolder
			noi iegitaddmd , folder(`"`folderStd'/`dir'"') comparefolder(`"`comparefolder'/`dir'"') `customFileRecurse' `manual'
		}
	}
}

end

*Write a README.md file when iegitaddmd finds an empty folder
cap program drop writePlaceholder
program define   writePlaceholder

qui {
	syntax , folder(string) newfilename(string) [customfiletocopy(string) manual]

	*If manual was used, get manual confirmation for each file
	if !missing("`manual'") {
		noi di ""
		global confirmation "" //Reset global

		*Keep aslking for input until the input is either Y, y, N, n or BREAK
		while (upper("${confirmation}") != "Y" & upper("${confirmation}") != "N" & "${confirmation}" != "BREAK") {
		  noi di as txt "{pstd}You are about to create a file [`newfilename'] in the folder [`folder']. If the folder does not exist it will be created. Do you want to do that? To confirm type {bf:Y} and hit enter, to abort type {bf:N} and hit enter. Type {bf:BREAK} and hit enter to stop the code.{p_end}", _request(confirmation)
		}
		*Copy user input to local
		local createfile = upper("${confirmation}")
		* If user wrote "BREAK" then exit the code
		if ("`createfile'" == "BREAK") error 1
	}

	*If "manual" were used and input was Y or if manual was not used, create the file
	if ("`createfile'" == "Y") | (missing("`manual'")) {

		*Recursively create folder if needed
		noi rmkdir, folder(`"`folder'"')

		*Copy custom file if custom file is used
		if !missing(`"`customfile'"') copy "`customfile'" `"`folder'/`newfilename'"', replace
		*Custom file is not used, so cretae defeault file
		else {
			*Create file
			tempname 	newHandle
			cap file close 	`newHandle'
			file open  	`newHandle' using "`folder'/`newfilename'", text write replace
			*Write the content to the file
			file write  `newHandle' ///
				"# Placeholder file" _n _n ///
				"This file has been created automatically by the command **iegitaddmd** from the Stata package [**ietoolkit**](https://worldbank.github.io/ietoolkit) to make GitHub sync this folder. GitHub does not sync empty folders or folders that only contain ignored files, but in research projects it is often important to share the full standardized folder structure along with the actual files. This command is intended to be used with **iefolder**, but it can be used in any folder structure intended to be shared on GitHub." _n _n ///
				"In recently started projects, it is typical to create data folders, script folders (do-files, r-files) and output folders, among others. The output folders are initially empty, but the script files might include a file path to them for later use. If an output folder is empty and another collaborator clones the repository, then the output folder expected by the scripts will not be included by GitHub in the cloned repository, and the script will not run properly." _n _n ///
				"## You should replace the content of this placeholder file" _n _n ///
				"The text in this file should be replaced with text that describes the intended use of this folder. This file is written in markdown (.md) format, which is suitable for GitHub syncing (unlike .doc/.docx). If the file is named *README.md*, GitHub automatically displays its contents when someone opens the containing folder on GitHub.com using a web browser." _n _n ///
				"If you are new to markup languages (markdown, html etc.) then this [Markdown Tutorial](https://www.markdowntutorial.com/) is a great place to start. If you have some experience with markup languages, then this [Markdown Cheat Sheet](https://guides.github.com/pdfs/markdown-cheatsheet-online.pdf) is a great place to start." _n _n ///
				"## Add similar files in other folders" _n _n ///
				"The Stata command **iegitaddmd** does not add anything to folders that already have content unless this is explicitly requested using the option `all`. It is best practice to create a *README.md* file in any folders that have content, for example, in folders whose purpose might not obvious to someone using the repository for the first time. Again, if the file is named *README.md*, then the content of the file will be shown in the browser when someone explores the repository on GitHub.com." _n _n ///
				"Another great use of a *README.md* file is to use it as a documentation on how the folder it sits in and its subfolders are organized,  where its content can be found, and where new content is meant to be saved. For example, if you have a folder called `/Baseline/`, then you can give a short description of the activities conducted during the baseline and where data, scripts and outputs related to it can be found." _n _n ///
				"## Removing this file" _n _n ///
				"Removing this file is not recommended, as GitHub may stop syncing the parent folder unless the folder now has other content. However, even when content is added and the file can be removed without breaking the GitHub functionality, our recommendation is to replace the content with more relevant content rather than deleting it." _n _n

			*Closing the file
			file close 		`newHandle'
		}
		*Output that the file was created
		noi di as result "{pstd}File [`folder'/`newfilename'] created.{p_end}"
	}
	*Manual was used and input was N, no file were creaetd
	else noi di as result "{pstd}No file or folder created.{p_end}"
}
end

*Write a README.md file when iegitaddmd finds an empty folder
cap program drop findLeafFolders
program define   findLeafFolders, rclass

qui {
	syntax, folder(string)

	local dlist : dir `"`folder'"' dirs  "*" , respectcase

	if missing(`"`dlist'"') {
		*This is a leaf, pass it back
		return local leaves `""`folder'""'
	}
	else {
		local return_leaves
		foreach dir of local dlist {
			*Recursive call on each subfolder
			findLeafFolders, folder(`"`folder'/`dir'"')
			local return_leaves `"`return_leaves' `r(leaves)'"'
		}
		return local leaves `"`return_leaves'"'
	}
}
end

*Recursively call parent folders in folderpath needed to be created until
* folder found that already exist, then creaet all subfolders.
cap program drop rmkdir
program define   rmkdir, rclass

qui {
	syntax, folder(string) [folderstocreate(string)]

	*Test if this folder exists
	mata : st_numscalar("r(dirExist)", direxists(`"`folder'"'))

	*Folder does not exist, find parent folder and make recursive call
	if (`r(dirExist)' == 0) {
		*Get the parent folder of folder
		local lastSlash = strpos(strreverse(`"`folder'"'),"/")
		local parentFolder = substr(`"`folder'"',1,strlen("`folder'")-`lastSlash')
		local thisFolder = substr(`"`folder'"', (-1 * `lastSlash')+1 ,.)

		*Make the coll recursivly on the parent folder and add this folder to folderstocreate()
		noi rmkdir , folder(`"`parentFolder'"') folderstocreate(`""`thisFolder'" `folderstocreate'"')
	}
	*Folder exist, create sub-folders to create if any
	else {
		*Create the folders needed
		foreach dir of local folderstocreate {
			mkdir "`folder'/`dir'"
			local folder "`folder'/`dir'"
		}
	}
}
end


ieduplicates : folder() missing in helpfile

The option folder() was only mentioned in the section where options are
described in detail, not in the syntax section. This is now corrected.

{smcl}
{* 1 Nov 2016}{...}
{hline}
help for {hi:ieduplicates}
{hline}

{title:Title}

{phang2}{cmdab:ieduplicates} {hline 2} Identify duplicates in ID variable and export them in
an Excel file that also can be used to correct the duplicates

{title:Syntax}

{phang2}
{cmdab:ieduplicates}
{it:ID_varname}
, {cmdab:fol:der(}{it:string}{cmd:)} {cmdab:unique:vars(}{it:varlist}{cmd:)}
[{cmdab:keep:vars(}{it:varlist}{cmd:)} {cmdab:tostringok} {cmdab:droprest}
{cmdab:nodaily} {cmdab:min:precision(}{it:numlist}{cmd:)]}

{phang2}where {it:ID_varname} is the variable that will be controlled for duplicates

{marker opts}{...}
{synoptset 28}{...}
{synopthdr:options}
{synoptline}
{synopt :{cmdab:fol:der(}{it:string}{cmd:)}}folder in which the duplicate report will be saved{p_end}
{synopt :{cmdab:unique:vars(}{it:varlist}{cmd:)}}variables used as unique ID within groups of duplicates in {it:ID_varname}{p_end}
{synopt :{cmdab:keep:vars(}{it:varlist}{cmd:)}}variables used to be included in the Excel report in addition to {it:ID_varname} and {cmdab:unique:vars()} {p_end}
{synopt :{cmdab:tostringok}}allows {it:ID_varname} to be recasted to string if required{p_end}
{synopt :{cmdab:droprest}}disables the requirement that duplicates must be explicitly deleted{p_end}
{synopt :{cmdab:nodaily}}disables daily back-up copies of the Excel report{p_end}
{synopt :{cmdab:min:precision(}{it:numlist}{cmd:)}}({it:rarely used}) manually set the precision when exporting and importing time variables to and from the Excel file{p_end}
{synoptline}

{title:Description}

{dlgtab:In brief:}
{pstd}{cmd:ieduplicates} outputs a report with any duplicates in {it:ID_varname} to an Excel file
and return the data set without those duplicates. Each time {cmd:ieduplicates} executes, it first
looks for an already created version of the Excel report, and applies any corrections already listed in it
before generating a new report. Note that there is no need import the corrections manually. This command
reads the corrections directly from the Excel file as long as the is saved at the same folder location
with the same file name.

{dlgtab:In more detail:}
{pstd}{cmd:ieduplicates} takes duplicates observations in {it:ID_varname} and export
them to an Excel report in directory {cmdab:fol:der(}{it:string}{cmd:)}. {it:ID_varname}
is per definition not unique in this Excel Report and {cmdab:unique:vars(}{it:varlist}{cmd:)}
needs to be specified in order to have a unique reference for each row in the Excel report. The
{it:varlist} in {cmdab:unique:vars(}{it:varlist}{cmd:)} must uniquely and fully identify all
observations in the Excel report, either on its own or together with {it:ID_varname}. {cmd:ieduplicates}
then returns the data set without these duplicates.

{pstd}The Excel report includes three columns called {it:correct}, {it:drop} and {it:newID}.
Each of them represents one way to correct the duplicates. If {it:correct} is indicated with
a "Yes" then that observation is kept unchanged, if {it:drop} is indicated with a "yes" then
that observation is deleted and if {it:newID} is indicated then that observation is assigned
a new ID using the value in column {it:newID}. After corrections are entered, the report should
be saved in the same location {cmdab:fol:der(}{it:string}{cmd:)} without any changes to its name.

{pstd}Before outputting a new report {cmd:ieduplicates} always checks if there already are an
Excel report with corrections and applies those corrections before generating a new report. It is
at this stage that {cmdab:unique:vars(}{it:varlist}{cmd:)} is required as it otherwise is impossible
to know which duplicate within a group of duplicates that should be corrected in which way.

{pstd}{cmd:ieduplicates} keeps only one observation if a group of duplicates are duplicates in
all variables across the data set without any action is needed in the Excel report. These cases
are not even exported to the Excel report.

{pstd}{cmdab:keep:vars(}{it:varlist}{cmd:)} allows the user to include more variables in the Excel report
that can help identifying each duplicate is supposed to be corrected. The report also includes two
columns {it:initials} and {it:notes}. Using these columns is not required but it is recommended to use {it:initials}
to keep track of who decided how to correct that duplicate and to use {it:notes} to document why
the correction was chosen. If {it:initials} and {it:notes} are used, then the Excel report also functions
as an excellent documentation of the correction made.

{space 4}{hline}

{title:Options}

{phang}{cmdab:fol:der(}{it:string}{cmd:)} specifies the folder where previous Excel
files will be looked for, and where the updated Excel Report will be exported. Note that
this folder needs to have a subfolder called {it:Daily} where the duplicate report
file is backed up daily.

{phang}{cmdab:unique:vars(}{it:varlist}{cmd:)} list variables that by themselves or together
with {it:ID_varname} uniquely identifies all observations. This varlist is required when the corrections are
imported back into Stata and merged with the original data set. Time variables
should always be avoided if possible in {cmdab:uniquevars()}. See option {cmdab:min:precision()} for
an explanation of why time variables should be avoided. Data that has been downloaded from
a server usually has a variable called "KEY" or similar. Such a variable would be optimal
for {cmdab:unique:vars(}{it:varlist}{cmd:)}.

{phang}{cmdab:keep:vars(}{it:varlist}{cmd:)} list variables to be included in the exported
Excel report. These variables can help team members identifying which observation to keep,
drop and assign a new ID to. For data integrity reasons, be careful not to export and share
Excel files including both identifying variables and names together with {it:ID_varname}.

{phang}{cmdab:tostringok} allows {it:ID_varname} to be turned into a string variable in case
{it:ID_varname} is numeric but a value listed in {it:newID} is non-numeric. Otherwise an error is generated.

{phang}{cmdab:droprest} disables the requirement that duplicates must be explicitly deleted.
The default is that if one of the duplicates in a group of duplicates has a
correction, then that correction is only valid if all other duplicates in that
group have a correction as well. For example, if there are four observation with
the same value for {it:ID_varname} and one is correct, one needs a new ID and
two are incorrect and should be deleted. Then the first one is indicated to be
kept in the {it:correct} column, the second one is given a new ID in {it:newID}
and the other two observations must be indicated for deletion in {it:drop}
unless {cmdab:droprest}. The first two corrections are not considered valid and
will cause an error in case if {cmdab:droprest} is not specified and the other
two observations are not explicitly indicated to be dropped. It is recommended
to not use {cmdab:droprest} and to manually indicate all deletions to avoid
mistakes, but this option exist for cases when that might be very inconvenient.

{phang}{cmdab:nodaily} disables the generation of daily back-up copies of the
Excel report. The default is that the command saves dated copies of the Excel
report in a sub-folder called Daily in the folder specified in {cmdab:folder()}. If
the folder Daily does not exist, then it either needs to be created or the
option {cmdab:nodaily} need to be specified in order to not cause an error.

{phang}{cmdab:min:precision(}{it:numlist}{cmd:)} is rarely used but can be used
to manually set the precision (in minutes) when exporting and importing a time
variable to and from the Excel report. Time variables should always be avoided
if possible in {cmdab:uniquevars()}, but sometimes they are the only option. While
Stata and Excel both keep a very high precision in time variables, they do so
slightly differently, and this can generate a difference of a few seconds after
a time variable was exported to Excel and then imported back to Stata. If the
time variable is used in {cmdab:uniquevars()}, then the time variable may no
longer be identical to its original value after it is imported back to Stata, and it
may therefore no longer be possible to use it to merge the Excel data to the correct Stata
observation. If this happens, then {cmdab:min:precision()} can be used to set the
precision manually. This should only be considered a solution of last resort,
as lowering the precision increases the risk the time variable no longer uniquely
identifies each observation. The typical user will never use this option.

{title:The Excel Report}

{pstd}A report of duplicates will be created in {cmdab:fol:der(}{it:string}{cmd:)}
if any duplicates in {it:ID_varname} were found. The folder listed in
{cmdab:fol:der(}{it:string}{cmd:)} must have a subfolder called {it:Daily}
where daily back-ups of the report are saved. If a report is back-uped already
that day, then that report will be overwritten.

{pstd}All duplicates in a group of duplicates must have a correction indicated. If
one or more duplicates are indicated as correct in {it:correct} or assigned a new
ID in {it:newID}, then all other duplicates with the same value in {it:ID_varname} must
be explicitly indicated for deletion. This requirement may (but probably
shouldn't) be disabled by option {cmdab:droprest}.

{dlgtab:Columns in Excel Report filled in automatically:}

{phang}{it:dupListID} stores an auto incremented duplicate list ID that is used
to maintain the sort order in the Excel Report regardless of how the data in memory
is sorted at the time {cmd:ieduplicates} is executed.

{phang}{it:dateListed} stores the date the duplicate was first identified.

{phang}{it:dateFixed} stores the date a valid correction was imported the first
time for that duplicate.

{dlgtab:Columns in Excel Report to be filled in manually by a user:}

{phang}{it:correct} is used to indicate that the duplicate should be kept. Valid values are
restricted to "yes" and "y" to reduce the risk of unintended entries. The values
are not sensitive to case. All valid values are changed to "yes" lower case when
imported. If {it:correct} is indicated then both {it:drop} and {it:newID} must be
left empty.

{phang}{it:drop} is used to indicate that the duplicate should be deleted. Valid values are
restricted to "yes" and "y" to reduce the risk of unintended entries. The values
are not sensitive to case. All valid values are changed to "yes" lower case when
imported. If {it:drop} is indicated then both {it:correct} and {it:newID} must be
left empty.

{phang}{it:newID} is used to assign a new ID values to a duplicate. If {it:ID_varname}
is a string then all values are valid for {it:newID}. If {it:ID_varname} is numeric then
only digits are valid, unless the option {cmdab:tostringok} is specified.
If {cmdab:tostringok} is specified and {it:newID} is non-numeric, then {it:ID_varname}
is recasted to a string variable. If {it:newID} is indicated then both {it:correct} and {it:drop} must be
left empty.

{phang}{it:initials} allows the team working with this data to keep track on who
decided on corrections.

{phang}{it:notes} allows the team working with this data to document the reason
for the duplicates and the why one type of correction was chosen over the others.

{dlgtab:Columns in Excel Report with data from the data set:}

{pstd}The columns above are followed by the values in {cmdab:unique:vars(}{it:varlist}{cmd:)}
and in {cmdab:keep:vars(}{it:varlist}{cmd:)}. These column keeps the name the
variables have in the data set. These variables can help the team to identify
which correction should be applied to which duplicate.

{space 4}{hline}

{marker results}{...}
{title:Stored results}

{pstd}
{cmdab:ieduplicates} stores the following results in {hi:r()}:

{synoptset 15 tabbed}{...}
{p2col 5 15 19 2: Scalars}{p_end}
{synopt:{cmd:r(numDup)}}number of unresolved duplicates{p_end}
{p2colreset}{...}

{pstd}
{cmd:r(numDup)} is intended to allow for the option to pause Stata in case unresolved duplicates
are found. See example 4 below for a code example on how to use {cmd:r(numDup)}. See {help pause} for instructions
on how to resume the execution of the code if Stata is paused.

{title:Examples}

{pstd}
{hi:Example 1.}

{phang2}{inp:ieduplicates HHID, folder(C:\myImpactEvaluation\baseline\data) uniquevars(KEY)}{p_end}

{pmore}Specified like this {cmdab:ieduplicates} start by looking for any corrections in any
duplicates report in the folder C:\myImpactEvaluation\baseline\data. If there is a report
with corrections those corrections are applied to the data set. Then the command looks for
unresolved duplicates in HHID and exports a new report if any duplicates were found. The data
set is returned without any of the unresolved duplicates. The variable KEY is used to separate
observations that are duplication in the ID var.

{phang}
{hi:Example 2.}

{phang2}{inp:ieduplicates HHID, folder(C:\myImpactEvaluation\baseline\data) keepvars(enumerator) uniquevars(KEY)}{p_end}

{pmore}Similar to the example above, but it also includes the variable enumerator in the Excel
report which is most likely helpful if the data set is collected through a household survey.

{phang}
{hi:Example 3.} Using {cmd:r(numDup)} to pause the execution of the code if
unresolved duplicates were found

{phang2}{inp:ieduplicates HHID, folder(C:\myImpactEvaluation\baseline\data) uniquevars(KEY)}{p_end}
{phang2}{inp:if (r(numDup) != 0) {c -(}}{p_end}
{phang3}{inp:pause}{p_end}
{phang2}{inp:{c )-}}{p_end}

{phang}
{hi:Example 4.} Using the Excel file. The table below could be the report generated in Example 2 above. Make the viewer window wider and reload the page if the table below does not display properly!

{col 3}{c TLC}{hline 116}{c TRC}
{col 3}{c |}{col 4}HHID{col 10}dupListID{col 21}dateListed{col 33}dateFixed{col 44}correct{col 53}drop{col 59}newID{col 65}initials{col 75}note{col 94}KEY{col 107}enumerator{col 120}{c |}
{col 3}{c LT}{hline 116}{c RT}
{col 3}{c |}{col 4}4321{col 10}1{col 21}27Dec2015{col 33}02Jan2016{col 44}yes{col 53}   {col 59}    {col 65}KB{col 75}double submission{col 94}{it:uniquevalue}{col 107}{it:keepvarvalue}{col 120}{c |}
{col 3}{c |}{col 4}4321{col 10}2{col 21}27Dec2015{col 33}02Jan2016{col 44}   {col 53}yes{col 59}    {col 65}KB{col 75}double submission{col 94}{it:uniquevalue}{col 107}{it:keepvarvalue}{col 120}{c |}
{col 3}{c |}{col 4}7365{col 10}3{col 21}03Jan2016{col 33}         {col 44}   {col 53}   {col 59}    {col 65}  {col 75}                 {col 94}{it:uniquevalue}{col 107}{it:keepvarvalue}{col 120}{c |}
{col 3}{c |}{col 4}7365{col 10}4{col 21}03Jan2016{col 33}         {col 44}   {col 53}   {col 59}    {col 65}  {col 75}                 {col 94}{it:uniquevalue}{col 107}{it:keepvarvalue}{col 120}{c |}
{col 3}{c |}{col 4}1145{col 10}5{col 21}03Jan2016{col 33}11Jan2016{col 44}   {col 53}   {col 59}1245{col 65}IB{col 75}incorrect id     {col 94}{it:uniquevalue}{col 107}{it:keepvarvalue}{col 120}{c |}
{col 3}{c |}{col 4}1145{col 10}6{col 21}03Jan2016{col 33}11Jan2016{col 44}yes{col 53}   {col 59}    {col 65}IB{col 75}correct id       {col 94}{it:uniquevalue}{col 107}{it:keepvarvalue}{col 120}{c |}
{col 3}{c |}{col 4}9834{col 10}7{col 21}11Jan2016{col 33}         {col 44}   {col 53}   {col 59}    {col 65}  {col 75}                 {col 94}{it:uniquevalue}{col 107}{it:keepvarvalue}{col 120}{c |}
{col 3}{c |}{col 4}9834{col 10}8{col 21}11Jan2016{col 33}         {col 44}   {col 53}   {col 59}    {col 65}  {col 75}                 {col 94}{it:uniquevalue}{col 107}{it:keepvarvalue}{col 120}{c |}
{col 3}{c BLC}{hline 116}{c BRC}

{pmore}The table above shows an example of an Excel report with 4 duplicates groups with
two duplicates in each groups. The duplicates in 4321 and in 1145 have both been corrected
but 7365 and 9834 are still unresolved. Before any observation was corrected, all observations had
{it:dateFixed}, {it:correct}, {it:drop}, {it:newID}, {it:initials} and {it:note} empty just like the observations for ID 7365 and 9834. {it:dateFixed}
is not updated by the user, the command adds this date the first time the correction is made.

{pmore}Observation with dupListID == 5 was found to have been
assigned the incorrect ID while the data was collected. This observation is assigned the correct ID in {it:newID}
and observation dupListID == 6 is indicated to be correct. Someone with initials IB made this
correction and made a note. This note can and should be more descriptive but is kept short in this example.

{pmore}Observations with dupListID == 1 and dupListID == 2 were identified as a duplicate submissions of the same
observation. One is kept and one is dropped, usually it does not matter which you keep and which you drop, but that should be confirmed.

{pmore}Both corrections described in the example would have been easily identified using this command's sister command {help iecompdup}.

{title:Acknowledgements}

{phang}I would like to acknowledge the help in testing and proofreading I received in relation to this command and help file from (in alphabetic order):{p_end}
{pmore}Mehrab Ali{break}Michell Dong{break}Paula Gonzales{break}Seungmin Lee

{title:Author}

{phang}Kristoffer Bjrkefur, The World Bank, DECIE

{phang}Please send bug-reports, suggestions and requests for clarifications
		 writing "ietools ieduplicates" in the subject line to:{break}
		 kbjarkefur@worldbank.org

{phang}You can also see the code, make comments to the code, see the version
		 history of the code, and submit additions or edits to the code through
		 the github repository of ietoolkit:{break}
		 {browse "https://github.com/worldbank/ietoolkit"}

{smcl}
{* 1 Nov 2016}{...}
{hline}
help for {hi:ieduplicates}
{hline}

{title:Title}

{phang2}{cmdab:ieduplicates} {hline 2} Identify duplicates in ID variable and export them in
an Excel file that also can be used to correct the duplicates

{title:Syntax}

{phang2}
{cmdab:ieduplicates}
{it:ID_varname}
, {cmdab:fol:der(}{it:string}{cmd:)} {cmdab:unique:vars(}{it:varlist}{cmd:)}
[{cmdab:keep:vars(}{it:varlist}{cmd:)} {cmdab:tostringok} {cmdab:droprest}
{cmdab:nodaily} {cmdab:min:precision(}{it:numlist}{cmd:)]}

{phang2}where {it:ID_varname} is the variable that will be controlled for duplicates

{marker opts}{...}
{synoptset 28}{...}
{synopthdr:options}
{synoptline}
{synopt :{cmdab:unique:vars(}{it:varlist}{cmd:)}}variables used as unique ID within groups of duplicates in {it:ID_varname}{p_end}
{synopt :{cmdab:keep:vars(}{it:varlist}{cmd:)}}variables used to be included in the Excel report in addition to {it:ID_varname} and {cmdab:unique:vars()} {p_end}
{synopt :{cmdab:tostringok}}allows {it:ID_varname} to be recasted to string if required{p_end}
{synopt :{cmdab:droprest}}disables the requirement that duplicates must be explicitly deleted{p_end}
{synopt :{cmdab:nodaily}}disables daily back-up copies of the Excel report{p_end}
{synopt :{cmdab:min:precision(}{it:numlist}{cmd:)}}({it:rarely used}) manually set the precision when exporting and importing time variables to and from the Excel file{p_end}
{synoptline}

{title:Description}

{dlgtab:In brief:}
{pstd}{cmd:ieduplicates} outputs a report with any duplicates in {it:ID_varname} to an Excel file
and return the data set without those duplicates. Each time {cmd:ieduplicates} executes, it first
looks for an already created version of the Excel report, and applies any corrections already listed in it
before generating a new report. Note that there is no need import the corrections manually. This command
reads the corrections directly from the Excel file as long as the is saved at the same folder location
with the same file name.

{dlgtab:In more detail:}
{pstd}{cmd:ieduplicates} takes duplicates observations in {it:ID_varname} and export
them to an Excel report in directory {cmdab:fol:der(}{it:string}{cmd:)}. {it:ID_varname}
is per definition not unique in this Excel Report and {cmdab:unique:vars(}{it:varlist}{cmd:)}
needs to be specified in order to have a unique reference for each row in the Excel report. The
{it:varlist} in {cmdab:unique:vars(}{it:varlist}{cmd:)} must uniquely and fully identify all
observations in the Excel report, either on its own or together with {it:ID_varname}. {cmd:ieduplicates}
then returns the data set without these duplicates.

{pstd}The Excel report includes three columns called {it:correct}, {it:drop} and {it:newID}.
Each of them represents one way to correct the duplicates. If {it:correct} is indicated with
a "Yes" then that observation is kept unchanged, if {it:drop} is indicated with a "yes" then
that observation is deleted and if {it:newID} is indicated then that observation is assigned
a new ID using the value in column {it:newID}. After corrections are entered, the report should
be saved in the same location {cmdab:fol:der(}{it:string}{cmd:)} without any changes to its name.

{pstd}Before outputting a new report {cmd:ieduplicates} always checks if there already are an
Excel report with corrections and applies those corrections before generating a new report. It is
at this stage that {cmdab:unique:vars(}{it:varlist}{cmd:)} is required as it otherwise is impossible
to know which duplicate within a group of duplicates that should be corrected in which way.

{pstd}{cmd:ieduplicates} keeps only one observation if a group of duplicates are duplicates in
all variables across the data set without any action is needed in the Excel report. These cases
are not even exported to the Excel report.

{pstd}{cmdab:keep:vars(}{it:varlist}{cmd:)} allows the user to include more variables in the Excel report
that can help identifying each duplicate is supposed to be corrected. The report also includes two
columns {it:initials} and {it:notes}. Using these columns is not required but it is recommended to use {it:initials}
to keep track of who decided how to correct that duplicate and to use {it:notes} to document why
the correction was chosen. If {it:initials} and {it:notes} are used, then the Excel report also functions
as an excellent documentation of the correction made.

{space 4}{hline}

{title:Options}

{phang}{cmdab:fol:der(}{it:string}{cmd:)} specifies the folder where previous Excel
files will be looked for, and where the updated Excel Report will be exported. Note that
this folder needs to have a subfolder called {it:Daily} where the duplicate report
file is backed up daily.

{phang}{cmdab:unique:vars(}{it:varlist}{cmd:)} list variables that by themselves or together
with {it:ID_varname} uniquely identifies all observations. This varlist is required when the corrections are
imported back into Stata and merged with the original data set. Time variables
should always be avoided if possible in {cmdab:uniquevars()}. See option {cmdab:min:precision()} for
an explanation of why time variables should be avoided. Data that has been downloaded from
a server usually has a variable called "KEY" or similar. Such a variable would be optimal
for {cmdab:unique:vars(}{it:varlist}{cmd:)}.

{phang}{cmdab:keep:vars(}{it:varlist}{cmd:)} list variables to be included in the exported
Excel report. These variables can help team members identifying which observation to keep,
drop and assign a new ID to. For data integrity reasons, be careful not to export and share
Excel files including both identifying variables and names together with {it:ID_varname}.

{phang}{cmdab:tostringok} allows {it:ID_varname} to be turned into a string variable in case
{it:ID_varname} is numeric but a value listed in {it:newID} is non-numeric. Otherwise an error is generated.

{phang}{cmdab:droprest} disables the requirement that duplicates must be explicitly deleted.
The default is that if one of the duplicates in a group of duplicates has a
correction, then that correction is only valid if all other duplicates in that
group have a correction as well. For example, if there are four observation with
the same value for {it:ID_varname} and one is correct, one needs a new ID and
two are incorrect and should be deleted. Then the first one is indicated to be
kept in the {it:correct} column, the second one is given a new ID in {it:newID}
and the other two observations must be indicated for deletion in {it:drop}
unless {cmdab:droprest}. The first two corrections are not considered valid and
will cause an error in case if {cmdab:droprest} is not specified and the other
two observations are not explicitly indicated to be dropped. It is recommended
to not use {cmdab:droprest} and to manually indicate all deletions to avoid
mistakes, but this option exist for cases when that might be very inconvenient.

{phang}{cmdab:nodaily} disables the generation of daily back-up copies of the
Excel report. The default is that the command saves dated copies of the Excel
report in a sub-folder called Daily in the folder specified in {cmdab:folder()}. If
the folder Daily does not exist, then it either needs to be created or the
option {cmdab:nodaily} need to be specified in order to not cause an error.

{phang}{cmdab:min:precision(}{it:numlist}{cmd:)} is rarely used but can be used
to manually set the precision (in minutes) when exporting and importing a time
variable to and from the Excel report. Time variables should always be avoided
if possible in {cmdab:uniquevars()}, but sometimes they are the only option. While
Stata and Excel both keep a very high precision in time variables, they do so
slightly differently, and this can generate a difference of a few seconds after
a time variable was exported to Excel and then imported back to Stata. If the
time variable is used in {cmdab:uniquevars()}, then the time variable may no
longer be identical to its original value after it is imported back to Stata, and it
may therefore no longer be possible to use it to merge the Excel data to the correct Stata
observation. If this happens, then {cmdab:min:precision()} can be used to set the
precision manually. This should only be considered a solution of last resort,
as lowering the precision increases the risk the time variable no longer uniquely
identifies each observation. The typical user will never use this option.

{title:The Excel Report}

{pstd}A report of duplicates will be created in {cmdab:fol:der(}{it:string}{cmd:)}
if any duplicates in {it:ID_varname} were found. The folder listed in
{cmdab:fol:der(}{it:string}{cmd:)} must have a subfolder called {it:Daily}
where daily back-ups of the report are saved. If a report is back-uped already
that day, then that report will be overwritten.

{pstd}All duplicates in a group of duplicates must have a correction indicated. If
one or more duplicates are indicated as correct in {it:correct} or assigned a new
ID in {it:newID}, then all other duplicates with the same value in {it:ID_varname} must
be explicitly indicated for deletion. This requirement may (but probably
shouldn't) be disabled by option {cmdab:droprest}.

{dlgtab:Columns in Excel Report filled in automatically:}

{phang}{it:dupListID} stores an auto incremented duplicate list ID that is used
to maintain the sort order in the Excel Report regardless of how the data in memory
is sorted at the time {cmd:ieduplicates} is executed.

{phang}{it:dateListed} stores the date the duplicate was first identified.

{phang}{it:dateFixed} stores the date a valid correction was imported the first
time for that duplicate.

{dlgtab:Columns in Excel Report to be filled in manually by a user:}

{phang}{it:correct} is used to indicate that the duplicate should be kept. Valid values are
restricted to "yes" and "y" to reduce the risk of unintended entries. The values
are not sensitive to case. All valid values are changed to "yes" lower case when
imported. If {it:correct} is indicated then both {it:drop} and {it:newID} must be
left empty.

{phang}{it:drop} is used to indicate that the duplicate should be deleted. Valid values are
restricted to "yes" and "y" to reduce the risk of unintended entries. The values
are not sensitive to case. All valid values are changed to "yes" lower case when
imported. If {it:drop} is indicated then both {it:correct} and {it:newID} must be
left empty.

{phang}{it:newID} is used to assign a new ID values to a duplicate. If {it:ID_varname}
is a string then all values are valid for {it:newID}. If {it:ID_varname} is numeric then
only digits are valid, unless the option {cmdab:tostringok} is specified.
If {cmdab:tostringok} is specified and {it:newID} is non-numeric, then {it:ID_varname}
is recasted to a string variable. If {it:newID} is indicated then both {it:correct} and {it:drop} must be
left empty.

{phang}{it:initials} allows the team working with this data to keep track on who
decided on corrections.

{phang}{it:notes} allows the team working with this data to document the reason
for the duplicates and the why one type of correction was chosen over the others.

{dlgtab:Columns in Excel Report with data from the data set:}

{pstd}The columns above are followed by the values in {cmdab:unique:vars(}{it:varlist}{cmd:)}
and in {cmdab:keep:vars(}{it:varlist}{cmd:)}. These column keeps the name the
variables have in the data set. These variables can help the team to identify
which correction should be applied to which duplicate.

{space 4}{hline}

{marker results}{...}
{title:Stored results}

{pstd}
{cmdab:ieduplicates} stores the following results in {hi:r()}:

{synoptset 15 tabbed}{...}
{p2col 5 15 19 2: Scalars}{p_end}
{synopt:{cmd:r(numDup)}}number of unresolved duplicates{p_end}
{p2colreset}{...}

{pstd}
{cmd:r(numDup)} is intended to allow for the option to pause Stata in case unresolved duplicates
are found. See example 4 below for a code example on how to use {cmd:r(numDup)}. See {help pause} for instructions
on how to resume the execution of the code if Stata is paused.

{title:Examples}

{pstd}
{hi:Example 1.}

{phang2}{inp:ieduplicates HHID, folder(C:\myImpactEvaluation\baseline\data) uniquevars(KEY)}{p_end}

{pmore}Specified like this {cmdab:ieduplicates} start by looking for any corrections in any
duplicates report in the folder C:\myImpactEvaluation\baseline\data. If there is a report
with corrections those corrections are applied to the data set. Then the command looks for
unresolved duplicates in HHID and exports a new report if any duplicates were found. The data
set is returned without any of the unresolved duplicates. The variable KEY is used to separate
observations that are duplication in the ID var.

{phang}
{hi:Example 2.}

{phang2}{inp:ieduplicates HHID, folder(C:\myImpactEvaluation\baseline\data) keepvars(enumerator) uniquevars(KEY)}{p_end}

{pmore}Similar to the example above, but it also includes the variable enumerator in the Excel
report which is most likely helpful if the data set is collected through a household survey.

{phang}
{hi:Example 3.} Using {cmd:r(numDup)} to pause the execution of the code if
unresolved duplicates were found

{phang2}{inp:ieduplicates HHID, folder(C:\myImpactEvaluation\baseline\data) uniquevars(KEY)}{p_end}
{phang2}{inp:if (r(numDup) != 0) {c -(}}{p_end}
{phang3}{inp:pause}{p_end}
{phang2}{inp:{c )-}}{p_end}

{phang}
{hi:Example 4.} Using the Excel file. The table below could be the report generated in Example 2 above. Make the viewer window wider and reload the page if the table below does not display properly!

{col 3}{c TLC}{hline 116}{c TRC}
{col 3}{c |}{col 4}HHID{col 10}dupListID{col 21}dateListed{col 33}dateFixed{col 44}correct{col 53}drop{col 59}newID{col 65}initials{col 75}note{col 94}KEY{col 107}enumerator{col 120}{c |}
{col 3}{c LT}{hline 116}{c RT}
{col 3}{c |}{col 4}4321{col 10}1{col 21}27Dec2015{col 33}02Jan2016{col 44}yes{col 53}   {col 59}    {col 65}KB{col 75}double submission{col 94}{it:uniquevalue}{col 107}{it:keepvarvalue}{col 120}{c |}
{col 3}{c |}{col 4}4321{col 10}2{col 21}27Dec2015{col 33}02Jan2016{col 44}   {col 53}yes{col 59}    {col 65}KB{col 75}double submission{col 94}{it:uniquevalue}{col 107}{it:keepvarvalue}{col 120}{c |}
{col 3}{c |}{col 4}7365{col 10}3{col 21}03Jan2016{col 33}         {col 44}   {col 53}   {col 59}    {col 65}  {col 75}                 {col 94}{it:uniquevalue}{col 107}{it:keepvarvalue}{col 120}{c |}
{col 3}{c |}{col 4}7365{col 10}4{col 21}03Jan2016{col 33}         {col 44}   {col 53}   {col 59}    {col 65}  {col 75}                 {col 94}{it:uniquevalue}{col 107}{it:keepvarvalue}{col 120}{c |}
{col 3}{c |}{col 4}1145{col 10}5{col 21}03Jan2016{col 33}11Jan2016{col 44}   {col 53}   {col 59}1245{col 65}IB{col 75}incorrect id     {col 94}{it:uniquevalue}{col 107}{it:keepvarvalue}{col 120}{c |}
{col 3}{c |}{col 4}1145{col 10}6{col 21}03Jan2016{col 33}11Jan2016{col 44}yes{col 53}   {col 59}    {col 65}IB{col 75}correct id       {col 94}{it:uniquevalue}{col 107}{it:keepvarvalue}{col 120}{c |}
{col 3}{c |}{col 4}9834{col 10}7{col 21}11Jan2016{col 33}         {col 44}   {col 53}   {col 59}    {col 65}  {col 75}                 {col 94}{it:uniquevalue}{col 107}{it:keepvarvalue}{col 120}{c |}
{col 3}{c |}{col 4}9834{col 10}8{col 21}11Jan2016{col 33}         {col 44}   {col 53}   {col 59}    {col 65}  {col 75}                 {col 94}{it:uniquevalue}{col 107}{it:keepvarvalue}{col 120}{c |}
{col 3}{c BLC}{hline 116}{c BRC}

{pmore}The table above shows an example of an Excel report with 4 duplicates groups with
two duplicates in each groups. The duplicates in 4321 and in 1145 have both been corrected
but 7365 and 9834 are still unresolved. Before any observation was corrected, all observations had
{it:dateFixed}, {it:correct}, {it:drop}, {it:newID}, {it:initials} and {it:note} empty just like the observations for ID 7365 and 9834. {it:dateFixed}
is not updated by the user, the command adds this date the first time the correction is made.

{pmore}Observation with dupListID == 5 was found to have been
assigned the incorrect ID while the data was collected. This observation is assigned the correct ID in {it:newID}
and observation dupListID == 6 is indicated to be correct. Someone with initials IB made this
correction and made a note. This note can and should be more descriptive but is kept short in this example.

{pmore}Observations with dupListID == 1 and dupListID == 2 were identified as a duplicate submissions of the same
observation. One is kept and one is dropped, usually it does not matter which you keep and which you drop, but that should be confirmed.

{pmore}Both corrections described in the example would have been easily identified using this command's sister command {help iecompdup}.

{title:Acknowledgements}

{phang}I would like to acknowledge the help in testing and proofreading I received in relation to this command and help file from (in alphabetic order):{p_end}
{pmore}Mehrab Ali{break}Michell Dong{break}Paula Gonzales{break}Seungmin Lee

{title:Author}

{phang}Kristoffer Bjrkefur, The World Bank, DECIE

{phang}Please send bug-reports, suggestions and requests for clarifications
		 writing "ietools ieduplicates" in the subject line to:{break}
		 kbjarkefur@worldbank.org

{phang}You can also see the code, make comments to the code, see the version
		 history of the code, and submit additions or edits to the code through
		 the github repository of ietoolkit:{break}
		 {browse "https://github.com/worldbank/ietoolkit"}