[iegitaddmd] small edits to test messages

*! version 6.2 31JAN2019 DIME Analytics dimeanalytics@worldbank.org

cap program drop   iegitaddmd
	program define iegitaddmd

qui {

		syntax , folder(string) [comparefolder(string) customfile(string) all skip replace manual]

		/******************************
		*******************************
			Test user input
		*******************************
		******************************/

		******************************
		*	Test input: folder

		*Test that folder exist
		mata : st_numscalar("r(dirExist)", direxists("`folder'"))

		if (`r(dirExist)' == 0) | (length("`folder'")<=10) {

			noi di as error `"{phang}The folder used in [folder(`folder')] does not exist or you have not entered the full path. For example, full paths on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}. Important: Specify the whole file path to the repository folder, not just {it:C:/} or {it:/user/} as that would create the placeholder file in every empty folder on your computer!{p_end}"'
			error 693
			exit
		}

		* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
		local folderStd			= subinstr(`"`folder'"',"\","/",.)
		local comparefolderStd	= subinstr(`"`comparefolder'"',"\","/",.)

		******************************
		*	Test input: customfile

		*Test that file exist if not using default file
		if "`customfile'" != "" {
			*File option used test if file exists
			capture confirm file "`customfile'"
			if _rc {
				*File does not exist, throw error
				noi di as error `"{phang}File "`customfile'" was not found. Remember that you must enter the full path. For example, on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}.{p_end}"'
				error 693
				exit
			}

			* Get the name of file (include path to keep )
			* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
			local customFileStd	= subinstr("`customfile'","\","/",.)
			*Find the last slash in the file name
			local lastSlash = strpos(strreverse(`"`customFileStd'"'),"/")
			*Use that position to get the file name. Multiply with minus one as we count from the back
			local customFileName = substr(`"`customFileStd'"', (-1 * `lastSlash') ,.)
			*Used for the recursive call of the command when there is a subfolder
			local customFileRecurse `"customfile(`customFileStd')"'

			*Path and name of thile regardless if customfile were used
			local newPlaceholderFile `"`customFileName'"'
		}
		else {
			*Path and name of thile regardless if customfile were not used
			local newPlaceholderFile "README.md"
		}

		******************************
		*	Test input: skip & replace

		*Options skip and replace cannot be used together
		if !missing("`skip'") & !missing("`replace'") {
			noi di as error `"{phang}The options skip and replace may not be used together.{p_end}"'
			error 198
			exit
		}

		/******************************
		*******************************
			List Files and folders
		*******************************
		******************************/

		******************************
		*	List all files and folders

		*List files, directories and other files
		local flist : dir `"`folderStd'"' files "*"	, respectcase
		local dlist : dir `"`folderStd'"' dirs  "*" , respectcase
		local olist : dir `"`folderStd'"' other "*"	, respectcase

		/******************************
		*******************************
			Exectute command without compare option
		*******************************
		******************************/

		if missing("`comparefolder'") {

			******************************
			*	Call command recursively on all sub-folders

			*Use the command on each subfolder to this folder (if any)
			foreach dir of local dlist {
				*Recursive call on each subfolder
				noi iegitaddmd , folder(`"`folderStd'/`dir'"') `all' `customFileRecurse' `skip' `replace' `manual'
			}

			******************************
			*	Evaluate if file will be crated in this folder

			*Test if all of those lists are empty, meaning there are
			* no folders or files in this folder
			if `"`flist'`dlist'`olist'"' == "" noi writePlaceholder, folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'

			*If the folder is not empty, test if option all were used.
			else if ("`all'" == "all") {

					*Test if a file with exactly that name is already used
					cap confirm file `"`folderStd'/`newPlaceholderFile'"'

					* No file with this name exists in this folder, create file without further ado
					if _rc != 0 noi writePlaceholder, folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'

					* File exist, but no sintructions on how to deal with this has been given, throw error
					else if missing("`skip'") & missing("`replace'")  {
						noi di as error `"{phang}The file  `newPlaceholderFile' already exists. Either remove option {it:all} or use either option {it:skip} or {it:replace}. See help file before using either of these options unless you are already familiar with them.{p_end}"'
						error 602
					}

					* File exist, and instruction is to replace, create files and overwrite as needed
					else if !missing("`replace'") noi writePlaceholder,  folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'

					* File exist, and instruction is to skip, do nothing
					else if !missing("`skip'") {
							*Nothing is done, this if-block is only included for readability and completion
					}

					* The code should nevere get to this point
					else  {
						noi di as error `"{phang}Coding error in if-chain in "All" block.Please report this error to dimeanalytics@worldbank.org or here: https://github.com/worldbank/ietoolkit/issues/new{p_end}"'
						error 602
					}
			}
			else {
				* Folder is not empty and option all is not used, so do nothing in this folder
			}
		}
		/******************************
		*******************************
			Exectute command with compare option
		*******************************
		******************************/

		else if !missing("`comparefolder'") {

			*List folders that are in comparefolder() but not in in folder()
			local comp_dlist : dir `"`comparefolderStd'"' dirs  "*" , respectcase
			local only_in_cdlist : list comp_dlist - dlist

			*Loop over all folders only in comparefolder()
			foreach dir of local only_in_cdlist {
				*Find all leaf folders. A leaf folder is a folder with no
				*sub-folder. Think of the folder structure as a tree, and
				*each folder with no sub-folders is an end of a branch where
				*you find leaves.
				noi findLeafFolders, folder(`"`comparefolderStd'/`dir'"')

				*Loop over all leaves and create accordingly
				foreach leaf in `r(leaves)' {
					*Leaf exist in compare folder, create the path to the leaf
					* to be created in the corresponding folder
					local folderleaf = subinstr(`"`leaf'"',`"`comparefolderStd'"', `"`folder'"', 1)

					*Create placeholder file in this new leaf
					noi writePlaceholder, folder(`"`folderleaf'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'
				}
			}

			*Recursivley do the same call on all sub-folders
			foreach dir of local dlist {
				*Recursive call on each subfolder
				noi iegitaddmd , folder(`"`folderStd'/`dir'"') comparefolder(`"`comparefolder'/`dir'"') `customFileRecurse' `manual'
			}
		}
}

end

*Write a README.md file when iegitaddmd finds an empty folder
cap program drop writePlaceholder
program define   writePlaceholder

qui {

	syntax , folder(string) newfilename(string) [customfiletocopy(string) manual]

	*If manual was used, get manual confirmation for each file
	if !missing("`manual'") {
		noi di ""
		global confirmation "" //Reset global

		*Keep aslking for input until the input is either Y, y, N, n or BREAK
		while (upper("${confirmation}") != "Y" & upper("${confirmation}") != "N" & "${confirmation}" != "BREAK") {
		  noi di as txt "{pstd}You are about to create a file [`newfilename'] in the folder [`folder']. If the folder does not exist it will be created. Do you want to do that? To confirm type {bf:Y} and hit enter, to abort type {bf:N} and hit enter. Type {bf:BREAK} and hit enter to stop the code.{p_end}", _request(confirmation)
		}
		*Copy user input to local
		local createfile = upper("${confirmation}")
		* If user wrote "BREAK" then exit the code
		if ("`createfile'" == "BREAK") error 1
	}

	*If "manual" were used and input was Y or if manual was not used, create the file
	if ("`createfile'" == "Y") | (missing("`manual'")) {

		*Recursively create folder if needed
		noi rmkdir, folder(`"`folder'"')

		*Copy custom file if custom file is used
		if !missing(`"`customfile'"') copy "`customfile'" `"`folder'/`newfilename'"', replace
		*Custom file is not used, so cretae defeault file
		else {
			*Create file
			tempname 	newHandle
			cap file close 	`newHandle'
			file open  	`newHandle' using "`folder'/`newfilename'", text write replace
			*Write the content to the file
			file write  `newHandle' ///
				"# Placeholder file" _n _n ///
				"This file has been created automatically by the command **iegitaddmd** from the Stata package [**ietoolkit**](https://worldbank.github.io/ietoolkit) to make GitHub sync this folder. GitHub does not sync empty folders or folders that only contain ignored files, but in research projects it is often important to share the full standardized folder structure along with the actual files. This command is intended to be used with **iefolder**, but it can be used in any folder structure intended to be shared on GitHub." _n _n ///
				"In recently started projects, it is typical to create data folders, script folders (do-files, r-files) and output folders, among others. The output folders are initially empty, but the script files might include a file path to them for later use. If an output folder is empty and another collaborator clones the repository, then the output folder expected by the scripts will not be included by GitHub in the cloned repository, and the script will not run properly." _n _n ///
				"## You should replace the content of this placeholder file" _n _n ///
				"The text in this file should be replaced with text that describes the intended use of this folder. This file is written in markdown (.md) format, which is suitable for GitHub syncing (unlike .doc/.docx). If the file is named *README.md*, GitHub automatically displays its contents when someone opens the containing folder on GitHub.com using a web browser." _n _n ///
				"If you are new to markup languages (markdown, html etc.) then this [Markdown Tutorial](https://www.markdowntutorial.com/) is a great place to start. If you have some experience with markup languages, then this [Markdown Cheat Sheet](https://guides.github.com/pdfs/markdown-cheatsheet-online.pdf) is a great place to start." _n _n ///
				"## Add similar files in other folders" _n _n ///
				"The Stata command **iegitaddmd** does not add anything to folders that already have content unless this is explicitly requested using the option `all`. It is best practice to create a *README.md* file in any folders that have content, for example, in folders whose purpose might not obvious to someone using the repository for the first time. Again, if the file is named *README.md*, then the content of the file will be shown in the browser when someone explores the repository on GitHub.com." _n _n ///
				"Another great use of a *README.md* file is to use it as a documentation on how the folder it sits in and its subfolders are organized,  where its content can be found, and where new content is meant to be saved. For example, if you have a folder called `/Baseline/`, then you can give a short description of the activities conducted during the baseline and where data, scripts and outputs related to it can be found." _n _n ///
				"## Removing this file" _n _n ///
				"Removing this file is not recommended, as GitHub may stop syncing the parent folder unless the folder now has other content. However, even when content is added and the file can be removed without breaking the GitHub functionality, our recommendation is to replace the content with more relevant content rather than deleting it." _n _n

			*Closing the file
			file close 		`newHandle'

		}
		*Uoutput that the file was created
		noi di as result "{pstd}File [`folder'/`newfilename'] created.{p_end}"
	}
	*Manual was used and input was N, no file were creaetd
	else noi di as result "{pstd}No file or folder created.{p_end}"
}
end

*Write a README.md file when iegitaddmd finds an empty folder
cap program drop findLeafFolders
program define   findLeafFolders, rclass

qui {

	syntax, folder(string)

	local dlist : dir `"`folder'"' dirs  "*" , respectcase

	if missing(`"`dlist'"') {
		*This is a leaf, pass it back
		return local leaves `""`folder'""'
	}
	else {
		local return_leaves
		foreach dir of local dlist {
			*Recursive call on each subfolder
			findLeafFolders, folder(`"`folder'/`dir'"')
			local return_leaves `"`return_leaves' `r(leaves)'"'
		}
		return local leaves `"`return_leaves'"'
	}
}
end

*Recursively call parent folders in folderpath needed to be created until
* folder found that already exist, then creaet all subfolders.
cap program drop rmkdir
program define   rmkdir, rclass

qui {

	syntax, folder(string) [folderstocreate(string)]

	*Test if this folder exists
	mata : st_numscalar("r(dirExist)", direxists(`"`folder'"'))

	*Folder does not exist, find parent folder and make recursive call
	if (`r(dirExist)' == 0) {

		*Get the parent folder of folder
		local lastSlash = strpos(strreverse(`"`folder'"'),"/")
		local parentFolder = substr(`"`folder'"',1,strlen("`folder'")-`lastSlash')
		local thisFolder = substr(`"`folder'"', (-1 * `lastSlash')+1 ,.)

		*Make the coll recursivly on the parent folder and add this folder to folderstocreate()
		noi rmkdir , folder(`"`parentFolder'"') folderstocreate(`""`thisFolder'" `folderstocreate'"')

	}
	*Folder exist, create sub-folders to create if any
	else {
		*Create the folders needed
		foreach dir of local folderstocreate {
			mkdir "`folder'/`dir'"
			local folder "`folder'/`dir'"
		}
	}
}
end

*! version 6.2 31JAN2019 DIME Analytics dimeanalytics@worldbank.org

cap program drop   iegitaddmd
	program define iegitaddmd

qui {

		syntax , folder(string) [comparefolder(string) customfile(string) all skip replace manual]

		/******************************
		*******************************
			Test user input
		*******************************
		******************************/

		******************************
		*	Test input: folder

		*Test that folder exist
		mata : st_numscalar("r(dirExist)", direxists("`folder'"))

		if (`r(dirExist)' == 0) | (length("`folder'")<=10) {

			noi di as error `"{phang}The "`folder'" folder does not exist or you have not entered the full path. For example, full paths on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}. Important: Specify the whole file path to the repository folder, not just {it:C:/} or {it:/user/} as that would create the placeholder file in every empty folder on your computer!{p_end}"'
			error 693
			exit
		}

		* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
		local folderStd			= subinstr(`"`folder'"',"\","/",.)
		local comparefolderStd	= subinstr(`"`comparefolder'"',"\","/",.)

		******************************
		*	Test input: customfile

		*Test that file exist if not using default file
		if "`customfile'" != "" {
			*File option used test if file exists
			capture confirm file "`customfile'"
			if _rc {
				*File does not exist, throw error
				noi di as error `"{phang}File "`customfile'" was not found. Remember that you must enter the full path. For example, on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}.{p_end}"'
				error 693
				exit
			}

			* Get the name of file (include path to keep )
			* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
			local customFileStd	= subinstr("`customfile'","\","/",.)
			*Find the last slash in the file name
			local lastSlash = strpos(strreverse(`"`customFileStd'"'),"/")
			*Use that position to get the file name. Multiply with minus one as we count from the back
			local customFileName = substr(`"`customFileStd'"', (-1 * `lastSlash') ,.)
			*Used for the recursive call of the command when there is a subfolder
			local customFileRecurse `"customfile(`customFileStd')"'

			*Path and name of thile regardless if customfile were used
			local newPlaceholderFile `"`customFileName'"'
		}
		else {
			*Path and name of thile regardless if customfile were not used
			local newPlaceholderFile "README.md"
		}

		******************************
		*	Test input: skip & replace

		*Options skip and replace cannot be used together
		if "`skip'" != "" & "`replace'" != "" {
			noi di as error `"{phang}The options skip and replace may not be used together.{p_end}"'
			error 198
			exit
		}

		/******************************
		*******************************
			List Files and folders
		*******************************
		******************************/

		******************************
		*	List all files and folders

		*List files, directories and other files
		local flist : dir `"`folderStd'"' files "*"	, respectcase
		local dlist : dir `"`folderStd'"' dirs  "*" , respectcase
		local olist : dir `"`folderStd'"' other "*"	, respectcase

		/******************************
		*******************************
			Exectute command without compare option
		*******************************
		******************************/

		if missing("`comparefolder'") {

			******************************
			*	Call command recursively on all sub-folders

			*Use the command on each subfolder to this folder (if any)
			foreach dir of local dlist {
				*Recursive call on each subfolder
				noi iegitaddmd , folder(`"`folderStd'/`dir'"') `all' `customFileRecurse' `skip' `replace' `manual'
			}

			******************************
			*	Evaluate if file will be crated in this folder

			*Test if all of those lists are empty, meaning there are
			* no folders or files in this folder
			if `"`flist'`dlist'`olist'"' == "" noi writePlaceholder, folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'

			*If the folder is not empty, test if option all were used.
			else if ("`all'" == "all") {

					*Test if a file with exactly that name is already used
					cap confirm file `"`folderStd'/`newPlaceholderFile'"'

					* No file with this name exists in this folder, create file without further ado
					if _rc != 0 noi writePlaceholder, folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'

					* File exist, but no sintructions on how to deal with this has been given, throw error
					else if missing("`skip'") & missing("`replace'")  {
						noi di as error `"{phang}The file  `newPlaceholderFile' already exists. Either remove option {it:all} or use either option {it:skip} or {it:replace}. See help file before using either of these options unless you are already familiar with them.{p_end}"'
						error 602
					}

					* File exist, and instruction is to replace, create files and overwrite as needed
					else if !missing("`replace'") noi writePlaceholder,  folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'

					* File exist, and instruction is to skip, do nothing
					else if !missing("`skip'") {
							*Nothing is done, this if-block is only included for readability and completion
					}

					* The code should nevere get to this point
					else  {
						noi di as error `"{phang}Coding error in if-chain in "All" block.Please report this error to dimeanalytics@worldbank.org or here: https://github.com/worldbank/ietoolkit/issues/new{p_end}"'
						error 602
					}
			}
			else {
				* Folder is not empty and option all is not used, so do nothing in this folder
			}
		}
		/******************************
		*******************************
			Exectute command with compare option
		*******************************
		******************************/

		else if !missing("`comparefolder'") {

			*List folders that are in comparefolder() but not in in folder()
			local comp_dlist : dir `"`comparefolderStd'"' dirs  "*" , respectcase
			local only_in_cdlist : list comp_dlist - dlist

			*Loop over all folders only in comparefolder()
			foreach dir of local only_in_cdlist {
				*Find all leaf folders. A leaf folder is a folder with no
				*sub-folder. Think of the folder structure as a tree, and
				*each folder with no sub-folders is an end of a branch where
				*you find leaves.
				noi findLeafFolders, folder(`"`comparefolderStd'/`dir'"')

				*Loop over all leaves and create accordingly
				foreach leaf in `r(leaves)' {
					*Leaf exist in compare folder, create the path to the leaf
					* to be created in the corresponding folder
					local folderleaf = subinstr(`"`leaf'"',`"`comparefolderStd'"', `"`folder'"', 1)

					*Create placeholder file in this new leaf
					noi writePlaceholder, folder(`"`folderleaf'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'
				}
			}

			*Recursivley do the same call on all sub-folders
			foreach dir of local dlist {
				*Recursive call on each subfolder
				noi iegitaddmd , folder(`"`folderStd'/`dir'"') comparefolder(`"`comparefolder'/`dir'"') `customFileRecurse' `manual'
			}
		}
}

end

*Write a README.md file when iegitaddmd finds an empty folder
cap program drop writePlaceholder
program define   writePlaceholder

qui {

	syntax , folder(string) newfilename(string) [customfiletocopy(string) manual]

	*If manual was used, get manual confirmation for each file
	if !missing("`manual'") {
		noi di ""
		global confirmation "" //Reset global

		*Keep aslking for input until the input is either Y, y, N, n or BREAK
		while (upper("${confirmation}") != "Y" & upper("${confirmation}") != "N" & "${confirmation}" != "BREAK") {
		  noi di as txt "{pstd}You are about to create a file [`newfilename'] in the folder [`folder']. If the folder does not exist it will be created. Do you want to do that? To confirm type {bf:Y} and hit enter, to abort type {bf:N} and hit enter. Type {bf:BREAK} and hit enter to stop the code.{p_end}", _request(confirmation)
		}
		*Copy user input to local
		local createfile = upper("${confirmation}")
		* If user wrote "BREAK" then exit the code
		if ("`createfile'" == "BREAK") error 1
	}

	*If "manual" were used and input was Y or if manual was not used, create the file
	if ("`createfile'" == "Y") | (missing("`manual'")) {

		*Recursively create folder if needed
		noi rmkdir, folder(`"`folder'"')

		*Copy custom file if custom file is used
		if !missing(`"`customfile'"') copy "`customfile'" `"`folder'/`newfilename'"', replace
		*Custom file is not used, so cretae defeault file
		else {
			*Create file
			tempname 	newHandle
			cap file close 	`newHandle'
			file open  	`newHandle' using "`folder'/`newfilename'", text write replace
			*Write the content to the file
			file write  `newHandle' ///
				"# Placeholder file" _n _n ///
				"This file has been created automatically by the command **iegitaddmd** from the Stata package [**ietoolkit**](https://worldbank.github.io/ietoolkit) to make GitHub sync this folder. GitHub does not sync empty folders or folders that only contain ignored files, but in research projects it is often important to share the full standardized folder structure along with the actual files. This command is intended to be used with **iefolder**, but it can be used in any folder structure intended to be shared on GitHub." _n _n ///
				"In recently started projects, it is typical to create data folders, script folders (do-files, r-files) and output folders, among others. The output folders are initially empty, but the script files might include a file path to them for later use. If an output folder is empty and another collaborator clones the repository, then the output folder expected by the scripts will not be included by GitHub in the cloned repository, and the script will not run properly." _n _n ///
				"## You should replace the content of this placeholder file" _n _n ///
				"The text in this file should be replaced with text that describes the intended use of this folder. This file is written in markdown (.md) format, which is suitable for GitHub syncing (unlike .doc/.docx). If the file is named *README.md*, GitHub automatically displays its contents when someone opens the containing folder on GitHub.com using a web browser." _n _n ///
				"If you are new to markup languages (markdown, html etc.) then this [Markdown Tutorial](https://www.markdowntutorial.com/) is a great place to start. If you have some experience with markup languages, then this [Markdown Cheat Sheet](https://guides.github.com/pdfs/markdown-cheatsheet-online.pdf) is a great place to start." _n _n ///
				"## Add similar files in other folders" _n _n ///
				"The Stata command **iegitaddmd** does not add anything to folders that already have content unless this is explicitly requested using the option `all`. It is best practice to create a *README.md* file in any folders that have content, for example, in folders whose purpose might not obvious to someone using the repository for the first time. Again, if the file is named *README.md*, then the content of the file will be shown in the browser when someone explores the repository on GitHub.com." _n _n ///
				"Another great use of a *README.md* file is to use it as a documentation on how the folder it sits in and its subfolders are organized,  where its content can be found, and where new content is meant to be saved. For example, if you have a folder called `/Baseline/`, then you can give a short description of the activities conducted during the baseline and where data, scripts and outputs related to it can be found." _n _n ///
				"## Removing this file" _n _n ///
				"Removing this file is not recommended, as GitHub may stop syncing the parent folder unless the folder now has other content. However, even when content is added and the file can be removed without breaking the GitHub functionality, our recommendation is to replace the content with more relevant content rather than deleting it." _n _n

			*Closing the file
			file close 		`newHandle'

		}
		*Uoutput that the file was created
		noi di as result "{pstd}File [`folder'/`newfilename'] created.{p_end}"
	}
	*Manual was used and input was N, no file were creaetd
	else noi di as result "{pstd}No file or folder created.{p_end}"
}
end

*Write a README.md file when iegitaddmd finds an empty folder
cap program drop findLeafFolders
program define   findLeafFolders, rclass

qui {

	syntax, folder(string)

	local dlist : dir `"`folder'"' dirs  "*" , respectcase

	if missing(`"`dlist'"') {
		*This is a leaf, pass it back
		return local leaves `""`folder'""'
	}
	else {
		local return_leaves
		foreach dir of local dlist {
			*Recursive call on each subfolder
			findLeafFolders, folder(`"`folder'/`dir'"')
			local return_leaves `"`return_leaves' `r(leaves)'"'
		}
		return local leaves `"`return_leaves'"'
	}
}
end

*Recursively call parent folders in folderpath needed to be created until
* folder found that already exist, then creaet all subfolders.
cap program drop rmkdir
program define   rmkdir, rclass

qui {

	syntax, folder(string) [folderstocreate(string)]

	*Test if this folder exists
	mata : st_numscalar("r(dirExist)", direxists(`"`folder'"'))

	*Folder does not exist, find parent folder and make recursive call
	if (`r(dirExist)' == 0) {

		*Get the parent folder of folder
		local lastSlash = strpos(strreverse(`"`folder'"'),"/")
		local parentFolder = substr(`"`folder'"',1,strlen("`folder'")-`lastSlash')
		local thisFolder = substr(`"`folder'"', (-1 * `lastSlash')+1 ,.)

		*Make the coll recursivly on the parent folder and add this folder to folderstocreate()
		noi rmkdir , folder(`"`parentFolder'"') folderstocreate(`""`thisFolder'" `folderstocreate'"')

	}
	*Folder exist, create sub-folders to create if any
	else {
		*Create the folders needed
		foreach dir of local folderstocreate {
			mkdir "`folder'/`dir'"
			local folder "`folder'/`dir'"
		}
	}
}
end


redo-sh.do: hide warning output from 'which' in some shells.

exec >&2
redo-ifchange t/shelltest.od

rm -rf $1.new $1/sh
mkdir $1.new

GOOD=
WARN=

for sh in dash sh ash ksh pdksh bash zsh busybox; do
	printf "Testing %s... " "$sh"
	FOUND=`which $sh 2>/dev/null` || { echo "missing"; continue; }

	# It's important for the file to actually be named 'sh'.  Some
	# shells (like bash and zsh) only go into POSIX-compatible mode if
	# they have that name.  If they're not in POSIX-compatible mode,
	# they'll fail the test.
	rm -f $1.new/sh
	ln -s $FOUND $1.new/sh

	set +e
	( cd t && ../$1.new/sh shelltest.od >/dev/null 2>&1 )
	RV=$?
	set -e

	case $RV in
		0) echo "good"; GOOD=$FOUND; break ;;
		42) echo "warnings"; [ -n "$WARN" ] || WARN=$FOUND ;;
		*) echo "failed" ;;
	esac
done

rm -rf $1 $1.new $3

if [ -n "$GOOD" ]; then
	echo "Selected perfect shell: $GOOD"
	mkdir $3
	ln -s $GOOD $3/sh
elif [ -n "$WARN" ]; then
	echo "Selected mostly good shell: $WARN"
	mkdir $3
	ln -s $WARN $3/sh
else
	echo "No good shells found!  Maybe install dash, bash, or zsh."
	exit 1
fi

exec >&2
redo-ifchange t/shelltest.od

rm -rf $1.new $1/sh
mkdir $1.new

GOOD=
WARN=

for sh in dash sh ash ksh pdksh bash zsh busybox; do
	printf "Testing %s... " "$sh"
	FOUND=`which $sh` || { echo "missing"; continue; }

	# It's important for the file to actually be named 'sh'.  Some
	# shells (like bash and zsh) only go into POSIX-compatible mode if
	# they have that name.  If they're not in POSIX-compatible mode,
	# they'll fail the test.
	rm -f $1.new/sh
	ln -s $FOUND $1.new/sh

	set +e
	( cd t && ../$1.new/sh shelltest.od >/dev/null 2>&1 )
	RV=$?
	set -e

	case $RV in
		0) echo "good"; GOOD=$FOUND; break ;;
		42) echo "warnings"; [ -n "$WARN" ] || WARN=$FOUND ;;
		*) echo "failed" ;;
	esac
done

rm -rf $1 $1.new $3

if [ -n "$GOOD" ]; then
	echo "Selected perfect shell: $GOOD"
	mkdir $3
	ln -s $GOOD $3/sh
elif [ -n "$WARN" ]; then
	echo "Selected mostly good shell: $WARN"
	mkdir $3
	ln -s $WARN $3/sh
else
	echo "No good shells found!  Maybe install dash, bash, or zsh."
	exit 1
fi


ietoolkit : update version locals

ietoolkit : update version locals

*! version 6.1 09112018 DIME Analytics dimeanalytics@worldbank.org

capture program drop ietoolkit
program ietoolkit, rclass

	* UPDATE THESE LOCALS FOR EACH NEW VERSION PUBLISHED
	local version "6.1"
	local versionDate "11NOV2018"

	syntax [anything]

	/**********************
		Error messages
	**********************/

	* Make sure that no arguments were passed
	if "`anything'" != "" {
		noi di as error "This command does not take any arguments, write only {it:ietoolkit}"
		error 198
	}

	/**********************
		Output
	**********************/

	* Prepare returned locals
	return local 	versiondate "`versionDate'"
	return scalar 	version		= `version'

	* Display output
	noi di ""
	noi di _col(4) "This version of ietoolkit installed is version " _col(54)"`version'"
	noi di _col(4) "This version of ietoolkit was released on " _col(54)"`versionDate'"

end

*! version 6.1 09112018 DIME Analytics dimeanalytics@worldbank.org

capture program drop ietoolkit
program ietoolkit, rclass

	* UPDATE THESE LOCALS FOR EACH NEW VERSION PUBLISHED
	local version "6.0"
	local versionDate "19OCT2018"

	syntax [anything]

	/**********************
		Error messages
	**********************/

	* Make sure that no arguments were passed
	if "`anything'" != "" {
		noi di as error "This command does not take any arguments, write only {it:ietoolkit}"
		error 198
	}

	/**********************
		Output
	**********************/

	* Prepare returned locals
	return local 	versiondate "`versionDate'"
	return scalar 	version		= `version'

	* Display output
	noi di ""
	noi di _col(4) "This version of ietoolkit installed is version " _col(54)"`version'"
	noi di _col(4) "This version of ietoolkit was released on " _col(54)"`versionDate'"

end


divide jobs among houses

divide jobs among houses

* the world is crumbling. solutions are being introduced. they wont pass without radical political change
* 1 billion divided equally between states
* 1 billion distributed on the basis of relative 16-25 y/o compared to total in all states
* 1 billion on relative unemployed compared to toal in all states
* 1 billion on the basis of the rleative # disadvantaged youth compared to total in all states

*STEPS:
* # of 16-25 y/o's per state
* total # of 16-25 y/os
* # of unemployed per state
* total # unemployed
* # disdvantaged youth
* total # disadvantaged youth

*then by puma w/in state
*
*tabulate age if statefip == 06 & puma == 3703 [fweight=perwt]
drop if year == 2013

gen youth = (age <= 25 & age >= 16)
gen disadvantaged = (poverty <= 100)
gen disadvantaged_youth = (youth & disadvantaged)
gen unemployed = (empstat == 2)
gen unemployedyouth = youth & unemployed

egen youth_in_hh = max(youth), by(serial)
egen unemployed_in_hh = max(unemployed), by(serial)
egen disadvantaged_hh = max(disadvantaged_youth), by(serial)

egen youth_state = total(perwt * youth), by(statefip)
egen youth_puma = total(perwt * youth), by(statefip puma)
*sum youth_puma if statefip == 06 & puma == 3703
*25371 youth in lancaster
gen youth_percentage = youth_puma / youth_state

egen unemployed_state = total(perwt * unemployed), by(statefip)
egen unemployed_puma = total(perwt * unemployed), by(statefip puma)

gen unemployed_percentage = unemployed_puma / unemployed_state

egen unemployed_youth_state = total(perwt * unemployedyouth), by(statefip)

*sum youth_puma if statefip == 06 & puma == 3703
*5797 youth in lancaster

egen disadvantaged_state = total(perwt * disadvantaged_youth), by(statefip)
egen disadvantaged_puma = total(perwt * disadvantaged_youth), by(statefip puma)

gen disadvantaged_percentage = disadvantaged_puma / disadvantaged_state

egen youth_total = total(perwt * youth)
egen unemployed_total = total(perwt * unemployed)
egen disadvantaged_total = total(perwt * disadvantaged_youth)

gen youth_percentage_state = youth_state/youth_total
gen unemployed_percentage_state = unemployed_state/unemployed_total
gen disadvantaged_percentage_state = disadvantaged_state/disadvantaged_total

gen state_money = 1000 * .02 + 1000 * youth_percentage_state + 1000 * unemployed_percentage_state + 1000 * disadvantaged_percentage_state

egen npumas = nvals(puma), by(statefip)

gen puma_money = (state_money/3) * (youth_percentage + unemployed_percentage + disadvantaged_percentage)
gen numjobspuma = puma_money / .0105 //.0105 is a 10,500 dollar per yr job ($15*30hrs*12weeks in summer and $15*10hrs*34weeks in schoolyr)
//alternatively, could be doled out as 48 weeks of $15/hr pay 15 hrs a week
gen numjobsstate = state_money / .0105

*figure out households w/ youth in them
egen householdwithyouth = max(youth), by(serial)
*youth hh's living in poverty
gen youthinpov = (householdwithyouth & (poverty <= 100) & pernum == 1)
tab youthinpov if pernum == 1 [fweight = perwt]
*162 jobs for 5287 households
save "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie5-1.dta", replace
keep if youthinpov
bysort statefip puma: gen runningwt = sum(hhwt)
keep serial pernum runningwt
joinby serial pernum using "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie5-1.dta", unmatched(both)
gen jobsleft = 0
gen prevwt = 0
replace prevwt = runningwt - hhwt if runningwt > numjobspuma
replace jobsleft = numjobspuma - prevwt if (prevwt < numjobspuma & prevwt > 0)
gen newhhwt = hhwt
expand 2 if jobsleft > 0, generate(duplicate)
replace newhhwt = jobsleft if duplicate == 1
replace newhhwt = hhwt-jobsleft if duplicate == 0

gen job_money = 0
replace job_money = 10500 if (youthinpov & pernum == 1) & ((jobsleft > 0 & duplicate == 1) | duplicate == 0)

gen familyincome = inctot
replace familyincome = ftotinc if ftotinc < 9999999
replace familyincome = 0 if familyincome < 0

gen newincome = familyincome+job_money

gen povline = 11670 + (numprec-1)*4060
gen outofpov = disadvantaged & (newincome > povline) & (newincome < 9999999)

egen outofpovfam = max(outofpov), by(serial)

sgini newincome if pernum == 1 & familyincome < 9999999 [fweight=hhwt]
sgini familyincome if pernum == 1 & familyincome < 9999999 [fweight=hhwt]

gen povgap = povline - familyincome if familyincome < povline
gen newpovgap = povline - newincome if newincome < povline

egen totalhh = sum(hhwt) if pernum == 1
egen p0 = sum(hhwt) if poverty < 100 & pernum == 1

egen p1 = sum((povgap*hhwt)/povline)
disp p1/totalhh
//.22842
egen newp1 = sum((newpovgap*hhwt)/povline)
disp newp1/totalhh
//.19363

egen p2 = sum((povgap/povline)^2*hhwt)
egen newp2 = sum((newpovgap/povline)^2*hhwt)

disp p2/totalhh
//.1659
disp newp2/totalhh
//.1356

disp p0[23]/totalhh
//.19710145

egen newp0 = sum(hhwt) if ((poverty < 100 & pernum == 1) & !(outofpov == 1 & pernum == 1))
disp newp0[23]/totalhh
//.1978997

egen totalhhstate = sum(hhwt) if pernum == 1, by(statefip)
egen p0state = sum(hhwt) if poverty < 100 & pernum == 1, by(statefip)
egen newp0state = sum(hhwt) if ((poverty < 100 & pernum == 1) & !(outofpov == 1 & pernum == 1)), by(statefip)

/*. keep if youthinpov
(2,915,463 observations deleted)

. bysort statefip puma: gen runningwt = sum(hhwt)

. keep serial pernum runningwt

. joinby serial pernum using "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie4-11.dta", unmatched(both)
*/

/* collapse (mean) state_money, by(statefip)
. export delimited using "/Users/braddv/Desktop/BERNIE/state_money.csv",
how to collapse without losing data? or i guess just use file again.
*/
//tab occ2010 if statefip==06 & youth [fweight=perwt] (california young jobs distribution)
//
/*collapse (mean) youth_state, by(statefip)
export delimited using "/Users/braddv/Desktop/BERNIE/youth_state.csv"*/
/*collapse (mean) disadvantaged_state, by(statefip)
export delimited using "/Users/braddv/Desktop/BERNIE/disadvantaged_state.csv"*/
/*collapse (mean) unemployed_state, by(statefip)
export delimited using "/Users/braddv/Desktop/BERNIE/unemployed_state.csv"*/
/*collapse (mean) unemployed_youth_state, by(statefip)
export delimited using "/Users/braddv/Desktop/BERNIE/unemployed_youth_state.csv"*/
/*collapse (mean) numjobsstate, by(statefip)
export delimited using "/Users/braddv/Desktop/BERNIE/numjobs_state.csv"*/

//tab job_money if pernum == 1 [fweight = hhwt]

* the world is crumbling. solutions are being introduced. they wont pass without radical political change
* 1 billion divided equally between states
* 1 billion distributed on the basis of relative 16-25 y/o compared to total in all states
* 1 billion on relative unemployed compared to toal in all states
* 1 billion on the basis of the rleative # disadvantaged youth compared to total in all states

*STEPS:
* # of 16-25 y/o's per state
* total # of 16-25 y/os
* # of unemployed per state
* total # unemployed
* # disdvantaged youth
* total # disadvantaged youth

*then by puma w/in state
*
*tabulate age if statefip == 06 & puma == 3703 [fweight=perwt]
drop if year == 2013

gen youth = (age <= 25 & age >= 16)
gen disadvantaged = (poverty <= 100)
gen disadvantaged_youth = (youth & disadvantaged)
gen unemployed = (empstat == 2)
gen unemployedyouth = youth & unemployed

egen youth_in_hh = max(youth), by(serial)
egen unemployed_in_hh = max(unemployed), by(serial)
egen disadvantaged_hh = max(disadvantaged_youth), by(serial)

egen youth_state = total(perwt * youth), by(statefip)
egen youth_puma = total(perwt * youth), by(statefip puma)
*sum youth_puma if statefip == 06 & puma == 3703
*25371 youth in lancaster
gen youth_percentage = youth_puma / youth_state

egen unemployed_state = total(perwt * unemployed), by(statefip)
egen unemployed_puma = total(perwt * unemployed), by(statefip puma)

gen unemployed_percentage = unemployed_puma / unemployed_state

egen unemployed_youth_state = total(perwt * unemployedyouth), by(statefip)

*sum youth_puma if statefip == 06 & puma == 3703
*5797 youth in lancaster

egen disadvantaged_state = total(perwt * disadvantaged_youth), by(statefip)
egen disadvantaged_puma = total(perwt * disadvantaged_youth), by(statefip puma)

gen disadvantaged_percentage = disadvantaged_puma / disadvantaged_state

egen youth_total = total(perwt * youth)
egen unemployed_total = total(perwt * unemployed)
egen disadvantaged_total = total(perwt * disadvantaged_youth)

gen youth_percentage_state = youth_state/youth_total
gen unemployed_percentage_state = unemployed_state/unemployed_total
gen disadvantaged_percentage_state = disadvantaged_state/disadvantaged_total

gen state_money = 1000 * .02 + 1000 * youth_percentage_state + 1000 * unemployed_percentage_state + 1000 * disadvantaged_percentage_state

egen npumas = nvals(puma), by(statefip)

gen puma_money = (state_money/3) * (youth_percentage + unemployed_percentage + disadvantaged_percentage)
gen numjobspuma = puma_money / .0105 //.0105 is a 10,500 dollar per yr job ($15*30hrs*12weeks in summer and $15*10hrs*34weeks in schoolyr)
//alternatively, could be doled out as 48 weeks of $15/hr pay 15 hrs a week
gen numjobsstate = state_money / .0105

*figure out households w/ youth in them
egen householdwithyouth = max(youth), by(serial)
*youth hh's living in poverty
gen youthinpov = (householdwithyouth & (poverty <= 100) & pernum == 1)
tab youthinpov if pernum == 1 [fweight = perwt]
*162 jobs for 5287 households
save "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie5-1.dta"
keep if youthinpov
bysort statefip puma: gen runningwt = sum(hhwt)
keep serial pernum runningwt
joinby serial pernum using "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie5-1.dta", unmatched(both)
gen job_money = 0
replace job_money = 10500 if youthinpov & pernum == 1 & runningwt < numjobspuma

gen familyincome = inctot
replace familyincome = ftotinc if ftotinc < 9999999
replace familyincome = 0 if familyincome < 0

gen newincome = familyincome+job_money

gen povline = 11670 + (numprec-1)*4060
gen outofpov = disadvantaged & (newincome > povline) & (newincome < 9999999)

egen outofpovfam = max(outofpov), by(serial)

sgini newincome if pernum == 1 & familyincome < 9999999 [fweight=hhwt]
sgini familyincome if pernum == 1 & familyincome < 9999999 [fweight=hhwt]

gen povgap = povline - familyincome if familyincome < povline
gen newpovgap = povline - newincome if newincome < povline

egen totalhh = sum(hhwt) if pernum == 1
egen p0 = sum(hhwt) if poverty < 100 & pernum == 1

egen p1 = sum((povgap*hhwt)/povline)
disp p1/totalhh
//.22842
egen newp1 = sum((newpovgap*hhwt)/povline)
disp newp1/totalhh
//.19363

egen p2 = sum((povgap/povline)^2*hhwt)
egen newp2 = sum((newpovgap/povline)^2*hhwt)

disp p2/totalhh
//.1659
disp newp2/totalhh
//.1356

disp p0[23]/totalhh
//.19710145

egen newp0 = sum(hhwt) if ((poverty < 100 & pernum == 1) & !(outofpov == 1 & pernum == 1))
disp newp0[23]/totalhh
//.1978997

egen totalhhstate = sum(hhwt) if pernum == 1, by(statefip)
egen p0state = sum(hhwt) if poverty < 100 & pernum == 1, by(statefip)
egen newp0state = sum(hhwt) if ((poverty < 100 & pernum == 1) & !(outofpov == 1 & pernum == 1)), by(statefip)

/*. keep if youthinpov
(2,915,463 observations deleted)

. bysort statefip puma: gen runningwt = sum(hhwt)

. keep serial pernum runningwt

. joinby serial pernum using "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie4-11.dta", unmatched(both)
*/

/* collapse (mean) state_money, by(statefip)
. export delimited using "/Users/braddv/Desktop/BERNIE/state_money.csv",
how to collapse without losing data? or i guess just use file again.
*/
//tab occ2010 if statefip==06 & youth [fweight=perwt] (california young jobs distribution)
//
/*collapse (mean) youth_state, by(statefip)
export delimited using "/Users/braddv/Desktop/BERNIE/youth_state.csv"*/
/*collapse (mean) disadvantaged_state, by(statefip)
export delimited using "/Users/braddv/Desktop/BERNIE/disadvantaged_state.csv"*/
/*collapse (mean) unemployed_state, by(statefip)
export delimited using "/Users/braddv/Desktop/BERNIE/unemployed_state.csv"*/
/*collapse (mean) unemployed_youth_state, by(statefip)
export delimited using "/Users/braddv/Desktop/BERNIE/unemployed_youth_state.csv"*/
/*collapse (mean) numjobsstate, by(statefip)
export delimited using "/Users/braddv/Desktop/BERNIE/numjobs_state.csv"*/

//tab job_money if pernum == 1 [fweight = hhwt]


Shuffle the binary classification example because the train/test split makes everything be one class, otherwise.

// Setup
sysuse auto

// Machine learning methods like SVM are a black-box and it is very easy to overfit them.
// To compensate, it is important to split data into training and testing sets, fit on
// the former and measure performance on the latter, so that performance measurements
// are not artificially inflated by data they've already seen.
local split = floor(_N/2)
local train = "1/`=`split'-1'"
local test = "`split'/`=_N'"

// But the proportion of classes in each set can be unbalanced.
// The reliable way to handle this is a stratified split;
// the quick and dirty way is a shuffle:
set seed 9876
gen u = uniform()
sort u

// Fit the classification model, with 'verbose' enabled, on the training set.
// Training cannot handle missing data; here we elide it, but usually you should impute.
svm foreign price-gear_ratio if !missing(rep78) in `train', v

// Predict on the test set.
// Unlike training, predict simply predicts missing if any data is missing.
predict P in `test'

// Compute error rate.
gen err = foreign != P in `test'
sum err in `test'

// Setup
sysuse auto

// Machine learning methods like SVM are a black-box and it is very easy to overfit them.
// To compensate, it is important to split data into training and testing sets, fit on
// the former and measure performance on the latter, so that performance measurements
// are not artificially inflated by data they've already seen.
local split = floor(_N/2)
local train = "1/`=`split'-1'"
local test = "`split'/`=_N'"

// Fit the classification model, with 'verbose' enabled, on the training set.
// Training cannot handle missing data; here we elide it, but usually you should impute.
svm foreign price-gear_ratio if !missing(rep78) in `train', v

// Predict on the test set.
// Unlike training, predict simply predicts missing if any data is missing.
predict P in `test'

// Compute error rate.
gen err = foreign != P in `test'
sum err in `test'


reasonable numbers

reasonable numbers

* 1 billion divided equally between states
* 1 billion distributed on the basis of relative 16-25 y/o compared to total in all states
* 1 billion on relative unemployed compared to toal in all states
* 1 billion on the basis of the rleative # disadvantaged youth compared to total in all states

*STEPS:
* # of 16-25 y/o's per state
* total # of 16-25 y/os
* # of unemployed per state
* total # unemployed
* # disdvantaged youth
* total # disadvantaged youth

*then by puma w/in state
*
*tabulate age if statefip == 06 & puma == 3703 [fweight=perwt]
use "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie08.dta", clear

drop if year == 2013
drop if gq == 4 | gq == 3
drop if statefip == 36 & puma == 3309
drop if statefip == 25 & puma == 1300

gen youth = (age <= 25 & age >= 16)
gen povline = 11670 + (famsize-1)*4060
gen disadvantaged = (ftotinc < povline)
gen unemployed = (empstat == 2)
gen unemployed_youth = youth * unemployed
gen disadvantaged_youth = youth * disadvantaged

egen youth_in_hh = max(youth), by(serial)
egen unemployed_in_hh = max(unemployed), by(serial)
egen disadvantaged_hh = max(disadvantaged), by(serial)

egen youth_in_fam = max(youth), by(serial famunit)
egen unemployed_in_fam = max(unemployed), by(serial famunit)
egen disadvantaged_fam = max(disadvantaged), by(serial famunit)
gen disadvantagedyouth_fam = disadvantaged_fam*youth_in_fam
gen unemployedyouth_fam = youth_in_fam*unemployed_youth

egen youth_state = total(perwt * youth), by(statefip)
egen youth_puma = total(perwt * youth), by(statefip puma)
*sum youth_puma if statefip == 06 & puma == 3703
*25371 youth in lancaster
gen youth_percentage = youth_puma / youth_state

egen unemployed_state = total(perwt * unemployed), by(statefip)
egen unemployed_puma = total(perwt * unemployed), by(statefip puma)

gen unemployed_percentage = unemployed_puma / unemployed_state

egen unemployed_youth_state = total(perwt * unemployedyouth), by(statefip)

*sum youth_puma if statefip == 06 & puma == 3703
*5797 youth in lancaster

egen disadvantaged_state = total(perwt * disadvantaged_youth), by(statefip)
egen disadvantaged_puma = total(perwt * disadvantaged_youth), by(statefip puma)

gen disadvantaged_percentage = disadvantaged_puma / disadvantaged_state

egen youth_total = total(perwt * youth)
egen unemployed_total = total(perwt * unemployed)
egen disadvantaged_total = total(perwt * disadvantaged_youth)

gen youth_percentage_state = youth_state/youth_total
gen unemployed_percentage_state = unemployed_state/unemployed_total
gen disadvantaged_percentage_state = disadvantaged_state/disadvantaged_total

gen state_money = 1000 * .02 + 1000 * youth_percentage_state + 1000 * unemployed_percentage_state + 1000 * disadvantaged_percentage_state

egen npumas = nvals(puma), by(statefip)

gen puma_money = (state_money/3) * (youth_percentage + unemployed_percentage + disadvantaged_percentage)
gen numjobspuma = puma_money / .0105 //.0105 is a 10,500 dollar per yr job ($15*30hrs*12weeks in summer and $15*10hrs*34weeks in schoolyr)
//alternatively, could be doled out as 48 weeks of $15/hr pay 15 hrs a week
gen numjobsstate = state_money / .0105

*gen numjobspumamin = puma_money / state_minwage
*gen numjobsstatemin = state_money / state_minwage

*figure out households w/ youth in them
egen householdwithyouth = max(youth), by(serial)
*youth hh's living in poverty
gen youthinpov = (householdwithyouth & (poverty <= 100) & pernum == 1)

egen familyheadnum = min(pernum), by(serial famunit)
gen familyhead = (familyheadnum == pernum)
gen familyheadyouthunemployed = familyhead*unemployedyouth_fam
gen familyheadyouthdisadvantaged = familyhead*disadvantagedyouth_fam

save "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie08-egen.dta", replace

use "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie08-egen.dta", clear
gen runningwt = 0
gen prevwt = 0
save "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie5-1.dta", replace
keep if familyheadyouthdisadvantaged
bysort statefip puma: gen pumaid = _n
bysort statefip puma: replace runningwt = sum(perwt)
replace prevwt = runningwt - perwt if runningwt > numjobspuma
gen jobsleft = numjobspuma - prevwt
replace jobsleft = 0 if jobsleft < 0
replace jobsleft = 0 if runningwt < jobsleft
gen jobrecipient = 0
replace jobrecipient = 1 if runningwt < numjobspuma | jobsleft > 0
keep serial pernum jobrecipient jobsleft
joinby serial pernum using "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie5-1.dta", unmatched(both)

replace jobrecipient = 0 if missing(jobrecipient)
replace jobsleft = 0 if missing(jobsleft)

gen newperwt = perwt
egen maxrunningwt = max(runningwt), by(statefip puma)

expand 2 if jobsleft > 0, generate(duplicate)
replace newperwt = int(jobsleft) if duplicate == 1
replace newperwt = max(int(perwt-jobsleft),0) if duplicate == 0 & jobsleft > 0
replace jobrecipient = 0 if duplicate == 0 & jobsleft > 0

egen jobsreceived = sum(jobrecipient*newperwt)

gen job_money = 0
replace job_money = 10500 if jobrecipient

gen numjobsafter = numjobspuma
replace numjobsafter = maxrunningwt if maxrunningwt < numjobspuma

gen familyincome = ftotinc

gen newincome = familyincome+job_money

gen outofpov = (ftotinc <= povline) & (newincome > povline) & (newincome < 9999999)

egen outofpovfam = max(outofpov), by(serial famunit)

*sgini newincome if familyhead == 1 & familyincome < 9999999 [fweight=newperwt]
*sgini familyincome if familyhead == 1 & familyincome < 9999999 [fweight=newperwt]

gen povgap = povline - familyincome if familyincome < povline
gen newpovgap = povline - newincome if newincome < povline

gen poor = familyincome < povline
gen newpoor = newincome < povline

tab poor [fweight=newperwt]
tab newpoor [fweight=newperwt]

egen moneyspent = sum(job_money*newperwt) if familyhead

tabulate job_money if familyhead [fweight=newperwt]

egen totalfam = sum(newperwt) if familyhead == 1
egen p0 = sum(newperwt) if poor & familyhead == 1
egen newp0 = sum(newperwt) if newpoor & familyhead == 1

egen p1 = sum((povgap*newperwt)/povline) if familyhead == 1
disp p1/totalfam
egen newp1 = sum((newpovgap*newperwt)/povline) if familyhead == 1
disp newp1/totalfam

egen p2 = sum((povgap/povline)^2*newperwt)
egen newp2 = sum((newpovgap/povline)^2*newperwt)

disp p2/totalfam
disp newp2/totalfam

disp p0[23]/totalfam

disp newp0[23]/totalfam

egen totalhhstate = sum(newperwt) if pernum == 1, by(statefip)
egen p0state = sum(newperwt) if poverty < 100 & pernum == 1, by(statefip)
egen newp0state = sum(newperwt) if ((poverty < 100 & pernum == 1) & !(outofpov == 1 & pernum == 1)), by(statefip)*/

/*. keep if youthinpov
(2,915,463 observations deleted)

. bysort statefip puma: gen runningwt = sum(hhwt)

. keep serial pernum runningwt

. joinby serial pernum using "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie4-11.dta", unmatched(both)
*/

/* collapse (mean) state_money, by(statefip)
. export delimited using "/Users/braddv/Desktop/BERNIE/state_money.csv",
how to collapse without losing data? or i guess just use file again.
*/
//tab occ2010 if statefip==06 & youth [fweight=perwt] (california young jobs distribution)
//
/*collapse (mean) youth_state, by(statefip)
export delimited using "/Users/braddv/Desktop/BERNIE/youth_state.csv"*/
/*collapse (mean) disadvantaged_state, by(statefip)
export delimited using "/Users/braddv/Desktop/BERNIE/disadvantaged_state.csv"*/
/*collapse (mean) unemployed_state, by(statefip)
export delimited using "/Users/braddv/Desktop/BERNIE/unemployed_state.csv"*/
/*collapse (mean) unemployed_youth_state, by(statefip)
export delimited using "/Users/braddv/Desktop/BERNIE/unemployed_youth_state.csv"*/
/*collapse (mean) numjobsstate, by(statefip)
export delimited using "/Users/braddv/Desktop/BERNIE/numjobs_state.csv"*/

//tab job_money if pernum == 1 [fweight = hhwt]

* 1 billion divided equally between states
* 1 billion distributed on the basis of relative 16-25 y/o compared to total in all states
* 1 billion on relative unemployed compared to toal in all states
* 1 billion on the basis of the rleative # disadvantaged youth compared to total in all states

*STEPS:
* # of 16-25 y/o's per state
* total # of 16-25 y/os
* # of unemployed per state
* total # unemployed
* # disdvantaged youth
* total # disadvantaged youth

*then by puma w/in state
*
*tabulate age if statefip == 06 & puma == 3703 [fweight=perwt]
use "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie08.dta", clear

drop if year == 2013
drop if gq == 4 | gq == 3

gen youth = (age <= 25 & age >= 16)
gen disadvantaged = (poverty <= 100)
gen unemployed = (empstat == 2)
gen unemployed_youth = youth * unemployed
gen disadvantaged_youth = youth * disadvantaged

egen youth_in_hh = max(youth), by(serial)
egen unemployed_in_hh = max(unemployed), by(serial)
egen disadvantaged_hh = max(disadvantaged), by(serial)

egen youth_in_fam = max(youth), by(serial famunit)
egen unemployed_in_fam = max(unemployed), by(serial famunit)
egen disadvantaged_fam = max(disadvantaged), by(serial famunit)
gen disadvantagedyouth_fam = disadvantaged_fam*youth_in_fam
gen unemployedyouth_fam = youth_in_fam*unemployed_youth

egen youth_state = total(perwt * youth), by(statefip)
egen youth_puma = total(perwt * youth), by(statefip puma)
*sum youth_puma if statefip == 06 & puma == 3703
*25371 youth in lancaster
gen youth_percentage = youth_puma / youth_state

egen unemployed_state = total(perwt * unemployed), by(statefip)
egen unemployed_puma = total(perwt * unemployed), by(statefip puma)

gen unemployed_percentage = unemployed_puma / unemployed_state

egen unemployed_youth_state = total(perwt * unemployedyouth), by(statefip)

*sum youth_puma if statefip == 06 & puma == 3703
*5797 youth in lancaster

egen disadvantaged_state = total(perwt * disadvantaged_youth), by(statefip)
egen disadvantaged_puma = total(perwt * disadvantaged_youth), by(statefip puma)

gen disadvantaged_percentage = disadvantaged_puma / disadvantaged_state

egen youth_total = total(perwt * youth)
egen unemployed_total = total(perwt * unemployed)
egen disadvantaged_total = total(perwt * disadvantaged_youth)

gen youth_percentage_state = youth_state/youth_total
gen unemployed_percentage_state = unemployed_state/unemployed_total
gen disadvantaged_percentage_state = disadvantaged_state/disadvantaged_total

gen state_money = 1000 * .02 + 1000 * youth_percentage_state + 1000 * unemployed_percentage_state + 1000 * disadvantaged_percentage_state

egen npumas = nvals(puma), by(statefip)

gen puma_money = (state_money/3) * (youth_percentage + unemployed_percentage + disadvantaged_percentage)
gen numjobspuma = puma_money / .0105 //.0105 is a 10,500 dollar per yr job ($15*30hrs*12weeks in summer and $15*10hrs*34weeks in schoolyr)
//alternatively, could be doled out as 48 weeks of $15/hr pay 15 hrs a week
gen numjobsstate = state_money / .0105

*gen numjobspumamin = puma_money / state_minwage
*gen numjobsstatemin = state_money / state_minwage

*figure out households w/ youth in them
egen householdwithyouth = max(youth), by(serial)
*youth hh's living in poverty
gen youthinpov = (householdwithyouth & (poverty <= 100) & pernum == 1)

egen familyheadnum = min(pernum), by(serial famunit)
gen familyhead = (familyheadnum == pernum)
gen familyheadyouthunemployed = familyhead*unemployedyouth_fam
gen familyheadyouthdisadvantaged = familyhead*disadvantagedyouth_fam

save "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie08-egen.dta", replace

use "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie08-egen.dta", clear
gen runningwt = 0
save "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie5-1.dta", replace
keep if familyheadyouthdisadvantaged
bysort statefip puma: replace runningwt = sum(perwt)
keep serial pernum runningwt
joinby serial pernum using "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie5-1.dta", unmatched(both)
gen jobsleft = 0
gen prevwt = 0
replace prevwt = runningwt - perwt if runningwt > numjobspuma
replace jobsleft = numjobspuma - prevwt if (prevwt < numjobspuma & prevwt > 0)
gen newhhwt = hhwt
gen newperwt = perwt
egen maxrunningwt = max(runningwt), by(statefip puma)

expand 2 if jobsleft > 0 & maxrunningwt > numjobspuma, generate(duplicate)
replace newperwt = int(jobsleft) if duplicate == 1
replace newperwt = max(int(perwt-jobsleft),0) if duplicate == 0

gen jobrecipient = 0
replace jobrecipient = 1 if ((runningwt < numjobspuma) | duplicate == 1)

gen job_money = 0
replace job_money = 10500 if ((runningwt < numjobspuma) | (duplicate == 1))
replace job_money = 10500 * (numjobspuma / maxrunningwt) if (maxrunningwt < numjobspuma & familyheadyouthdisadvantaged)

gen numjobsafter = numjobspuma
replace numjobsafter = maxrunningwt if maxrunningwt < numjobspuma

gen familyincome = ftotinc

gen newincome = familyincome+job_money

gen povline = 11670 + (famsize-1)*4060
gen outofpov = (ftotinc <= povline) & (newincome > povline) & (newincome < 9999999)

egen outofpovfam = max(outofpov), by(serial famunit)

sgini newincome if familyhead == 1 & familyincome < 9999999 [fweight=newperwt]
sgini familyincome if familyhead == 1 & familyincome < 9999999 [fweight=newperwt]

gen povgap = povline - familyincome if familyincome < povline
gen newpovgap = povline - newincome if newincome < povline

gen poor = familyincome < povline
gen newpoor = newincome < povline

tab poor [fweight=newperwt]
tab newpoor [fweight=newperwt]

egen moneyspent = sum(job_money*newperwt) if familyhead

tabulate job_money if familyhead [fweight=newperwt]

/*egen totalfam = sum(newperwt) if familyhead == 1
egen p0 = sum(newperwt) if poor

egen p1 = sum((povgap*newperwt)/povline)
disp p1/totalfam
egen newp1 = sum((newpovgap*newperwt)/povline)
disp newp1/totalfam

egen p2 = sum((povgap/povline)^2*newperwt)
egen newp2 = sum((newpovgap/povline)^2*newperwt)

disp p2/totalfam
disp newp2/totalfam

disp p0[23]/totalfam

egen newp0 = sum(newperwt) if newpoor
disp newp0[23]/totalfam

egen totalhhstate = sum(newperwt) if pernum == 1, by(statefip)
egen p0state = sum(newperwt) if poverty < 100 & pernum == 1, by(statefip)
egen newp0state = sum(newperwt) if ((poverty < 100 & pernum == 1) & !(outofpov == 1 & pernum == 1)), by(statefip)*/

/*. keep if youthinpov
(2,915,463 observations deleted)

. bysort statefip puma: gen runningwt = sum(hhwt)

. keep serial pernum runningwt

. joinby serial pernum using "/Users/braddv/Desktop/BERNIE/employamericansnow/bernie4-11.dta", unmatched(both)
*/

/* collapse (mean) state_money, by(statefip)
. export delimited using "/Users/braddv/Desktop/BERNIE/state_money.csv",
how to collapse without losing data? or i guess just use file again.
*/
//tab occ2010 if statefip==06 & youth [fweight=perwt] (california young jobs distribution)
//
/*collapse (mean) youth_state, by(statefip)
export delimited using "/Users/braddv/Desktop/BERNIE/youth_state.csv"*/
/*collapse (mean) disadvantaged_state, by(statefip)
export delimited using "/Users/braddv/Desktop/BERNIE/disadvantaged_state.csv"*/
/*collapse (mean) unemployed_state, by(statefip)
export delimited using "/Users/braddv/Desktop/BERNIE/unemployed_state.csv"*/
/*collapse (mean) unemployed_youth_state, by(statefip)
export delimited using "/Users/braddv/Desktop/BERNIE/unemployed_youth_state.csv"*/
/*collapse (mean) numjobsstate, by(statefip)
export delimited using "/Users/braddv/Desktop/BERNIE/numjobs_state.csv"*/

//tab job_money if pernum == 1 [fweight = hhwt]


Run mkdocs only after generating the rest of the doc files.

This avoids a mkdocs bug where mkdocs will (rarely) see redo temp files
in its directory list, but they're gone by the time it tries to open
them, so it aborts with an exception.

redo-ifchange doc.list
sed 's/\.md/.1/' <doc.list |
xargs redo-ifchange

# mkdocs foolishly tries to process every file in this directory, which
# leads it to try to open temp files produced by the above redo-ifchange
# if it runs in parallel with those jobs.  So don't run it until they
# finish.
redo-ifchange mkdocs

redo-ifchange doc.list
sed 's/\.md/.1/' <doc.list |
xargs redo-ifchange mkdocs


Add MaximumPrecision

Add MaximumPrecision

#!/usr/bin/env python3
from collections import OrderedDict, defaultdict
import os
import json
import yaml
import re
from pathlib import Path
import redo
redo.ifchange('comap.py'); import comap

project = redo.base
project_path = Path('projects') / project

with redo.ifchange(evaluation_config=project_path / 'evaluation-config.yaml') \
        as files:
    evaluation_config = yaml.load(files['evaluation_config'])

databases = OrderedDict(evaluation_config['databases'])
outcome_ids = evaluation_config['outcome_ids']
concept_filenames = {
    outcome_id: '{}.{}.concepts.json'.format(project, outcome_id)
    for outcome_id in outcome_ids
}
child_concept_filenames = {
    outcome_id: '{}.{}.child-concepts.json'.format(project, outcome_id)
    for outcome_id in outcome_ids
}
casedef_paths = {
    outcome_id: project_path / 'case-definitions' / (outcome_id + '.yaml')
    for outcome_id in outcome_ids
}

with redo.ifchange(coding_systems='config/coding_systems.yaml',
                   references=(project_path / 'reference.yaml').as_posix(),
                   concepts=concept_filenames,
                   child_concepts=child_concept_filenames,
                   casedefs=casedef_paths) as files:
    coding_systems = yaml.load(files['coding_systems'])
    references = yaml.load(files['references'])
    concepts_by_outcome = {}
    for outcome_id, f in files['concepts'].items():
        concepts_by_outcome[outcome_id] = json.load(f)
    child_concepts_by_outcome = {}
    for outcome_id, f in files['child_concepts'].items():
        child_concepts_by_outcome[outcome_id] = json.load(f).items()
    outcomes = {
        outcome_id: yaml.load(f)
        for outcome_id, f in files['casedefs'].items()
    }

class Variation(object):

    """Identity"""

    def __init__(self):
        self.outcome_id = None
        self.coding_system = None

    def description(self):
        return self.__doc__

    def vary_code(self, code, coding_system):
        """Called on each code of generated concepts and the reference before
        before vary_codes and vary_concepts_and_mapping"""
        return code

    def vary_concepts_and_mapping(self, concepts, mapping, outcome_id):
        """Called for each outcome_id and database before vary_codes. For
        concept-level variations."""
        return concepts, mapping

    def vary_codes(self, generated, reference, coding_systems):
        """Code-level variations."""
        return generated, reference

class MaximumRecall(Variation):

    """Maximize recall"""

    def __init__(self):
        self.client = comap.ComapClient()

    def vary_concepts_and_mapping(self, concepts, mapping, outcome_id):
        "Return concepts that contain all codes in `mapping`."
        cuis = set()
        for database_id in mapping.keys():
            if not mapping[database_id]:
                continue
            codes = set(mapping[database_id])
            coding_system = databases[database_id]
            cui_codes = cosynonym_codes(codes, coding_system)
            for code in codes:
                # Find the `cui` that implicates `code` with the least number of irrelevant codes
                cuis_with_count = \
                    [(cui, len(codes - codes1))
                     for cui, codes1 in cui_codes.items()
                     if code in codes1]
                if cuis_with_count:
                    key = lambda x: x[1]
                    cui, _ = sorted(cuis_with_count, key=key)[0]
                    cuis.add(cui)
        concepts = self.client.umls_concepts(cuis, coding_systems)
        return concepts, mapping

class MaximiumPrecision(Variation):

    """Maximize precision"""

    def __init__(self):
        self.client = comap.ComapClient()

    def vary_concepts_and_mapping(self, concepts, mapping, outcome_id):
        "Return concepts such that code is not in `mapping`."
        all_cui_codes = defaultdict(lambda: defaultdict(set))
        for database_id in mapping.keys():
            if not mapping[database_id]:
                continue
            coding_system = databases[database_id]
            codes = set(mapping[database_id])
            cui_codes = cosynonym_codes(codes, coding_system)
            for cui, codes in cui_codes.items():
                for code in codes:
                    all_cui_codes[cui][database_id].add(code)
        # Find the `cui` where all codes are in the reference
        cuis = set()
        for cui, all_codes in all_cui_codes.items():
            if all(mapping[database_id] is not None and \
                   all_codes[database_id].issubset(mapping[database_id])
                   for database_id, coding_system in databases.items()):
                cuis.add(cui)
        concepts = self.client.umls_concepts(cuis, coding_systems)
        return concepts, mapping

def cosynonym_codes(codes, coding_system):

    if not codes:
        return defaultdict(set)

    original_coding_system = coding_system
    if coding_system == 'RCD2':
        coding_system = 'RCD'
        read_translation, read_backtranslation = comap.translation_read2_to_read3(codes)
        codes = set(code for codes in read_translation.values() for code in codes)

    # ... WHERE cui IN (SELECT cui FROM MRCONSO WHERE code in (...)) was too slow, separate
    cuis_query = """
        SELECT DISTINCT cui FROM MRCONSO
        WHERE sab = %s AND code IN ({placeholders})
    """.format(placeholders=', '.join(['%s'] * len(codes)))

    cuis_for_codes = []
    with comap.umls_db.cursor() as cursor:
        try:
            cursor.execute(cuis_query, tuple([coding_system] + list(codes)))
        except:
            raise
        for row in cursor.fetchall():
            cuis_for_codes.append(row[0])

    if not cuis_for_codes:
        return defaultdict(set)

    cui_code_query = """
        SELECT DISTINCT cui, code FROM MRCONSO
        WHERE cui IN ({placeholders})
    """.format(placeholders=', '.join(['%s'] * len(cuis_for_codes)))

    cui_codes = defaultdict(set)
    with comap.umls_db.cursor() as cursor:
        cursor.execute(cui_code_query, tuple(cuis_for_codes))
        for row in cursor.fetchall():
            cui, code = row
            cui_codes[cui].add(code)

    if original_coding_system == 'RCD2':
        cui_codes = {
            cui: set(code2
                     for code3 in codes3
                     for code2 in read_backtranslation[code3])
            for cui, codes3 in cui_codes.items()
        }
    return cui_codes

class VariationChain(Variation):

    """Chain of variations that are applied to the input left-to-right."""

    def __init__(self, *variations):
        self.variations = variations

    def set_context(self, *args, **kwargs):
        for variation in self.variations:
            variation.set_context(*args, **kwargs)

    def description(self):
        res = '; '.join(v.description() for v in self.variations)
        if res:
            return res[0].upper() + res[1:]
        else:
            return ""

    def vary_code(self, code, coding_system):
        for variation in self.variations:
            code = variation.vary_code(code, coding_system)
        return code

    def vary_codes(self, generated, reference, coding_system):
        for variation in self.variations:
            generated, reference = variation.vary_codes(generated, reference, coding_system)
        return generated, reference

    def vary_concepts_and_mapping(self, concepts, mapping, outcome_id):
        for variation in self.variations:
            concepts, mapping = variation.vary_concepts_and_mapping(concepts, mapping, outcome_id)
        return concepts, mapping

class CodeNormalization(Variation):

    def __init__(self, *coding_system_prefixes):
        """`coding_systems` is None or a list of prefixes and this
        normalization is applied only if the list contains a prefix of the
        current coding systems."""
        self.coding_system_prefixes = coding_system_prefixes \
            or [''] # A prefix of any coding system

    def vary_code(self, code, coding_system):
        if any(coding_system.startswith(prefix)
               for prefix in self.coding_system_prefixes):
            return self.normalize(code, coding_system)
        else:
            return code

class NormalizeCodeCase(CodeNormalization):

    """Normalize code case in ICD, ICPC"""

    def __init__(self):
        super().__init__('ICD', 'ICPC')

    def normalize(self, code, coding_system):
        return code.upper()

class NormalizeCodeSuffixDot(CodeNormalization):

    """Drop suffix dots in codes in ICPC"""

    def __init__(self):
        super().__init__('ICPC')

    def normalize(self, code, coding_system):
        if code[-1] == '.':
            return code[:-1]
        else:
            return code

class NormalizeCodeXDD(CodeNormalization):

    """Normalize codes ICD and ICPC to 3 letters"""

    def __init__(self):
        super().__init__('ICD', 'ICPC')

    def normalize(self, code, coding_system):
        code = code.upper()
        m1 = re.match(comap.RegexHierarchy.TYPES['LDD'], code)
        try:
            return m1.group('parent')
        except:
            m2 = re.match(comap.RegexHierarchy.TYPES['DDD'], code)
            try:
                return m2.group('parent')
            except:
                return code

class NormalizeRead2Codes(CodeNormalization):

    """Normalize READ2 codes to 5-byte"""

    READ_RE = re.compile(r'(?P<code>[A-Z\d][A-Za-z0-9.]{4}).*')

    def __init__(self):
        super().__init__('RCD2')

    def normalize(self, code, coding_system):
        m = self.READ_RE.match(code)
        assert m, "Not a READ2 code: {}".format(code)
        return m.group('code')

class IncludeRelatedCodes(Variation):

    """Include related codes"""

    hierarchies = {
        'ICD9CM': comap.RegexHierarchy('DDD'),
        'ICD10CM': comap.RegexHierarchy('LDD'),
        'ICPC': comap.RegexHierarchy('LDD'),
        'RCD2': comap.Read2Hierarchy(),
    }

    def __init__(self, direction):
        """ direction : 'FN-FP' | 'FN' """
        self.direction = direction

    def description(self):
        return super().description() + ' ({})'.format(self.direction)

    def vary_codes(self, generated, reference, coding_system):
        hierarchy = self.hierarchies[coding_system]
        codes = comap.confusion_matrix(generated, reference)
        related_codes, derivated_codes = {
            'FN': comap.include_related,
            'FN-FP': comap.include_related_bidirect,
        }[self.direction](hierarchy, codes=codes)
        return comap.from_confusion_matrix(related_codes)

class IncludeChildConcepts(Variation):

    """Include child concepts"""

    def vary_concepts_and_mapping(self, concepts, mapping, outcome_id):
        cuis = [c['cui'] for c in concepts]
        child_concepts_by_parent = child_concepts_by_outcome[outcome_id]
        child_concepts = [
            child_concept
            for parent_id, concepts_for_parent in child_concepts_by_parent
            for child_concept in concepts_for_parent
            if child_concept not in cuis
        ]
        return concepts + child_concepts, mapping

class IncludeRelatedConcepts(Variation):

    """Include related concepts"""

    def __init__(self, *relations):
        self.relations = list(relations)
        self.client = comap.ComapClient()

    def description(self):
        return super().description() + ' ({})'.format(', '.join(self.relations))

    def vary_concepts_and_mapping(self, concepts, mapping, outcome_id):
        cuis = [c['cui'] for c in concepts]
        coding_systems = list(databases.values())
        related = self.client.related(cuis, self.relations, coding_systems)
        concepts_and_related = {c['cui']: c for c in concepts}
        for cui in related.keys():
            for rel in related[cui].keys():
                for new_concept in related[cui][rel]:
                    new_cui = new_concept['cui']
                    codes = set((sourceConcept['id'], sourceConcept['codingSystem'])
                                for sourceConcept in new_concept['sourceConcepts'])
                    if new_cui not in concepts_and_related:
                        concepts_and_related[new_cui] = new_concept
        return list(concepts_and_related.values()), mapping

def create_results(references, concepts_by_outcome, variations):

    # mapping ::= { database_id: { code } | None }

    # { outcome_id: mapping }
    mappings = {}
    for outcome_id in outcome_ids:
        mappings[outcome_id] = {}
        for database in databases.keys():
            mapping = references[database]['mappings'][outcome_id]
            if 'inclusion' in mapping:
                codes = set(mapping['inclusion'] + mapping.get('exclusion', []))
            else:
                codes = None
            mappings[outcome_id][database] = codes

    # { outcome_id: { variation_id: { "concepts": { concept }, "mapping": mapping } } }
    varied_by_outcome = {}
    for outcome_id in outcome_ids:
        varied_by_outcome[outcome_id] = {}
        concepts = concepts_by_outcome[outcome_id]
        reference_codes = OrderedDict([
            (database_id, mappings[outcome_id][database_id])
            for database_id in databases.keys()
        ])
        for variation_id, variation in variations.items():
            # Map codes in generated concepts with variation.vary_code
            concepts1 = []
            for concept in concepts_by_outcome[outcome_id]:
                concept1 = concept.copy()
                concept1['sourceConcepts'] = []
                for source_concept in concept['sourceConcepts']:
                    source_concept1 = source_concept.copy()
                    id, coding_system = source_concept['id'], source_concept['codingSystem']
                    source_concept1['id'] = variation.vary_code(id, coding_system)
                    concept1['sourceConcepts'].append(source_concept1)
                concepts1.append(concept1)
            # Map reference codes with variation.vary_code
            reference_codes1 = OrderedDict()
            for database_id in databases.keys():
                if reference_codes[database_id] is None:
                    reference_codes1[database_id] = None
                else:
                    reference_codes1[database_id] = [
                        variation.vary_code(code, databases[database_id])
                        for code in mappings[outcome_id][database_id]
                    ]
            concepts2, reference_codes2 = \
                variation.vary_concepts_and_mapping(concepts1, reference_codes1, outcome_id)
            assert concepts2 is not None, (outcome_id, variation_id)
            varied_by_outcome[outcome_id][variation_id] = {
                'concepts': concepts2,
                'reference-codes': reference_codes2,
            }

    res = OrderedDict()
    for outcome_id in outcome_ids:
        res[outcome_id] = OrderedDict()
        for database_id, coding_system in databases.items():
            res[outcome_id][database_id] = OrderedDict([])
            mapping = mappings[outcome_id][database_id]
            if mapping is not None:
                res[outcome_id][database_id]['variations'] = OrderedDict()
                for variation_id, variation in variations.items():
                    varied = varied_by_outcome[outcome_id][variation_id]
                    generated_codes = [
                        variation.vary_code(code['id'], coding_system)
                        for concept in varied['concepts']
                        for code in concept['sourceConcepts']
                        if code['codingSystem'] == coding_system
                    ]
                    reference_codes = [
                        variation.vary_code(code, coding_system)
                        for code in varied['reference-codes'][database_id]
                    ]

                    varied_generated_codes, varied_reference_codes = \
                        variation.vary_codes(generated_codes, reference_codes, coding_system)

                    varied_codes = comap.confusion_matrix(generated=varied_generated_codes,
                                                          reference=varied_reference_codes)

                    measures = comap.measures(codes=varied_codes)

                    res[outcome_id][database_id]['variations'][variation_id] = \
                        OrderedDict([
                            ('name', variation.description()),
                            ('comparison', OrderedDict([
                                ('codes', varied_codes),
                                ('measures', measures),
                            ])),
                    ])
            comment = references[database_id]['mappings'][outcome_id].get('comment')
            if comment is not None:
                res[outcome_id][database_id]['comment'] = comment
    return res

default_variations = VariationChain(NormalizeCodeCase(), NormalizeCodeSuffixDot(), NormalizeRead2Codes())
variations = OrderedDict([
    ('baseline', default_variations),
    ('maximum-recall', VariationChain(default_variations, MaximumRecall())),
    ('maximum-precision', VariationChain(default_variations, MaximiumPrecision())),
    ('3-letter-codes', VariationChain(default_variations, NormalizeCodeXDD())),
    ('related-FN', VariationChain(default_variations, IncludeRelatedCodes('FN'))),
    ('related-FN-FP', VariationChain(default_variations, IncludeRelatedCodes('FN-FP'))),
    ('related-concepts-RN', VariationChain(default_variations, IncludeRelatedConcepts('RN'))),
    ('related-concepts-RN-RB', VariationChain(default_variations, IncludeRelatedConcepts('RN', 'RB'))),
    ('related-concepts-CHD', VariationChain(default_variations, IncludeRelatedConcepts('CHD'))),
    ('related-concepts-CHD-PAR', VariationChain(default_variations, IncludeRelatedConcepts('CHD', 'PAR'))),
    ('hyponyms', VariationChain(default_variations, IncludeChildConcepts())),
    ('child-concepts-related-FN-FP', VariationChain(default_variations, IncludeChildConcepts(), IncludeRelatedCodes('FN-FP'))),
])

evaluation = create_results(references, concepts_by_outcome, variations)

# Output results as YAML

with redo.output() as f:
    yaml.dump(evaluation, f)

if os.getenv('DOCTEST'):
    import doctest
    doctest.testmod()
    doctest.testmod(comap)

#!/usr/bin/env python3
from collections import OrderedDict, defaultdict
import os
import json
import yaml
import re
from pathlib import Path
import redo
redo.ifchange('comap.py'); import comap

project = redo.base
project_path = Path('projects') / project

with redo.ifchange(evaluation_config=project_path / 'evaluation-config.yaml') \
        as files:
    evaluation_config = yaml.load(files['evaluation_config'])

databases = OrderedDict(evaluation_config['databases'])
outcome_ids = evaluation_config['outcome_ids']
concept_filenames = {
    outcome_id: '{}.{}.concepts.json'.format(project, outcome_id)
    for outcome_id in outcome_ids
}
child_concept_filenames = {
    outcome_id: '{}.{}.child-concepts.json'.format(project, outcome_id)
    for outcome_id in outcome_ids
}
casedef_paths = {
    outcome_id: project_path / 'case-definitions' / (outcome_id + '.yaml')
    for outcome_id in outcome_ids
}

with redo.ifchange(coding_systems='config/coding_systems.yaml',
                   references=(project_path / 'reference.yaml').as_posix(),
                   concepts=concept_filenames,
                   child_concepts=child_concept_filenames,
                   casedefs=casedef_paths) as files:
    coding_systems = yaml.load(files['coding_systems'])
    references = yaml.load(files['references'])
    concepts_by_outcome = {}
    for outcome_id, f in files['concepts'].items():
        concepts_by_outcome[outcome_id] = json.load(f)
    child_concepts_by_outcome = {}
    for outcome_id, f in files['child_concepts'].items():
        child_concepts_by_outcome[outcome_id] = json.load(f).items()
    outcomes = {
        outcome_id: yaml.load(f)
        for outcome_id, f in files['casedefs'].items()
    }

class Variation(object):

    """Identity"""

    def __init__(self):
        self.outcome_id = None
        self.coding_system = None

    def description(self):
        return self.__doc__

    def vary_code(self, code, coding_system):
        """Called on each code of generated concepts and the reference before
        before vary_codes and vary_concepts_and_mapping"""
        return code

    def vary_concepts_and_mapping(self, concepts, mapping, outcome_id):
        """Called for each outcome_id and database before vary_codes. For
        concept-level variations."""
        return concepts, mapping

    def vary_codes(self, generated, reference, coding_systems):
        """Code-level variations."""
        return generated, reference

class MaximumSensitivityVariation(Variation):

    """Maximium sensitivity"""

    def __init__(self):
        self.client = comap.ComapClient()

    def vary_concepts_and_mapping(self, concepts, mapping, outcome_id):
        cuis = set()
        for database_id in mapping.keys():
            coding_system = databases[database_id]
            codes = mapping[database_id]
            if coding_system == 'RCD2':
                coding_system = 'RCD'
                read_translation = comap.translation_read2_to_read3(codes)
                codes = set(code for codes in read_translation.values() for code in codes)
            if not codes:
                continue
            # ... WHERE cui IN (SELECT ... is too slow!
            # query = """
            #     SELECT DISTINCT cui, code FROM MRCONSO
            #     WHERE cui IN (SELECT DISTINCT cui FROM MRCONSO
            #                   WHERE sab = %s AND code IN ({placeholders}))
            #     AND sab = %s
            # """.format(placeholders=', '.join(['%s'] * len(codes)))
            # args = [coding_system] + list(codes) + [coding_system]
            cuis_query = """
                SELECT DISTINCT cui FROM MRCONSO
                WHERE sab = %s AND code IN ({placeholders})
            """.format(placeholders=', '.join(['%s'] * len(codes)))
            with comap.umls_db.cursor() as cursor:
                cursor.execute(cuis_query, tuple([coding_system] + list(codes)))
                cuis_for_codes = []
                for row in cursor.fetchall():
                    cuis_for_codes.append(row[0])
            query = """
                SELECT DISTINCT cui, code FROM MRCONSO
                WHERE cui IN ({placeholders})
            """.format(placeholders=', '.join(['%s'] * len(cuis_for_codes)))
            with comap.umls_db.cursor() as cursor:
                cursor.execute(query, tuple(cuis_for_codes))
                cui_codes = defaultdict(set)
                for row in cursor.fetchall():
                    cui, code = row
                    cui_codes[cui].add(code)
            for code in codes:
                # Find the `cui` that implicates `code` with the least number of irrelevant codes
                cuis_with_count = \
                    [(cui, len(set(codes) - set(codes1)))
                     for cui, codes1 in cui_codes.items()
                     if code in codes1]
                if cuis_with_count:
                    key = lambda x: x[1]
                    cui, _ = sorted(cuis_with_count, key=key)[0]
                    cuis.add(cui)
        return self.client.umls_concepts(cuis, coding_systems), mapping

class VariationChain(Variation):

    """Chain of variations that are applied to the input left-to-right."""

    def __init__(self, *variations):
        self.variations = variations

    def set_context(self, *args, **kwargs):
        for variation in self.variations:
            variation.set_context(*args, **kwargs)

    def description(self):
        res = '; '.join(v.description() for v in self.variations)
        if res:
            return res[0].upper() + res[1:]
        else:
            return ""

    def vary_code(self, code, coding_system):
        for variation in self.variations:
            code = variation.vary_code(code, coding_system)
        return code

    def vary_codes(self, generated, reference, coding_system):
        for variation in self.variations:
            generated, reference = variation.vary_codes(generated, reference, coding_system)
        return generated, reference

    def vary_concepts_and_mapping(self, concepts, mapping, outcome_id):
        for variation in self.variations:
            concepts, mapping = variation.vary_concepts_and_mapping(concepts, mapping, outcome_id)
        return concepts, mapping

class CodeNormalization(Variation):

    def __init__(self, *coding_system_prefixes):
        """`coding_systems` is None or a list of prefixes and this
        normalization is applied only if the list contains a prefix of the
        current coding systems."""
        self.coding_system_prefixes = coding_system_prefixes \
            or [''] # A prefix of any coding system

    def vary_code(self, code, coding_system):
        if any(coding_system.startswith(prefix)
               for prefix in self.coding_system_prefixes):
            return self.normalize(code, coding_system)
        else:
            return code

class NormalizeCodeCase(CodeNormalization):

    """Normalize code case in ICD, ICPC"""

    def __init__(self):
        super().__init__('ICD', 'ICPC')

    def normalize(self, code, coding_system):
        return code.upper()

class NormalizeCodeSuffixDot(CodeNormalization):

    """Drop suffix dots in codes in ICPC"""

    def __init__(self):
        super().__init__('ICPC')

    def normalize(self, code, coding_system):
        if code[-1] == '.':
            return code[:-1]
        else:
            return code

class NormalizeCodeXDD(CodeNormalization):

    """Normalize codes ICD and ICPC to 3 letters"""

    def __init__(self):
        super().__init__('ICD', 'ICPC')

    def normalize(self, code, coding_system):
        code = code.upper()
        m1 = re.match(comap.RegexHierarchy.TYPES['LDD'], code)
        try:
            return m1.group('parent')
        except:
            m2 = re.match(comap.RegexHierarchy.TYPES['DDD'], code)
            try:
                return m2.group('parent')
            except:
                return code

class NormalizeRead2Codes(CodeNormalization):

    """Normalize READ2 codes to 5-byte"""

    READ_RE = re.compile(r'(?P<code>[A-Z\d][A-Za-z0-9.]{4}).*')

    def __init__(self):
        super().__init__('RCD2')

    def normalize(self, code, coding_system):
        m = self.READ_RE.match(code)
        assert m, "Not a READ2 code: {}".format(code)
        return m.group('code')

class IncludeRelatedCodes(Variation):

    """Include related codes"""

    hierarchies = {
        'ICD9CM': comap.RegexHierarchy('DDD'),
        'ICD10CM': comap.RegexHierarchy('LDD'),
        'ICPC': comap.RegexHierarchy('LDD'),
        'RCD2': comap.Read2Hierarchy(),
    }

    def __init__(self, direction):
        """ direction : 'FN-FP' | 'FN' """
        self.direction = direction

    def description(self):
        return super().description() + ' ({})'.format(self.direction)

    def vary_codes(self, generated, reference, coding_system):
        hierarchy = self.hierarchies[coding_system]
        codes = comap.confusion_matrix(generated, reference)
        related_codes, derivated_codes = {
            'FN': comap.include_related,
            'FN-FP': comap.include_related_bidirect,
        }[self.direction](hierarchy, codes=codes)
        return comap.from_confusion_matrix(related_codes)

class IncludeChildConcepts(Variation):

    """Include child concepts"""

    def vary_concepts_and_mapping(self, concepts, mapping, outcome_id):
        cuis = [c['cui'] for c in concepts]
        child_concepts_by_parent = child_concepts_by_outcome[outcome_id]
        child_concepts = [
            child_concept
            for parent_id, concepts_for_parent in child_concepts_by_parent
            for child_concept in concepts_for_parent
            if child_concept not in cuis
        ]
        return concepts + child_concepts, mapping

class IncludeRelatedConcepts(Variation):

    """Include related concepts"""

    def __init__(self, *relations):
        self.relations = list(relations)
        self.client = comap.ComapClient()

    def description(self):
        return super().description() + ' ({})'.format(', '.join(self.relations))

    def vary_concepts_and_mapping(self, concepts, mapping, outcome_id):
        cuis = [c['cui'] for c in concepts]
        coding_systems = list(databases.values())
        related = self.client.related(cuis, self.relations, coding_systems)
        concepts_and_related = {c['cui']: c for c in concepts}
        for cui in related.keys():
            for rel in related[cui].keys():
                for new_concept in related[cui][rel]:
                    new_cui = new_concept['cui']
                    codes = set((sourceConcept['id'], sourceConcept['codingSystem'])
                                for sourceConcept in new_concept['sourceConcepts'])
                    if new_cui not in concepts_and_related:
                        concepts_and_related[new_cui] = new_concept
        return list(concepts_and_related.values()), mapping

def create_results(references, concepts_by_outcome, variations):

    # mapping ::= { database_id: { code } | None }

    # { outcome_id: mapping }
    mappings = {}
    for outcome_id in outcome_ids:
        mappings[outcome_id] = {}
        for database in databases.keys():
            mapping = references[database]['mappings'][outcome_id]
            if 'inclusion' in mapping:
                codes = set(mapping['inclusion'] + mapping.get('exclusion', []))
            else:
                codes = None
            mappings[outcome_id][database] = codes

    # { outcome_id: { variation_id: { "concepts": { concept }, "mapping": mapping } } }
    varied_by_outcome = {}
    for outcome_id in outcome_ids:
        varied_by_outcome[outcome_id] = {}
        concepts = concepts_by_outcome[outcome_id]
        reference_codes = OrderedDict([
            (database_id, mappings[outcome_id][database_id])
            for database_id in databases.keys()
        ])
        for variation_id, variation in variations.items():
            # Map codes in generated concepts with variation.vary_code
            concepts1 = []
            for concept in concepts_by_outcome[outcome_id]:
                concept1 = concept.copy()
                concept1['sourceConcepts'] = []
                for source_concept in concept['sourceConcepts']:
                    source_concept1 = source_concept.copy()
                    id, coding_system = source_concept['id'], source_concept['codingSystem']
                    source_concept1['id'] = variation.vary_code(id, coding_system)
                    concept1['sourceConcepts'].append(source_concept1)
                concepts1.append(concept1)
            # Map reference codes with variation.vary_code
            reference_codes1 = OrderedDict()
            for database_id in databases.keys():
                if reference_codes[database_id] is None:
                    reference_codes1[database_id] = None
                else:
                    reference_codes1[database_id] = [
                        variation.vary_code(code, databases[database_id])
                        for code in mappings[outcome_id][database_id]
                    ]
            concepts2, reference_codes2 = \
                variation.vary_concepts_and_mapping(concepts1, reference_codes1, outcome_id)
            assert concepts2 is not None, (outcome_id, variation_id)
            varied_by_outcome[outcome_id][variation_id] = {
                'concepts': concepts2,
                'reference-codes': reference_codes2,
            }

    res = OrderedDict()
    for outcome_id in outcome_ids:
        res[outcome_id] = OrderedDict()
        for database_id, coding_system in databases.items():
            res[outcome_id][database_id] = OrderedDict([])
            mapping = mappings[outcome_id][database_id]
            if mapping is not None:
                res[outcome_id][database_id]['variations'] = OrderedDict()
                for variation_id, variation in variations.items():
                    varied = varied_by_outcome[outcome_id][variation_id]
                    generated_codes = [
                        variation.vary_code(code['id'], coding_system)
                        for concept in varied['concepts']
                        for code in concept['sourceConcepts']
                        if code['codingSystem'] == coding_system
                    ]
                    reference_codes = [
                        variation.vary_code(code, coding_system)
                        for code in varied['reference-codes'][database_id]
                    ]
                    varied_generated_codes, varied_reference_codes = \
                        variation.vary_codes(generated_codes, reference_codes, coding_system)
                    varied_codes = comap.confusion_matrix(generated=varied_generated_codes,
                                                          reference=varied_reference_codes)
                    measures = comap.measures(codes=varied_codes)

                    res[outcome_id][database_id]['variations'][variation_id] = \
                        OrderedDict([
                            ('name', variation.description()),
                            ('comparison', OrderedDict([
                                ('codes', varied_codes),
                                ('measures', measures),
                            ])),
                    ])
            comment = references[database_id]['mappings'][outcome_id].get('comment')
            if comment is not None:
                res[outcome_id][database_id]['comment'] = comment
    return res

default_variations = VariationChain(NormalizeCodeCase(), NormalizeCodeSuffixDot(), NormalizeRead2Codes())
variations = OrderedDict([
    ('baseline', default_variations),
    ('maximum-sensitivity', VariationChain(default_variations, MaximumSensitivityVariation())),
    ('3-letter-codes', VariationChain(default_variations, NormalizeCodeXDD())),
    ('related-FN', VariationChain(default_variations, IncludeRelatedCodes('FN'))),
    ('related-FN-FP', VariationChain(default_variations, IncludeRelatedCodes('FN-FP'))),
    ('related-concepts-RN', VariationChain(default_variations, IncludeRelatedConcepts('RN'))),
    ('related-concepts-RN-RB', VariationChain(default_variations, IncludeRelatedConcepts('RN', 'RB'))),
    ('related-concepts-CHD', VariationChain(default_variations, IncludeRelatedConcepts('CHD'))),
    ('related-concepts-CHD-PAR', VariationChain(default_variations, IncludeRelatedConcepts('CHD', 'PAR'))),
    ('hyponyms', VariationChain(default_variations, IncludeChildConcepts())),
    ('child-concepts-related-FN-FP', VariationChain(default_variations, IncludeChildConcepts(), IncludeRelatedCodes('FN-FP'))),
])

evaluation = create_results(references, concepts_by_outcome, variations)

# Output results as YAML

with redo.output() as f:
    yaml.dump(evaluation, f)

if os.getenv('DOCTEST'):
    import doctest
    doctest.testmod()
    doctest.testmod(comap)