[iegitaddmd] tests for the comparefolder() option

*! version 6.2 31JAN2019 DIME Analytics dimeanalytics@worldbank.org

cap program drop   iegitaddmd
	program define iegitaddmd

qui {

		syntax , folder(string) [comparefolder(string) customfile(string) all skip replace manual]

		/******************************
		*******************************
			Test user input
		*******************************
		******************************/

		******************************
		*	Test input: folder

		*Test that folder exist
		mata : st_numscalar("r(dirExist)", direxists("`folder'"))

		if (`r(dirExist)' == 0) | (length("`folder'")<=10) {

			noi di as error `"{phang}The folder used in [folder(`folder')] does not exist or you have not entered the full path. For example, full paths on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}. Important: Specify the whole file path to the repository folder, not just {it:C:/} or {it:/user/} as that would create the placeholder file in every empty folder on your computer!{p_end}"'
			error 693
			exit
		}

		* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
		local folderStd			= subinstr(`"`folder'"',"\","/",.)

		******************************
		*	Test input: comparefolder

		if !missing("`comparefolder'") {
			mata : st_numscalar("r(dirExist)", direxists("`comparefolder'"))

			if (`r(dirExist)' == 0) | (length("`comparefolder'")<=10) {

				noi di as error `"{phang}The folder used in [comaparefolder(`comparefolder')] does not exist or you have not entered the full path. For example, full paths on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}. Important: Specify the whole file path to the repository folder, not just {it:C:/} or {it:/user/} as that would create the placeholder file in every empty folder on your computer!{p_end}"'
				error 693
				exit
			}

			* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
			local comparefolderStd	= subinstr(`"`comparefolder'"',"\","/",.)

			*Get the name of the last folder in both folder() and comparefolder()
			local thisLastSlash = strpos(strreverse(`"`folder'"'),"/")
			local thisFolder 	= substr(`"`folder'"', (-1 * `thisLastSlash')+1 ,.)
			local compLastSlash = strpos(strreverse(`"`comparefolder'"'),"/")
			local compFolder 	= substr(`"`comparefolder'"', (-1 * `compLastSlash')+1 ,.)

			*The last folder name should always be the same for folder() and
			* comparefolder() otherwise there it is likely that the to paths
			* point to differnet starting points in the two fodler trees that
			* are to be compared.
			if ("`compFolder'" != "`thisFolder'") {
				noi di as error `"{phang}The last folder [`thisFolder'] in [folder(`folder')] is not identical to the last folder [`compFolder'] in [comparefolder(`comparefolder')]. This is an indication that the the two fodler trees to be compared are not two versions of the same folder tree.{p_end}"'
				error 693
				exit
			}
		}

		******************************
		*	Test input: customfile

		*Test that file exist if not using default file
		if "`customfile'" != "" {
			*File option used test if file exists
			capture confirm file "`customfile'"
			if _rc {
				*File does not exist, throw error
				noi di as error `"{phang}File "`customfile'" was not found. Remember that you must enter the full path. For example, on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}.{p_end}"'
				error 693
				exit
			}

			* Get the name of file (include path to keep )
			* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
			local customFileStd	= subinstr("`customfile'","\","/",.)
			*Find the last slash in the file name
			local lastSlash = strpos(strreverse(`"`customFileStd'"'),"/")
			*Use that position to get the file name. Multiply with minus one as we count from the back
			local customFileName = substr(`"`customFileStd'"', (-1 * `lastSlash') ,.)
			*Used for the recursive call of the command when there is a subfolder
			local customFileRecurse `"customfile(`customFileStd')"'

			*Path and name of thile regardless if customfile were used
			local newPlaceholderFile `"`customFileName'"'
		}
		else {
			*Path and name of thile regardless if customfile were not used
			local newPlaceholderFile "README.md"
		}

		******************************
		*	Test input: skip & replace

		*Options skip and replace cannot be used together
		if !missing("`skip'") & !missing("`replace'") {
			noi di as error `"{phang}The options skip and replace may not be used together.{p_end}"'
			error 198
			exit
		}

		if !missing("`comparefolder'") & (!missing("`all'") | !missing("`skip'") | !missing("`replace'")) {
			noi di as error `"{phang}The options all, skip and replace may not be used when option comparefolder() is used.{p_end}"'
			error 198
			exit
		}

		/******************************
		*******************************
			List Files and folders
		*******************************
		******************************/

		******************************
		*	List all files and folders

		*List files, directories and other files
		local flist : dir `"`folderStd'"' files "*"	, respectcase
		local dlist : dir `"`folderStd'"' dirs  "*" , respectcase
		local olist : dir `"`folderStd'"' other "*"	, respectcase

		/******************************
		*******************************
			Exectute command without compare option
		*******************************
		******************************/

		if missing("`comparefolder'") {

			******************************
			*	Call command recursively on all sub-folders

			*Use the command on each subfolder to this folder (if any)
			foreach dir of local dlist {
				*Recursive call on each subfolder
				noi iegitaddmd , folder(`"`folderStd'/`dir'"') `all' `customFileRecurse' `skip' `replace' `manual'
			}

			******************************
			*	Evaluate if file will be crated in this folder

			*Test if all of those lists are empty, meaning there are
			* no folders or files in this folder
			if `"`flist'`dlist'`olist'"' == "" noi writePlaceholder, folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'

			*If the folder is not empty, test if option all were used.
			else if ("`all'" == "all") {

					*Test if a file with exactly that name is already used
					cap confirm file `"`folderStd'/`newPlaceholderFile'"'

					* No file with this name exists in this folder, create file without further ado
					if _rc != 0 noi writePlaceholder, folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'

					* File exist, but no sintructions on how to deal with this has been given, throw error
					else if missing("`skip'") & missing("`replace'")  {
						noi di as error `"{phang}The file  `newPlaceholderFile' already exists. Either remove option {it:all} or use either option {it:skip} or {it:replace}. See help file before using either of these options unless you are already familiar with them.{p_end}"'
						error 602
					}

					* File exist, and instruction is to replace, create files and overwrite as needed
					else if !missing("`replace'") noi writePlaceholder,  folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'

					* File exist, and instruction is to skip, do nothing
					else if !missing("`skip'") {
							*Nothing is done, this if-block is only included for readability and completion
					}

					* The code should nevere get to this point
					else  {
						noi di as error `"{phang}Coding error in if-chain in "All" block.Please report this error to dimeanalytics@worldbank.org or here: https://github.com/worldbank/ietoolkit/issues/new{p_end}"'
						error 602
					}
			}
			else {
				* Folder is not empty and option all is not used, so do nothing in this folder
			}
		}
		/******************************
		*******************************
			Exectute command with compare option
		*******************************
		******************************/

		else if !missing("`comparefolder'") {

			*List folders that are in comparefolder() but not in in folder()
			local comp_dlist : dir `"`comparefolderStd'"' dirs  "*" , respectcase
			local only_in_cdlist : list comp_dlist - dlist

			*Loop over all folders only in comparefolder()
			foreach dir of local only_in_cdlist {
				*Find all leaf folders. A leaf folder is a folder with no
				*sub-folder. Think of the folder structure as a tree, and
				*each folder with no sub-folders is an end of a branch where
				*you find leaves.
				noi findLeafFolders, folder(`"`comparefolderStd'/`dir'"')

				*Loop over all leaves and create accordingly
				foreach leaf in `r(leaves)' {
					*Leaf exist in compare folder, create the path to the leaf
					* to be created in the corresponding folder
					local folderleaf = subinstr(`"`leaf'"',`"`comparefolderStd'"', `"`folder'"', 1)

					*Create placeholder file in this new leaf
					noi writePlaceholder, folder(`"`folderleaf'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'
				}
			}

			*Recursivley do the same call on all sub-folders
			foreach dir of local dlist {
				*Recursive call on each subfolder
				noi iegitaddmd , folder(`"`folderStd'/`dir'"') comparefolder(`"`comparefolder'/`dir'"') `customFileRecurse' `manual'
			}
		}
}

end

*Write a README.md file when iegitaddmd finds an empty folder
cap program drop writePlaceholder
program define   writePlaceholder

qui {

	syntax , folder(string) newfilename(string) [customfiletocopy(string) manual]

	*If manual was used, get manual confirmation for each file
	if !missing("`manual'") {
		noi di ""
		global confirmation "" //Reset global

		*Keep aslking for input until the input is either Y, y, N, n or BREAK
		while (upper("${confirmation}") != "Y" & upper("${confirmation}") != "N" & "${confirmation}" != "BREAK") {
		  noi di as txt "{pstd}You are about to create a file [`newfilename'] in the folder [`folder']. If the folder does not exist it will be created. Do you want to do that? To confirm type {bf:Y} and hit enter, to abort type {bf:N} and hit enter. Type {bf:BREAK} and hit enter to stop the code.{p_end}", _request(confirmation)
		}
		*Copy user input to local
		local createfile = upper("${confirmation}")
		* If user wrote "BREAK" then exit the code
		if ("`createfile'" == "BREAK") error 1
	}

	*If "manual" were used and input was Y or if manual was not used, create the file
	if ("`createfile'" == "Y") | (missing("`manual'")) {

		*Recursively create folder if needed
		noi rmkdir, folder(`"`folder'"')

		*Copy custom file if custom file is used
		if !missing(`"`customfile'"') copy "`customfile'" `"`folder'/`newfilename'"', replace
		*Custom file is not used, so cretae defeault file
		else {
			*Create file
			tempname 	newHandle
			cap file close 	`newHandle'
			file open  	`newHandle' using "`folder'/`newfilename'", text write replace
			*Write the content to the file
			file write  `newHandle' ///
				"# Placeholder file" _n _n ///
				"This file has been created automatically by the command **iegitaddmd** from the Stata package [**ietoolkit**](https://worldbank.github.io/ietoolkit) to make GitHub sync this folder. GitHub does not sync empty folders or folders that only contain ignored files, but in research projects it is often important to share the full standardized folder structure along with the actual files. This command is intended to be used with **iefolder**, but it can be used in any folder structure intended to be shared on GitHub." _n _n ///
				"In recently started projects, it is typical to create data folders, script folders (do-files, r-files) and output folders, among others. The output folders are initially empty, but the script files might include a file path to them for later use. If an output folder is empty and another collaborator clones the repository, then the output folder expected by the scripts will not be included by GitHub in the cloned repository, and the script will not run properly." _n _n ///
				"## You should replace the content of this placeholder file" _n _n ///
				"The text in this file should be replaced with text that describes the intended use of this folder. This file is written in markdown (.md) format, which is suitable for GitHub syncing (unlike .doc/.docx). If the file is named *README.md*, GitHub automatically displays its contents when someone opens the containing folder on GitHub.com using a web browser." _n _n ///
				"If you are new to markup languages (markdown, html etc.) then this [Markdown Tutorial](https://www.markdowntutorial.com/) is a great place to start. If you have some experience with markup languages, then this [Markdown Cheat Sheet](https://guides.github.com/pdfs/markdown-cheatsheet-online.pdf) is a great place to start." _n _n ///
				"## Add similar files in other folders" _n _n ///
				"The Stata command **iegitaddmd** does not add anything to folders that already have content unless this is explicitly requested using the option `all`. It is best practice to create a *README.md* file in any folders that have content, for example, in folders whose purpose might not obvious to someone using the repository for the first time. Again, if the file is named *README.md*, then the content of the file will be shown in the browser when someone explores the repository on GitHub.com." _n _n ///
				"Another great use of a *README.md* file is to use it as a documentation on how the folder it sits in and its subfolders are organized,  where its content can be found, and where new content is meant to be saved. For example, if you have a folder called `/Baseline/`, then you can give a short description of the activities conducted during the baseline and where data, scripts and outputs related to it can be found." _n _n ///
				"## Removing this file" _n _n ///
				"Removing this file is not recommended, as GitHub may stop syncing the parent folder unless the folder now has other content. However, even when content is added and the file can be removed without breaking the GitHub functionality, our recommendation is to replace the content with more relevant content rather than deleting it." _n _n

			*Closing the file
			file close 		`newHandle'

		}
		*Uoutput that the file was created
		noi di as result "{pstd}File [`folder'/`newfilename'] created.{p_end}"
	}
	*Manual was used and input was N, no file were creaetd
	else noi di as result "{pstd}No file or folder created.{p_end}"
}
end

*Write a README.md file when iegitaddmd finds an empty folder
cap program drop findLeafFolders
program define   findLeafFolders, rclass

qui {

	syntax, folder(string)

	local dlist : dir `"`folder'"' dirs  "*" , respectcase

	if missing(`"`dlist'"') {
		*This is a leaf, pass it back
		return local leaves `""`folder'""'
	}
	else {
		local return_leaves
		foreach dir of local dlist {
			*Recursive call on each subfolder
			findLeafFolders, folder(`"`folder'/`dir'"')
			local return_leaves `"`return_leaves' `r(leaves)'"'
		}
		return local leaves `"`return_leaves'"'
	}
}
end

*Recursively call parent folders in folderpath needed to be created until
* folder found that already exist, then creaet all subfolders.
cap program drop rmkdir
program define   rmkdir, rclass

qui {

	syntax, folder(string) [folderstocreate(string)]

	*Test if this folder exists
	mata : st_numscalar("r(dirExist)", direxists(`"`folder'"'))

	*Folder does not exist, find parent folder and make recursive call
	if (`r(dirExist)' == 0) {

		*Get the parent folder of folder
		local lastSlash = strpos(strreverse(`"`folder'"'),"/")
		local parentFolder = substr(`"`folder'"',1,strlen("`folder'")-`lastSlash')
		local thisFolder = substr(`"`folder'"', (-1 * `lastSlash')+1 ,.)

		*Make the coll recursivly on the parent folder and add this folder to folderstocreate()
		noi rmkdir , folder(`"`parentFolder'"') folderstocreate(`""`thisFolder'" `folderstocreate'"')

	}
	*Folder exist, create sub-folders to create if any
	else {
		*Create the folders needed
		foreach dir of local folderstocreate {
			mkdir "`folder'/`dir'"
			local folder "`folder'/`dir'"
		}
	}
}
end

*! version 6.2 31JAN2019 DIME Analytics dimeanalytics@worldbank.org

cap program drop   iegitaddmd
	program define iegitaddmd

qui {

		syntax , folder(string) [comparefolder(string) customfile(string) all skip replace manual]

		/******************************
		*******************************
			Test user input
		*******************************
		******************************/

		******************************
		*	Test input: folder

		*Test that folder exist
		mata : st_numscalar("r(dirExist)", direxists("`folder'"))

		if (`r(dirExist)' == 0) | (length("`folder'")<=10) {

			noi di as error `"{phang}The folder used in [folder(`folder')] does not exist or you have not entered the full path. For example, full paths on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}. Important: Specify the whole file path to the repository folder, not just {it:C:/} or {it:/user/} as that would create the placeholder file in every empty folder on your computer!{p_end}"'
			error 693
			exit
		}

		* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
		local folderStd			= subinstr(`"`folder'"',"\","/",.)
		local comparefolderStd	= subinstr(`"`comparefolder'"',"\","/",.)

		******************************
		*	Test input: customfile

		*Test that file exist if not using default file
		if "`customfile'" != "" {
			*File option used test if file exists
			capture confirm file "`customfile'"
			if _rc {
				*File does not exist, throw error
				noi di as error `"{phang}File "`customfile'" was not found. Remember that you must enter the full path. For example, on most Windows computers it starts with {it:C:/} and on most Mac computers with {it:/user/}.{p_end}"'
				error 693
				exit
			}

			* Get the name of file (include path to keep )
			* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
			local customFileStd	= subinstr("`customfile'","\","/",.)
			*Find the last slash in the file name
			local lastSlash = strpos(strreverse(`"`customFileStd'"'),"/")
			*Use that position to get the file name. Multiply with minus one as we count from the back
			local customFileName = substr(`"`customFileStd'"', (-1 * `lastSlash') ,.)
			*Used for the recursive call of the command when there is a subfolder
			local customFileRecurse `"customfile(`customFileStd')"'

			*Path and name of thile regardless if customfile were used
			local newPlaceholderFile `"`customFileName'"'
		}
		else {
			*Path and name of thile regardless if customfile were not used
			local newPlaceholderFile "README.md"
		}

		******************************
		*	Test input: skip & replace

		*Options skip and replace cannot be used together
		if !missing("`skip'") & !missing("`replace'") {
			noi di as error `"{phang}The options skip and replace may not be used together.{p_end}"'
			error 198
			exit
		}

		/******************************
		*******************************
			List Files and folders
		*******************************
		******************************/

		******************************
		*	List all files and folders

		*List files, directories and other files
		local flist : dir `"`folderStd'"' files "*"	, respectcase
		local dlist : dir `"`folderStd'"' dirs  "*" , respectcase
		local olist : dir `"`folderStd'"' other "*"	, respectcase

		/******************************
		*******************************
			Exectute command without compare option
		*******************************
		******************************/

		if missing("`comparefolder'") {

			******************************
			*	Call command recursively on all sub-folders

			*Use the command on each subfolder to this folder (if any)
			foreach dir of local dlist {
				*Recursive call on each subfolder
				noi iegitaddmd , folder(`"`folderStd'/`dir'"') `all' `customFileRecurse' `skip' `replace' `manual'
			}

			******************************
			*	Evaluate if file will be crated in this folder

			*Test if all of those lists are empty, meaning there are
			* no folders or files in this folder
			if `"`flist'`dlist'`olist'"' == "" noi writePlaceholder, folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'

			*If the folder is not empty, test if option all were used.
			else if ("`all'" == "all") {

					*Test if a file with exactly that name is already used
					cap confirm file `"`folderStd'/`newPlaceholderFile'"'

					* No file with this name exists in this folder, create file without further ado
					if _rc != 0 noi writePlaceholder, folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'

					* File exist, but no sintructions on how to deal with this has been given, throw error
					else if missing("`skip'") & missing("`replace'")  {
						noi di as error `"{phang}The file  `newPlaceholderFile' already exists. Either remove option {it:all} or use either option {it:skip} or {it:replace}. See help file before using either of these options unless you are already familiar with them.{p_end}"'
						error 602
					}

					* File exist, and instruction is to replace, create files and overwrite as needed
					else if !missing("`replace'") noi writePlaceholder,  folder(`"`folderStd'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'

					* File exist, and instruction is to skip, do nothing
					else if !missing("`skip'") {
							*Nothing is done, this if-block is only included for readability and completion
					}

					* The code should nevere get to this point
					else  {
						noi di as error `"{phang}Coding error in if-chain in "All" block.Please report this error to dimeanalytics@worldbank.org or here: https://github.com/worldbank/ietoolkit/issues/new{p_end}"'
						error 602
					}
			}
			else {
				* Folder is not empty and option all is not used, so do nothing in this folder
			}
		}
		/******************************
		*******************************
			Exectute command with compare option
		*******************************
		******************************/

		else if !missing("`comparefolder'") {

			*List folders that are in comparefolder() but not in in folder()
			local comp_dlist : dir `"`comparefolderStd'"' dirs  "*" , respectcase
			local only_in_cdlist : list comp_dlist - dlist

			*Loop over all folders only in comparefolder()
			foreach dir of local only_in_cdlist {
				*Find all leaf folders. A leaf folder is a folder with no
				*sub-folder. Think of the folder structure as a tree, and
				*each folder with no sub-folders is an end of a branch where
				*you find leaves.
				noi findLeafFolders, folder(`"`comparefolderStd'/`dir'"')

				*Loop over all leaves and create accordingly
				foreach leaf in `r(leaves)' {
					*Leaf exist in compare folder, create the path to the leaf
					* to be created in the corresponding folder
					local folderleaf = subinstr(`"`leaf'"',`"`comparefolderStd'"', `"`folder'"', 1)

					*Create placeholder file in this new leaf
					noi writePlaceholder, folder(`"`folderleaf'"') newfilename("`newPlaceholderFile'") customfiletocopy(`"`customFileStd'"') `manual'
				}
			}

			*Recursivley do the same call on all sub-folders
			foreach dir of local dlist {
				*Recursive call on each subfolder
				noi iegitaddmd , folder(`"`folderStd'/`dir'"') comparefolder(`"`comparefolder'/`dir'"') `customFileRecurse' `manual'
			}
		}
}

end

*Write a README.md file when iegitaddmd finds an empty folder
cap program drop writePlaceholder
program define   writePlaceholder

qui {

	syntax , folder(string) newfilename(string) [customfiletocopy(string) manual]

	*If manual was used, get manual confirmation for each file
	if !missing("`manual'") {
		noi di ""
		global confirmation "" //Reset global

		*Keep aslking for input until the input is either Y, y, N, n or BREAK
		while (upper("${confirmation}") != "Y" & upper("${confirmation}") != "N" & "${confirmation}" != "BREAK") {
		  noi di as txt "{pstd}You are about to create a file [`newfilename'] in the folder [`folder']. If the folder does not exist it will be created. Do you want to do that? To confirm type {bf:Y} and hit enter, to abort type {bf:N} and hit enter. Type {bf:BREAK} and hit enter to stop the code.{p_end}", _request(confirmation)
		}
		*Copy user input to local
		local createfile = upper("${confirmation}")
		* If user wrote "BREAK" then exit the code
		if ("`createfile'" == "BREAK") error 1
	}

	*If "manual" were used and input was Y or if manual was not used, create the file
	if ("`createfile'" == "Y") | (missing("`manual'")) {

		*Recursively create folder if needed
		noi rmkdir, folder(`"`folder'"')

		*Copy custom file if custom file is used
		if !missing(`"`customfile'"') copy "`customfile'" `"`folder'/`newfilename'"', replace
		*Custom file is not used, so cretae defeault file
		else {
			*Create file
			tempname 	newHandle
			cap file close 	`newHandle'
			file open  	`newHandle' using "`folder'/`newfilename'", text write replace
			*Write the content to the file
			file write  `newHandle' ///
				"# Placeholder file" _n _n ///
				"This file has been created automatically by the command **iegitaddmd** from the Stata package [**ietoolkit**](https://worldbank.github.io/ietoolkit) to make GitHub sync this folder. GitHub does not sync empty folders or folders that only contain ignored files, but in research projects it is often important to share the full standardized folder structure along with the actual files. This command is intended to be used with **iefolder**, but it can be used in any folder structure intended to be shared on GitHub." _n _n ///
				"In recently started projects, it is typical to create data folders, script folders (do-files, r-files) and output folders, among others. The output folders are initially empty, but the script files might include a file path to them for later use. If an output folder is empty and another collaborator clones the repository, then the output folder expected by the scripts will not be included by GitHub in the cloned repository, and the script will not run properly." _n _n ///
				"## You should replace the content of this placeholder file" _n _n ///
				"The text in this file should be replaced with text that describes the intended use of this folder. This file is written in markdown (.md) format, which is suitable for GitHub syncing (unlike .doc/.docx). If the file is named *README.md*, GitHub automatically displays its contents when someone opens the containing folder on GitHub.com using a web browser." _n _n ///
				"If you are new to markup languages (markdown, html etc.) then this [Markdown Tutorial](https://www.markdowntutorial.com/) is a great place to start. If you have some experience with markup languages, then this [Markdown Cheat Sheet](https://guides.github.com/pdfs/markdown-cheatsheet-online.pdf) is a great place to start." _n _n ///
				"## Add similar files in other folders" _n _n ///
				"The Stata command **iegitaddmd** does not add anything to folders that already have content unless this is explicitly requested using the option `all`. It is best practice to create a *README.md* file in any folders that have content, for example, in folders whose purpose might not obvious to someone using the repository for the first time. Again, if the file is named *README.md*, then the content of the file will be shown in the browser when someone explores the repository on GitHub.com." _n _n ///
				"Another great use of a *README.md* file is to use it as a documentation on how the folder it sits in and its subfolders are organized,  where its content can be found, and where new content is meant to be saved. For example, if you have a folder called `/Baseline/`, then you can give a short description of the activities conducted during the baseline and where data, scripts and outputs related to it can be found." _n _n ///
				"## Removing this file" _n _n ///
				"Removing this file is not recommended, as GitHub may stop syncing the parent folder unless the folder now has other content. However, even when content is added and the file can be removed without breaking the GitHub functionality, our recommendation is to replace the content with more relevant content rather than deleting it." _n _n

			*Closing the file
			file close 		`newHandle'

		}
		*Uoutput that the file was created
		noi di as result "{pstd}File [`folder'/`newfilename'] created.{p_end}"
	}
	*Manual was used and input was N, no file were creaetd
	else noi di as result "{pstd}No file or folder created.{p_end}"
}
end

*Write a README.md file when iegitaddmd finds an empty folder
cap program drop findLeafFolders
program define   findLeafFolders, rclass

qui {

	syntax, folder(string)

	local dlist : dir `"`folder'"' dirs  "*" , respectcase

	if missing(`"`dlist'"') {
		*This is a leaf, pass it back
		return local leaves `""`folder'""'
	}
	else {
		local return_leaves
		foreach dir of local dlist {
			*Recursive call on each subfolder
			findLeafFolders, folder(`"`folder'/`dir'"')
			local return_leaves `"`return_leaves' `r(leaves)'"'
		}
		return local leaves `"`return_leaves'"'
	}
}
end

*Recursively call parent folders in folderpath needed to be created until
* folder found that already exist, then creaet all subfolders.
cap program drop rmkdir
program define   rmkdir, rclass

qui {

	syntax, folder(string) [folderstocreate(string)]

	*Test if this folder exists
	mata : st_numscalar("r(dirExist)", direxists(`"`folder'"'))

	*Folder does not exist, find parent folder and make recursive call
	if (`r(dirExist)' == 0) {

		*Get the parent folder of folder
		local lastSlash = strpos(strreverse(`"`folder'"'),"/")
		local parentFolder = substr(`"`folder'"',1,strlen("`folder'")-`lastSlash')
		local thisFolder = substr(`"`folder'"', (-1 * `lastSlash')+1 ,.)

		*Make the coll recursivly on the parent folder and add this folder to folderstocreate()
		noi rmkdir , folder(`"`parentFolder'"') folderstocreate(`""`thisFolder'" `folderstocreate'"')

	}
	*Folder exist, create sub-folders to create if any
	else {
		*Create the folders needed
		foreach dir of local folderstocreate {
			mkdir "`folder'/`dir'"
			local folder "`folder'/`dir'"
		}
	}
}
end


phraseology

phraseology

{smcl}
{* *! version 0.0.1  28may2015}{...}
{vieweralsosee "[R] regress" "mansection R regress"}{...}
{viewerjumpto "Syntax" "svm##syntax"}{...}
{viewerjumpto "Description" "svm##description"}{...}
{viewerjumpto "Options" "svm##options"}{...}
{viewerjumpto "Examples" "svm##examples"}{...}
{viewerjumpto "Stored results" "svm##results"}{...}
{viewerjumpto "Gotchas" "svm##gotchas"}{...}
{viewerjumpto "Copyright" "svm##copyright"}{...}
{viewerjumpto "Authors" "svm##authors"}{...}
{viewerjumpto "References" "svm##references"}{...}
{...}{* NB: these hide the newlines }
{...}
{...}
{title:Title}

{p2colset 5 18 20 2}{...}
{p2col :{cmd:svm} {hline 2}}Support Vector Machines{p_end}
{p2colreset}{...}

{marker syntax}{...}
{title:Syntax}

{p 8 16 2}
{help svm##svm:svm} {depvar} {indepvars} {ifin}{* [{it:{help svm##weight:weight}}] } [{cmd:,} {it:options}]

{p 8 16 2}
{help svm##svm:svm} {indepvars} {ifin}{* [{it:{help svm##weight:weight}}] }, type({help svm##one_class:one_class}) [{it:options}]

{synoptset 20 tabbed}{...}
{synopthdr}
{synoptline}
{syntab:Model}
{synopt :{opth t:ype(svm##type:type)}}Type of model to fit: {opt c_svc}, {opt nu_svc}, {opt one_class}, {opt epsilon_svr}, or {opt nu_svr}. Default: {cmd:type(c_svc)}{p_end}
{synopt :{opth k:ernel(svm##kernel:kernel)}}SVM kernel function to use: {opt linear}, {opt poly}, {opt rbf}, {opt sigmoid}, or {opt precomputed}. Default: {cmd:kernel(rbf)}{p_end}

{synopt :{opth deg:ree(svm##degree:degree)}}For {opt poly} kernels, the degree of the polynomial to use. Default: cubic ({cmd:degree(3)}){p_end}

{* XXX the division between 'tuning' and 'model' parameters is hazy; e.g. you could in theory cross-validate to choose degree (and people do this with neural networks), or even to choose the kernel . hmmmmm}{...}

{syntab:Tuning}
{synopt :{opth c:(svm##c:c)}}For {opt c_svc}, {opt epsilon_svr} and {opt nu_svr} SVMs, this is a regularization parameter which weights the slack variables. Should be > 0. Default: {cmd:c(1)}{p_end}
{synopt :{opth eps:ilon(svm##epsilon:eps)}}For {opt epsilon_svr} SVMs, the margin of error allowed for to count something as a support vector. Default: {cmd:p(0.1)}{p_end}
{synopt :{opth nu:(svm##nu:nu)}}For {opt nu_svc}, {opt one_class}, and {opt nu_svr} SVMs, tunes the number of supports vectors. Should be in (0, 1]. Default: {cmd:nu(0.5)}{p_end}

{synopt :{opth gamma:(svm##gamma:gamma)}}For {opt poly}, {opt rbf} and {opt sigmoid} kernels, a scaling factor for the linear part of the kernel. Default: {cmd:gamma(1/[# {indepvars}])}{p_end}
{synopt :{opth coef0:(svm##coef0:coef0)}}For {opt poly} and {opt sigmoid} kernels, a bias ("intercept") term for the linear part of the kernel. Default: {cmd:coef0(0)}{p_end}

{synopt :{opt shrink:ing}}Whether to use {help svm##shrinking:shrinkage} heuristics to improve the fit. Default: disabled{p_end}

{syntab:Features}
{synopt :{opt norm:alize}}Whether to {help svm##normalize:center and scale} the data. NOT IMPLEMENTED. Default: disabled{p_end}
{synopt :{opt prob:ability}}Whether to {help svm##probability:prepare} for "predict, prob" during estimation. Only applicable to classification problems. Default: disabled{p_end}
{synopt :{opth sv:(svm##sv:newvarname)}}If given, an indicator variable to generate to mark each row as a support vector or not. Default: disabled{p_end}

{syntab:Performance}
{synopt :{opth tol:erance(svm##tolerance:tol)}}The stopping tolerance used to decide when convergence has happened. Default: {cmd:epsilon(0.001)}{p_end}
{synopt :{opt v:erbose}}Turns on {help svm##verbose:verbose mode}. NOT IMPLEMENTED. Default: disabled{p_end}
{synopt :{opth cache:_size(svm##cache_size:cache)}}The size of the RAM cache used during fitting, in megabytes. Default: 100MB ({cmd:cache_size(100)}){p_end}

{synoptline}
{pstd}All variables must be numeric, including categorical variables,
which may appear to be strings if they have have {help label values:value labels} attached, but are probably actually stored as {help byte}s.
If you in fact have categories stored in string variables use {help encode}.{p_end}
INCLUDE help fvvarlist

{p 8 16 2}
{help svm##predict:predict} {newvar} {ifin}, [{cmdab:prob:ability}]

{synoptset 20 tabbed}{...}
{synopthdr}
{synoptline}
{synopt :{cmdab:prob:ability}}If specified, estimate class probabilities for each observation. This only makes sense for classification problems.{p_end}

{p 8 16 2}
{help svm##export:svm_export} using {filename}

{p 8 16 2}
{help svm##export:svm_import} using {filename}

{synoptset 20 tabbed}{...}
{synopthdr}
{synoptline}
{synopt :{cmd: using}}The filename to export or import a fitted libsvm model from, conventionally ending in '.model'.{p_end}

{marker description}{...}
{title:Description}

{pstd}
{cmd:svm} fits a support vector machine (SVM) model.
SVM is not one, but several, variant models each based upon the principles of
splitting hyperplanes and the culling of unimportant observations.
The name comes from these two parts:
each observation in a dataset can be thought of as a (typically high-dimensional) linear algebra vector,
and the {it:support vectors} are those observations which the algorithm detects are critical to the fit.
The non-support observations are ignored after fitting is done, which makes SVM very memory efficient.
See {help svm##svmtutorial:this SVM tutorial} for a gentle introduction to the method.

{pstd}
This package is a thin wrapper for the widely deployed {help svm##libsvm:libsvm},
so in addition to the Stata package, {bf:libsvm must be installed}.
On Windows, libsvm.dll is bundled with the package,
and you can find it in your {help adopath} (try {cmd:findfile libsvm.dll} to verify this).
On OS X, libsvm is available in both {browse "https://brew.sh":brew} and {browse "https://www.macports.org":macports}.
On Linux, search for libsvm in your distribution's package manager.
You can also compile and install libsvm from source, which you would need to do if your package manager has an older or no version of libsvm.
If you are having plugin load errors, please {help svm##authors:contact the authors},
as we want to make the experience as smooth as possible for our users.

{pstd}
The thinness of this wrapper is an intentional feature which means work done under
Stata-SVM should be replicable with other libsvm wrappers such as
{browse "http://weka.wikispaces.com/LibSVM":Weka} or {browse "http://scikit-learn.org/stable/modules/svm.html":sklearn}, or {browse "":}.
Please do {help svm##authors:send us feature requests},
but unless they are Stata-specific we would rather try to
get them into libsvm first so that everyone can benefit.

{pstd}
Not all combinations of options are valid, and you will get an error if you specify an invalid combination.
Amongst valid combinations, not all options and datasets will give good results.
Our goal is to have sane defaults,
so that the only choice you need to make is what {help svm##type:type} and {help svm##kernel:kernel} to use,
but to offer access to all parameters so they can be tweaked for difficult datasets.
The {help svm##libsvmguide:libsvm guide} covers issues in parameter specification.

{pstd}
If this manual leaves you confused, refer to the authoratative source:
the libsvm {browse "http://www.csie.ntu.edu.tw/~cjlin/libsvm/faq.html":FAQ},
{browse "https://github.com/cjlin1/libsvm/blob/master/README":README},
and {help svm##libsvm:implementation paper}.
Then please write us with suggestions for clarification.

{marker options}{...}
{title:Options}

{marker svm}{...}
{dlgtab:svm}{* this is a misuse of dlgtab because I have no corresponding dialog, but it drastically helps readability }

{pstd}
{cmd:svm} fits an SVM model, fitting {depvar} to {indepvars} except under {opt type(one_class)} which only uses {indepvars}.

{* MODEL PARAMS: }
{phang}
{marker type}{...}
{opt type} specifies what type of SVM model to run.{p_end}
{pmore}{opt c_svc} and {opt nu_svc} perform classification.{p_end}
{pmore2}{depvar} should be a variable containing categories.
If you try to use floating point values with classification you
will find that they are truncated mercilessly to their integer parts,
so you may need to recode your categories first.
If you end up with almost as many classes as observations,
you have probably used a continuous {depvar} and
should use regression instead.{p_end}

{pmore2}Multiclass classification is automatically handled using the {browse "http://en.wikipedia.org/wiki/Multiclass_classification":one-against-one} method.
You do not need to do anything special to invoke it.{p_end}

{marker one_class}{...}
{pmore}{opt one_class} separates outliers from the bulk of the data.{p_end}
{pmore2}{opt one_class} does not take a {depvar} because it is a form of unsupervised learning: all the information it uses is in the predictors themselves.{p_end}
{pmore2}Its predictions give 1 for "data" and -1 for "outlier".{p_end}
{pmore3}{bf:Tip:} You may use the same {varlist} as with the other types:
 this just interprets your {depvar} as one of the {indepvars},
 giving {opt one_class} more information to work with.{p_end}

{pmore}{opt epsilon_svr} and {opt nu_svr} perform regression.{p_end}
{pmore2}{depvar} should be a variable containing continuous values.{p_end}
{pmore2}While you can use this with discrete {depvar}s, it is more common to use it with continuous ones.{p_end}
{pmore2}See {help svm##svr_tutorial:the SVR tutorial} for more details.{p_end}

{pmore}To learn about the {opt c}/{opt nu} distinction, see {help svm##nusvm:Chen and Lin's ν-SVM tutorial}.{p_end}

{pmore}{opt type} is case insensitive.{p_end}

{phang}
{marker kernel}{...}
{opt kernel} gives a kernel function to use.
The basic SVM idea is to find a linear boundary in high-dimensional space (a hyperplane):
for classification, this is a boundary between two classes;
for regression it is a line {help svm##p:near} which predictions are expected to cluster--much like in {help regess:OLS}.
Much real data is not straight but the {browse "https://en.wikipedia.org/wiki/Kernel_Method":kernel trick} makes SVM
flexible by making a space non-linear under a high-dimensional mapping.
See {browse "https://www.youtube.com/watch?v=3liCbRZPrZA"} for a visualization of this process.
The part that makes it a {it:trick} is that the fit can be done efficiently without constructing the high-dimensional points,
as the estimation only cares about a scalar value <u,v>, the output of the kernel function for coefficients u on support vector v,
which is used in scoring the fit [CITATION NEEDED],
and for certain kernels this value can be computed straight
from the original data without doing the expensive high-dimensional mapping at all.{p_end}

{pmore}Kernels available in this implementation are:{p_end}
{pmore2}{opt linear} means the linear kernel you are probably familiar with from {help regress:OLS}: u'*v{p_end}
{pmore2}{opt poly} is (gamma*u'*v + coef0)^degree{p_end}
{pmore2}{opt rbf} stands for Radial Basis Functions, and treats the coefficients as a mean to smoothly approach in a ball, with the form exp(-gamma*|u-v|^2); this kernel tends to be good at [...].{p_end}
{pmore2}{opt sigmoid} is a kernel which bends the linear kernel to fit in -1 to 1, similar to {help logistic} regression: tanh(gamma*u'*v + coef0){p_end}
{pmore2}{opt precomputed} assumes that {depvar} is actually a list of precomputed kernel values. With effort, you can use this to use custom kernels with your data [TODO].{p_end}

{pmore}{opt kernel} is case insensitive.{p_end}

{phang}
{marker degree}{...}
{opt degree} selects the degree of the polynomial used by the {opt poly} kernel.
Setting this too high will result in overfitting. Setting it too low may result in non-convergence.
[TODO: tips about choosing this]

{* TUNING PARAMS: }
{phang}
{marker c}{...}
{opt c} weights (regularizes) the slack variables used in {opt c_svc}, {opt epsilon_svr} and {opt nu_svr}.
If your fit is poor, try cranking this up.
{* XXX NOT TRUE: This will pre-multiply any sample-specific {opt weight}s. }

{phang}
{marker epsilon}{...}
{opt epsilon} is the margin of error allowed by {opt epsilon_svr}.
Larger makes your fit more able to incorporate more observations, but can lead to underfitting.
Smaller can lead to overfitting.

{phang}
{marker nu}{...}
{opt nu} is used in the nu variants.
The nu variants are a reparamaterization of regular SVM which lets you directly tune,
using {opt nu}, the tradeoff between error and computation.
{opt nu} is an upper bound on the fraction of training errors and a lower bound of the
fraction of support vectors, so that [TODO smaller means ... and larger means .....]
See {help svm##nusvm:the ν-SVM tutorial} for details.
{* ..wait... this doesn't make any sense. nu == 0.1 means there are at most 10% (training) errors and at least 10% are support vectors.}{...}
{*      nu = 0.9 means there are at most 90% errors and at least 90% SVs.   you should always choose 0, then, to get perfect prediction and zero memory usage}{...}

{phang}
{marker gamma}{...}
{opt gamma} is used in the non-linear kernels as a scaling factor for the linear part, as seen above.
[TODO: tips about choosing this]

{phang}
{marker coef0}{...}
{opt coef0} similarly is used in the non-linear kernels as a pseudo-intercept term.
Except it is not used in the {opt rbf} kernel as {opt rbf} is essentially a distance function, and biasing would be pointless.[???]
[TODO: tips about choosing this]

{phang}
{marker shrinking}{...}
{opt shrinking} invokes the shrinkage heuristics [TODO: what does this mean? shrinkage estimation?],
which reduce can improve the fit by trading bias for variance: by giving up accuracy in the estimates,
variance of the estimates can be reduced, which is sometimes better overall.

{* FEATURE PARAMS: }
{marker normalize}{...}
{phang}
{opt normalize} instructs the estimation to first center and scale the data
as SVM tends to be very sensitive to scaling issues.
This normalizes all data to [0,1] using min-max normalization, as suggested in the {help svm##libsvmguide:libsvm guide}.
Normalization creates temporary variables, so you may prefer to preprocess the data yourself---destructively and in-place---to save time on re-estimations and memory for variables,
especially if you are bumping up against your Stata system limits. You may find {cmd:ssc install norm} helpful.

{phang}
{marker probability}{...}
{opt probability} enables the use of "{help svm##predict_prob:predict, prob}".
This requires precomputing class-against-class probability estimates with a 5-fold cross-validation,
and so takes a great deal of additional CPU and RAM.

{phang}
{marker sv}{...}
{opt sv} records in the given variable a boolean indicating whether each observation was a support vector or not.

{* PERFORMANCE PARAMS: }
{phang}
{marker tolerance}{...}
{opt tolerance} is the stopping tolerance used by the numerical optimizer. You could widen this if you are finding convergence is slow,
 but be aware that this usually non-convergence is a deeper problem in the interaction of data, kernel, and tuning parameters.

{phang}
{marker verbose}{...}
{opt verbose} enables output from the low level libsvm code for the duration of the operation.

{phang}
{marker cache_size}{...}
{opt cache_size} controls a time-memory tradeoff during estimation.
Value is how many megabytes (MB) of RAM to set aside for caching data points as the optimizer is iterated.
Generally, more is faster, at least until you run out of RAM or cause your machine to start swapping.

{...}
{...}

{marker predict}{...}
{dlgtab:predict}

{pstd}After training you can ask svm to {cmd:predict} what the category (classification) or outcome value (regression)
      should be for each given observation. Results are placed into {newvar}.{p_end}
{pstd}{newvar} must not exist, so if you want to repredict your choices are {cmd:drop {newvar}} or to pick a new name, e.g. {cmd:predict {newvar}2}.{p_end}

{marker predict_prob}{...}
{phang}For classification ({opt c_svc}, {opt nu_svc}) problems, {opt probability} requests, for each observation, the probability of it being each class.
{newvar} is used as a stem for the new columns.
This option is not valid for other SVM types.{p_end}

{phang}
{opt verbose}: see {help svm##verbose:svm, verbose}.

{pstd}
Prediction implicitly uses the same {indepvars} as during estimation, so be careful about renaming or dropping variables.
{...}
{...}

{marker export}{...}
{dlgtab:import/export}

{pstd}
libsvm has an ad-hoc format it uses to save trained models.
The command line programs {cmd:svm-train} and {cmd:svm-predict} which come with libsvm communicate via it.
We support this format for interoperability.

{pstd}
{cmd:svm_export} will write a fitted model to disk.
This can only be run if there is one in memory, either from {help svm##svm:svm} or {help svm##export:svm_import}.
Conventionally you should name your file with '.model' for a file extension.

{pstd}
{cmd:svm_import} will load a model from disk, replacing any previous in-memory fit.
When you import, [TODO: some properties] will be missing because the import was done without reference to any dataset.

{phang}
{opt verbose}: see {help svm##verbose:svm, verbose}.

{pstd}
Do not confuse these commands with {cmd:import_svmlight} and {cmd:export_svmlight}.

{marker examples}{...}
{title:Examples:  classification}

        {hline}
{phang2}{cmd:. webuse highschool, clear}{p_end}
{phang2}{cmd:. svm gender}{p_end}
{phang2}{cmd:. ereturn list}{p_end}
{phang2}{cmd:. predict P, prob}{p_end}
        {hline}

{title:Examples:  regression, suppressing sv_coef}

        {hline}
{phang2}{cmd:. sysuse auto, clear}{p_end}
        {hline}

{title:Examples:  classification with tuning}

        {hline}
{phang2}{cmd:. webuse regsmpl, clear}{p_end}
        {hline}

{title:Examples:  viewing support vectors}

--- crosstab sv against category and compare to nSV

{title:Examples:  forecasting}

        {hline}
--- train on one set of data, generate more observations, predict
        {hline}

{title:Examples:  custom kernel}

        {hline}
--- use hacks + kernel(precomputed)
        {hline}

{title:Example:  weighted regression}

{pstd}
Not currently implemented.

{marker results}{...}
{title:Stored results}

{pstd}
{cmd:svm} and {cmd:svm_import} stores the following in {cmd:e()}:

{synoptset 20 tabbed}{...}
{* Note: svm_model components left unexposed: }{...}
{*      - SV, the actual SVs found. If you want this in a new fit, you can use the sv() option; the use case of inspecting old SVs on an imported fit is rare, we assume, and we do not support it; (they will still be loaded to memory }{...}
{*      - sv_indices, the indexes of which support vectors were found; exposed indirectly with the sv() option }{...}
{*      - probA, probB, the coefficients used for predict, prob; these are not, by themselves, interesting }{...}
{*      - label, the "labels" of the classes (which are the integers libsvm casts out of the initial dataset; exported with strLabels to be used for labelling rho and sv_coef, but otherwise not directly interesting and  }{...}
{*      - nSV, the number of SVs per class; this is only interesting for classifications, and it duplicates what you can get out of "tab `e(depvar)' SV" }{...}
{*      - free_sv, internal libsvm flag which is a hack to stretch svm_model to handle creation from both svm_train() and svm_import() }{...}
{p2col 5 20 24 2: Scalars}{p_end}
{synopt:{cmd:e(N)}}number of observations{p_end}
{synopt:{cmd:e(N_class)}}number of classes, in a classification problem. {opt 2} in a regression problem.{p_end}
{synopt:{cmd:e(N_SV)}}number of support vectors.
If {opt e(N_SV)}/{opt e(N)} is close to 100% your fit is inefficient; perhaps you need to adjust your {help svm##kernel:kernel}.
{p_end}

{synoptset 20 tabbed}{...}
{p2col 5 20 24 2: Macros}{p_end}
{synopt:{cmd:e(cmd)}}"{cmd:svm}"{p_end}
{synopt:{cmd:e(cmdline)}}command as typed{p_end}
{synopt:{cmd:e(depvar)}}name of dependent variable{p_end}
{synopt:{cmd:e(title)}}title in estimation output{p_end}
{synopt:{cmd:e(model)}}"{cmd:svm}"{p_end}
{synopt:{cmd:e(svm_type)}}SVM type string, as above{p_end}
{synopt:{cmd:e(svm_kernel)}}kernel string, as above{p_end}
{synopt:{cmd:e(predict)}}program used to implement {cmd:predict}{p_end}
{synopt:{cmd:e(levels)}}list of the classes detected, in the order they were detected, if applicable{p_end}
{* {synopt:{cmd:e(estat_cmd)}}program used to implement {cmd:estat}{p_end} }{...}

{synoptset 20 tabbed}{...}
{p2col 5 20 24 2: Matrices}({help svm##gotchas:may be missing}){p_end}
{synopt:{cmd:e(sv_coef)}}The coefficients of the support vectors for each fitted hyperplane. [TODO]{p_end}
{synopt:{cmd:e(rho)}}The intercept term for each fitted hyperplane. It is lower-triangular and {cmd:e(N_class)}^2 large, with each entry [i,j] representing the hyperplane between class i and class j.{p_end}

{* XXX this section title should be more formal}{...}
{marker gotchas}{...}
{title:Gotchas}

{pstd}
{bf:Memory Limits}: The cheaper versions of Stata allow only allow less variables and smaller matrices to be used.
As machine learning problems typically are on very large datasets,
it is easy to inadvertently instruct this package to construct more columns or larger matrices than you can afford.
If you overflow {help maxvar}, you will receive an error, the operation will fail, and the dataset will be left untouched.
If you overflow {help matsize}, the matrix that overflowed will be missing, but operation will otherwise succeed.

{pmore}
If Stata's memory limits are an impossible hurdle,
your best option is to give up on Stata and switching to libsvm's companion {cmd:svm-train} program.
This will have been installed with the libsvm package if you used a package manager, or
you can get it {browse "http://www.csie.ntu.edu.tw/~cjlin/cgi-bin/libsvm.cgi?+http://www.csie.ntu.edu.tw/~cjlin/libsvm+zip":from its authors};
You can use {help svmlight:export svmlight} to extract your dataset for use with {cmd:svm-train}.

{marker copyright}{...}
{title:Copyright}

{pstd}
The wrapper is licensed:

{pmore}
The MIT License (MIT)

{pmore}
Except where otherwise noted in the code, Copyright (c) 2015
Nick Guenther <nguenthe@uwaterloo.ca>, Matthias Schonlau <schonlau@uwaterloo.ca>

{pmore}
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

{pmore}
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

{pmore}
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

{pstd}
libsvm is licensed:

{pmore}
Copyright (c) 2000-2014 Chih-Chung Chang and Chih-Jen Lin
All rights reserved.

{pmore}
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

{pmore}
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.

{pmore}
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.

{pmore}
3. Neither name of copyright holders nor the names of its contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.

{pmore}
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

{marker authors}{...}
{title:Authors}

{pstd}
Though the license does not obligate you in any way to do so, if you find
this software useful we would be curious and appreciative to hear about your
adventures in machine learning with Stata.{p_end}
{pmore}Thank you.

{pstd}You can contact us at{p_end}
{pmore}* Nick Guenther <nguenthe@uwaterloo.ca>{p_end}
{pmore}* Matthias Schonlau <schonlau@uwaterloo.ca>{p_end}

{marker references}{...}
{title:References}

{marker sourcecode}{...}
{phang}
Guenther, Nick and Schonlau, Matthias. 2015.
{it:Stata-SVM}.
{browse "https://github.com/kousu/statasvm/"}.
{p_end}

{marker svmtutorial}{...}
{phang}
Bennett, Kristin P., and Colin Campbell. 2000.
{it:Support Vector Machines: Hype or Hallelujah?}
SIGKDD Explor. Newsl. 2.2: 1–13.
{browse "http://www.svms.org/tutorials/BennettCampbell2000.pdf"}.
{p_end}

{marker libsvm}{...}
{phang}
Chang, Chih-Chung and Lin, Chih-Jen. 2011.
{it:LIBSVM: a library for support vector machines.}
ACM Transactions on Intelligent Systems and Technology, 2:27:1--27:27.
{browse "http://www.csie.ntu.edu.tw/~cjlin/papers/libsvm.pdf"}.
Software available at {browse "http://www.csie.ntu.edu.tw/~cjlin/libsvm"}
{p_end}

{marker libsvmguide}{...}
{phang}
Hsu, Chih-Wei, Chang, Chih-Chung, and Lin, Chih-Jen. April 15, 2010.
{it:A Practical Guide to Support Vector Classification}.
{browse "http://www.csie.ntu.edu.tw/~cjlin/papers/guide/guide.pdf"}.
{p_end}

{marker svr_tutorial}{...}
{phang}
Smola, Alex J., and Schölkopf, Bernhard. 2004.
{it:A tutorial on support vector regression}.
Statistics and Computing 14.3: 199–222.
{* This one is behind a paywall, so the best we can do is a give a DOI link }{...}
{browse "http://dx.doi.org/10.1023/b:stco.0000035301.49549.88"}.
{p_end}

{marker nusvm}{...}
{phang}
Chen, Pai-Hsuen, Lin Chih-Jen, and Schölkopf, Bernhard. 2005.
{it:A Tutorial on ν-Support Vector Machines}.
Applied Stochastic Models in Business and Industry 21.2: 111–136.
{browse "http://www.csie.ntu.edu.tw/~cjlin/papers/nusvmtutorial.pdf"}.
{p_end}

{smcl}
{* *! version 0.0.1  28may2015}{...}
{vieweralsosee "[R] regress" "mansection R regress"}{...}
{viewerjumpto "Syntax" "svm##syntax"}{...}
{viewerjumpto "Description" "svm##description"}{...}
{viewerjumpto "Options" "svm##options"}{...}
{viewerjumpto "Examples" "svm##examples"}{...}
{viewerjumpto "Stored results" "svm##results"}{...}
{viewerjumpto "Gotchas" "svm##gotchas"}{...}
{viewerjumpto "Copyright" "svm##copyright"}{...}
{viewerjumpto "Authors" "svm##authors"}{...}
{viewerjumpto "References" "svm##references"}{...}
{...}{* NB: these hide the newlines }
{...}
{...}
{title:Title}

{p2colset 5 18 20 2}{...}
{p2col :{cmd:svm} {hline 2}}Support Vector Machines{p_end}
{p2colreset}{...}

{marker syntax}{...}
{title:Syntax}

{p 8 16 2}
{help svm##svm:svm} {depvar} {indepvars} {ifin}{* [{it:{help svm##weight:weight}}] } [{cmd:,} {it:options}]

{p 8 16 2}
{help svm##svm:svm} {indepvars} {ifin}{* [{it:{help svm##weight:weight}}] }, type({help svm##one_class:one_class}) [{it:options}]

{synoptset 20 tabbed}{...}
{synopthdr}
{synoptline}
{syntab:Model}
{synopt :{opth t:ype(svm##type:type)}}Type of model to fit: {opt c_svc}, {opt nu_svc}, {opt one_class}, {opt epsilon_svr}, or {opt nu_svr}. Default: {cmd:type(c_svc)}{p_end}
{synopt :{opth k:ernel(svm##kernel:kernel)}}SVM kernel function to use: {opt linear}, {opt poly}, {opt rbf}, {opt sigmoid}, or {opt precomputed}. Default: {cmd:kernel(rbf)}{p_end}

{synopt :{opth deg:ree(svm##degree:degree)}}For {opt poly} kernels, the degree of the polynomial to use. Default: cubic ({cmd:degree(3)}){p_end}

{* XXX the division between 'tuning' and 'model' parameters is hazy; e.g. you could in theory cross-validate to choose degree (and people do this with neural networks), or even to choose the kernel . hmmmmm}{...}

{syntab:Tuning}
{synopt :{opth c:(svm##c:c)}}For {opt c_svc}, {opt epsilon_svr} and {opt nu_svr} SVMs, this is a regularization parameter which weights the slack variables. Should be > 0. Default: {cmd:c(1)}{p_end}
{synopt :{opth eps:ilon(svm##epsilon:eps)}}For {opt epsilon_svr} SVMs, the margin of error allowed for to count something as a support vector. Default: {cmd:p(0.1)}{p_end}
{synopt :{opth nu:(svm##nu:nu)}}For {opt nu_svc}, {opt one_class}, and {opt nu_svr} SVMs, tunes the number of supports vectors. Should be in (0, 1]. Default: {cmd:nu(0.5)}{p_end}

{synopt :{opth gamma:(svm##gamma:gamma)}}For {opt poly}, {opt rbf} and {opt sigmoid} kernels, a scaling factor for the linear part of the kernel. Default: {cmd:gamma(1/[# {indepvars}])}{p_end}
{synopt :{opth coef0:(svm##coef0:coef0)}}For {opt poly} and {opt sigmoid} kernels, a bias ("intercept") term for the linear part of the kernel. Default: {cmd:coef0(0)}{p_end}

{synopt :{opt shrink:ing}}Whether to use {help svm##shrinking:shrinkage} heuristics to improve the fit. Default: disabled{p_end}

{syntab:Features}
{synopt :{opt norm:alize}}Whether to {help svm##normalize:center and scale} the data. NOT IMPLEMENTED. Default: disabled{p_end}
{synopt :{opt prob:ability}}Whether to {help svm##probability:prepare} for "predict, prob" during estimation. Only applicable to classification problems. Default: disabled{p_end}
{synopt :{opth sv:(svm##sv:newvarname)}}If given, an indicator variable to generate to mark each row as a support vector or not. Default: disabled{p_end}

{syntab:Performance}
{synopt :{opth tol:erance(svm##tolerance:tol)}}The stopping tolerance used to decide when convergence has happened. Default: {cmd:epsilon(0.001)}{p_end}
{synopt :{opt v:erbose}}Turns on {help svm##verbose:verbose mode}. NOT IMPLEMENTED. Default: disabled{p_end}
{synopt :{opth cache:_size(svm##cache_size:cache)}}The size of the RAM cache used during fitting, in megabytes. Default: 100MB ({cmd:cache_size(100)}){p_end}

{synoptline}
{pstd}All variables must be numeric, including categorical variables,
which may appear to be strings if they have have {help label values:value labels} attached, but are probably actually stored as {help byte}s.
If you in fact have categories stored in string variables use {help encode}.{p_end}
INCLUDE help fvvarlist

{p 8 16 2}
{help svm##predict:predict} {newvar} {ifin}, [{cmdab:prob:ability}]

{synoptset 20 tabbed}{...}
{synopthdr}
{synoptline}
{synopt :{cmdab:prob:ability}}If specified, estimate class probabilities for each observation. This only makes sense for classification problems.{p_end}

{p 8 16 2}
{help svm##export:svm_export} using {filename}

{p 8 16 2}
{help svm##export:svm_import} using {filename}

{synoptset 20 tabbed}{...}
{synopthdr}
{synoptline}
{synopt :{cmd: using}}The filename to export or import a fitted libsvm model from, conventionally ending in '.model'.{p_end}

{marker description}{...}
{title:Description}

{pstd}
{cmd:svm} fits a support vector machine (SVM) model.
SVM is not one, but several, variant models each based upon the principles of
splitting hyperplanes and the culling of unimportant observations.
The name comes from these two parts:
each observation in a dataset can be thought of as a (typically high-dimensional) linear algebra vector,
and the {it:support vectors} are those observations which the algorithm detects are critical to the fit.
The non-support observations are ignored after fitting is done, which makes SVM very memory efficient.
See {help svm##svmtutorial:this SVM tutorial} for a gentle introduction to the method.

{pstd}
This package is a thin wrapper for the widely deployed {help svm##libsvm:libsvm},
so in addition to the Stata package, {bf:libsvm must be installed}.
On Windows, libsvm.dll is bundled with the package,
and you can find it in your {help adopath} (try {cmd:findfile libsvm.dll} to verify this).
On OS X, libsvm is available in both {browse "https://brew.sh":brew} and {browse "https://www.macports.org":macports}.
On Linux, search for libsvm in your distribution's package manager.
You can also compile and install libsvm from source, which you would need to do if your package manager has an older or no version of libsvm.
If you are having plugin load errors, please {help svm##authors:contact the authors},
as we want to make the experience as smooth as possible for our users.

{pstd}
The thinness of this wrapper is an intentional feature which means work done under
Stata-SVM should be replicable with other libsvm wrappers such as
{browse "http://weka.wikispaces.com/LibSVM":Weka} or {browse "http://scikit-learn.org/stable/modules/svm.html":sklearn}, or {browse "":}.
Please do {help svm##authors:send us feature requests},
but unless they are Stata-specific we would rather try to
get them into libsvm first so that everyone can benefit.

{pstd}
Not all combinations of options are valid, and you will get an error if you specify an invalid combination.
Amongst valid combinations, not all options and datasets will give good results.
Our goal is to have sane defaults,
so that the only choice you need to make is what {help svm##type:type} and {help svm##kernel:kernel} to use,
but to offer access to all parameters so they can be tweaked for difficult datasets.
The {help svm##libsvmguide:libsvm guide} covers issues in parameter specification.

{pstd}
If this manual leaves you confused, refer to the authoratative source:
the libsvm {browse "http://www.csie.ntu.edu.tw/~cjlin/libsvm/faq.html":FAQ},
{browse "https://github.com/cjlin1/libsvm/blob/master/README":README},
and {help svm##libsvm:implementation paper}.
Then please write us with suggestions for clarification.

{marker options}{...}
{title:Options}

{marker svm}{...}
{dlgtab:svm}{* this is a misuse of dlgtab because I have no corresponding dialog, but it drastically helps readability }

{pstd}
{cmd:svm} does SVM model training, otherwise known as fitting or estimation (depending on your statistical background),
of {depvar} on predictors {indepvars}.

{* MODEL PARAMS: }
{phang}
{marker type}{...}
{opt type} specifies what subtype of SVM to run.{p_end}
{pmore}{opt c_svc} and {opt nu_svc} perform classification.{p_end}
{pmore2}{depvar} should be a variable containing categories.
If you try to use floating point values with classification you
will find that they are truncated mercilessly to their integer parts,
so you may need to recode your categories first.
If you end up with almost as many classes as observations,
you have probably used a continuous {depvar} and
should use regression instead.{p_end}

{pmore2}Multiclass classification is automatically handled using the {browse "http://en.wikipedia.org/wiki/Multiclass_classification":one-against-one} method.
You do not need to do anything special to invoke it.{p_end}

{marker one_class}{...}
{pmore}{opt one_class} separates outliers from the bulk of the data.{p_end}
{pmore2}{opt one_class} does not take a {depvar} because it is a form of unsupervised learning: all the information it uses is in the predictors themselves.{p_end}
{pmore2}Its predictions give 1 for "data" and -1 for "outlier".{p_end}
{pmore3}{bf:Tip:} You may use the same {varlist} as with the other types:
 this just interprets your {depvar} as one of the {indepvars},
 giving {opt one_class} more information to work with.{p_end}

{pmore}{opt epsilon_svr} and {opt nu_svr} perform regression.{p_end}
{pmore2}{depvar} should be a variable containing continuous values.{p_end}
{pmore2}While you can use this with discrete {depvar}s, it is more common to use it with continuous ones.{p_end}
{pmore2}See {help svm##svr_tutorial:the SVR tutorial} for more details.{p_end}

{pmore}To learn about the {opt c}/{opt nu} distinction, see {help svm##nusvm:Chen and Lin's ν-SVM tutorial}.{p_end}

{pmore}{opt type} is case insensitive.{p_end}

{phang}
{marker kernel}{...}
{opt kernel} gives a kernel function to use.
The basic SVM idea is to find a linear boundary in high-dimensional space (a hyperplane):
for classification, this is a boundary between two classes;
for regression it is a line {help svm##p:near} which predictions are expected to cluster--much like in {help regess:OLS}.
Much real data is not straight but the {browse "https://en.wikipedia.org/wiki/Kernel_Method":kernel trick} makes SVM
flexible by making a space non-linear under a high-dimensional mapping.
See {browse "https://www.youtube.com/watch?v=3liCbRZPrZA"} for a visualization of this process.
The part that makes it a {it:trick} is that the fit can be done efficiently without constructing the high-dimensional points,
as the estimation only cares about a scalar value <u,v>, the output of the kernel function for coefficients u on support vector v,
which is used in scoring the fit [CITATION NEEDED],
and for certain kernels this value can be computed straight
from the original data without doing the expensive high-dimensional mapping at all.{p_end}

{pmore}Kernels available in this implementation are:{p_end}
{pmore2}{opt linear} means the linear kernel you are probably familiar with from {help regress:OLS}: u'*v{p_end}
{pmore2}{opt poly} is (gamma*u'*v + coef0)^degree{p_end}
{pmore2}{opt rbf} stands for Radial Basis Functions, and treats the coefficients as a mean to smoothly approach in a ball, with the form exp(-gamma*|u-v|^2); this kernel tends to be good at [...].{p_end}
{pmore2}{opt sigmoid} is a kernel which bends the linear kernel to fit in -1 to 1, similar to {help logistic} regression: tanh(gamma*u'*v + coef0){p_end}
{pmore2}{opt precomputed} assumes that {depvar} is actually a list of precomputed kernel values. With effort, you can use this to use custom kernels with your data [TODO].{p_end}

{pmore}{opt kernel} is case insensitive.{p_end}

{phang}
{marker degree}{...}
{opt degree} selects the degree of the polynomial used by the {opt poly} kernel.
Setting this too high will result in overfitting. Setting it too low may result in non-convergence.
[TODO: tips about choosing this]

{* TUNING PARAMS: }
{phang}
{marker c}{...}
{opt c} weights (regularizes) the slack variables used in {opt c_svc}, {opt epsilon_svr} and {opt nu_svr}.
If your fit is poor, try cranking this up.
{* XXX NOT TRUE: This will pre-multiply any sample-specific {opt weight}s. }

{phang}
{marker epsilon}{...}
{opt epsilon} is the margin of error allowed by {opt epsilon_svr}.
Larger makes your fit more able to incorporate more observations, but can lead to underfitting.
Smaller can lead to overfitting.

{phang}
{marker nu}{...}
{opt nu} is used in the nu variants.
The nu variants are a reparamaterization of regular SVM which lets you directly tune,
using {opt nu}, the tradeoff between error and computation.
{opt nu} is an upper bound on the fraction of training errors and a lower bound of the
fraction of support vectors, so that [TODO smaller means ... and larger means .....]
See {help svm##nusvm:the ν-SVM tutorial} for details.
{* ..wait... this doesn't make any sense. nu == 0.1 means there are at most 10% (training) errors and at least 10% are support vectors.}{...}
{*      nu = 0.9 means there are at most 90% errors and at least 90% SVs.   you should always choose 0, then, to get perfect prediction and zero memory usage}{...}

{phang}
{marker gamma}{...}
{opt gamma} is used in the non-linear kernels as a scaling factor for the linear part, as seen above.
[TODO: tips about choosing this]

{phang}
{marker coef0}{...}
{opt coef0} similarly is used in the non-linear kernels as a pseudo-intercept term.
Except it is not used in the {opt rbf} kernel as {opt rbf} is essentially a distance function, and biasing would be pointless.[???]
[TODO: tips about choosing this]

{phang}
{marker shrinking}{...}
{opt shrinking} invokes the shrinkage heuristics [TODO: what does this mean? shrinkage estimation?],
which reduce can improve the fit by trading bias for variance: by giving up accuracy in the estimates,
variance of the estimates can be reduced, which is sometimes better overall.

{* FEATURE PARAMS: }
{marker normalize}{...}
{phang}
{opt normalize} instructs the estimation to first center and scale the data
as SVM tends to be very sensitive to scaling issues.
This normalizes all data to [0,1] using min-max normalization, as suggested in the {help svm##libsvmguide:libsvm guide}.
Normalization creates temporary variables, so you may prefer to preprocess the data yourself---destructively and in-place---to save time on re-estimations and memory for variables,
especially if you are bumping up against your Stata system limits. You may find {cmd:ssc install norm} helpful.

{phang}
{marker probability}{...}
{opt probability} enables the use of "{help svm##predict_prob:predict, prob}".
This requires precomputing class-against-class probability estimates with a 5-fold cross-validation,
and so takes a great deal of additional CPU and RAM.

{phang}
{marker sv}{...}
{opt sv} records in the given variable a boolean indicating whether each observation was a support vector or not.

{* PERFORMANCE PARAMS: }
{phang}
{marker tolerance}{...}
{opt tolerance} is the stopping tolerance used by the numerical optimizer. You could widen this if you are finding convergence is slow,
 but be aware that this usually non-convergence is a deeper problem in the interaction of data, kernel, and tuning parameters.

{phang}
{marker verbose}{...}
{opt verbose} enables output from the low level libsvm code for the duration of the operation.

{phang}
{marker cache_size}{...}
{opt cache_size} controls a time-memory tradeoff during estimation.
Value is how many megabytes (MB) of RAM to set aside for caching data points as the optimizer is iterated.
Generally, more is faster, at least until you run out of RAM or cause your machine to start swapping.

{...}
{...}

{marker predict}{...}
{dlgtab:predict}

{pstd}After training you can ask svm to {cmd:predict} what the category (classification) or outcome value (regression)
      should be for each given observation. Results are placed into {newvar}.{p_end}
{pstd}{newvar} must not exist, so if you want to repredict your choices are {cmd:drop {newvar}} or to pick a new name, e.g. {cmd:predict {newvar}2}.{p_end}

{marker predict_prob}{...}
{phang}For classification ({opt c_svc}, {opt nu_svc}) problems, {opt probability} requests, for each observation, the probability of it being each class.
{newvar} is used as a stem for the new columns.
This option is not valid for other SVM types.{p_end}

{phang}
{opt verbose}: see {help svm##verbose:svm, verbose}.

{pstd}
Prediction implicitly uses the same {indepvars} as during estimation, so be careful about renaming or dropping variables.
{...}
{...}

{marker export}{...}
{dlgtab:import/export}

{pstd}
libsvm has an ad-hoc format it uses to save trained models.
The command line programs {cmd:svm-train} and {cmd:svm-predict} which come with libsvm communicate via it.
We support this format for interoperability.

{pstd}
{cmd:svm_export} will write a fitted model to disk.
This can only be run if there is one in memory, either from {help svm##svm:svm} or {help svm##export:svm_import}.
Conventionally you should name your file with '.model' for a file extension.

{pstd}
{cmd:svm_import} will load a model from disk, replacing any previous in-memory fit.
When you import, [TODO: some properties] will be missing because the import was done without reference to any dataset.

{phang}
{opt verbose}: see {help svm##verbose:svm, verbose}.

{pstd}
Do not confuse these commands with {cmd:import_svmlight} and {cmd:export_svmlight}.

{marker examples}{...}
{title:Examples:  classification}

        {hline}
{phang2}{cmd:. webuse highschool, clear}{p_end}
{phang2}{cmd:. svm gender}{p_end}
{phang2}{cmd:. ereturn list}{p_end}
{phang2}{cmd:. predict P, prob}{p_end}
        {hline}

{title:Examples:  regression, suppressing sv_coef}

        {hline}
{phang2}{cmd:. sysuse auto, clear}{p_end}
        {hline}

{title:Examples:  classification with tuning}

        {hline}
{phang2}{cmd:. webuse regsmpl, clear}{p_end}
        {hline}

{title:Examples:  viewing support vectors}

--- crosstab sv against category and compare to nSV

{title:Examples:  forecasting}

        {hline}
--- train on one set of data, generate more observations, predict
        {hline}

{title:Examples:  custom kernel}

        {hline}
--- use hacks + kernel(precomputed)
        {hline}

{title:Example:  weighted regression}

{pstd}
Not currently implemented.

{marker results}{...}
{title:Stored results}

{pstd}
{cmd:svm} and {cmd:svm_import} stores the following in {cmd:e()}:

{synoptset 20 tabbed}{...}
{* Note: svm_model components left unexposed: }{...}
{*      - SV, the actual SVs found. If you want this in a new fit, you can use the sv() option; the use case of inspecting old SVs on an imported fit is rare, we assume, and we do not support it; (they will still be loaded to memory }{...}
{*      - sv_indices, the indexes of which support vectors were found; exposed indirectly with the sv() option }{...}
{*      - probA, probB, the coefficients used for predict, prob; these are not, by themselves, interesting }{...}
{*      - label, the "labels" of the classes (which are the integers libsvm casts out of the initial dataset; exported with strLabels to be used for labelling rho and sv_coef, but otherwise not directly interesting and  }{...}
{*      - nSV, the number of SVs per class; this is only interesting for classifications, and it duplicates what you can get out of "tab `e(depvar)' SV" }{...}
{*      - free_sv, internal libsvm flag which is a hack to stretch svm_model to handle creation from both svm_train() and svm_import() }{...}
{p2col 5 20 24 2: Scalars}{p_end}
{synopt:{cmd:e(N)}}number of observations{p_end}
{synopt:{cmd:e(N_class)}}number of classes, in a classification problem. {opt 2} in a regression problem.{p_end}
{synopt:{cmd:e(N_SV)}}number of support vectors.
If {opt e(N_SV)}/{opt e(N)} is close to 100% your fit is inefficient; perhaps you need to adjust your {help svm##kernel:kernel}.
{p_end}

{synoptset 20 tabbed}{...}
{p2col 5 20 24 2: Macros}{p_end}
{synopt:{cmd:e(cmd)}}"{cmd:svm}"{p_end}
{synopt:{cmd:e(cmdline)}}command as typed{p_end}
{synopt:{cmd:e(depvar)}}name of dependent variable{p_end}
{synopt:{cmd:e(title)}}title in estimation output{p_end}
{synopt:{cmd:e(model)}}"{cmd:svm}"{p_end}
{synopt:{cmd:e(svm_type)}}SVM type string, as above{p_end}
{synopt:{cmd:e(svm_kernel)}}kernel string, as above{p_end}
{synopt:{cmd:e(predict)}}program used to implement {cmd:predict}{p_end}
{synopt:{cmd:e(levels)}}list of the classes detected, in the order they were detected, if applicable{p_end}
{* {synopt:{cmd:e(estat_cmd)}}program used to implement {cmd:estat}{p_end} }{...}

{synoptset 20 tabbed}{...}
{p2col 5 20 24 2: Matrices}({help svm##gotchas:may be missing}){p_end}
{synopt:{cmd:e(sv_coef)}}The coefficients of the support vectors for each fitted hyperplane. [TODO]{p_end}
{synopt:{cmd:e(rho)}}The intercept term for each fitted hyperplane. It is lower-triangular and {cmd:e(N_class)}^2 large, with each entry [i,j] representing the hyperplane between class i and class j.{p_end}

{* XXX this section title should be more formal}{...}
{marker gotchas}{...}
{title:Gotchas}

{pstd}
{bf:Memory Limits}: The cheaper versions of Stata allow only allow less variables and smaller matrices to be used.
As machine learning problems typically are on very large datasets,
it is easy to inadvertently instruct this package to construct more columns or larger matrices than you can afford.
If you overflow {help maxvar}, you will receive an error, the operation will fail, and the dataset will be left untouched.
If you overflow {help matsize}, the matrix that overflowed will be missing, but operation will otherwise succeed.

{pmore}
If Stata's memory limits are an impossible hurdle,
your best option is to give up on Stata and switching to libsvm's companion {cmd:svm-train} program.
This will have been installed with the libsvm package if you used a package manager, or
you can get it {browse "http://www.csie.ntu.edu.tw/~cjlin/cgi-bin/libsvm.cgi?+http://www.csie.ntu.edu.tw/~cjlin/libsvm+zip":from its authors};
You can use {help svmlight:export svmlight} to extract your dataset for use with {cmd:svm-train}.

{marker copyright}{...}
{title:Copyright}

{pstd}
The wrapper is licensed:

{pmore}
The MIT License (MIT)

{pmore}
Except where otherwise noted in the code, Copyright (c) 2015
Nick Guenther <nguenthe@uwaterloo.ca>, Matthias Schonlau <schonlau@uwaterloo.ca>

{pmore}
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

{pmore}
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

{pmore}
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

{pstd}
libsvm is licensed:

{pmore}
Copyright (c) 2000-2014 Chih-Chung Chang and Chih-Jen Lin
All rights reserved.

{pmore}
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

{pmore}
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.

{pmore}
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.

{pmore}
3. Neither name of copyright holders nor the names of its contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.

{pmore}
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

{marker authors}{...}
{title:Authors}

{pstd}
Though the license does not obligate you in any way to do so, if you find
this software useful we would be curious and appreciative to hear about your
adventures in machine learning with Stata.{p_end}
{pmore}Thank you.

{pstd}You can contact us at{p_end}
{pmore}* Nick Guenther <nguenthe@uwaterloo.ca>{p_end}
{pmore}* Matthias Schonlau <schonlau@uwaterloo.ca>{p_end}

{marker references}{...}
{title:References}

{marker sourcecode}{...}
{phang}
Guenther, Nick and Schonlau, Matthias. 2015.
{it:Stata-SVM}.
{browse "https://github.com/kousu/statasvm/"}.
{p_end}

{marker svmtutorial}{...}
{phang}
Bennett, Kristin P., and Colin Campbell. 2000.
{it:Support Vector Machines: Hype or Hallelujah?}
SIGKDD Explor. Newsl. 2.2: 1–13.
{browse "http://www.svms.org/tutorials/BennettCampbell2000.pdf"}.
{p_end}

{marker libsvm}{...}
{phang}
Chang, Chih-Chung and Lin, Chih-Jen. 2011.
{it:LIBSVM: a library for support vector machines.}
ACM Transactions on Intelligent Systems and Technology, 2:27:1--27:27.
{browse "http://www.csie.ntu.edu.tw/~cjlin/papers/libsvm.pdf"}.
Software available at {browse "http://www.csie.ntu.edu.tw/~cjlin/libsvm"}
{p_end}

{marker libsvmguide}{...}
{phang}
Hsu, Chih-Wei, Chang, Chih-Chung, and Lin, Chih-Jen. April 15, 2010.
{it:A Practical Guide to Support Vector Classification}.
{browse "http://www.csie.ntu.edu.tw/~cjlin/papers/guide/guide.pdf"}.
{p_end}

{marker svr_tutorial}{...}
{phang}
Smola, Alex J., and Schölkopf, Bernhard. 2004.
{it:A tutorial on support vector regression}.
Statistics and Computing 14.3: 199–222.
{* This one is behind a paywall, so the best we can do is a give a DOI link }{...}
{browse "http://dx.doi.org/10.1023/b:stco.0000035301.49549.88"}.
{p_end}

{marker nusvm}{...}
{phang}
Chen, Pai-Hsuen, Lin Chih-Jen, and Schölkopf, Bernhard. 2005.
{it:A Tutorial on ν-Support Vector Machines}.
Applied Stochastic Models in Business and Industry 21.2: 111–136.
{browse "http://www.csie.ntu.edu.tw/~cjlin/papers/nusvmtutorial.pdf"}.
{p_end}


ENH: estmat2nparray.ado add alias, add unused single row/col string

*
* Save estimation results and matrices to a Python module
*
* Based on mat2nparray by Skipper
* Changes by Josef
*
* changes
* -------
* write also column and row names of matrices to py module
* replace missing values by np.nan in matrices
* make namelist optional
* add estimation results from e(), e(scalars) and e(macros), not the matrices in e
* make estimation result optional
* add aliases for params_table
* don't split col or row names if only 1 - changed my mind: always list

* Issues
* ------
* row and colum names if only a single row or column - list or string

capture program drop estmat2nparray
program define estmat2nparray
    version 11.0
    syntax [namelist(min=1)], SAVing(str) [ Format(str) APPend REPlace NOEst]
    if "`format'"=="" local format "%16.0g"
    local saving: subinstr local saving "." ".", count(local ext)
    if !`ext' local saving "`saving'.py"
    tempname myfile
    file open `myfile' using "`saving'", write text `append' `replace'
    file write `myfile' "import numpy as np" _n _n

	/* get results from e()*/
	if "`noest'" == "" {
		file write `myfile' "est = dict(" _n

		local escalars : e(scalars)
		foreach ii in `escalars'{
			file write `myfile' "           " "`ii'" " = " "`e(`ii')'" "," _n
		}

		local emacros : e(macros)
		foreach ii in `emacros' {
			file write `myfile' "           " "`ii'" " = "  `"""'   "`e(`ii')'" `"""' ","   _n
		}
		file write `myfile' "          )" _n _n
	}
	/* end write e()*/

    foreach mat of local namelist {
        mkarray `mat' `myfile' `format'
    }
    file write `myfile' "class Bunch(dict):" _n
    file write `myfile' "    def __init__(self, **kw):" _n
    file write `myfile' "        dict.__init__(self, kw)" _n
    file write `myfile' "        self.__dict__  = self" _n _n

	if "`noest'" == "" {
		file write `myfile' "        for i,att in enumerate(['params', 'bse', 'tvalues', 'pvalues']):" _n
		file write `myfile' "            self[att] = self.params_table[:,i]" _n _n
	}
	file write `myfile' "" _n
    file write `myfile' "results = Bunch(" _n
    foreach mat of local namelist {
        file write `myfile' "                `mat'=`mat', " _n
		file write `myfile' "                `mat'_colnames=`mat'_colnames, " _n
		file write `myfile' "                `mat'_rownames=`mat'_rownames, " _n
    }

	if "`noest'" == "" {
		file write `myfile' "                **est" _n
	}
    file write `myfile' "                )" _n _n
file close `myfile'
end

capture program drop mkarray
program define mkarray

    args mat myfile fmt
    local nrows = rowsof(`mat')
    local ncols = colsof(`mat')
    local i 1
    local j 1
    file write `myfile' "`mat' = np.array(["
    local justifyn = length("`mat' = np.array([")
    forvalues i=1/`nrows' {
        forvalues j = 1/`ncols' {
            if `i' > 1 | `j' > 1 { // then we need to indent
                forvalues k=1/`justifyn' {
                    file write `myfile' " "
                }
            }
            if `i' < `nrows' | `j' < `ncols' {
			    if mi(`mat'[`i',`j']) {
					file write `myfile' "np.nan" ", " _n
				}
				else {
					file write `myfile' `fmt' (`mat'[`i',`j']) ", " _n
				}
            }
            else {
				if mi(`mat'[`i',`j']) {
					file write `myfile' "np.nan"
				}
				else {
					file write `myfile' `fmt' (`mat'[`i',`j'])
				}
            }
        }
    }

    if `nrows' == 1 | `ncols' == 1 {
        file write `myfile' "])" _n _n
    }
    else {
        file write `myfile' "]).reshape(`nrows',`ncols')" _n _n
    }
	capture drop colnms
	local colnms: coln `mat'
	capture file write `myfile' "`mat'_colnames = '" "`colnms'" "'"
	* always split -> return list for single column
	* set > 1 to avoid list for single column
	if `ncols' > 0 {
        file write `myfile' ".split()" _n _n
    }
    else {
		file write `myfile' _n _n
    }

	capture drop rownms
	local rownms: rown `mat'
	capture file write `myfile' "`mat'_rownames = '" "`rownms'" "'"
	* always split -> return list for single row
	if `nrows' > 0 {
        file write `myfile' ".split()" _n _n
    }
    else {
		file write `myfile' _n _n
	}
end

*
* Save estimation results and matrices to a Python module
*
* Based on mat2nparray by Skipper
* Changes by Josef
*
* changes
* -------
* write also column and row names of matrices to py module
* replace missing values by np.nan in matrices
* make namelist optional
* add estimation results from e(), e(scalars) and e(macros), not the matrices in e
* make estimation result optional

capture program drop estmat2nparray
program define estmat2nparray
    version 11.0
    syntax [namelist(min=1)], SAVing(str) [ Format(str) APPend REPlace NOEst]
    if "`format'"=="" local format "%16.0g"
    local saving: subinstr local saving "." ".", count(local ext)
    if !`ext' local saving "`saving'.py"
    tempname myfile
    file open `myfile' using "`saving'", write text `append' `replace'
    file write `myfile' "import numpy as np" _n _n

	/* get results from e()*/
	if "`noest'" == "" {
		file write `myfile' "est = dict(" _n

		local escalars : e(scalars)
		foreach ii in `escalars'{
			file write `myfile' "           " "`ii'" " = " "`e(`ii')'" "," _n
		}

		local emacros : e(macros)
		foreach ii in `emacros' {
			file write `myfile' "           " "`ii'" " = "  `"""'   "`e(`ii')'" `"""' ","   _n
		}
		file write `myfile' "          )" _n _n
	}
	/* end write e()*/

    foreach mat of local namelist {
        mkarray `mat' `myfile' `format'
    }
    file write `myfile' "class Bunch(dict):" _n
    file write `myfile' "    def __init__(self, **kw):" _n
    file write `myfile' "        dict.__init__(self, kw)" _n
    file write `myfile' "        self.__dict__  = self" _n _n _n
    file write `myfile' "results = Bunch(" _n
    foreach mat of local namelist {
        file write `myfile' "                `mat'=`mat', " _n
		file write `myfile' "                `mat'_colnames=`mat'_colnames, " _n
		file write `myfile' "                `mat'_rownames=`mat'_rownames, " _n
    }

	if "`noest'" == "" {
		file write `myfile' "                **est" _n
	}
    file write `myfile' "                )" _n _n
file close `myfile'
end

capture program drop mkarray
program define mkarray

    args mat myfile fmt
    local nrows = rowsof(`mat')
    local ncols = colsof(`mat')
    local i 1
    local j 1
    file write `myfile' "`mat' = np.array(["
    local justifyn = length("`mat' = np.array([")
    forvalues i=1/`nrows' {
        forvalues j = 1/`ncols' {
            if `i' > 1 | `j' > 1 { // then we need to indent
                forvalues k=1/`justifyn' {
                    file write `myfile' " "
                }
            }
            if `i' < `nrows' | `j' < `ncols' {
			    if mi(`mat'[`i',`j']) {
					file write `myfile' "np.nan" ", " _n
				}
				else {
					file write `myfile' `fmt' (`mat'[`i',`j']) ", " _n
				}
            }
            else {
				if mi(`mat'[`i',`j']) {
					file write `myfile' "np.nan"
				}
				else {
					file write `myfile' `fmt' (`mat'[`i',`j'])
				}
            }
        }
    }

    if `nrows' == 1 | `ncols' == 1 {
        file write `myfile' "])" _n _n
    }
    else {
        file write `myfile' "]).reshape(`nrows',`ncols')" _n _n
    }
	capture drop colnms
	local colnms: coln `mat'
	*gen str `col_names' = "`colnms'"
	*file write `myfile' "# " `col_names' _n _n
	capture file write `myfile' "`mat'_colnames = '" "`colnms'" "'.split()" _n _n

	capture drop colnms
	local rownms: rown `mat'
	capture file write `myfile' "`mat'_rownames = '" "`rownms'" "'.split()" _n _n
end


added numchanges var

added numchanges var

program makedv

use ${stash}mergemaster1, clear

// unit = fips, geoid10 or tileid

local unit `0'
local unit "geoid10"

drop if `unit' == "NA"

egen unitid = group(`unit')
genvar unitid

end

program genvar

local unit `1'

// useful vars
bysort uid: gen numusercontrib = _N
bysort uid: egen minmonth = min(month)

bysort `unit' uid month: gen tmp1 = (_n==1)
bysort `unit' uid: egen nummonth = total(tmp1)
drop tmp1

// a) contribs
bysort `unit' month: gen numcontrib = _N
bysort `unit' month: egen numchanges = total(num_change)

// b) users
bysort `unit' month uid: gen tmp1 = (_n==1)
bysort `unit' month: egen numuser = total(tmp1)
drop tmp1

// c) serious users
// 90 percentile == 18, 95 is 56

bysort `unit' month uid: gen tmp1 = (_n==1)*(numuserc > 18)
bysort `unit' month: egen numserious90 = total(tmp1)

bysort `unit' month uid: gen tmp2 = (_n==1)*(numuserc > 56)
bysort `unit' month: egen numserious95 = total(tmp2)
drop tmp1 tmp2

// d) new users

// new users
bysort `unit' month uid: gen tmp1 = (_n==1) * (month==minmonth)
bysort `unit' month: egen numnewusers = total(tmp1)
drop tmp1

// new committed users
bysort `unit' month uid: gen tmp1 = (_n==1) * (month==minmonth) * (nummonth >= 6)
bysort `unit' month: egen numnewusers6 = total(tmp1)
drop tmp1

// new super users
bysort `unit' month uid: gen tmp1 = (_n==1) * (month==minmonth) * (numuserc >= 18)
bysort `unit' month: egen numnewusers90 = total(tmp1)
drop tmp1

end

program makedv

use ${stash}mergemaster1, clear

// unit = fips, geoid10 or tileid

local unit `0'
* local unit "fips geoid10"

egen unitid = group(`unit')
genvar unitid

end

program genvar

local unit `1'

// useful vars
bysort uid: gen numusercontrib = _N
bysort uid: egen minmonth = min(month)

bysort `unit' uid month: gen tmp1 = (_n==1)
bysort `unit' uid: egen nummonth = total(tmp1)
drop tmp1

// a) contribs
bysort `unit' month: gen numcontrib = _N

// b) users
bysort `unit' month uid: gen tmp1 = (_n==1)
bysort `unit' month: egen numuser = total(tmp1)
drop tmp1

// c) serious users
// 90 percentile == 18, 95 is 56

bysort `unit' month uid: gen tmp1 = (_n==1)*(numuserc > 18)
bysort `unit' month: egen numserious90 = total(tmp1)

bysort `unit' month uid: gen tmp2 = (_n==1)*(numuserc > 56)
bysort `unit' month: egen numserious95 = total(tmp2)
drop tmp1 tmp2

// d) new users

// new users
bysort `unit' month uid: gen tmp1 = (_n==1) * (month==minmonth)
bysort `unit' month: egen numnewusers = total(tmp1)
drop tmp1

// new committed users
bysort `unit' month uid: gen tmp1 = (_n==1) * (month==minmonth) * (nummonth >= 6)
bysort `unit' month: egen numnewusers6 = total(tmp1)
drop tmp1

// new super users
bysort `unit' month uid: gen tmp1 = (_n==1) * (month==minmonth) * (numuserc >= 18)
bysort `unit' month: egen numnewusers90 = total(tmp1)
drop tmp1

end

// testing

bysort geoid: egen avgtreat = mean(treat)

codebook uaname if avg > 0 & avg < 1