iefodler : empty line befoer output

iefodler : empty line befoer output

cap program drop 	iefolder
	program define	iefolder

qui {

	syntax anything, PROJectfolder(string) [ABBreviation(string)]

	***Todo
	*Test that old master do file exist

	*Create an empty line before error message or output
	noi di ""

	/***************************************************

		Parse input

	***************************************************/

	*Parse out the sub-command
	gettoken subcommand item : anything

	*Parse out the item type and the item name
	gettoken itemType itemName : item

	*Make sure that the item name is only one word
	gettoken itemName rest : itemName

	*Clean up the input
	local subcomand = trim("`subcommand'")
	local itemType 	= trim("`itemType'")
	local itemName 	= trim("`itemName'")

	*noi di "SubCommand 	`subcommand'"
	*noi di "ItemType 	`itemType'"
	*noi di "ItemName	`itemName'"

	if "`rest'" != "" {

		noi di as error "{pstd}You have specified to many words in: [{it:iefolder `subcommand' `itemType' `itemName'`rest'}]. Spaces are not allowed in the {it:itemname}. Use underscores or camel case.{p_end}"
		error 198
	}

	 /***************************************************

		Test input

	***************************************************/

	local sub_commands "new"
	local itemTypes "project round master"

	*Test if subcommand is valid
	if `:list subcommand in sub_commands' == 0 {

		noi di as error "{phang}You have not used a valid subcommand. See help file for details.{p_end}"
		error 198
	}

	*Test if item type is valid
	if `:list itemType in itemTypes' == 0 {

		noi di as error "{phang}You have not used a valid item type. See help file for details.{p_end}"
		error 198
	}

	*Test that item name is used when item type is anything but project
	else if ("`itemType'" != "project" & "`itemName'" == "" ) {

		noi di as error "{phang}You must specify a name of the `itemType'. See help file for details.{p_end}"
		error 198
	}

	di "All tests ok"

	/***************************************************

		Start making updates to the project folder

	***************************************************/

	*Create a temporary textfile
	tempname 	newHandle
	tempfile	newTextFile

	cap file close 	`newHandle'
		file open  	`newHandle' using "`newTextFile'", text write append

	*Create a global pointing to the main data folder
	global dataWorkFolder 	"`projectfolder'/DataWork"
	global encryptFolder 	"$dataWorkFolder/EncryptedData"

	if "`subcommand'" == "new" {

		di "Subcommand: New"

		*Creating a new project
		if "`itemType'" == "project" {

			di "ItemType: Project"
			iefolder_newProject "`projectfolder'" `newHandle'
			noi di "{pstd}Command ran succesfully, a new DataWork folder was created here created:{p_end}"
			noi di "{pstd}1) [${dataWorkFolder}]{p_end}"

		}
		*Creating a new round
		else if "`itemType'" == "round" {

			*Use the full item name if abbrevaition was not specified
			if "`abbreviation'" == "" local abbreviation "`itemName'"

			di "ItemType: round"
			iefolder_newRound `newHandle' "`itemName'" "`abbreviation'"

			noi di "{pstd}Command ran succesfully, for the round `itemName' the following folders and master dofile were created:{p_end}"
			noi di "{pstd}1) [${`abbreviation'}]{p_end}"
			noi di "{pstd}2) [${encryptFolder}/`itemName' Encrypted Data]{p_end}"
			noi di "{pstd}3) [${`abbreviation'}/`itemName'_MasterDofile.do]{p_end}"
		}
		*Creating a new level of observation for master data set
		else if "`itemType'" == "master" {

			di "ItemType: master/unitofobs"
			di `"iefolder_newMaster	`newHandle' "`itemName'""'
			iefolder_newMaster	`newHandle' "`itemName'"

			noi di "{pstd}Command ran succesfully, for the master `itemName' the following folders were created:{p_end}"
			noi di "{pstd}1) [${dataWorkFolder}/MasterData/`itemName']{p_end}"
			noi di "{pstd}2) [${encryptFolder}/IDMasterKey/`itemName']{p_end}"
		}
	}

	*Closing the new main master dofile handle
	file close 		`newHandle'

	*Copy the new master dofile from the tempfile to the original position
	copy "`newTextFile'"  "$dataWorkFolder/Project_MasterDofile.do" , replace
}
end

	/***************************************************

		Here are all functions for combination of
		subcommand and item type

	***************************************************/

cap program drop 	iefolder_newProject
	program define	iefolder_newProject

		args projectfolder newHandle

		*Test if folder where to create new folder exist
		checkFolderExists "`projectfolder'" "parent"

		*Test that the new folder does not already exist
		checkFolderExists "$dataWorkFolder" "new"

		mkdir "$dataWorkFolder"

		******************************
		*Writing master do file header
		******************************

		*Write intro part with description of project,
		mdofle_p0 `newHandle' project

		*Write flolder globals section header and the root folders
		mdofle_p1 `newHandle' "`projectfolder'"

		*Write globals section header and the root folders
		mdofle_p2 `newHandle'

		*Write section that runs sub-master dofiles
		mdofle_p3 `newHandle' project

end

cap program drop 	iefolder_newRound
	program define	iefolder_newRound

	args subHandle rndName rndAbb

	di "new round command"

	*Test if folder where to create new folder exist
	checkFolderExists "$dataWorkFolder" "parent"

	*Test that the new folder does not already exist
	checkFolderExists "$dataWorkFolder/`rndName'" "new"

	*Old file reference
	tempname 	oldHandle
	local 		oldTextFile 	"$dataWorkFolder/Project_MasterDofile.do"

	file open `oldHandle' using `"`oldTextFile'"', read
	file read `oldHandle' line

	*Locals needed for the section devider
	local partNum 		= 0 //Keeps track of the part number

	while r(eof)==0 {

		*Do not interpret macros
		local line : subinstr local line "\$"  "\\$"
		local line : subinstr local line "\`"  "\\`"

		parseReadLine `"`line'"'

		if `r(ief_line)' == 0 {

			*This is a regular line of code. It should be copied as-is. No action needed.
			file write	`subHandle' `"`line'"' _n

		}
		else if `r(ief_line)' == 1 {

			**This is a line of code with a devisor written by
			* iefodler. New additions to a section are always added right before an end of section line.

			*return list

			**This is NOT an end of section line. Nothing will be written here
			* but we test that there is no confict with previous names
			if `r(ief_end)' == 0 {

					*Test if it is an end of globals section as we are writing a new
					if "`r(sectionName)'" == "monitor" {

					*Write devisor for this section
					writeDevisor 			`subHandle' 1 RoundGlobals rounds `rndName' `rndAbb'

					*Write the globals to the master do file and create the folders
					newRndFolderAndGlobals 	`rndName' `rndAbb' "dataWorkFolder" `subHandle'

					*Write an empty line before the end devisor
					file write		`subHandle' "" _n

				}
				else {

					*Test that the folder name about to be used is not already in use
					if "`r(itemName)'" == "`rndName'" {
						noi di as error "{phang}The new round name `rndName' is already in use. No new folders are creaetd and the master do-files has not been changed.{p_end}"
						error 507
					}

					*Test that the folder name about to be used is not already in use
					if "`r(itemAbb)'" == "`rndAbb'" {
						noi di as error "{phang}The new round abbreviation `rndAbb' is already in use. No new folders are creaetd and the master do-files has not been changed.{p_end}"
						error 507
					}

				}

				*Copy the line as is
				file write		`subHandle' `"`line'"' _n
			}

			**This is an end of section line. We will add the new content here
			* before writing the end of section line
			else if `r(ief_end)' == 1 {

				*Test if it is an end of globals section as we are writing a new
				if "`r(partName)'" == "End_RunDofiles" {

					*Write devisor for this section
					writeDevisor 			`subHandle' 3 RunDofiles `rndName' `rndAbb'

					*Write the
					file write  `subHandle' ///
						_col(4)"if (0) { //Change the 0 to 1 to run the `rndName' master dofile" _n ///
						_col(8) `"do ""' _char(36) `"`rndAbb'/`rndName'_MasterDofile.do" "' _n ///
						_col(4)"}" ///
						_n

					*Write an empty line before the end devisor
					file write		`subHandle' "" _n

				}

				*Write the end of section line
				file write		`subHandle' `"`line'"' _n
			}
		}

		*Read next file and repeat the while loop
		file read `oldHandle' line
	}

	*Close the old file.
	file close `oldHandle'

end

cap program drop 	iefolder_newMaster
	program define	iefolder_newMaster

	args subHandle obsName

	di "new master command"

	global mastData 		"$dataWorkFolder/MasterData"
	global mastDataIDKey 	"$encryptFolder/IDMasterKey"

	*********************************************
	*Test if folder can be created where expected
	*********************************************

	*Test if folder where to create new folder exist
	checkFolderExists "$mastDataIDKey" "parent"

	*Test that the new folder does not already exist
	checkFolderExists "$mastDataIDKey/`obsName'" "new"

	*Test if folder where to create new fodler exist
	checkFolderExists "$mastData" "parent"

	*Test that the new folder does not already exist
	checkFolderExists "$mastData/`obsName'" "new"

	*************
	*create folder in masterdata

	*Old file reference
	tempname 	oldHandle
	local 		oldTextFile 	"$dataWorkFolder/Project_MasterDofile.do"

	file open `oldHandle' using `"`oldTextFile'"', read
	file read `oldHandle' line

	*Locals needed for the section devider
	local partNum 		= 0 //Keeps track of the part number

	while r(eof)==0 {

		*Do not interpret macros
		local line : subinstr local line "\$"  "\\$"
		local line : subinstr local line "\`"  "\\`"

		parseReadLine `"`line'"'

		if `r(ief_line)' == 1 & `r(ief_line)' == 1 & `r(ief_end)' == 0  & "`r(sectionName)'" == "rawData" {

			*Create master data folder and add global to folder in master do file
			cap createFolderWriteGlobal "`obsName'"  "mastData"  	mastData_`obsName'	`subHandle'

			if _rc == 693 {

				*If that folder exist, problem
				noi di as error "{phang}could not create new folder, folder name might already exist "
				error _rc

			}
			else if _rc != 0 {

				*Unexpected error, run the exact command name again and throw the error to user
				createFolderWriteGlobal "`obsName'"  "mastData"  	mastData_`obsName'	`subHandle'
			}

			file write		`subHandle' `"`line'"' _n

		}
		else {

			*Copy the line as is
			file write		`subHandle' `"`line'"' _n

		}

		*Read next file and repeat the while loop
		file read `oldHandle' line
	}

	*Close the old file.
	file close `oldHandle'

	*Create master data subfolders
	createFolderWriteGlobal "MasterDataSets"  	"mastData_`obsName'"  masterDataSets
	createFolderWriteGlobal "Dofiles"  			"mastData_`obsName'"  mastDataDo

		*************
	*create folder in encrypred ID key master

	createFolderWriteGlobal "`obsName'"  		"mastDataIDKey"  			mastData_E_`obsName'
	createFolderWriteGlobal "Data"  			"mastData_E_`obsName'"  	mastData_E_data
	createFolderWriteGlobal "Sampling"  		"mastData_E_`obsName'"  	mastData_E_Samp
	createFolderWriteGlobal "Treatment"  		"mastData_E_`obsName'"  	mastData_E_Treat

end

cap program drop 	iefolder
	program define	iefolder

qui {

	syntax anything, PROJectfolder(string) [ABBreviation(string)]

	***Todo
	*Test that a folder called DataWork already exist (unless new project)
	*Test that old master do file exist
	*allow to specifiy other place for master do file

	/***************************************************

		Parse input

	***************************************************/

	*Parse out the sub-command
	gettoken subcommand item : anything

	*Parse out the item type and the item name
	gettoken itemType itemName : item

	*Make sure that the item name is only one word
	gettoken itemName rest : itemName

	*Clean up the input
	local subcomand = trim("`subcommand'")
	local itemType 	= trim("`itemType'")
	local itemName 	= trim("`itemName'")

	*noi di "SubCommand 	`subcommand'"
	*noi di "ItemType 	`itemType'"
	*noi di "ItemName	`itemName'"

	if "`rest'" != "" {

		noi di as error "{pstd}You have specified to many words in: [{it:iefolder `subcommand' `itemType' `itemName'`rest'}]. Spaces are not allowed in the {it:itemname}. Use underscores or camel case.{p_end}"
		error 198
	}

	 /***************************************************

		Test input

	***************************************************/

	local sub_commands "new"
	local itemTypes "project round master"

	*Test if subcommand is valid
	if `:list subcommand in sub_commands' == 0 {

		noi di as error "{phang}You have not used a valid subcommand. See help file for details.{p_end}"
		error 198
	}

	*Test if item type is valid
	if `:list itemType in itemTypes' == 0 {

		noi di as error "{phang}You have not used a valid item type. See help file for details.{p_end}"
		error 198
	}

	*Test that item name is used when item type is anything but project
	else if ("`itemType'" != "project" & "`itemName'" == "" ) {

		noi di as error "{phang}You must specify a name of the `itemType'. See help file for details.{p_end}"
		error 198
	}

	di "All tests ok"

	/***************************************************

		Start making updates to the project folder

	***************************************************/

	*Create a temporary textfile
	tempname 	newHandle
	tempfile	newTextFile

	cap file close 	`newHandle'
		file open  	`newHandle' using "`newTextFile'", text write append

	*Create a global pointing to the main data folder
	global dataWorkFolder 	"`projectfolder'/DataWork"
	global encryptFolder 	"$dataWorkFolder/EncryptedData"

	if "`subcommand'" == "new" {

		di "Subcommand: New"

		*Creating a new project
		if "`itemType'" == "project" {

			di "ItemType: Project"
			iefolder_newProject "`projectfolder'" `newHandle'
			noi di "{pstd}Command ran succesfully, a new DataWork folder was created here created:{p_end}"
			noi di "{pstd}1) [${dataWorkFolder}]{p_end}"

		}
		*Creating a new round
		else if "`itemType'" == "round" {

			*Use the full item name if abbrevaition was not specified
			if "`abbreviation'" == "" local abbreviation "`itemName'"

			di "ItemType: round"
			iefolder_newRound `newHandle' "`itemName'" "`abbreviation'"

			noi di "{pstd}Command ran succesfully, for the round `itemName' the following folders and master dofile were created:{p_end}"
			noi di "{pstd}1) [${`abbreviation'}]{p_end}"
			noi di "{pstd}2) [${encryptFolder}/`itemName' Encrypted Data]{p_end}"
			noi di "{pstd}3) [${`abbreviation'}/`itemName'_MasterDofile.do]{p_end}"
		}
		*Creating a new level of observation for master data set
		else if "`itemType'" == "master" {

			di "ItemType: master/unitofobs"
			di `"iefolder_newMaster	`newHandle' "`itemName'""'
			iefolder_newMaster	`newHandle' "`itemName'"

			noi di "{pstd}Command ran succesfully, for the master `itemName' the following folders were created:{p_end}"
			noi di "{pstd}1) [${dataWorkFolder}/MasterData/`itemName']{p_end}"
			noi di "{pstd}2) [${encryptFolder}/IDMasterKey/`itemName']{p_end}"
		}
	}

	*Closing the new main master dofile handle
	file close 		`newHandle'

	*Copy the new master dofile from the tempfile to the original position
	copy "`newTextFile'"  "$dataWorkFolder/Project_MasterDofile.do" , replace
}
end

	/***************************************************

		Here are all functions for combination of
		subcommand and item type

	***************************************************/

cap program drop 	iefolder_newProject
	program define	iefolder_newProject

		args projectfolder newHandle

		*Test if folder where to create new folder exist
		checkFolderExists "`projectfolder'" "parent"

		*Test that the new folder does not already exist
		checkFolderExists "$dataWorkFolder" "new"

		mkdir "$dataWorkFolder"

		******************************
		*Writing master do file header
		******************************

		*Write intro part with description of project,
		mdofle_p0 `newHandle' project

		*Write flolder globals section header and the root folders
		mdofle_p1 `newHandle' "`projectfolder'"

		*Write globals section header and the root folders
		mdofle_p2 `newHandle'

		*Write section that runs sub-master dofiles
		mdofle_p3 `newHandle' project

end

cap program drop 	iefolder_newRound
	program define	iefolder_newRound

	args subHandle rndName rndAbb

	di "new round command"

	*Test if folder where to create new folder exist
	checkFolderExists "$dataWorkFolder" "parent"

	*Test that the new folder does not already exist
	checkFolderExists "$dataWorkFolder/`rndName'" "new"

	*Old file reference
	tempname 	oldHandle
	local 		oldTextFile 	"$dataWorkFolder/Project_MasterDofile.do"

	file open `oldHandle' using `"`oldTextFile'"', read
	file read `oldHandle' line

	*Locals needed for the section devider
	local partNum 		= 0 //Keeps track of the part number

	while r(eof)==0 {

		*Do not interpret macros
		local line : subinstr local line "\$"  "\\$"
		local line : subinstr local line "\`"  "\\`"

		parseReadLine `"`line'"'

		if `r(ief_line)' == 0 {

			*This is a regular line of code. It should be copied as-is. No action needed.
			file write	`subHandle' `"`line'"' _n

		}
		else if `r(ief_line)' == 1 {

			**This is a line of code with a devisor written by
			* iefodler. New additions to a section are always added right before an end of section line.

			*return list

			**This is NOT an end of section line. Nothing will be written here
			* but we test that there is no confict with previous names
			if `r(ief_end)' == 0 {

					*Test if it is an end of globals section as we are writing a new
					if "`r(sectionName)'" == "monitor" {

					*Write devisor for this section
					writeDevisor 			`subHandle' 1 RoundGlobals rounds `rndName' `rndAbb'

					*Write the globals to the master do file and create the folders
					newRndFolderAndGlobals 	`rndName' `rndAbb' "dataWorkFolder" `subHandle'

					*Write an empty line before the end devisor
					file write		`subHandle' "" _n

				}
				else {

					*Test that the folder name about to be used is not already in use
					if "`r(itemName)'" == "`rndName'" {
						noi di as error "{phang}The new round name `rndName' is already in use. No new folders are creaetd and the master do-files has not been changed.{p_end}"
						error 507
					}

					*Test that the folder name about to be used is not already in use
					if "`r(itemAbb)'" == "`rndAbb'" {
						noi di as error "{phang}The new round abbreviation `rndAbb' is already in use. No new folders are creaetd and the master do-files has not been changed.{p_end}"
						error 507
					}

				}

				*Copy the line as is
				file write		`subHandle' `"`line'"' _n
			}

			**This is an end of section line. We will add the new content here
			* before writing the end of section line
			else if `r(ief_end)' == 1 {

				*Test if it is an end of globals section as we are writing a new
				if "`r(partName)'" == "End_RunDofiles" {

					*Write devisor for this section
					writeDevisor 			`subHandle' 3 RunDofiles `rndName' `rndAbb'

					*Write the
					file write  `subHandle' ///
						_col(4)"if (0) { //Change the 0 to 1 to run the `rndName' master dofile" _n ///
						_col(8) `"do ""' _char(36) `"`rndAbb'/`rndName'_MasterDofile.do" "' _n ///
						_col(4)"}" ///
						_n

					*Write an empty line before the end devisor
					file write		`subHandle' "" _n

				}

				*Write the end of section line
				file write		`subHandle' `"`line'"' _n
			}
		}

		*Read next file and repeat the while loop
		file read `oldHandle' line
	}

	*Close the old file.
	file close `oldHandle'

end

cap program drop 	iefolder_newMaster
	program define	iefolder_newMaster

	args subHandle obsName

	di "new master command"

	global mastData 		"$dataWorkFolder/MasterData"
	global mastDataIDKey 	"$encryptFolder/IDMasterKey"

	*********************************************
	*Test if folder can be created where expected
	*********************************************

	*Test if folder where to create new folder exist
	checkFolderExists "$mastDataIDKey" "parent"

	*Test that the new folder does not already exist
	checkFolderExists "$mastDataIDKey/`obsName'" "new"

	*Test if folder where to create new fodler exist
	checkFolderExists "$mastData" "parent"

	*Test that the new folder does not already exist
	checkFolderExists "$mastData/`obsName'" "new"

	*************
	*create folder in masterdata

	*Old file reference
	tempname 	oldHandle
	local 		oldTextFile 	"$dataWorkFolder/Project_MasterDofile.do"

	file open `oldHandle' using `"`oldTextFile'"', read
	file read `oldHandle' line

	*Locals needed for the section devider
	local partNum 		= 0 //Keeps track of the part number

	while r(eof)==0 {

		*Do not interpret macros
		local line : subinstr local line "\$"  "\\$"
		local line : subinstr local line "\`"  "\\`"

		parseReadLine `"`line'"'

		if `r(ief_line)' == 1 & `r(ief_line)' == 1 & `r(ief_end)' == 0  & "`r(sectionName)'" == "rawData" {

			*Create master data folder and add global to folder in master do file
			cap createFolderWriteGlobal "`obsName'"  "mastData"  	mastData_`obsName'	`subHandle'

			if _rc == 693 {

				*If that folder exist, problem
				noi di as error "{phang}could not create new folder, folder name might already exist "
				error _rc

			}
			else if _rc != 0 {

				*Unexpected error, run the exact command name again and throw the error to user
				createFolderWriteGlobal "`obsName'"  "mastData"  	mastData_`obsName'	`subHandle'
			}

			file write		`subHandle' `"`line'"' _n

		}
		else {

			*Copy the line as is
			file write		`subHandle' `"`line'"' _n

		}

		*Read next file and repeat the while loop
		file read `oldHandle' line
	}

	*Close the old file.
	file close `oldHandle'

	*Create master data subfolders
	createFolderWriteGlobal "MasterDataSets"  	"mastData_`obsName'"  masterDataSets
	createFolderWriteGlobal "Dofiles"  			"mastData_`obsName'"  mastDataDo

		*************
	*create folder in encrypred ID key master

	createFolderWriteGlobal "`obsName'"  		"mastDataIDKey"  			mastData_E_`obsName'
	createFolderWriteGlobal "Data"  			"mastData_E_`obsName'"  	mastData_E_data
	createFolderWriteGlobal "Sampling"  		"mastData_E_`obsName'"  	mastData_E_Samp
	createFolderWriteGlobal "Treatment"  		"mastData_E_`obsName'"  	mastData_E_Treat

end


changes.do

changes.do

clear
set more off

global path "/mnt/nfs6/wikipedia.proj/jmp/"
global rawosm "/mnt/nfs6/wikipedia.proj/jmp/rawdata/osm/"
global rawmaps "/mnt/nfs6/wikipedia.proj/jmp/rawdata/maps/"
global rawtrips "/mnt/nfs6/wikipedia.proj/jmp/rawdata/trips/"
global stash "/mnt/nfs6/wikipedia.proj/jmp/rawdata/stash/"
global myestimates "/mnt/nfs6/wikipedia.proj/jmp/jmpscripts/stata/estimates/"
global tables "/mnt/nfs6/wikipedia.proj/jmp/jmpscripts/stata/tables/"

cd `path'

********************************************
*************** datasets ********************
********************************************

insheet using ${rawosm}ne_final_head.csv, clear

insheet using ${rawosm}ne_final_sed.csv, clear
insheet using ${rawosm}splitfinal/ne_final_x01.csv, clear

renamevar

keepvar

gennewvar

//// programlib

program drop gennewvar
program gennewvar

egen tileid = group(tilename)
maketimevar timestamp

bysort month tileid: gen sumamenity = _N
bysort month tileid: gen tag = (_n==1)

end

program drop maketimevar
program maketimevar

gen tstamp_stata = clock(`1', "YMD#hms#")
format tstamp_stata %tc

gen tstamp_date = dofc(tstamp_stata)
format tstamp_date %td

gen month = mofd(tstamp_date)
format month %tm

end

program drop keepvar
program keepvar

// drop non US
drop if fips == "NA" & geoid10 == "NA"
drop if otype == "v1"

drop if import_uuid != ""

gen keeptag = 0

local keeplist "building amenity highway maxspeed gnisfcode gnisfeatureid oneway waterway natural place leisure foot access wheelchair cycleway addr sidewalk import_uuid"

foreach x in `keeplist' {
    capture replace keeptag = 1 if `x' != ""
    capture replace keeptag = 1 if `x' != .
}

drop if keeptag == 0
drop keeptag

replace amenity = leisure if leisure != ""  & amenity == ""
replace amenity = place if place != "" & amenity == ""

gen isgnis = ((gnisfeat != "") | (gnisfc != "")) & version == "1"
drop if isgnis == 1
drop isgnis gnisf*

drop if amenity == ""

drop otype lon lat lon_tmp lat_tmp x_tmp y_tmp mergevar import_uuid building waterway natural oneway maxspeed place leisure foot access wheelchair cycleway nhdfcode addr sidewalk

end

program drop renamevar
program renamevar
rename v1 otype
rename v2 id
rename v3 lon
rename v4 lat
rename v5 version
rename v6 timestamp
rename v7 changeset
rename v8 uid
rename v9 user
rename v10 highway
rename v11 name
rename v12 amenity
rename v13 building
rename v14 waterway
rename v15 natural
rename v16 oneway
rename v17 maxspeed
rename v18 place
rename v19 leisure
rename v20 foot
rename v21 access
rename v22 wheelchair
rename v23 cycleway
rename v24 gnisfeatureid
rename v25 gnisfcode
rename v26 nhdfcode
rename v27 addr
rename v28 import_uuid
rename v29 sidewalk
rename v30 lon_tmp
rename v31 lat_tmp
rename v32 tilename
rename v33 geoid10
rename v34 msaname
rename v35 x_tmp
rename v36 y_tmp
rename v37 fips
rename v38 countyname
rename v39 class
rename v40 mergevar
end

clear
set more off

global path "/mnt/nfs6/wikipedia.proj/jmp/"
global rawosm "/mnt/nfs6/wikipedia.proj/jmp/rawdata/osm/"
global rawmaps "/mnt/nfs6/wikipedia.proj/jmp/rawdata/maps/"
global rawtrips "/mnt/nfs6/wikipedia.proj/jmp/rawdata/trips/"
global stash "/mnt/nfs6/wikipedia.proj/jmp/rawdata/stash/"
global myestimates "/mnt/nfs6/wikipedia.proj/jmp/jmpscripts/stata/estimates/"
global tables "/mnt/nfs6/wikipedia.proj/jmp/jmpscripts/stata/tables/"

cd `path'

********************************************
*************** datasets ********************
********************************************

insheet using ${rawosm}ne_final_head.csv, clear
insheet using ${rawosm}ne_final_sed.csv, clear

renamevar

keepvar

gennewvar

//// programlib

program drop gennewvar
program gennewvar

egen tileid = group(tilename)
maketimevar timestamp

local list "amenity gnisfcode gnisfeatureid place leisure"

foreach x in `list'{
    bysort month tileid: gen
}

end

program drop maketimevar
program maketimevar

gen tstamp_stata = clock(`1', "YMD#hms#")
format tstamp_stata %tc

gen tstamp_date = dofc(tstamp_stata)
format tstamp_date %td

gen month = mofd(tstamp_date)
format month %tm

end

program drop keepvar
program keepvar

// drop non US
drop if fips == "NA" & geoid10 == "NA"
drop if otype == "v1"

drop if import_uuid != ""

gen keeptag = 0

local keeplist "building amenity highway maxspeed gnisfcode gnisfeatureid oneway waterway natural place leisure foot access wheelchair cycleway addr sidewalk import_uuid"

foreach x in `keeplist' {
    capture replace keeptag = 1 if `x' != ""
    capture replace keeptag = 1 if `x' != .
}

drop if keeptag == 0
drop keeptag

drop otype id lon lat lon_tmp lat_tmp x_tmp y_tmp mergevar import_uuid

replace amenity = leisure if leisure != ""  & amenity == ""
replace amenity = place if place != "" & amenity == ""

gen isgnis = ((gnisfeat != "") | (gnisfc != "")) & version == "1"
drop if isgnis == 1
drop isgnis gnisf*

drop if amenity == ""

end

program drop renamevar
program renamevar
rename v1 otype
rename v2 id
rename v3 lon
rename v4 lat
rename v5 version
rename v6 timestamp
rename v7 changeset
rename v8 uid
rename v9 user
rename v10 highway
rename v11 name
rename v12 amenity
rename v13 building
rename v14 waterway
rename v15 natural
rename v16 oneway
rename v17 maxspeed
rename v18 place
rename v19 leisure
rename v20 foot
rename v21 access
rename v22 wheelchair
rename v23 cycleway
rename v24 gnisfeatureid
rename v25 gnisfcode
rename v26 nhdfcode
rename v27 addr
rename v28 import_uuid
rename v29 sidewalk
rename v30 lon_tmp
rename v31 lat_tmp
rename v32 tilename
rename v33 geoid10
rename v34 msaname
rename v35 x_tmp
rename v36 y_tmp
rename v37 fips
rename v38 countyname
rename v39 class
rename v40 mergevar
end


minor changes

minor changes

program makemeanline

use ${stash}panelfips, clear

* testing
*local var numcontri
*local time time
*local varlabel Contributions
*local cutoff 2011

*gen quarter = qofd(dofm(month))
*format quarter %tq
*gen year = yofd(dofm(month))

local var `1'
local time `2'
local cutoff `3'
local varlabel `4'

drop if year > `cutoff'

collapse (mean) mean=`var' (semean) se=`var' , by(`time' treat)

** replace mean = ln(mean*100000)

sort `time' treat
label variable mean "`varlabel'"

tw (connected mean `time' if treat == 0, msize(small) ) (connected mean `time' if treat == 1, msize(small)), legend(order(2 "TIGER Counties" 1 "Control Counties" )) xtitle("Quarter") title("`varlabel'") xline(191)

graph export ${tables}meanline_`var'.eps, replace
shell epstopdf ${tables}meanline_`var'.eps

end

program makemeanline

use ${stash}panelfips, clear

* testing
* local var numcontri
* local time quarter
* local varlabel Contributions
* local cutoff 2011

gen quarter = qofd(dofm(month))
format quarter %tq
* gen year = yofd(dofm(month))

local var `1'
local time `2'
local cutoff `3'
local varlabel `4'

drop if year > `cutoff'

collapse (mean) mean=`var' (semean) se=`var' , by(`time' treat)

** replace mean = ln(mean*100000)

sort `time' treat
label variable mean "`varlabel'"

tw (connected mean `time' if treat == 0, msize(small) ) (connected mean `time' if treat == 1, msize(small)), legend(order(2 "TIGER Counties" 1 "Control Counties" )) xtitle("Quarter") title("`varlabel'") xline(191)

graph export ${tables}meanline_`var'.eps, replace
shell epstopdf ${tables}meanline_`var'.eps

end


variation_id in XLS evaluation

variation_id in XLS evaluation

#!/usr/bin/env python3
from collections import OrderedDict
import yaml
from pathlib import Path
import pandas as pd
import redo
import comap # Just for the YAML converters

project = redo.base
project_path = Path('projects') / project

with redo.ifchange(evaluation_config=project_path / 'evaluation-config.yaml',
                   evaluation=redo.target.replace('.xls', '.yaml')) as files:
    evaluation_config = yaml.load(files['evaluation_config'])
    evaluations = yaml.load(files['evaluation'])

databases = OrderedDict(evaluation_config['databases'])
outcome_ids = evaluation_config['outcome_ids']

casedef_paths = {
    outcome_id: project_path / 'case-definitions' / (outcome_id + '.yaml')
    for outcome_id in outcome_ids
}

with redo.ifchange(casedefs=casedef_paths) as files:
    outcomes = {
        outcome_id: yaml.load(f)
        for outcome_id, f in files['casedefs'].items()
    }

def evaluations_to_xls(filename, evaluations, outcomes=None, databases=None):

    """
    evaluations ::= { outcome_id: { database: {variations?: {variation_id: variation}, comment?: str} } }
    variation = { name: str, comparison: comparison }
    comparison ::= { codes: { TP, FP, FN: codes }, measures: { recall, precision: float } }
    """

    outcome_ids = list(evaluations.keys())
    database_names = list(evaluations[outcome_ids[0]].keys()
                          if databases is not None else
                          databases.keys())
    for database_name in database_names:
        if 'variations' in evaluations[outcome_ids[0]][database_name]:
            variations = evaluations[outcome_ids[0]][database_name]['variations']
            assert type(variations) == OrderedDict, type(variations)
            variation_ids = list(variations.keys())
            variation_names = {
                variation_id: variations[variation_id]['name']
                for variation_id in variations.keys()
            }
            break

    columns_per_database = ['TP', 'FP', 'FN', 'recall', 'precision']
    columns = pd.MultiIndex.from_tuples([
        (database_name
         if databases is None else
         '{} ({})'.format(database_name, databases[database_name]),
         c)
        for database_name in database_names
        for c in columns_per_database
    ] + [('Average over databases', 'recall'), ('Average over databases', 'precision')])

    str_of_list = lambda v: '{}: {}'.format(len(v), ' '.join(str(v0) for v0 in v))
    index, rows = [], []

    for variation_id in variation_ids:
        if variation_ids.index(variation_id):
            index.extend(['', ''])
            rows.extend([[None] * len(columns)] * 2)
        index.append(variation_id)
        rows.append([variation_names[variation_id]] + [None] * (len(columns) - 1))
        measures_over_outcomes = {
            database_name: pd.DataFrame(columns=['recall', 'precision'])
            for database_name in database_names
        }
        for outcome_id in outcome_ids:
            row = []
            measures_over_databases = pd.DataFrame(columns=['recall', 'precision'])
            for database_name in database_names:
                for_database = evaluations[outcome_id][database_name]
                if 'variations' in for_database:
                    variation = for_database['variations'][variation_id]
                    codes, measures = (variation['comparison'][key] for key in ('codes', 'measures'))
                    measures_over_databases.ix[database_name] = measures
                    measures_over_outcomes[database_name].ix[outcome_id] = measures
                    row.extend([
                        str_of_list(codes['TP']),
                        str_of_list(codes['FP']),
                        str_of_list(codes['FN']),
                        measures['recall'],
                        measures['precision'],
                    ])
                else:
                    row.extend([for_database.get('comment')] + [None] * (len(columns_per_database) - 1))
            if len(measures_over_databases):
                average = measures_over_databases.sum() / len(measures_over_databases)
                row.extend([average['recall'], average['precision']])
            else:
                row.extend([None, None])
            index.append(outcomes[outcome_id]['name'] if outcomes else outcome_id)
            rows.append(row)

        row = []
        for database_name in database_names:
            measures = measures_over_outcomes[database_name]
            average = measures.sum() / len(measures)
            row.extend([None] * (len(columns_per_database)-2) + [average['recall'], average['precision']])
        index.append('Average over outcomes')
        rows.append(row)

    writer = pd.ExcelWriter(filename)
    df = pd.DataFrame(rows, index=pd.Index(index), columns=columns)
    df.to_excel(writer, float_format='%.2f')
    writer.save()

evaluations_to_xls(redo.temp, evaluations,
                   databases=databases,
                   outcomes=outcomes)

#!/usr/bin/env python3
from collections import OrderedDict
import yaml
from pathlib import Path
import pandas as pd
import redo
import comap # Just for the YAML converters

project = redo.base
project_path = Path('projects') / project

with redo.ifchange(evaluation_config=project_path / 'evaluation-config.yaml',
                   evaluation=redo.target.replace('.xls', '.yaml')) as files:
    evaluation_config = yaml.load(files['evaluation_config'])
    evaluations = yaml.load(files['evaluation'])

databases = OrderedDict(evaluation_config['databases'])
outcome_ids = evaluation_config['outcome_ids']

casedef_paths = {
    outcome_id: project_path / 'case-definitions' / (outcome_id + '.yaml')
    for outcome_id in outcome_ids
}

with redo.ifchange(casedefs=casedef_paths) as files:
    outcomes = {
        outcome_id: yaml.load(f)
        for outcome_id, f in files['casedefs'].items()
    }

def evaluations_to_xls(filename, evaluations, outcomes=None, databases=None):

    """
    evaluations ::= { outcome_id: { database: {variations?: {variation_id: variation}, comment?: str} } }
    variation = { name: str, comparison: comparison }
    comparison ::= { codes: { TP, FP, FN: codes }, measures: { recall, precision: float } }
    """

    outcome_ids = list(evaluations.keys())
    database_names = list(evaluations[outcome_ids[0]].keys()
                          if databases is not None else
                          databases.keys())
    for database_name in database_names:
        if 'variations' in evaluations[outcome_ids[0]][database_name]:
            variations = evaluations[outcome_ids[0]][database_name]['variations']
            assert type(variations) == OrderedDict, type(variations)
            variation_ids = list(variations.keys())
            variation_names = {
                variation_id: variations[variation_id]['name']
                for variation_id in variations.keys()
            }
            break

    columns_per_database = ['TP', 'FP', 'FN', 'recall', 'precision']
    columns = pd.MultiIndex.from_tuples([
        (database_name
         if databases is None else
         '{} ({})'.format(database_name, databases[database_name]),
         c)
        for database_name in database_names
        for c in columns_per_database
    ] + [('Average over databases', 'recall'), ('Average over databases', 'precision')])

    str_of_list = lambda v: '{}: {}'.format(len(v), ' '.join(str(v0) for v0 in v))
    index, rows = [], []

    for variation_id in variation_ids:
        if variation_ids.index(variation_id):
            index.extend(['', ''])
            rows.extend([[None] * len(columns)] * 2)
        index.append(variation_names[variation_id])
        rows.append([None] * len(columns))
        measures_over_outcomes = {
            database_name: pd.DataFrame(columns=['recall', 'precision'])
            for database_name in database_names
        }
        for outcome_id in outcome_ids:
            row = []
            measures_over_databases = pd.DataFrame(columns=['recall', 'precision'])
            for database_name in database_names:
                for_database = evaluations[outcome_id][database_name]
                if 'variations' in for_database:
                    variation = for_database['variations'][variation_id]
                    codes, measures = (variation['comparison'][key] for key in ('codes', 'measures'))
                    measures_over_databases.ix[database_name] = measures
                    measures_over_outcomes[database_name].ix[outcome_id] = measures
                    row.extend([
                        str_of_list(codes['TP']),
                        str_of_list(codes['FP']),
                        str_of_list(codes['FN']),
                        measures['recall'],
                        measures['precision'],
                    ])
                else:
                    row.extend([for_database.get('comment')] + [None] * (len(columns_per_database) - 1))
            if len(measures_over_databases):
                average = measures_over_databases.sum() / len(measures_over_databases)
                row.extend([average['recall'], average['precision']])
            else:
                row.extend([None, None])
            index.append(outcomes[outcome_id]['name'] if outcomes else outcome_id)
            rows.append(row)

        row = []
        for database_name in database_names:
            measures = measures_over_outcomes[database_name]
            average = measures.sum() / len(measures)
            row.extend([None] * (len(columns_per_database)-2) + [average['recall'], average['precision']])
        index.append('Average over outcomes')
        rows.append(row)

    writer = pd.ExcelWriter(filename)
    df = pd.DataFrame(rows, index=pd.Index(index), columns=columns)
    df.to_excel(writer, float_format='%.2f')
    writer.save()

evaluations_to_xls(redo.temp, evaluations,
                   databases=databases,
                   outcomes=outcomes)


[ietoolkit] update meta data in command ietoolkit

*! version 6.3 5NOV2019 DIME Analytics dimeanalytics@worldbank.org

capture program drop ietoolkit
program ietoolkit, rclass

	* UPDATE THESE LOCALS FOR EACH NEW VERSION PUBLISHED
	local version "6.3"
	local versionDate "5NOV2019"

	syntax [anything]

	/**********************
		Error messages
	**********************/

	* Make sure that no arguments were passed
	if "`anything'" != "" {
		noi di as error "This command does not take any arguments, write only {it:ietoolkit}"
		error 198
	}

	/**********************
		Output
	**********************/

	* Prepare returned locals
	return local 	versiondate "`versionDate'"
	return scalar 	version		= `version'

	* Display output
	noi di ""
	noi di _col(4) "This version of ietoolkit installed is version " _col(54)"`version'"
	noi di _col(4) "This version of ietoolkit was released on " _col(54)"`versionDate'"

end

*! version 6.3 5NOV2019 DIME Analytics dimeanalytics@worldbank.org

capture program drop ietoolkit
program ietoolkit, rclass

	* UPDATE THESE LOCALS FOR EACH NEW VERSION PUBLISHED
	local version "6.2"
	local versionDate "31JAN2019"

	syntax [anything]

	/**********************
		Error messages
	**********************/

	* Make sure that no arguments were passed
	if "`anything'" != "" {
		noi di as error "This command does not take any arguments, write only {it:ietoolkit}"
		error 198
	}

	/**********************
		Output
	**********************/

	* Prepare returned locals
	return local 	versiondate "`versionDate'"
	return scalar 	version		= `version'

	* Display output
	noi di ""
	noi di _col(4) "This version of ietoolkit installed is version " _col(54)"`version'"
	noi di _col(4) "This version of ietoolkit was released on " _col(54)"`versionDate'"

end


minor format

minor format