ieduplicates : fix #166 improve error message

*! version 6.0 19OCT2018 DIME Analytics dimeanalytics@worldbank.org

	capture program drop ieduplicates
	program ieduplicates , rclass

	qui {

		syntax varname ,  FOLder(string) UNIQUEvars(varlist) [KEEPvars(varlist) tostringok droprest nodaily SUFfix(string)]

		version 11.0

		*Add version of Stata fix
		//Make sure that keepvars are still saved if saved if the duplicates file
		*	is generated on a subset of the data. For example, duplicates from
		*	version 1, 2 . If the command is run on only version 1. Then values
		*	for keeepvars in version 2 and 3 are dropped and not reloaded as those
		*	obs are not in current memory

		**Test that observations have not been deleted from the report before readind
		* it. Deleted in a way that the report does not make sense. Provide an error
		* message to this that is more informative.

		preserve

			/***********************************************************************
			************************************************************************

				Section 1 - Set up locals needed in data

				Saving a version of the data to be used before merging corrections
				back to the original data set and before correcting duplicates

			************************************************************************
			***********************************************************************/

			*Tempfiles to be uses
			tempfile originalData preppedReport datawithreportmerged dataToReturn

			** Save a version of original data that can be brought
			*  back withing the preserve/restore section
			save 	`originalData'

			* Create a local with todays date to date the reports
			local  date = subinstr(c(current_date)," ","",.)

			*Put idvar in a local with a more descriptive name
			local idvar `varlist'

			** Making one macro with all variables that will be
			*  imported and exported from the Excel file
			local argumentVars `idvar' `uniquevars' `keepvars'

			* Create a list of the variables created by this command to put in the report
			local excelVars dupListID dateListed dateFixed correct drop newID initials notes

			* Test that nu variable with the name needed for the excel report already exist in the data set
			foreach excelvar of local excelVars {
				cap confirm variable `excelvar'
				if _rc == 0 {
					*Variable exist, output error
					noi display as error "{phang}The data set already have a variable called {inp:`excelvar'} which is a name that this command requires to be availible. Please change the name of the variable already in the data set. Future versions of this command will allow the user to rename the variables used by this command.{p_end}"
					noi di ""
					error 198
					exit
				}
			}

			/***********************************************************************
			************************************************************************

				Section 2 - Test unique vars

				Test the unique vars so that they are identifying the data set and
				are not in a time format that might get corrupted by exporting to
				and importing from Excel. This is needed as the uniquevars are needed
				to merge the correct correction to the correct duplicat

			************************************************************************
			***********************************************************************/

			*Test that the unique vars fully and uniquely identifies the data set
			cap isid `uniquevars'
			if _rc {

				noi display as error "{phang}The variable(s) listed in uniquevars() does not uniquely and fully identifies all observations in the data set.{p_end}"
				isid `uniquevars'
				error 198
				exit
			}

			** Test that no unique vars are time format. Time values might be corrupted
			*  and changed a tiny bit when importing and exporting to Excel, which make merge not possible
			foreach uniquevar of local uniquevars {

				*Test if format is time format
				local format : format `uniquevar'
				if substr("`format'",1,2) == "%t" {

					noi display as error `"{phang}The variable {inp:`uniquevar'} listed in {inp:uniquevars()} is using time format which is not allowed. Stata and Excel stores and displays time slightly differently which can lead to small changes in the value when the value is imported and exported between Stata and Excel, and then the variable can no longer be used to merge the report back to the original data. Use another variable or create a string variable out of your time variable using this code: {inp: generate `uniquevar'_str = string(`uniquevar',"%tc")}.{p_end}"'
					noi di ""
					error 198
					exit
				}
			}

			/***********************************************************************
			************************************************************************

				Section 3 - Test input from Excel file

				If Excel repirt exist, import it and test for invalid corrections
				made in the excel report.

			************************************************************************
			***********************************************************************/

			/******************
				Section 3.1
				Check if report exists from before.
			******************/

			*Check if file exist. Suffix option can be used to create multiple reports in the same folder
			cap confirm file "`folder'/iedupreport`suffix'.xlsx"

			if !_rc {
				local fileExists 1
			}
			else {
				local fileExists 0
			}

			/******************
				Section 3.2
				If report exists, load file and check input, otherwise skip to section 4
			******************/

			if `fileExists' {

				*Load excel file. Load all vars as string and use metadata from Section 1
				import excel "`folder'/iedupreport`suffix'.xlsx"	, clear firstrow

				** All excelVars but dupListID and newID should be string. dupListID
				*  should be numeric and the type of newID should be based on the user input
				foreach excelvar of local excelVars {
					if !inlist("`excelvar'", "dupListID", "newID") {

						* Make original ID var string
						tostring `excelvar' , replace
						replace  `excelvar' = "" if `excelvar' == "."
					}
				}

				/******************
					Section 3.2.1
					Make sure that the ID variable and the uniquevars are
					not changed since last report.
				******************/

				*Copy a list of all variables in the imported report to the local existingExcelVars
				ds
				local existingExcelVars  `r(varlist)'

				*Test that the ID variable is in the imported report
				if `:list idvar in existingExcelVars' == 0 {

					noi display as error "{phang}The ID variable `idvar' does not exist in the previously exported Excle file. If you renamed the ID variable you need to rename it manually in the Excel report or start a new Excel report by renaming or moving the original report, then run the command again and create a new file and manually copy any corrections from the old file to the new. If you changed the ID varaible you need to start with a new report.{p_end}"
					noi di ""
					error 111
					exit

				}

				*Test that the unique variables are in the imported report
				if `:list uniquevars in existingExcelVars' == 0 {

					noi display as error "{phang}One or more unique variables in [`uniquevars'] do not exist in the previously exported Excel file. If you renamed or changed any variable used in uniquevars(), you need to start over with a new file. Rename or move the already existing file. Create a new file and carefully copy any corrections from the old file to the new.{p_end}"
					noi di ""
					error 111
					exit
				}

				/******************
					Section 3.3
					Make sure input is correct
				******************/

				*Temporary variables needed for checking input
				tempvar multiInp inputNotYes maxMultiInp notDrop yesCorrect groupNumCorrect toDrop anyCorrection groupAnyCorrection

				*Locals indicating in which ways input is incorrect (if any)
				local local_multiInp 		0
				local local_multiCorr		0
				local local_inputNotYes		0
				local local_notDrop			0

				/******************
					Section 3.3.1
					Make sure input is yes or y for the correct and drop columns
				******************/

				* Make string input lower case and change "y" to "yes"
				replace correct = lower(correct)
				replace drop 	= lower(drop)
				replace correct = "yes" if correct 	== "y"
				replace drop 	= "yes" if drop 	== "y"

				*Check that variables are either empty or "yes"
				gen `inputNotYes' = !((correct  == "yes" | correct == "") & (drop  == "yes" | drop == ""))

				*Set local to 1 if error should be outputted
				cap assert `inputNotYes' == 0
				if _rc local local_inputNotYes 	1

				/******************
					Section 3.3.2
					Make sure there are not too many corrections for a single observation
				******************/

				* Count the number of corrections (correct drop newID) per
				* observation. Only one correction per observation is allowed.
				egen `multiInp' = rownonmiss(correct drop newID), strok

				*Check that all rows have at most one correction
				cap assert `multiInp' == 0 | `multiInp' == 1

				*Error will be outputted below
				if _rc local local_multiInp 	1

				/******************
					Section 3.3.3
					Test that maximum one duplicate per duplicate group is indicated as correct

				******************/

				*Generate dummy if correct column is set to yes
				gen `yesCorrect' = (correct == "yes")

				*Count number of duplicates within duplicates where that dummy is 1
				bys `idvar' : egen `groupNumCorrect' =  total(`yesCorrect')

				*Test if more than 1 duplicate in that duplicate group is correct
				count if `groupNumCorrect' > 1

				*Output error is more than one duplicate in a duplicate group is yes
				if `r(N)' != 0 local local_multiCorr 1

				/******************
					Section 3.3.4
					Make sure that either option droprest is specified, or that
					drop was correctly indicated for all observations. i.e.; if
					correct or newID was indicated for at least one duplicate in
					a duplicate group, then all other observations should be
					indicated as drop (unless droprest is specified)

				******************/

				*Generate dummy if there is any correction for this observation
				gen `anyCorrection' = !missing(correct) | !missing(newID)

				*Count number of observations with any correction in suplicates group
				bys `idvar' : egen `groupAnyCorrection' =  total(`anyCorrection')

				*Create dummy that indicates each place this error happens
				gen `notDrop' = (missing(drop) & `groupAnyCorrection' > 0 & `anyCorrection' == 0)

				* Check if option droprest is specified
				if "`droprest'" == "" {

					** If option droprest is not used, then all observations in a duplicate
					*  group where at least one observation has a correction must have a
					*  correction or have drop set to yes.
					cap assert `notDrop' == 0

					*Error will be outputted below
					if _rc local local_notDrop 	1

				}
				else {

					** Option -droprest- specified. Drop will be changed to yes
					*  for any observations without drop or any other correction
					*  explicitly specified if the observation is in a duplicate
					*  group with at least one observation has a correction
					replace drop 	= "yes" if `notDrop' == 1

				}

				/******************
					Section 3.4
					Throw errors if any of the tests were not passed
				******************/

				*Output errors if errors in the report were detected
				if `local_multiInp' == 1 | `local_inputNotYes' == 1 | `local_notDrop' == 1 | `local_multiCorr' == 1  {
					noi {
						di ""
						di ""
						di as error "{phang}{ul:The corrections made in the Excel report has the following errors:}{p_end}"
						di ""

						*Error multiple input
						if `local_multiInp' == 1 {
							display as error "{phang}The following observations have more than one correction. Only one correction (correct, drop or newID) per row is allowed{p_end}"
							list `idvar' dupListID correct drop newID `uniquevars' if `multiInp' > 1
							di ""
						}

						*Error multiple correct
						if `local_multiCorr' == 1 {
							display as error "{phang}The following observations are in a duplicate group where more than one observation is listed as correct. Only one observation per duplicate group can be correct{p_end}"
							list `idvar' dupListID correct drop newID `uniquevars' if `groupNumCorrect' > 1
							di ""
						}

						*Error in incorrect string
						if `local_inputNotYes' == 1 {
							display as error "{phang}The following observations have an answer in either correct or drop that is neither yes nor y{p_end}"
							list `idvar' dupListID correct drop `uniquevars' if `inputNotYes' == 1
							di ""
						}

						*Error is not specfied as drop
						if `local_notDrop' == 1 {
							display as error "{phang}The following observations are not explicitly indicated as drop while other duplicates in the same duplicate group are corrected. Either manually indicate as drop or see option droprest{p_end}"
							list `idvar' dupListID correct drop newID `uniquevars' if `notDrop' == 1
							di ""
						}

						*Same error for any incorrect input

						di as error "{phang}Since there was at least one error in the Excel report no corrections have been made to the duplicates in the data set. Please address the errors above and then run the command again.{p_end}"
						error 198
						exit
					}
				}

				/******************
					Section 3.5
					Save the prepared report to be used later
				******************/

				*Keep only the variables needed for matching and variables used for input in the Excel file
				keep 	`idvar' `uniquevars' `excelVars' `groupAnyCorrection'

				*Save imported data set with all corrections
				save	`preppedReport'
			}

			/***********************************************************************
			************************************************************************

				Section 4 - Merge report to original data

				Merge corrections with original data, test that there are no
				obs in report that are not in main data, and save this data
				in temp file

			************************************************************************
			***********************************************************************/

			*Re-load original data
			use `originalData', clear

			* Merge original data with imported Excel file (if Excel file exists)
			if `fileExists'  {

				*Create a tempvar for merging results
				tempvar iedup_merge

				*Merge the corrections with the data set
				merge 1:1 `uniquevars' using `preppedReport', generate(`iedup_merge')

				*Make sure that obsrevations listed in the duplicate report is still in the data set
				cap assert `iedup_merge' != 2

				*Display error message if assertion is not true and some duplicates in the Excel file are no longer in the data set
				if _rc {

					display as error "{phang}One or several observations in the Excel report are no longer found in the data set. Always run ieduplicates on the raw data set that include all the duplicates, both new duplicates and those you have already identified. After removing duplicates, save the data set using a different name. You might also recieve this error if you are using an old ieduplicates Excel report on a new data set.{p_end}"
					error 9
					exit
				}

				*Explicitly drop temporary variable. Temporary variables might
				*be exported to excel so delete explicitly before that. Only
				*using tempvar here to create a name with no conflicts
				drop `iedup_merge'

			}

			*save the data set to be used when correcting the data
			save `datawithreportmerged'

			/***********************************************************************
			************************************************************************

				Section 5 - Generate the Excel report

				Test if there are duplicates in ID var. If any duplicates exist,
				then update the Excel file with new and unaddressed cases

			************************************************************************
			***********************************************************************/

			/******************
				Section 5.1
				Test if there are any duplicates in ID var
			******************/

			* Generate variable that is not 0
			* if observation is a duplicate
			tempvar dup
			duplicates tag `idvar', gen(`dup')

			*Test if there are any duplicates
			cap assert `dup'==0
			if _rc {

				/******************
					Section 5.2
					Keep only duplicates for the report
				******************/

				*Keep if observation is part of duplicate group
				keep if `dup' != 0

				if `fileExists'  {
					* If Excel file exists keep excel vars and
					* variables passed as arguments in the
					* command
					keep 	`argumentVars' `excelVars'
				}
				else {
					* Keep only variables passed as arguments in
					* the command and the string ID var as no Excel file exists
					keep 	`argumentVars'

					*Generate the excel variables used for indicating correction
					foreach excelvar of local excelVars {

						*Create all variables apart from dupListID as string vars
						if inlist("`excelvar'", "dupListID", "newID") {
							gen `excelvar' = .
						}
						else {
							gen `excelvar' = ""
						}
					}
				}

				/******************
					Section 5.3
					Update the excel vars that are not updated manually
				******************/

				* Generate a local that is 1 if there are new duplicates
				local unaddressedNewExcel 0
				count if missing(dateFixed)
				if `r(N)' > 0 local unaddressedNewExcel 1

				/******************
					Section 5.3.1 Date variables
				******************/

				* Add date first time duplicate was identified
				replace dateListed 	= "`date'" if missing(dateListed)

				** Add today's date to variable dateFixed if dateFixed
				*  is empty and at least one correction is added
				replace dateFixed 	= "`date'" if missing(dateFixed) & (!missing(correct) | !missing(drop) | !missing(newID))

				/******************
					Section 5.3.2 Duplicate report list ID
				******************/

				** Sort after dupListID and after ID var for
				*  duplicates currently without dupListID
				sort dupListID `idvar'

				** Assign dupListID 1 to the top row if no duplicate
				*  list IDs have been generated so far.
				replace dupListID = 1 if _n == 1 & missing(dupListID)

				** Generate new IDs based on the row above instead of directly
				*  from the row number. That prevents duplicates in the list in
				*  case an observation is deleted. The first observation with
				*  missing value will have an ID that is one digit higher than
				*  the highest ID already in the list
				replace dupListID = dupListID[_n - 1] + 1 if missing(dupListID)

				/******************
					Section 5.4
					Keep and order the variables and output the Excel files
				******************/

				* If cases unaddressed then update the Excel file
				if `unaddressedNewExcel'  {

					keep 	`argumentVars' `excelVars'
					order	`idvar' `excelVars' `uniquevars' `keepvars'

					if "`daily'" == "" {

						*Returns 0 if folder does not exist, 1 if it does
						mata : st_numscalar("r(dirExist)", direxists("`folder'/Daily"))

						** If the daily folder is not created, just create it
						if `r(dirExist)' == 0  {

							*Create the folder since it does not exist
							mkdir "`folder'/Daily"
						}

						*Export the daily file
						cap export excel using "`folder'/Daily/iedupreport`suffix'_`date'.xlsx"	, firstrow(variables) replace nolabel

						*Print error if daily report cannot be saved
						if _rc {

							display as error "{phang}The Daily copy could not be saved to the `folder'/Daily folder. Make sure to close any old daily copy or see the option nodaily.{p_end}"
							error 603
							exit
						}

						*Prepare local for output
						local daily_output " and a daily copy have been saved to the Daily folder"
					}

					*Export main report
					export excel using "`folder'/iedupreport`suffix'.xlsx"	, firstrow(variables) replace  nolabel

					*Produce output
					noi di `"{phang}Excel file created at: {browse "`folder'/iedupreport`suffix'.xlsx":`folder'/iedupreport`suffix'.xlsx}`daily_output'.{p_end}"'
					noi di ""
				}
			}

		/***********************************************************************
		************************************************************************

			Section 6

			Update the data set and with the new corrections.

		************************************************************************
		***********************************************************************/

		* Load the original data set merged with correction. Duplicates
		* in all variables are already dropped in this data set
		use 	`datawithreportmerged', clear

		* If excel file exists, apply any corrections indicated (if any)
		if `fileExists' {

			/******************
				Section 6.1
				Drop duplicates listed for drop
			******************/

			drop if drop == "yes"

			/******************
				Section 6.2
				Update new ID. ID var can be either numeric or
				string. All numbers can be made strings but not
				all strings can be numeric. Therefore this
				section is complicated.
			******************/

			/******************
				Section 6.2.1
				ID var in original file is string. Either
				newID was imported as string or the variable
				is made string. Easy.
			******************/

			*Test if there are any corrections by new ID
			cap assert missing(newID)
			if _rc {

				local idtype 	: type `idvar'
				local idtypeNew : type newID

				*If ID var is string but newID is not, then just make it string
				if substr("`idtype'",1,3) == "str" & substr("`idtypeNew'",1,3) != "str" {

					tostring newID , replace
					replace  newID = "" if newID == "."
				}

				*If ID var is numeric but the newID is loaded as string
				else if substr("`idtype'",1,3) != "str" & substr("`idtypeNew'",1,3) == "str" {

					* Check if [tostringok] is specificed:
					if "`tostringok'" != "" {

						* Make original ID var string
						tostring `idvar' , replace
						replace  `idvar' = "" if `idvar' == "."

					}

					* Error, IDvar cannot be updated
					else {

						* Create a local with all non-numeric values
						levelsof newID if missing(real(newID)), local(NaN_values) clean

						* Output error message
						di as error "{phang}The ID varaible `idvar' is numeric but newID has thes non-numeric values: `NaN_values'. Update newID to only contain numeric values or see option tostringok.{p_end}"
						error 109
						exit
					}
				}

				*After making sure that type is ok, update the IDs
				replace `idvar' = newID if !missing(newID)

				/******************
					Section 6.3
					Test that values in newID
					were neither used twice
					nor already existed
				******************/

				*Test if there are any duplicates after corrections (excluding observations in duplicate groups not yet corrected)
				tempvar newDup
				duplicates tag `idvar' if `groupAnyCorrection' != 0, generate(`newDup')

				*Consider missing values as no new duplicates, make conditions below easier
				replace `newDup' = 0 if missing(`newDup')

				cap assert `newDup' == 0
				if _rc {

					levelsof `idvar' if `newDup' != 0  , local(newDuplist)
					di as error "{phang}No corrections from the report are applied as it would lead to new duplicates in the following value(s): `newDuplist'.{p_end}"
					error 198
					exit
				}
			}

			/******************
				Section 6.4
				Drop Excel vars
			******************/

			drop `excelVars'
		}

		/***********************************************************************
		************************************************************************

			Section 7

			Return the data set without duplicates and
			output information regarding unresolved duplicates.

		************************************************************************
		***********************************************************************/

		* Generate a variable that is 1 if the observation is a duplicate in varlist
		tempvar dropDup
		duplicates tag `idvar',  gen(`dropDup')
		* Generate a list of the IDs that are still duplicates
		levelsof `idvar' 			if `dropDup' != 0 , local(dup_ids) clean
		* Drop the duplicates (they are exported in Excel)
		drop 						if `dropDup' != 0

		* Test if varlist is now uniquely and fully identifying the data set
		cap isid `idvar'
		if _rc {

			di as error "{phang}The data set is not returned with `idvar' uniquely and fully identifying the data set. Please report this bug to kbjarkefur@worldbank.org{p_end}"
			error 119
			exit

		}

		*Count number of duplicate groups still in the data
		local numDup	= `:list sizeof dup_ids'

		if `numDup' == 0 {
			noi di	"{phang}There are no unresolved duplicates in this data set. The data set is returned with `idvar' uniquely and fully identifying the data set.{p_end}"
		}
		else {
			noi di	"{phang}There are `numDup' duplicated IDs still unresolved. IDs still containing duplicates: `dup_ids'. The unresolved duplicate observations were exported in the Excel file. The data set is returned without those duplicates and with `idvar' uniquely and fully identifying the data set.{p_end}"
		}

		return scalar numDup	= `numDup'

		/***********************************************************************
		************************************************************************

			Section 8

			Save data set to be returned outside preserve/restore.
			Preserve/restore is used so that original data is returned
			in case an error is thrown.

		************************************************************************
		***********************************************************************/

		save 	`dataToReturn'

		restore

		** Using restore above to return the data to
		*  the orignal data set in case of error.
		use `dataToReturn', clear

	}
end

*! version 6.0 19OCT2018 DIME Analytics dimeanalytics@worldbank.org

	capture program drop ieduplicates
	program ieduplicates , rclass

	qui {

		syntax varname ,  FOLder(string) UNIQUEvars(varlist) [KEEPvars(varlist) tostringok droprest nodaily SUFfix(string)]

		version 11.0

		*Add version of Stata fix
		//Make sure that keepvars are still saved if saved if the duplicates file
		*	is generated on a subset of the data. For example, duplicates from
		*	version 1, 2 . If the command is run on only version 1. Then values
		*	for keeepvars in version 2 and 3 are dropped and not reloaded as those
		*	obs are not in current memory

		**Test that observations have not been deleted from the report before readind
		* it. Deleted in a way that the report does not make sense. Provide an error
		* message to this that is more informative.

		preserve

			/***********************************************************************
			************************************************************************

				Section 1 - Set up locals needed in data

				Saving a version of the data to be used before merging corrections
				back to the original data set and before correcting duplicates

			************************************************************************
			***********************************************************************/

			*Tempfiles to be uses
			tempfile originalData preppedReport datawithreportmerged dataToReturn

			** Save a version of original data that can be brought
			*  back withing the preserve/restore section
			save 	`originalData'

			* Create a local with todays date to date the reports
			local  date = subinstr(c(current_date)," ","",.)

			*Put idvar in a local with a more descriptive name
			local idvar `varlist'

			** Making one macro with all variables that will be
			*  imported and exported from the Excel file
			local argumentVars `idvar' `uniquevars' `keepvars'

			* Create a list of the variables created by this command to put in the report
			local excelVars dupListID dateListed dateFixed correct drop newID initials notes

			* Test that nu variable with the name needed for the excel report already exist in the data set
			foreach excelvar of local excelVars {
				cap confirm variable `excelvar'
				if _rc == 0 {
					*Variable exist, output error
					noi display as error "{phang}The data set already have a variable called {inp:`excelvar'} which is a name that this command requires to be availible. Please change the name of the variable already in the data set. Future versions of this command will allow the user to rename the variables used by this command.{p_end}"
					noi di ""
					error 198
					exit
				}
			}

			/***********************************************************************
			************************************************************************

				Section 2 - Test unique vars

				Test the unique vars so that they are identifying the data set and
				are not in a time format that might get corrupted by exporting to
				and importing from Excel. This is needed as the uniquevars are needed
				to merge the correct correction to the correct duplicat

			************************************************************************
			***********************************************************************/

			*Test that the unique vars fully and uniquely identifies the data set
			cap isid `uniquevars'
			if _rc {

				noi display as error "{phang}The variable(s) listed in uniquevars() does not uniquely and fully identifies all observations in the data set.{p_end}"
				isid `uniquevars'
				error 198
				exit
			}

			** Test that no unique vars are time format. Time values might be corrupted
			*  and changed a tiny bit when importing and exporting to Excel, which make merge not possible
			foreach uniquevar of local uniquevars {

				*Test if format is time format
				local format : format `uniquevar'
				if substr("`format'",1,2) == "%t" {

					noi display as error `"{phang}The variable {inp:`uniquevar'} listed in {inp:uniquevars()} is using time format which is not allowed. Stata and Excel stores and displays time slightly differently which can lead to small changes in the value when the value is imported and exported between Stata and Excel, and then the variable can no longer be used to merge the report back to the original data. Use another variable or create a string variable out of your time variable using this code: {inp: generate `uniquevar'_str = string(`uniquevar',"%tc")}.{p_end}"'
					noi di ""
					error 198
					exit
				}
			}

			/***********************************************************************
			************************************************************************

				Section 3 - Test input from Excel file

				If Excel repirt exist, import it and test for invalid corrections
				made in the excel report.

			************************************************************************
			***********************************************************************/

			/******************
				Section 3.1
				Check if report exists from before.
			******************/

			*Check if file exist. Suffix option can be used to create multiple reports in the same folder
			cap confirm file "`folder'/iedupreport`suffix'.xlsx"

			if !_rc {
				local fileExists 1
			}
			else {
				local fileExists 0
			}

			/******************
				Section 3.2
				If report exists, load file and check input, otherwise skip to section 4
			******************/

			if `fileExists' {

				*Load excel file. Load all vars as string and use metadata from Section 1
				import excel "`folder'/iedupreport`suffix'.xlsx"	, clear firstrow

				** All excelVars but dupListID and newID should be string. dupListID
				*  should be numeric and the type of newID should be based on the user input
				foreach excelvar of local excelVars {
					if !inlist("`excelvar'", "dupListID", "newID") {

						* Make original ID var string
						tostring `excelvar' , replace
						replace  `excelvar' = "" if `excelvar' == "."
					}
				}

				/******************
					Section 3.2.1
					Make sure that the ID variable and the uniquevars are
					not changed since last report.
				******************/

				*Copy a list of all variables in the imported report to the local existingExcelVars
				ds
				local existingExcelVars  `r(varlist)'

				*Test that the ID variable is in the imported report
				if `:list idvar in existingExcelVars' == 0 {

					noi display as error "{phang}The ID variable `idvar' does not exist in the previously exported Excle file. If you renamed the ID variable you need to rename it manually in the Excel report or start a new Excel report by renaming or moving the original report, then run the command again and create a new file and manually copy any corrections from the old file to the new. If you changed the ID varaible you need to start with a new report.{p_end}"
					noi di ""
					error 111
					exit

				}

				*Test that the unique variables are in the imported report
				if `:list uniquevars in existingExcelVars' == 0 {

					noi display as error "{phang}One or more unique variables in [`uniquevars'] do not exist in the previously exported Excel file. If you renamed or changed any variable used in uniquevars(), you need to start over with a new file. Rename or move the already existing file. Create a new file and carefully copy any corrections from the old file to the new.{p_end}"
					noi di ""
					error 111
					exit
				}

				/******************
					Section 3.3
					Make sure input is correct
				******************/

				*Temporary variables needed for checking input
				tempvar multiInp inputNotYes maxMultiInp notDrop yesCorrect groupNumCorrect toDrop anyCorrection groupAnyCorrection

				*Locals indicating in which ways input is incorrect (if any)
				local local_multiInp 		0
				local local_multiCorr		0
				local local_inputNotYes		0
				local local_notDrop			0

				/******************
					Section 3.3.1
					Make sure input is yes or y for the correct and drop columns
				******************/

				* Make string input lower case and change "y" to "yes"
				replace correct = lower(correct)
				replace drop 	= lower(drop)
				replace correct = "yes" if correct 	== "y"
				replace drop 	= "yes" if drop 	== "y"

				*Check that variables are either empty or "yes"
				gen `inputNotYes' = !((correct  == "yes" | correct == "") & (drop  == "yes" | drop == ""))

				*Set local to 1 if error should be outputted
				cap assert `inputNotYes' == 0
				if _rc local local_inputNotYes 	1

				/******************
					Section 3.3.2
					Make sure there are not too many corrections for a single observation
				******************/

				* Count the number of corrections (correct drop newID) per
				* observation. Only one correction per observation is allowed.
				egen `multiInp' = rownonmiss(correct drop newID), strok

				*Check that all rows have at most one correction
				cap assert `multiInp' == 0 | `multiInp' == 1

				*Error will be outputted below
				if _rc local local_multiInp 	1

				/******************
					Section 3.3.3
					Test that maximum one duplicate per duplicate group is indicated as correct

				******************/

				*Generate dummy if correct column is set to yes
				gen `yesCorrect' = (correct == "yes")

				*Count number of duplicates within duplicates where that dummy is 1
				bys `idvar' : egen `groupNumCorrect' =  total(`yesCorrect')

				*Test if more than 1 duplicate in that duplicate group is correct
				count if `groupNumCorrect' > 1

				*Output error is more than one duplicate in a duplicate group is yes
				if `r(N)' != 0 local local_multiCorr 1

				/******************
					Section 3.3.4
					Make sure that either option droprest is specified, or that
					drop was correctly indicated for all observations. i.e.; if
					correct or newID was indicated for at least one duplicate in
					a duplicate group, then all other observations should be
					indicated as drop (unless droprest is specified)

				******************/

				*Generate dummy if there is any correction for this observation
				gen `anyCorrection' = !missing(correct) | !missing(newID)

				*Count number of observations with any correction in suplicates group
				bys `idvar' : egen `groupAnyCorrection' =  total(`anyCorrection')

				*Create dummy that indicates each place this error happens
				gen `notDrop' = (missing(drop) & `groupAnyCorrection' > 0 & `anyCorrection' == 0)

				* Check if option droprest is specified
				if "`droprest'" == "" {

					** If option droprest is not used, then all observations in a duplicate
					*  group where at least one observation has a correction must have a
					*  correction or have drop set to yes.
					cap assert `notDrop' == 0

					*Error will be outputted below
					if _rc local local_notDrop 	1

				}
				else {

					** Option -droprest- specified. Drop will be changed to yes
					*  for any observations without drop or any other correction
					*  explicitly specified if the observation is in a duplicate
					*  group with at least one observation has a correction
					replace drop 	= "yes" if `notDrop' == 1

				}

				/******************
					Section 3.4
					Throw errors if any of the tests were not passed
				******************/

				*Output errors if errors in the report were detected
				if `local_multiInp' == 1 | `local_inputNotYes' == 1 | `local_notDrop' == 1 | `local_multiCorr' == 1  {
					noi {
						di ""
						di ""
						di as error "{phang}{ul:The corrections made in the Excel report has the following errors:}{p_end}"
						di ""

						*Error multiple input
						if `local_multiInp' == 1 {
							display as error "{phang}The following observations have more than one correction. Only one correction (correct, drop or newID) per row is allowed{p_end}"
							list `idvar' dupListID correct drop newID `uniquevars' if `multiInp' > 1
							di ""
						}

						*Error multiple correct
						if `local_multiCorr' == 1 {
							display as error "{phang}The following observations are in a duplicate group where more than one observation is listed as correct. Only one observation per duplicate group can be correct{p_end}"
							list `idvar' dupListID correct drop newID `uniquevars' if `groupNumCorrect' > 1
							di ""
						}

						*Error in incorrect string
						if `local_inputNotYes' == 1 {
							display as error "{phang}The following observations have an answer in either correct or drop that is neither yes nor y{p_end}"
							list `idvar' dupListID correct drop `uniquevars' if `inputNotYes' == 1
							di ""
						}

						*Error is not specfied as drop
						if `local_notDrop' == 1 {
							display as error "{phang}The following observations are not explicitly indicated as drop while other duplicates in the same duplicate group are corrected. Either manually indicate as drop or see option droprest{p_end}"
							list `idvar' dupListID correct drop newID `uniquevars' if `notDrop' == 1
							di ""
						}

						*Same error for any incorrect input
						error 198
						exit
					}
				}

				/******************
					Section 3.5
					Save the prepared report to be used later
				******************/

				*Keep only the variables needed for matching and variables used for input in the Excel file
				keep 	`idvar' `uniquevars' `excelVars' `groupAnyCorrection'

				*Save imported data set with all corrections
				save	`preppedReport'
			}

			/***********************************************************************
			************************************************************************

				Section 4 - Merge report to original data

				Merge corrections with original data, test that there are no
				obs in report that are not in main data, and save this data
				in temp file

			************************************************************************
			***********************************************************************/

			*Re-load original data
			use `originalData', clear

			* Merge original data with imported Excel file (if Excel file exists)
			if `fileExists'  {

				*Create a tempvar for merging results
				tempvar iedup_merge

				*Merge the corrections with the data set
				merge 1:1 `uniquevars' using `preppedReport', generate(`iedup_merge')

				*Make sure that obsrevations listed in the duplicate report is still in the data set
				cap assert `iedup_merge' != 2

				*Display error message if assertion is not true and some duplicates in the Excel file are no longer in the data set
				if _rc {

					display as error "{phang}One or several observations in the Excel report are no longer found in the data set. Always run ieduplicates on the raw data set that include all the duplicates, both new duplicates and those you have already identified. After removing duplicates, save the data set using a different name. You might also recieve this error if you are using an old ieduplicates Excel report on a new data set.{p_end}"
					error 9
					exit
				}

				*Explicitly drop temporary variable. Temporary variables might
				*be exported to excel so delete explicitly before that. Only
				*using tempvar here to create a name with no conflicts
				drop `iedup_merge'

			}

			*save the data set to be used when correcting the data
			save `datawithreportmerged'

			/***********************************************************************
			************************************************************************

				Section 5 - Generate the Excel report

				Test if there are duplicates in ID var. If any duplicates exist,
				then update the Excel file with new and unaddressed cases

			************************************************************************
			***********************************************************************/

			/******************
				Section 5.1
				Test if there are any duplicates in ID var
			******************/

			* Generate variable that is not 0
			* if observation is a duplicate
			tempvar dup
			duplicates tag `idvar', gen(`dup')

			*Test if there are any duplicates
			cap assert `dup'==0
			if _rc {

				/******************
					Section 5.2
					Keep only duplicates for the report
				******************/

				*Keep if observation is part of duplicate group
				keep if `dup' != 0

				if `fileExists'  {
					* If Excel file exists keep excel vars and
					* variables passed as arguments in the
					* command
					keep 	`argumentVars' `excelVars'
				}
				else {
					* Keep only variables passed as arguments in
					* the command and the string ID var as no Excel file exists
					keep 	`argumentVars'

					*Generate the excel variables used for indicating correction
					foreach excelvar of local excelVars {

						*Create all variables apart from dupListID as string vars
						if inlist("`excelvar'", "dupListID", "newID") {
							gen `excelvar' = .
						}
						else {
							gen `excelvar' = ""
						}
					}
				}

				/******************
					Section 5.3
					Update the excel vars that are not updated manually
				******************/

				* Generate a local that is 1 if there are new duplicates
				local unaddressedNewExcel 0
				count if missing(dateFixed)
				if `r(N)' > 0 local unaddressedNewExcel 1

				/******************
					Section 5.3.1 Date variables
				******************/

				* Add date first time duplicate was identified
				replace dateListed 	= "`date'" if missing(dateListed)

				** Add today's date to variable dateFixed if dateFixed
				*  is empty and at least one correction is added
				replace dateFixed 	= "`date'" if missing(dateFixed) & (!missing(correct) | !missing(drop) | !missing(newID))

				/******************
					Section 5.3.2 Duplicate report list ID
				******************/

				** Sort after dupListID and after ID var for
				*  duplicates currently without dupListID
				sort dupListID `idvar'

				** Assign dupListID 1 to the top row if no duplicate
				*  list IDs have been generated so far.
				replace dupListID = 1 if _n == 1 & missing(dupListID)

				** Generate new IDs based on the row above instead of directly
				*  from the row number. That prevents duplicates in the list in
				*  case an observation is deleted. The first observation with
				*  missing value will have an ID that is one digit higher than
				*  the highest ID already in the list
				replace dupListID = dupListID[_n - 1] + 1 if missing(dupListID)

				/******************
					Section 5.4
					Keep and order the variables and output the Excel files
				******************/

				* If cases unaddressed then update the Excel file
				if `unaddressedNewExcel'  {

					keep 	`argumentVars' `excelVars'
					order	`idvar' `excelVars' `uniquevars' `keepvars'

					if "`daily'" == "" {

						*Returns 0 if folder does not exist, 1 if it does
						mata : st_numscalar("r(dirExist)", direxists("`folder'/Daily"))

						** If the daily folder is not created, just create it
						if `r(dirExist)' == 0  {

							*Create the folder since it does not exist
							mkdir "`folder'/Daily"
						}

						*Export the daily file
						cap export excel using "`folder'/Daily/iedupreport`suffix'_`date'.xlsx"	, firstrow(variables) replace nolabel

						*Print error if daily report cannot be saved
						if _rc {

							display as error "{phang}The Daily copy could not be saved to the `folder'/Daily folder. Make sure to close any old daily copy or see the option nodaily.{p_end}"
							error 603
							exit
						}

						*Prepare local for output
						local daily_output " and a daily copy have been saved to the Daily folder"
					}

					*Export main report
					export excel using "`folder'/iedupreport`suffix'.xlsx"	, firstrow(variables) replace  nolabel

					*Produce output
					noi di `"{phang}Excel file created at: {browse "`folder'/iedupreport`suffix'.xlsx":`folder'/iedupreport`suffix'.xlsx}`daily_output'.{p_end}"'
					noi di ""
				}
			}

		/***********************************************************************
		************************************************************************

			Section 6

			Update the data set and with the new corrections.

		************************************************************************
		***********************************************************************/

		* Load the original data set merged with correction. Duplicates
		* in all variables are already dropped in this data set
		use 	`datawithreportmerged', clear

		* If excel file exists, apply any corrections indicated (if any)
		if `fileExists' {

			/******************
				Section 6.1
				Drop duplicates listed for drop
			******************/

			drop if drop == "yes"

			/******************
				Section 6.2
				Update new ID. ID var can be either numeric or
				string. All numbers can be made strings but not
				all strings can be numeric. Therefore this
				section is complicated.
			******************/

			/******************
				Section 6.2.1
				ID var in original file is string. Either
				newID was imported as string or the variable
				is made string. Easy.
			******************/

			*Test if there are any corrections by new ID
			cap assert missing(newID)
			if _rc {

				local idtype 	: type `idvar'
				local idtypeNew : type newID

				*If ID var is string but newID is not, then just make it string
				if substr("`idtype'",1,3) == "str" & substr("`idtypeNew'",1,3) != "str" {

					tostring newID , replace
					replace  newID = "" if newID == "."
				}

				*If ID var is numeric but the newID is loaded as string
				else if substr("`idtype'",1,3) != "str" & substr("`idtypeNew'",1,3) == "str" {

					* Check if [tostringok] is specificed:
					if "`tostringok'" != "" {

						* Make original ID var string
						tostring `idvar' , replace
						replace  `idvar' = "" if `idvar' == "."

					}

					* Error, IDvar cannot be updated
					else {

						* Create a local with all non-numeric values
						levelsof newID if missing(real(newID)), local(NaN_values) clean

						* Output error message
						di as error "{phang}The ID varaible `idvar' is numeric but newID has thes non-numeric values: `NaN_values'. Update newID to only contain numeric values or see option tostringok.{p_end}"
						error 109
						exit
					}
				}

				*After making sure that type is ok, update the IDs
				replace `idvar' = newID if !missing(newID)

				/******************
					Section 6.3
					Test that values in newID
					were neither used twice
					nor already existed
				******************/

				*Test if there are any duplicates after corrections (excluding observations in duplicate groups not yet corrected)
				tempvar newDup
				duplicates tag `idvar' if `groupAnyCorrection' != 0, generate(`newDup')

				*Consider missing values as no new duplicates, make conditions below easier
				replace `newDup' = 0 if missing(`newDup')

				cap assert `newDup' == 0
				if _rc {

					levelsof `idvar' if `newDup' != 0  , local(newDuplist)
					di as error "{phang}No corrections from the report are applied as it would lead to new duplicates in the following value(s): `newDuplist'.{p_end}"
					error 198
					exit
				}
			}

			/******************
				Section 6.4
				Drop Excel vars
			******************/

			drop `excelVars'
		}

		/***********************************************************************
		************************************************************************

			Section 7

			Return the data set without duplicates and
			output information regarding unresolved duplicates.

		************************************************************************
		***********************************************************************/

		* Generate a variable that is 1 if the observation is a duplicate in varlist
		tempvar dropDup
		duplicates tag `idvar',  gen(`dropDup')
		* Generate a list of the IDs that are still duplicates
		levelsof `idvar' 			if `dropDup' != 0 , local(dup_ids) clean
		* Drop the duplicates (they are exported in Excel)
		drop 						if `dropDup' != 0

		* Test if varlist is now uniquely and fully identifying the data set
		cap isid `idvar'
		if _rc {

			di as error "{phang}The data set is not returned with `idvar' uniquely and fully identifying the data set. Please report this bug to kbjarkefur@worldbank.org{p_end}"
			error 119
			exit

		}

		*Count number of duplicate groups still in the data
		local numDup	= `:list sizeof dup_ids'

		if `numDup' == 0 {
			noi di	"{phang}There are no unresolved duplicates in this data set. The data set is returned with `idvar' uniquely and fully identifying the data set.{p_end}"
		}
		else {
			noi di	"{phang}There are `numDup' duplicated IDs still unresolved. IDs still containing duplicates: `dup_ids'. The unresolved duplicate observations were exported in the Excel file. The data set is returned without those duplicates and with `idvar' uniquely and fully identifying the data set.{p_end}"
		}

		return scalar numDup	= `numDup'

		/***********************************************************************
		************************************************************************

			Section 8

			Save data set to be returned outside preserve/restore.
			Preserve/restore is used so that original data is returned
			in case an error is thrown.

		************************************************************************
		***********************************************************************/

		save 	`dataToReturn'

		restore

		** Using restore above to return the data to
		*  the orignal data set in case of error.
		use `dataToReturn', clear

	}
end


fix returned local/scalar #124

fix returned local/scalar #124

*! version 5.4 15DEC2017  DIME Analytics lcardosodeandrad@worldbank.org

capture program drop ietoolkit, rclass
program ietoolkit, rclass

	* UPDATE THESE LOCALS FOR EACH NEW VERSION PUBLISHED
	local version "5.4"
	local versionDate "15DEC2017"

	syntax [anything]

	/**********************
		Error messages
	**********************/

	* Make sure that no arguments were passed
	if "`anything'" != "" {
		noi di as error "This command does not take any arguments, write only {it:ietoolkit}"
		error 198
	}

	/**********************
		Output
	**********************/

	* Prepare returned locals
	return local 	versiondate "`versionDate'"
	return scalar 	version		= `version'

	* Display output
	noi di ""
	noi di _col(4) "This version of ietoolkit installed is version " _col(54)"`version'"
	noi di _col(4) "This version of ietoolkit was released on " _col(54)"`versionDate'"

end

*! version 5.4 15DEC2017  DIME Analytics lcardosodeandrad@worldbank.org

capture program drop ietoolkit, rclass
program ietoolkit, rclass

	* UPDATE THESE LOCALS FOR EACH NEW VERSION PUBLISHED
	local version "5.4"
	local versionDate "15DEC2017"

	syntax [anything]

	/**********************
		Error messages
	**********************/

	* Make sure that no arguments were passed
	if "`anything'" != "" {
		noi di as error "This command does not take any arguments, write only {it:ietoolkit}"
		error 198
	}

	/**********************
		Output
	**********************/

	* Prepare returned locals
	return local diffvars 	`"`difference'"'
	return scalar nummatch	= `:list sizeof match'

	* Display output
	noi di ""
	noi di _col(4) "This version of ietoolkit installed is version " _col(54)"`version'"
	noi di _col(4) "This version of ietoolkit was released on " _col(54)"`versionDate'"

end


remove legacy code

remove legacy code

clear
clear matrix
cd "C:\Users\Anna Goodman\Dropbox\GitHub"
*cd "C:\Users\annag\Dropbox\npct"

global geography = "msoa" // "msoa" or "lsoa"

* create directories by purpose/geography in the path_temp_scenario file [when move this to R, do that in R]

	/***********************
	* PREPARE GEOG LOOK UP FILES
	***********************
		import delimited "pct-inputs\02_intermediate\x_temporary_files\unzip\OA11_LSOA11_MSOA11_LAD11_EW_LUv2.csv", varnames(1) clear
		drop oa11cd
		duplicates drop
		saveold "pct-inputs\02_intermediate\x_temporary_files\scenario_building\LSOA11_MSOA11_LAD11_EW_LUv2.dta", replace

		foreach x in msoa lsoa {
		use "pct-inputs\02_intermediate\x_temporary_files\scenario_building\LSOA11_MSOA11_LAD11_EW_LUv2.dta", clear
		rename lso11anm lsoa11nm
		rename `x'11cd geo_code
		rename `x'11nm geo_name
		rename lad11cd lad11cd
		rename lad11nm lad_name
		keep geo_code geo_name lad11cd lad_name
		duplicates drop
		saveold "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\\`x'\geo_code_lookup.dta", replace
		}

	***********************
	* PREPARE MSOA AND LSOA TO HAVE SAME STARTING FORMATS (INPUT 1)
	***********************
	** LSOA
	*	import delimited "pct-inputs\02_intermediate\x_temporary_files\unzip\WM12EW[CT0489]_lsoa.csv", clear
use "C:\Users\Anna Goodman\Dropbox\1 - Phys Act_1-PA main\2015_PCT_largefiles\1a_dataoriginal\Census2011\Flow_Level\LSOA_Age_Sex_Safeguarded\WM12EW[CT0489]_lsoa.dta" , clear
		* RENAME
			rename areaofusualresidence geo_code_o
			rename areaofworkplace geo_code_d
			rename allmethods_allsexes_age16plus all
			rename workathome_allsexes_age16plus from_home
			rename underground_allsexes_age16plus light_rail
			rename train_allsexes_age16plus train
			rename bus_allsexes_age16plus bus
			rename taxi_allsexes_age16plus taxi
			rename motorcycle_allsexes_age16plus motorbike
			rename carorvan_allsexes_age16plus car_driver
			rename passenger_allsexes_age16plus car_passenger
			rename bicycle_allsexes_age16plus bicycle
			rename onfoot_allsexes_age16plus foot
			rename othermethod_allsexes_age16plus other
			rename allmethods_male_age16plus all_male
			rename allmethods_female_age16plus all_female
			rename bicycle_male_age16plus bicycle_male
			rename bicycle_female_age16plus bicycle_female
			order geo_code_o geo_code_d all from_home light_rail train bus taxi motorbike car_driver car_passenger bicycle foot other /*
				*/ all_male all_female bicycle_male bicycle_female
			keep geo_code_o - bicycle_female
		* COLLAPSE OTHER (CONSISTENT LSOA-MSOA)
			gen geo_code_dtemp=substr(geo_code_d,1,1)
			replace geo_code_d="OutsideEW" if geo_code_dtemp=="S" | geo_code_dtemp=="N" | geo_code_d=="OD0000002" | geo_code_d=="OD0000004"
				* Make 'OutsideEW' if S = scotland, N = northern ireland, OD0000002 = offshore installation, OD0000004 = otherwise overseas
			drop geo_code_dtemp
			foreach var of varlist all-other all_male-bicycle_female {
			bysort geo_code_o geo_code_d: egen `var'temp=sum(`var')
			replace `var'=`var'temp
			drop `var'temp
			}
			duplicates drop
		* SAVE
			compress
			saveold "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\lsoa\ODpairs_starting.dta", replace

	** PREPARE SEX DATA FOR MSOA
		* MERGE IN MSOA CODES
			use "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\lsoa\ODpairs_starting.dta", clear
			drop all - other
			rename geo_code_o lsoa11cd
			merge m:1 lsoa11cd using "pct-inputs\02_intermediate\x_temporary_files\scenario_building\LSOA11_MSOA11_LAD11_EW_LUv2.dta", keepus(msoa11cd) nogen
			rename msoa11cd geo_code_o
			drop lsoa11cd
			rename geo_code_d lsoa11cd
			merge m:1 lsoa11cd using "pct-inputs\02_intermediate\x_temporary_files\scenario_building\LSOA11_MSOA11_LAD11_EW_LUv2.dta", keepus(msoa11cd) nogen
			rename msoa11cd geo_code_d
			replace geo_code_d=lsoa11cd if geo_code_d==""
			drop lsoa11cd
		* MAKE MSOA AVERAGES
			foreach var of varlist all_male-bicycle_female {
			bysort geo_code_o geo_code_d: egen `var'temp=sum(`var')
			replace `var'=`var'temp
			drop `var'temp
			}
			order geo_code_o geo_code_d all_male all_female bicycle_male bicycle_female
			duplicates drop
			compress
			saveold "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\msoa\t2w_sex.dta", replace

	** MSOA
		import delimited "pct-inputs\02_intermediate\x_temporary_files\unzip\wu03ew_V2.csv", clear
		* RENAME
			rename areaofresidence geo_code_o
			rename areaofworkplace geo_code_d
			rename allcategoriesmethodoftraveltowor all
			rename workmainlyatorfromhome from_home
			rename undergroundmetrolightrailtram light_rail
			rename busminibusorcoach bus
			rename motorcyclescooterormoped motorbike
			rename drivingacarorvan car_driver
			rename passengerinacarorvan car_passenger
			rename onfoot foot
			rename othermethodoftraveltowork other
		* COLLAPSE OTHER (CONSISTENT LSOA-MSOA)
			gen geo_code_dtemp=substr(geo_code_d,1,1)
			replace geo_code_d="OutsideEW" if geo_code_dtemp=="S" | geo_code_dtemp=="N" | geo_code_d=="OD0000002" | geo_code_d=="OD0000004"
				* Make 'OutsideEW' if S = scotland, N = northern ireland, OD0000002 = offshore installation, OD0000004 = otherwise overseas
			**replace geo_code_d="OutsideEW" if geo_code_dtemp=="S" | geo_code_dtemp=="0" | geo_code_dtemp=="1" | geo_code_dtemp=="2" | geo_code_dtemp=="3" | geo_code_dtemp=="4" | geo_code_dtemp=="5" | geo_code_dtemp=="9" | geo_code_d=="OD0000002"
				* previous wu03bew_msoa_v1 safeguarded: Make 'OutsideEW' if S = scotland, 01-59 = foreign countries, 9xx = Northern Ireland, OD0000002 = offshore installation,
			drop geo_code_dtemp
			foreach var of varlist all-other {
			bysort geo_code_o geo_code_d: egen `var'temp=sum(`var')
			replace `var'=`var'temp
			drop `var'temp
			}
			duplicates drop
		* MERGE IN BY SEX
			merge 1:1 geo_code_o geo_code_d using "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\msoa\t2w_sex.dta", nogen
			count if ( all_male+ all_female)!=all // CHECK ZERO
		* SAVE
			compress
			saveold "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\msoa\ODpairs_starting.dta", replace

	**********************************
	* PREPARE HEAT MORTALITY RATE FILE (INPUT 2 - PRE-PROCESSED IN EXCEL)
	**********************************
		foreach x in msoa lsoa {
		import excel "pct-inputs\01_raw\03_health_data\commute\LA_WeightedMortality16-74_2014.xls", sheet("LA_Cyclists_MortRate") firstrow case(lower) clear
		keep if geographycode!=""
		keep geography geographycode mortrate_govtarget mortrate_gendereq mortrate_dutch
		* FIX PLACES WITH SMALL AREAS MERGED
			gen temp=1
			recode temp 1=2 if geography=="Cornwall,Isles of Scilly"
			recode temp 1=2 if geography=="Westminster,City of London"
			expand temp
			bysort geography: gen littlen=_n
			replace geographycode="E06000053" if geographycode=="E06000052" & littlen==2 	// ISLES OF SCILLY, NOT CORNWALL
			replace geography="Cornwall" if geographycode=="E06000052"
			replace geography="Isles of Scilly" if geographycode=="E06000053"
			replace geographycode="E09000001" if geographycode=="E09000033" & littlen==2	// CITY OF LONDON, NOT WESTMINSTER
			replace geography="Westminster" if geographycode=="E09000033"
			replace geography="City of London" if geographycode=="E09000001"
		* FIX PLACES WITH DIFFERENT CODES (2014 vs 2011 LA codes)
			replace geographycode="E07000097" if geographycode=="E07000242"
			replace geographycode="E08000020" if geographycode=="E08000037"
			replace geographycode="E06000048" if geographycode=="E06000057"
			replace geographycode="E07000100" if geographycode=="E07000240"
			replace geographycode="E07000101" if geographycode=="E07000243"
			replace geographycode="E07000104" if geographycode=="E07000241"
		rename geographycode lad11cd
		* SAVE
		merge 1:m lad11cd using "pct-inputs\02_intermediate\x_temporary_files\scenario_building\LSOA11_MSOA11_LAD11_EW_LUv2.dta" , keepus(`x'11cd lad11nm) nogen
		tab geography if geography!=lad11nm	// CHECK NOTHING
		rename `x'11cd geo_code
		order geo_code mortrate_govtarget mortrate_gendereq mortrate_dutch
		keep geo_code - mortrate_dutch
		compress
		duplicates drop
		saveold "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\\`x'\mortrate.dta", replace
		}
	*/
	**********************************
	* PREPARE CYCLE STREETS DATA - SAVE AS STATA (INPUT 3)
	**********************************
		import delimited "pct-inputs\02_intermediate\02_travel_data\commute\\$geography\\rfrq_all_data.csv", clear
		saveold "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\\$geography\rfrq_all_data.dta", replace

	*****************
	** MERGE OD DATA
	*****************
		* OPEN AND LIMIT TO COMMUTERS
			use "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\\$geography\ODpairs_starting.dta", clear
			drop if geo_code_d=="OD0000001"
				* drop if work mainly at home

		* MERGE IN MORT RATES (DATASETS D2)
			rename geo_code_o geo_code
			merge m:1 geo_code using "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\\$geography\mortrate.dta", nogen
			rename geo_code geo_code_o

		* MERGE IN CYCLE STREETS VARIABLES (DATASET D3)
			rename geo_code_o geo_code1
			rename geo_code_d geo_code2
			merge 1:1 geo_code1 geo_code2 using "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\\$geography\rfrq_all_data.dta", keepus(e_dist_km rf_dist_km rf_avslope_perc)
			drop if _merge==2
			drop _merge
			rename geo_code1 geocodetemp
			rename geo_code2 geo_code1
			rename geocodetemp geo_code2
			merge 1:1 geo_code1 geo_code2 using "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\\$geography\rfrq_all_data.dta", keepus(e_dist_km rf_dist_km rf_avslope_perc) update
			drop if _merge==2
			drop _merge
			rename geo_code1 geo_code_d
			rename geo_code2 geo_code_o

	*****************
	** STEP 1: DEFINE FLOW TYPE AND COLLAPSE TYPE 4 FLOWS TO 'OTHER'
	*****************
		* GENERATE FLOW TYPE
			gen flowtype=.
			gen geo_code_dtemp=substr(geo_code_d,1,1)
			recode flowtype .=1 if (geo_code_dtemp=="E" | geo_code_dtemp=="W") & rf_dist_km!=. & geo_code_o!=geo_code_d
			recode flowtype .=2 if geo_code_o==geo_code_d
			recode flowtype .=3 if geo_code_d=="OD0000003"
			recode flowtype .=4 if (geo_code_dtemp=="E" | geo_code_dtemp=="W") & rf_dist_km==.
			recode flowtype .=4 if geo_code_d=="OutsideEW"
			label def flowtypelab 1 "England/Wales between-zone" 2 "Within zone" 3 "No fixed place" 4 "Over max dist or outside England/Wales, offshore installation", modify
			label val flowtype flowtypelab
			drop geo_code_dtemp

		* COLLAPSE 'OTHER' FLOW TYPES
			replace geo_code_d="Other" if flowtype==4
			foreach var of varlist all- other all_male-bicycle_female {
			bysort geo_code_o geo_code_d: egen `var'temp=sum(`var')
			replace `var'=`var'temp
			drop `var'temp
			}
			duplicates drop

	*****************
	** STEP 2: ASSIGN VALUES IF DISTANCE/HILLINESS UNKNOWN
	*****************
		** ASSIGN ESTIMATED HILLINESS + DISTANCE VALUES IF START AND END IN SAME PLACE
			by geo_code_o (rf_dist_km), sort: gen littlen=_n
			foreach x in rf_dist_km rf_avslope_perc {
			gen `x'temp=`x'
			replace `x'temp=. if littlen>3
			bysort geo_code_o: egen `x'temp2=mean(`x'temp)
			replace `x'=`x'temp2 if flowtype==2 & `x'==.
			}
			replace rf_dist_km=rf_dist_km/3 if flowtype==2
				* DISTANCE = A THIRD OF THE MEAN DISTANCE OF SHORTEST 3 FLOWS
				* HILLINESS = MEAN HILLINESS OF SHORTEST 3 FLOWS
			recode rf_dist_km .=0.79 if (geo_code_o=="E02006781" | geo_code_o=="E01019077") & geo_code_o==geo_code_d 		//ISLES OF SCILLY [msoa/lsoa] - ESTIMATED FROM DISTANCE DISTRIBUTION
			recode rf_avslope_perc .=0.2 if (geo_code_o=="E02006781" | geo_code_o=="E01019077") & geo_code_o==geo_code_d //ISLES OF SCILLY [msoa/lsoa] - ESTIMATED FROM CYCLE STREET
			replace e_dist_km=0 if flowtype==2
			drop littlen rf_dist_kmtemp rf_dist_kmtemp2 rf_avslope_perctemp rf_avslope_perctemp2

		** ASSIGN DISTANCE *AMONG CYCLISTS* VALUES IF NO FIXED PLACE : MEAN DIST AMONG CYCLISTS TRAVELLING <15KM
			gen cyc_dist_km=rf_dist_km
			foreach x in 15 30 {
			gen rf_dist_km`x'=rf_dist_km
			replace rf_dist_km`x'=. if rf_dist_km>`x' | flowtype>2
			gen bicycle`x'=bicycle
			replace bicycle`x'=. if rf_dist_km`x'==.
			}
			bysort geo_code_o: egen numnrf_dist_km15=sum(rf_dist_km15*bicycle15)
			bysort geo_code_o: egen denrf_dist_km15=sum(bicycle15)
			gen meanrf_dist_km15=numnrf_dist_km15/denrf_dist_km15
			replace cyc_dist_km=meanrf_dist_km15 if flowtype==3

		** ASSIGN DISTANCE *AMONG CYCLISTS* VALUES IF OUTSIDE ENG/WALES OR >30KM: MEAN DIST AMONG CYCLISTS TRAVELLING <30KM
			egen numnrf_dist_km30=sum(rf_dist_km30*bicycle30)
			egen denrf_dist_km30=sum(bicycle30)
			gen meanrf_dist_km30=numnrf_dist_km30/denrf_dist_km30
			replace cyc_dist_km=meanrf_dist_km30 if flowtype==4
			replace cyc_dist_km=meanrf_dist_km30 if flowtype==3 & cyc_dist_km==.	// USE 30KM if NO LOCAL CYCLIST GOING <15KM

	** SAVE RELEVANT VARIABLES (USE THIS TO FIT INDIVIDUAL MODEL)
		order geo_code_o geo_code_d /*
			*/ all-other flowtype all_male- bicycle_female mortrate_govtarget mortrate_gendereq mortrate_dutch /*
			*/ e_dist_km rf_dist_km rf_avslope_perc cyc_dist_km
		keep geo_code_o-cyc_dist_km
		compress
		saveold "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\\$geography\ODpairs_process2.1.dta", replace
		*use "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\\$geography\ODpairs_process2.1.dta", clear

	*****************
	** STEP 3A: CALCULATE PROPENSITY TO CYCLE [model derived from MSOA trips]
	*****************
		** MODEL FITTING FOR TRIPS <30KM
			* INPUT PARAMETERS
				gen rf_dist_kmsq=rf_dist_km^2
				gen rf_dist_kmsqrt=sqrt(rf_dist_km)
				gen ned_rf_avslope_perc=rf_avslope_perc-0.97
				gen interact=rf_dist_km*ned_rf_avslope_perc
				gen interactsqrt=rf_dist_kmsqrt*ned_rf_avslope_perc

			* FIT REGRESSION EQUATION
				gen pred_base= /*
					*/ -3.959 + (-0.5963 * rf_dist_km) + (1.866 * rf_dist_kmsqrt) + (0.008050 * rf_dist_kmsq) + (-0.2710 * ned_rf_avslope_perc) + (0.009394 * rf_dist_km*ned_rf_avslope_perc) + (-0.05135 * rf_dist_kmsqrt*ned_rf_avslope_perc)
				gen bdutch = 2.523+(-0.07626*rf_dist_km)					// FROM DUTCH NTS
				replace bdutch=. if flowtype==3
				gen bebike= (0.05710*rf_dist_km)+(-0.0001087*rf_dist_kmsq)	// PARAMETERISED FROM DUTCH TRAVEL SURVEY, ASSUMING NO DIFFERENCE AT 0 DISTANCE
				replace bebike=bebike+(0.1812 *ned_rf_avslope_perc)	// SWISS TRAVEL SURVEY

				gen pred_dutch= pred_base + bdutch
				gen pred_ebike= pred_dutch + bebike
				foreach x in base dutch ebike {
				replace pred_`x'=exp(pred_`x')/(1+exp(pred_`x'))
				}

		** MODEL FITTING FOR TRIPS WITH NO FIXED PLACE
			* INPUT PARAMETERS
				foreach x in pred_base bdutch bebike { // WEIGHTED AVERAGE OF COMMUTERS IN FLOWTYPE 1 & "
				gen `x'temp=`x'
				replace `x'temp=. if flowtype!=1 & flowtype!=2	// WEIGHTED AVERAGE OF COMMUTERS IN FLOWTYPE 1 & "
				gen alltemp=all
				replace alltemp=. if `x'temp==.
				bysort geo_code_o: egen nummean`x'=sum(`x'temp*alltemp)
				bysort geo_code_o: egen denmean`x'=sum(alltemp)
				gen mean`x'=nummean`x'/denmean`x'
				drop `x'temp alltemp nummean`x' denmean`x'
				}
				gen meanpred_basesq=meanpred_base^2
				gen meanpred_basesqrt=meanpred_base^0.5

			* FIT REGRESSION EQUATION
				gen pred2_base= -6.399 + (184.0 * meanpred_basesq) + (10.36 * meanpred_basesqrt)
				gen pred2_dutch= pred2_base + meanbdutch
				gen pred2_ebike= pred2_dutch + meanbebike
				foreach x in base dutch ebike {
				replace pred2_`x'=exp(pred2_`x')/(1+exp(pred2_`x'))
				replace pred_`x'=pred2_`x' if flowtype==3
				drop pred2_`x'
				}

		** DROP INTERMEDIARY VARIABLES
			drop rf_dist_kmsq rf_dist_kmsqrt ned_rf_avslope_perc interact interactsqrt meanpred_base bdutch bebike meanbdutch meanbebike meanpred_basesq meanpred_basesqrt

	*****************
	** PART 3B: APPLY SCENARIOS TO OD DATA
	*****************
		** CALCULATE NO. CYCLISTS IN EACH SCENARIO
			gen nocyclists_slc=0
			gen nocyclists_sic=nocyclists_slc-bicycle

			gen govtarget_slc=bicycle+(pred_base*all)
			replace govtarget_slc=all if govtarget_slc>all & govtarget_slc!=. // MAXIMUM PERCENT CYCLISTS IS 100%
			gen govtarget_sic=govtarget_slc-bicycle
			order govtarget_slc, before(govtarget_sic)

			gen gendereq_slc=(bicycle_male*(1 + (all_female/all_male)))
			replace gendereq_slc=all if gendereq_slc>all & gendereq_slc!=. // [not needed] MAXIMUM PERCENT CYCLISTS IS 100%
			*tab all if female==0 | male==0
			replace gendereq_slc=bicycle if all_female==0 	// [not needed] NO CHANGE IF NO FEMALES IN FLOW
			replace gendereq_slc=bicycle if all_male==0 	// NO CHANGE IF NO MALES IN FLOW
			replace gendereq_slc=bicycle if gendereq_slc<bicycle 	// NO CHANGE IF SLC < BASELINE
			gen gendereq_sic=gendereq_slc-bicycle

			foreach x in dutch ebike {
			gen `x'_slc=pred_`x'*all
			replace `x'_slc=all if `x'_slc>all & `x'_slc!=. // MAXIMUM PERCENT CYCLISTS IS 100%
			*tab all if `x'_slc<bicycle
			replace `x'_slc=bicycle if `x'_slc<bicycle 		 // MINIMUM NO. CYCLISTS IS BASELINE
			gen `x'_sic=`x'_slc-bicycle
			}
			foreach x in govtarget gendereq dutch ebike {
			replace `x'_slc=bicycle if geo_code_d=="Other"	// NO INCREASE AMONG FLOWS OUT OF SCOPE AS TOO LONG/OutsideEW ETC
			replace `x'_sic=0 if geo_code_d=="Other"
			}

		** CALCULATE % NON-CYCLISTS MADE CYCLISTS IN EACH SCENARIO: TURN THAT % AWAY FROM WALKING
			foreach x in nocyclists {
			gen pchange_`x'=(all-`x'_slc)/(all-bicycle)
				/*
					gen pcycle=bicycle/all
					gen pfoot=foot/all
					gen pcar_driver=car_driver/all
					sum pfoot pcar_driver pcycle if pcycle>0.5 & pcycle<1 [fw=all]
					disp 0.131/(1-0.5826)		// 31% NON-CYCLE FLOWS WALKING IN HIGH CYCLE AREAS
					disp 0.147/(1-0.5826)	// 35% NON-CYCLE FLOWS DRIVING IN HIGH CYCLE AREAS
					*/
			gen `x'_slw=foot*pchange_`x'					// most flows - scale walking according to %change
			replace `x'_slw=((all-`x'_slc)*0.31) if bicycle==all	// Flows with pure bicycles at baseline - make walking 13% of new flows
			gen `x'_siw=`x'_slw-foot
			gen `x'_sld=car_driver*pchange_`x'
			replace `x'_sld=((all-`x'_slc)*0.35) if bicycle==all	// Flows with pure bicycles at baseline - make driving 44% of new flows
			gen `x'_sid=`x'_sld-car_driver
			order `x'_slw `x'_siw `x'_sld `x'_sid, after(`x'_sic)
			}

			foreach x in govtarget gendereq dutch ebike {
			gen pchange_`x'=(all-`x'_slc)/(all-bicycle) 	// % change in non-cycle modes
			recode pchange_`x' .=1 if all==bicycle 			// make 1 (i.e. no change) if everyone in the flow cycles
			gen `x'_slw=foot*pchange_`x'
			gen `x'_siw=`x'_slw-foot
			gen `x'_sld=car_driver*pchange_`x'
			gen `x'_sid=`x'_sld-car_driver
			order `x'_slw `x'_siw `x'_sld `x'_sid, after(`x'_sic)
			}

		** DROP INTERMEDIARY VARIABLES
			compress
			drop pred_base pred_dutch pred_ebike
			drop pchange_nocyclists pchange_govtarget pchange_gendereq pchange_dutch pchange_ebike

	*****************
	** STEP 4: DO HEAT
	*****************
		* INPUT PARAMETERS
			gen cyclecommute_tripspertypicalweek = 7.17
			gen cspeed = 14
			gen wspeed = 4.8
			gen ebikespeed = 15.8
			gen ebikemetreduction = 0.648
			recode cyc_dist_km min/4.9999999=.06 5/9.9999999=.11 10/19.999999=.17 20/max=.23, gen(percentebike_dutch)
			recode cyc_dist_km min/4.9999999=.71 5/19.9999999=.92 20/max=1, gen(percentebike_ebike)
			gen crr_heat=0.9
			gen cdur_ref_heat=100
			gen wrr_heat=0.89
			gen wdur_ref_heat=168
			gen mortrate_nocyclists=mortrate_govtarget
			gen mortrate_ebike=mortrate_dutch
			gen vsl=1855315		// VALUE IN POUNDS

		* DURATION OF CYCLING/WALKING
			gen cdur_obs = 60*((cyc_dist_km*cyclecommute_tripspertypicalweek)/cspeed) // TIME CYCLING PER WEEK IN MINUTES AMONG NEW CYCLISTS
			gen cdur_obs_dutch=((1-percentebike_dutch)*cdur_obs)+(percentebike_dutch*cdur_obs*ebikemetreduction*(cspeed/ebikespeed))
			gen cdur_obs_ebike=((1-percentebike_ebike)*cdur_obs)+(percentebike_ebike*cdur_obs*ebikemetreduction*(cspeed/ebikespeed))

			gen wdur_obs = 60*((cyc_dist_km*cyclecommute_tripspertypicalweek)/wspeed) // TIME WALKING PER WEEK IN MINUTES AMONG THOSE NOW SWITCHING TO CYCLING
		*	drop cyclecommute_tripspertypicalweek cspeed wspeed ebiketimereduction ebikemetreduction percentebike_dutch percentebike_ebike
		*	compress

		* MORTALITY PROTECTION
			gen cprotection_govtarget_heat= (1-crr_heat)*(cdur_obs/cdur_ref_heat)	// SCALE RR DEPENDING ON HOW DURATION IN THIS POP COMPARES TO REF
			gen cprotection_nocyclists_heat=cprotection_govtarget_heat
			gen cprotection_gendereq_heat=cprotection_govtarget_heat
			gen cprotection_dutch_heat= (1-crr_heat)*(cdur_obs_dutch/cdur_ref_heat)
			gen cprotection_ebike_heat= (1-crr_heat)*(cdur_obs_ebike/cdur_ref_heat)
			foreach x in nocyclists govtarget gendereq dutch ebike {
			recode cprotection_`x'_heat 0.45/max=0.45
			}
			gen wprotection_heat= (1-wrr_heat)*(wdur_obs/wdur_ref_heat)
			recode wprotection_heat 0.30/max=0.30

		* DEATHS AND VALUES
			foreach x in nocyclists govtarget gendereq dutch ebike {
			gen `x'_sic_death_heat=`x'_sic*mortrate_`x'*cprotection_`x'_heat*-1
			gen `x'_siw_death_heat=`x'_siw*mortrate_`x'*wprotection_heat*-1
			gen `x'_sideath_heat=`x'_sic_death_heat+`x'_siw_death_heat
			gen long `x'_sivalue_heat=`x'_sideath_heat*vsl*-1
			drop `x'_sic_death_heat `x'_siw_death_heat
			}
			gen base_sldeath_heat=-1*nocyclists_sideath_heat	// BASELINE LEVEL IS INVERSE OF 'NO CYCLISTS' SCENARIO INCREASE
			gen base_slvalue_heat=-1*nocyclists_sivalue_heat
			foreach x in govtarget gendereq dutch ebike {
			gen `x'_sldeath_heat=`x'_sideath_heat+base_sldeath_heat
			gen long `x'_slvalue_heat=`x'_sivalue_heat+base_slvalue_heat
			order `x'_sideath_heat `x'_sivalue_heat, after(`x'_slvalue_heat)
			}

		* DROP INTERMEDIARY VARIABLES
			drop mortrate_govtarget mortrate_gendereq mortrate_dutch cyclecommute_tripspertypicalweek - wprotection_heat
			drop nocyclists_sideath_heat nocyclists_sivalue_heat

	*****************
	** STEP 5: DO CO2 EMISSIONS CALCS
	*****************
		gen cyclecommute_tripsperweek=5.24
		gen co2kg_km=0.186
		foreach x in nocyclists govtarget gendereq dutch ebike {
		gen long `x'_sicartrips=`x'_sid * cyclecommute_tripsperweek * 52.2	// NO CYCLISTS * COMMUTE PER DAY
		gen long `x'_sico2=`x'_sid * cyc_dist_km * cyclecommute_tripsperweek * 52.2 * co2kg_km 	// NO CYCLISTS * DIST * COMMUTE PER DAY * CO2 EMISSIONS FACTOR
		}
		gen base_slco2=-1*nocyclists_sico2	// BASELINE LEVEL IS INVERSE OF 'NO CYCLISTS' SCENARIO INCREASE
		foreach x in govtarget gendereq dutch ebike {
		gen long `x'_slco2=`x'_sico2+base_slco2
		order `x'_sicartrips `x'_slco2 , before(`x'_sico2)
		}
		drop nocyclists* cyclecommute_tripsperweek co2kg_km cyc_dist_km

	*****************
	** FINISH: MAKE BIDIRECTIONAL OD AND SAVE TEMPORARY DATASET, PRE-AGGREGATION
	*****************
		gen osub1=substr(geo_code_o,1,1)
		gen dsub1=substr(geo_code_d,1,1)
		gen osub2=substr(geo_code_o,2,.)
		gen dsub2=substr(geo_code_d,2,.)
		destring osub2 dsub2, replace force
		gen homefirst=.
		recode homefirst .=1 if (osub1=="E" & dsub1!="E")
		recode homefirst .=1 if (osub1=="W" & dsub1!="E" & dsub1!="W")
		recode homefirst .=0 if (osub1=="W" & dsub1=="E")
		recode homefirst .=1 if (osub1=="E" & dsub1=="E" & osub2<= dsub2)
		recode homefirst .=1 if (osub1=="W" & dsub1=="W" & osub2<= dsub2)
		recode homefirst .=0 if (osub1=="E" & dsub1=="E" & osub2>dsub2)
		recode homefirst .=0 if (osub1=="W" & dsub1=="W" & osub2>dsub2)
		gen geo_code1=geo_code_o
		replace geo_code1=geo_code_d if homefirst==0
		gen geo_code2=geo_code_d
		replace geo_code2=geo_code_o if homefirst==0
		drop osub1 dsub1 osub2 dsub2 homefirst

		order geo_code1 geo_code2
		compress
		saveold "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\\$geography\ODpairs_process2.5.dta", replace

	*****************
	** PART 3A.1: AGGREGATE TO ZONE LEVEL
	*****************
		use "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\\$geography\ODpairs_process2.5.dta", clear
		* AGGREGATE UP AREA FIGURES
			foreach var of varlist all- other govtarget_slc- ebike_sico2 {
			bysort geo_code_o: egen a_`var'=sum(`var')
			}
		* PERCENT TRIPS AND TRIP HILLINESS
			recode rf_dist_km min/9.9999=1 10/max=0, gen(rf_u10km_dist)
			recode rf_u10km_dist .=0 if geo_code_d=="Other"
				* NB keep as missing if no fixed work place - implicitly assume they have same distribution as everyone else. This exclusion is comparable to what ONS do ()
			gen all_u10km_dist=all
			replace all_u10km_dist=. if rf_u10km_dist==.
			bysort geo_code_o: egen rf_u10km_dist_numerator=sum(rf_u10km_dist*all_u10km_dist)
			bysort geo_code_o: egen rf_u10km_dist_denominator=sum(all_u10km_dist)
			gen a_perc_rf_dist_u10km = 100*rf_u10km_dist_numerator/rf_u10km_dist_denominator

			gen rf_u10km_avslope_perc=rf_avslope_perc
			replace rf_u10km_avslope_perc=. if rf_u10km_dist!=1
			gen all_u10km_avslope=all
			replace all_u10km_avslope=. if rf_u10km_avslope==.
			bysort geo_code_o: egen rf_u10km_avslope_numerator=sum(rf_u10km_avslope_perc*all_u10km_avslope)
			bysort geo_code_o: egen rf_u10km_avslope_denominator=sum(all_u10km_avslope)
			gen a_avslope_perc_u10km = rf_u10km_avslope_numerator/rf_u10km_avslope_denominator

		* AREA FILE KEEP/RENAME + MERGE IN NAMES/LA + ORDER
			keep geo_code_o a_*
			rename geo_code_o geo_code
			rename a_* *
			drop from_home
			duplicates drop
			merge 1:1 geo_code using "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\\$geography\geo_code_lookup.dta", nogen
			order geo_code geo_name lad11cd lad_name all light_rail- other

		* SAVE
			export delimited using "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\\$geography\z_all_attributes_unrounded.csv", replace

	*****************
	** PART 3A.1: AGGREGATE TO LA & PCT REGION LEVEL [USE LSOA]
	*****************
	** LA
		import delimited using "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\lsoa\z_all_attributes_unrounded.csv", clear
		drop *cartrips
		* AGGREGATE
			foreach var of varlist all- other govtarget_slc- ebike_sico2 {
			bysort lad11cd: egen a_`var'=sum(`var')
			}
			foreach var of varlist perc_rf_dist_u10km avslope_perc_u10km {
			bysort lad11cd: egen temp_`var'=sum(`var'*all)
			gen a_`var'=temp_`var'/a_all
			}
			keep lad11cd lad_name a_*
			rename a_* *
			duplicates drop
		* CHANGE UNITS
			foreach x in base_sl govtarget_sl govtarget_si gendereq_sl gendereq_si dutch_sl dutch_si ebike_sl ebike_si{
			replace `x'value_heat=`x'value_heat/1000000 // convert to millions of pounds
			replace `x'co2=`x'co2/1000	// convert to tonnes
			}
		export delimited using "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\lad_all_attributes_unrounded.csv", replace
	** REGION
		import delimited "pct-inputs\01_raw\01_geographies\pct_regions\pct_regions_lad_lookup.csv", varnames(1) clear
		save "pct-inputs\02_intermediate\x_temporary_files\scenario_building\pct_regions_lad_lookup.dta", replace
		import delimited using "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\lad_all_attributes_unrounded.csv", clear
		merge 1:1 lad11cd using "pct-inputs\02_intermediate\x_temporary_files\scenario_building\pct_regions_lad_lookup.dta", keepus(region_name) nogen
		* AGGREGATE
			foreach var of varlist all- other govtarget_slc- ebike_sico2 {
			bysort region_name: egen a_`var'=sum(`var')
			}
			keep region_name a_*
			rename a_* *
			duplicates drop
		* PERCENTAGES FOR INTERFACE
			foreach var of varlist bicycle govtarget_slc gendereq_slc dutch_slc ebike_slc {
			gen `var'_perc=round(`var'*100/all, 1)
			order `var'_perc, after(`var')
			}
			keep region_name *perc
			list if bicycle_perc==.
			drop if bicycle_perc==. // should be nothing
		export delimited using "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\pct_regions_all_attributes.csv", replace

	*****************
	** PART 3B.1: AGGREGATE TO FLOW LEVEL
	*****************
		use "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\\$geography\ODpairs_process2.5.dta", clear
		drop *cartrips
		* IDENTIFY VARIABLES WHERE 'ALL' IS TOO SMALL (for lsoa combine <3 flows to 'under 3')
			bysort geo_code1 geo_code2: egen f_all_temp=sum(all)
			replace geo_code_d="Under 3" if f_all_temp<3 & "$geography"=="lsoa"
			replace geo_code1=geo_code_o if geo_code_d=="Under 3"
			replace geo_code2=geo_code_d if geo_code_d=="Under 3"
			foreach var of varlist e_dist_km rf_dist_km rf_avslope_perc {
			replace `var'=. if geo_code_d=="Under 3"
			}
			gen id = geo_code1+" "+geo_code2
			drop f_all_temp
		* AGGREGATE UP FLOW FIGURES
			foreach var of varlist all- other govtarget_slc- ebike_sico2 {
			bysort id: egen f_`var'=sum(`var')
			}
		* FLOW FILE KEEP/RENAME + MERGE IN NAMES/LA + ORDER
			keep id geo_code1 geo_code2 f_* e_dist_km rf_dist_km rf_avslope_perc
			rename f_* *
			drop from_home
			duplicates drop
			forval i=1/2 {
			rename geo_code`i' geo_code
			merge m:1 geo_code using "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\\$geography\geo_code_lookup.dta"
			drop if _m==2
			drop _m
			foreach x in geo_code geo_name lad11cd lad_name {
			rename `x' `x'`i'
			}
			}
			foreach x in geo_name2 lad11cd2 lad_name2 {
			replace `x'="No fixed place" if geo_code2=="OD0000003"
			replace `x'="Other" if geo_code2=="Other"
			replace `x'="Under 3" if geo_code2=="Under 3"
			}
			order id geo_code1 geo_code2 geo_name1 geo_name2 lad11cd1 lad11cd2 lad_name1 lad_name2 all light_rail- other govtarget_slc- ebike_sico2
		* MERGE IN OTHER CS DATA (BETWEEN-LINES ONLY)
			merge 1:1 id using "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\\$geography\rfrq_all_data.dta"
			drop if _m==2
			count if _m!=3 & geo_code1!=geo_code2 & geo_code2!="Other" & geo_code2!="OD0000003" & geo_code2!="Under 3" // should be none
			drop _m
		* SAVE
			export delimited using "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\\$geography\od_all_attributes_unrounded.csv", replace

	*****************
	** PART 3B.2: FILE FOR RASTER WITH NUMBERS ROUNDED (BUT CORRECT TOTALS MAINTAINED), AND NO FILTERING OF LINES <3
	*****************
		use "pct-inputs\02_intermediate\x_temporary_files\scenario_building\commute\\$geography\ODpairs_process2.5.dta", clear
		* SUBSET BY DISTANCE AND TO SCENARIO VARIABLES
			keep if rf_dist_km<20 & geo_code1!=geo_code2

			gen id = geo_code1+" "+geo_code2
			keep id bicycle *_slc
		* AGGREGATE UP FLOW FIGURES
			foreach var of varlist bicycle- ebike_slc {
			bysort id: egen f_`var'=sum(`var')
			}
			keep id f_*
			rename f_* *
			duplicates drop
		* ROUND + MOVE SOME CYCLISTS 0 TO 1, TO GET TOTAL CORRECT NUMBER
			set seed 20170121
			gen random=uniform()
			foreach var of varlist govtarget_slc gendereq_slc dutch_slc ebike_slc {
			rename `var' `var'_orig
			gen `var'=round(`var'_orig)
			total `var'_orig `var' if `var'_orig<1.5
			matrix A=r(table)
			gen `var'_diff = round(A[1,1]-A[1,2]) // COUNT THE DIFFERENCE BETWEEN NO CYCLISTS ROUNDED VS NOT ROUNDED AMONG THOSE WHERE NOT ROUNDED IS <1.5
			sort `var' random
			gen littlen=_n
			recode `var' 0=1 if littlen <=`var'_diff // ROUND SOME 0 TO 1 SO THAT TOTAL NO. <1.5 IS CORRECT
			drop littlen
			total `var'_orig `var'
			matrix A=r(table)
			di "`var': " round((100*(1-A[1,2]/A[1,1])),0.01) "%"
			}
		* LIMIT TO THOSE WITH ANY CYCLING, AND SAVE
			egen sumcycle=rowtotal(bicycle govtarget_slc gendereq_slc dutch_slc ebike_slc)
			drop if sumcycle==0
			keep id bicycle govtarget_slc gendereq_slc dutch_slc ebike_slc
			sort id
			export delimited using "pct-inputs\02_intermediate\02_travel_data\commute\\$geography\od_raster_attributes.csv", replace


redo: revert to '//' substring replacement

#!/bin/bash
# automaton
# (c) 2013 Justin Gottula
# The source code of this project is distributed under the terms of the
# simplified BSD license. See the LICENSE file for details.

exec >&2

TARGET=$1
TARGET_BASE=$2
OUTPUT=$3

CC="ccache avr-gcc"
AS="avr-as"
LD="avr-ld"
NM="avr-nm"
OBJCOPY="avr-objcopy"
OBJDUMP="avr-objdump"

PROJ_NAME="automaton"

MCU="atmega324a"
CPU_KHZ="20000"

DEFINES=("F_CPU=${CPU_KHZ}000UL")
INC_DIRS=()
LIB_DIRS=()
LIBS=()

CFLAGS_CPU=("-mmcu=$MCU" -fpack-struct -ffreestanding)
CFLAGS_DEBUG=(-ggdb)
CFLAGS_OPT=(-Os -flto -fuse-linker-plugin -fwhole-program)
CFLAGS_WARN=(-Wall -Wextra -Wno-unused-function)
CFLAGS_ETC=(-fno-diagnostics-show-caret)

CFLAGS="-std=gnu11 ${CFLAGS_CPU[@]} ${CFLAGS_DEBUG[@]} ${CFLAGS_OPT[@]} \
${CFLAGS_WARN[@]} ${CFLAGS_ETC[@]}"
CFLAGS_COMPILE="$CFLAGS ${DEFINES[@]/#/-D} ${INC_DIRS[@]/#/-I}"
CFLAGS_LINK="$CFLAGS ${LIB_DIRS[@]/#/-L} ${LIBS[@]/#/-l}"

SOURCES=$(find src -type f -iname '*.c')
OBJECTS=${SOURCES//.c/.o}

OUT_ELF="out/$PROJ_NAME.elf"
OUT_BIN="out/$PROJ_NAME.bin"
OUT_HEX="out/$PROJ_NAME.hex"
OUT_MAP="out/$PROJ_NAME.map"
OUT_DUMP="out/$PROJ_NAME.dump"

case "$TARGET" in
all)
	redo-ifchange $OUT_ELF $OUT_BIN $OUT_HEX $OUT_MAP $OUT_DUMP
	;;
$OUT_ELF)
	redo-ifchange $OBJECTS
	$CC $CFLAGS_LINK -o $OUTPUT $OBJECTS
	;;
$OUT_BIN)
	redo-ifchange $OUT_ELF
	$OBJCOPY -O binary $OUT_ELF $OUTPUT
	;;
$OUT_HEX)
	redo-ifchange $OUT_ELF
	$OBJCOPY -O ihex $OUT_ELF $OUTPUT
	;;
$OUT_MAP)
	redo-ifchange $OUT_ELF
	$NM -Cn $OUT_ELF >$OUTPUT
	;;
$OUT_DUMP)
	redo-ifchange $OUT_ELF
	$OBJDUMP -aCdfhpS $OUT_ELF >$OUTPUT
	;;
*.o)
	$CC $CFLAGS_COMPILE -o${TARGET//.o/.dep} -MM -MG ${TARGET//.o/.c}

	read DEPS <${TARGET//.o/.dep}
	redo-ifchange ${DEPS#*:}

	$CC $CFLAGS_COMPILE -o$OUTPUT -c ${TARGET//.o/.c}
	;;
clean)
	rm -rf $(find out -type f)
	rm -rf $(find src -type f -iname *.o)
	rm -rf $(find src -type f -iname *.dep)
	;;
load)
	redo-ifchange $OUT_HEX
	sudo avrdude -p ${MCU/atmega/m} -c usbasp -B 460800 -U flash:w:$OUT_HEX
	;;
*)
	echo "unknown target '$TARGET'"
	exit 1
	;;
esac

#!/bin/bash
# automaton
# (c) 2013 Justin Gottula
# The source code of this project is distributed under the terms of the
# simplified BSD license. See the LICENSE file for details.

exec >&2

TARGET=$1
TARGET_BASE=$2
OUTPUT=$3

CC="ccache avr-gcc"
AS="avr-as"
LD="avr-ld"
NM="avr-nm"
OBJCOPY="avr-objcopy"
OBJDUMP="avr-objdump"

PROJ_NAME="automaton"

MCU="atmega324a"
CPU_KHZ="20000"

DEFINES=("F_CPU=${CPU_KHZ}000UL")
INC_DIRS=()
LIB_DIRS=()
LIBS=()

CFLAGS_CPU=("-mmcu=$MCU" -fpack-struct -ffreestanding)
CFLAGS_DEBUG=(-ggdb)
CFLAGS_OPT=(-Os -flto -fuse-linker-plugin -fwhole-program)
CFLAGS_WARN=(-Wall -Wextra -Wno-unused-function)
CFLAGS_ETC=(-fno-diagnostics-show-caret)

CFLAGS="-std=gnu11 ${CFLAGS_CPU[@]} ${CFLAGS_DEBUG[@]} ${CFLAGS_OPT[@]} \
${CFLAGS_WARN[@]} ${CFLAGS_ETC[@]}"
CFLAGS_COMPILE="$CFLAGS ${DEFINES[@]/#/-D} ${INC_DIRS[@]/#/-I}"
CFLAGS_LINK="$CFLAGS ${LIB_DIRS[@]/#/-L} ${LIBS[@]/#/-l}"

SOURCES=$(find src -type f -iname '*.c')
OBJECTS=${SOURCES/%.c/.o}

OUT_ELF="out/$PROJ_NAME.elf"
OUT_BIN="out/$PROJ_NAME.bin"
OUT_HEX="out/$PROJ_NAME.hex"
OUT_MAP="out/$PROJ_NAME.map"
OUT_DUMP="out/$PROJ_NAME.dump"

case "$TARGET" in
all)
	redo-ifchange $OUT_ELF $OUT_BIN $OUT_HEX $OUT_MAP $OUT_DUMP
	;;
$OUT_ELF)
	redo-ifchange $OBJECTS
	$CC $CFLAGS_LINK -o $OUTPUT $OBJECTS
	;;
$OUT_BIN)
	redo-ifchange $OUT_ELF
	$OBJCOPY -O binary $OUT_ELF $OUTPUT
	;;
$OUT_HEX)
	redo-ifchange $OUT_ELF
	$OBJCOPY -O ihex $OUT_ELF $OUTPUT
	;;
$OUT_MAP)
	redo-ifchange $OUT_ELF
	$NM -Cn $OUT_ELF >$OUTPUT
	;;
$OUT_DUMP)
	redo-ifchange $OUT_ELF
	$OBJDUMP -aCdfhpS $OUT_ELF >$OUTPUT
	;;
*.o)
	$CC $CFLAGS_COMPILE -o${TARGET/%.o/.dep} -MM -MG ${TARGET/%.o/.c}

	read DEPS <${TARGET/%.o/.dep}
	redo-ifchange ${DEPS#*:}

	$CC $CFLAGS_COMPILE -o$OUTPUT -c ${TARGET/%.o/.c}
	;;
clean)
	rm -rf $(find out -type f)
	rm -rf $(find src -type f -iname *.o)
	rm -rf $(find src -type f -iname *.dep)
	;;
load)
	redo-ifchange $OUT_HEX
	sudo avrdude -p ${MCU/atmega/m} -c usbasp -B 460800 -U flash:w:$OUT_HEX
	;;
*)
	echo "unknown target '$TARGET'"
	exit 1
	;;
esac


Update 00_SetupFolderGlobals.do

Update 00_SetupFolderGlobals.do

/*-------------------------------------------------------------------------------
# Name:		00_SetupFolderGlobals
# Purpose:	Create series of folders Food for Uganda Vulnerability Analysis
# Author:	Tim Essam, Ph.D.
# Created:	10/31/2014; 02/19/2015.
# Owner:	USAID GeoCenter | OakStream Systems, LLC
# License:	MIT License
# Ado(s):	see below
#-------------------------------------------------------------------------------
*/

/* RUN/READ ME FIRST -- Make directories for the study.
 1. Requires root name for file folder structure
 2. Requires branch names for sub-folders
 3. Sets global macros for the study; These are used through the do files.
 4. TODO -- Script DOES NOT copy in raw data at this point.
 5. TODO -- Create program folder for calling custom programs.
 6. TODO -- Convert setup .do file into a program (inputs are project path & folder name)
*/

* install the confirm directory ado if not already installed
* list all known user-written .ados needed for project
local required_ados confirmdir mvfiles fs spatgsa  adolist labellist winsor2
foreach x of local required_ados {
	capture findfile `x'.ado
		if _rc==601 {
			cap ssc install `x'
		}
		else disp in yellow "`x' currently installed."
	}
*end

* Determine path for the study
* global projectpath "U:/"
*global projectpath "C:/Users/Tim/Documents/"
global projectpath "C:/Users/t/Documents/"
cd "$projectpath"

* Run a macro to set up study folder (needs to be modified)
* Name the file path below -- replace "nigerlsms" with your folder name
local pFolder Uganda
foreach dir in `pFolder' {
	confirmdir "`dir'"
	if `r(confirmdir)'==170 {
		mkdir "`dir'"
		display in yellow "Project directory named: `dir' created"
		}
	else disp as error "`dir' already exists, not created."
	cd "$projectpath/`dir'"
	}
* end

* Run initially to set up folder structure
* Choose your folders to set up as the local macro `folders'
local folders Rawdata Stata Datain Log Output Dataout Excel PDF Word Graph GIS Export R Python Programs Sensitive_Data FinalProducts
foreach dir in `folders' {
	confirmdir "`dir'"
	if `r(confirmdir)'==170 {
			mkdir "`dir'"
			disp in yellow "`dir' successfully created."
		}
	else disp as error "`dir' already exists. Skipped to next folder."
}
*end

/*---------------------------------
# Set Globals based on path above #
-----------------------------------*/
global date $S_DATE
local dir `c(pwd)'
global path "`dir'"
global pathdo "`dir'/Stata"
global pathdo2 "C:/Users/t/Documents/GitHub/Uganda/Stata"
global pathlog  "`dir'/Log"
global pathin "`dir'/Datain"
global pathout "`dir'/Dataout"
global pathgraph "`dir'/Graph"
global pathxls "`dir'/Excel"
global pathreg "`dir'/Output"
global pathgis "`dir'/GIS"
global pathraw "`dir'/Rawdata"
global pathexport "`dir'/Export"
global pathR "`dir'/R"
global pathPython "`dir'/Python"
global pathProgram "`dir'/Program"
global pathPdrive "P:/GeoCenter/GIS/Projects/LVA/Uganda"
global pathSensitive "`dir'/Sensitive_Data"
global pathProducts "`dir'/FinalProducts"
* Project macros are defined as:
macro list

* Add in subfolders for GIS and Final Products Folders
cd "$pathgis"
local gisfolders FeatureClass Shapefiles csv Layers mxd Rasters scratch
foreach dir in `gisfolders' {
	confirmdir "`dir'"
	if `r(confirmdir)'==170 {
			mkdir "`dir'"
			disp in yellow "`dir' successfully created."
		}
	else disp as error "`dir' already exists. Skipped to next folder."
}
*end

cd "$pathProducts"
local subfolders AI PDFs PosterJPG PPTJPG
foreach dir in `subfolders' {
	confirmdir "`dir'"
	if `r(confirmdir)'==170 {
			mkdir "`dir'"
			disp in yellow "`dir' successfully created."
		}
	else disp as error "`dir' already exists. Skipped to next folder."
}
*end

/*------------------------------------------------------------
# Manually copy raw data  into Datain Folder #

/*-------------------------------------------------------------------------------
# Name:		00_SetupFolderGlobals
# Purpose:	Create series of folders Food for Uganda Vulnerability Analysis
# Author:	Tim Essam, Ph.D.
# Created:	10/31/2014; 02/19/2015.
# Owner:	USAID GeoCenter | OakStream Systems, LLC
# License:	MIT License
# Ado(s):	see below
#-------------------------------------------------------------------------------
*/

/* RUN/READ ME FIRST -- Make directories for the study.
 1. Requires root name for file folder structure
 2. Requires branch names for sub-folders
 3. Sets global macros for the study; These are used through the do files.
 4. TODO -- Script DOES NOT copy in raw data at this point.
 5. TODO -- Create program folder for calling custom programs.
 6. TODO -- Convert setup .do file into a program (inputs are project path & folder name)
*/

* install the confirm directory ado if not already installed
* list all known user-written .ados needed for project
local required_ados confirmdir mvfiles fs spatgsa  adolist labellist winsor2
foreach x of local required_ados {
	capture findfile `x'.ado
		if _rc==601 {
			cap ssc install `x'
		}
		else disp in yellow "`x' currently installed."
	}
*end

* Determine path for the study
* global projectpath "U:/"
* global projectpath "C:/Users/Tim/Documents/"
global projectpath "C:/Users/t/Documents/"
cd "$projectpath"

* Run a macro to set up study folder (needs to be modified)
* Name the file path below -- replace "nigerlsms" with your folder name
local pFolder Uganda
foreach dir in `pFolder' {
	confirmdir "`dir'"
	if `r(confirmdir)'==170 {
		mkdir "`dir'"
		display in yellow "Project directory named: `dir' created"
		}
	else disp as error "`dir' already exists, not created."
	cd "$projectpath/`dir'"
	}
* end

* Run initially to set up folder structure
* Choose your folders to set up as the local macro `folders'
local folders Rawdata Stata Datain Log Output Dataout Excel PDF Word Graph GIS Export R Python Programs Sensitive_Data FinalProducts
foreach dir in `folders' {
	confirmdir "`dir'"
	if `r(confirmdir)'==170 {
			mkdir "`dir'"
			disp in yellow "`dir' successfully created."
		}
	else disp as error "`dir' already exists. Skipped to next folder."
}
*end

/*---------------------------------
# Set Globals based on path above #
-----------------------------------*/
global date $S_DATE
local dir `c(pwd)'
global path "`dir'"
global pathdo "`dir'/Stata"
global pathdo2 "`dir'/Github\Uganda\Stata"
global pathlog  "`dir'/Log"
global pathin "`dir'/Datain"
global pathout "`dir'/Dataout"
global pathgraph "`dir'/Graph"
global pathxls "`dir'/Excel"
global pathreg "`dir'/Output"
global pathgis "`dir'/GIS"
global pathraw "`dir'/Rawdata"
global pathexport "`dir'/Export"
global pathR "`dir'/R"
global pathPython "`dir'/Python"
global pathProgram "`dir'/Program"
global pathPdrive "P:/GeoCenter/GIS/Projects/LVAM\Uganda"
global pathSensitive "`dir'/Sensitive_Data"
global pathProducts "`dir'/FinalProducts"
* Project macros are defined as:
macro list

* Add in subfolders for GIS and Final Products Folders
cd "$pathgis"
local gisfolders FeatureClass Shapefiles csv Layers mxd Rasters scratch
foreach dir in `gisfolders' {
	confirmdir "`dir'"
	if `r(confirmdir)'==170 {
			mkdir "`dir'"
			disp in yellow "`dir' successfully created."
		}
	else disp as error "`dir' already exists. Skipped to next folder."
}
*end

cd "$pathProducts"
local subfolders AI PDFs PosterJPG PPTJPG
foreach dir in `subfolders' {
	confirmdir "`dir'"
	if `r(confirmdir)'==170 {
			mkdir "`dir'"
			disp in yellow "`dir' successfully created."
		}
	else disp as error "`dir' already exists. Skipped to next folder."
}
*end

/*------------------------------------------------------------
# Manually copy raw data  into Datain Folder #