* MAJOR fix for field parsing

* MAJOR fix for field parsing

INLIST=false

cat $1.linklist | while read LINE; do
    LINEMARKER=`echo $LINE | grep --only-matching --perl-regexp '^....(?= )' || echo`
    test "$LINEMARKER" = "HTML" && test "$INLIST" = "true" && echo '
' >> $3
    test "$LINEMARKER" = "HTML" && HTML=`echo $LINE | grep --only-matching --perl-regexp '(?<=^.... ).*$'`
    test "$LINEMARKER" = "HTML" && echo \ \ \ \ $HTML >> $3
    test "$LINEMARKER" = "HTML" && INLIST=false && continue

    TIMECODE=`echo $LINE | cut -d " " -f1`
    URL=`echo $LINE | cut -d " " -f2 | tr -d "<>"`
    TEXT=`echo $LINE | cut -d " " -f3-`
    test "$INLIST" = "false" && echo '
' >> $3
    cat << EOF >> $3

            $TIMECODE
            $TEXT

EOF
    INLIST=true
done

cat << EOF >> $3

EOF

INLIST=false

cat $1.linklist | while read LINE; do
    LINEMARKER=`echo $LINE | grep --only-matching --perl-regexp '^....(?= )' || echo`
    test "$LINEMARKER" = "HTML" && test "$INLIST" = "true" && echo '
' >> $3
    test "$LINEMARKER" = "HTML" && HTML=`echo $LINE | grep --only-matching --perl-regexp '(?<=^.... ).*$'`
    test "$LINEMARKER" = "HTML" && echo \ \ \ \ $HTML >> $3
    test "$LINEMARKER" = "HTML" && INLIST=false && continue

    TIMECODE=`echo $LINE | grep --only-matching --perl-regexp ^.*:..(?= \<)`
    URL=`echo $LINE | grep --only-matching --perl-regexp '(?<= \<).*:.*(?=\> )'`;
    TEXT=`echo $LINE | grep --only-matching --perl-regexp '(?<=\> ).*$'`
    test "$INLIST" = "false" && echo '
' >> $3
    cat << EOF >> $3

            $TIMECODE
            $TEXT

EOF
    INLIST=true
done

cat << EOF >> $3

EOF


fixed winsorize weird behaviour

fixed winsorize weird behaviour

burned by version; fixed; to note

cap pr drop winsorize
pr def winsorize, rclass
	version 10
	syntax varlist [, GENerate(name) AT(numlist max = 2) LIMits(numlist)]
	loc ncutpoints : list sizeof at
	if `ncutpoints' > 2 {
		di "Cannot have more than 2 cutpoints"
		error
	}
	else if `ncutpoints' > 0 {
		if `ncutpoints' != 1 {
			tokenize `at'
			loc upper `1'
			loc lower `2'
		}
		else {
			loc lower 0
			loc upper `at'
		}
	}
	else {
		loc lower 0
		loc upper 99
	}
	if "`limits'" != "" {
		loc lower `limits'
		loc upper `=1-`limits''
	}
	if "`generate'" != "" {
		loc targetvar `generate'
		g `targetvar' = `varlist'
	}
	else {
		loc targetvar `varlist'
	}
	if `lower' != 0 {
		_pctile `varlist' , p(`lower' `upper')
		loc ll = r(r1)
		loc ul = r(r2)
		replace `targetvar' = `ll' if `varlist' == `ll'
	}
	else {
		_pctile `varlist' , p(`upper')
		loc ul = r(r1)
		loc ll "."
	}
	replace `targetvar' = `ul' if `varlist' > `ul'
	return scalar lower = `ll'
	return scalar upper = `ul'
	return local targetvar `targetvar'
end

cap pr drop winsorize
pr def winsorize, rclass
	version 6
	syntax varlist [, GENerate(name) AT(numlist max = 2) LIMits(numlist)]
	loc ncutpoints : list sizeof at
	if `ncutpoints' > 2 {
		di "Cannot have more than 2 cutpoints"
		error
	}
	else if `ncutpoints' > 0 {
		if `ncutpoints' != 1 {
			tokenize `at'
			loc upper `1'
			loc lower `2'
		}
		else {
			loc lower 0
			loc upper `at'
		}
	}
	else {
		loc lower 0
		loc upper 99
	}
	if "`limits'" == "" {
		loc lower `limits'
		loc upper `=1-`limits''
	}
	if "`generate'" != "" {
		loc targetvar `generate'
		g `targetvar' = `varlist'
	}
	else {
		loc targetvar `varlist'
	}
	if `lower' != 0 {
		_pctile `varlist' , p(`lower' `upper')
		loc ll = r(r1)
		loc ul = r(r2)
		replace `targetvar' = `ll' if `varlist' == `ll'
	}
	else {
		_pctile `varlist' , p(`upper')
		loc ul = r(r1)
		loc ll "."
	}
	replace `targetvar' = `ul' if `varlist' > `ul'
	return scalar lower = `ll'
	return scalar upper = `ul'
	return local targetvar `targetvar'
end


Forgot to update build script

Forgot to update build script

redo-ifchange bin/e
redo-ifchange roms/mlm
redo-ifchange roms/ngaro

redo-ifchange bin/e
redo-ifchange roms/mlm
redo-ifchange roms/vm


fix issue #198 - dash does not work in early Stata, use hyphen

*! version 6.2 31JAN2019 DIME Analytics dimeanalytics@worldbank.org

	capture program drop ieboilstart
	program ieboilstart , rclass

	qui {

		syntax ,  Versionnumber(string) [noclear maxvar(numlist) matsize(numlist) Quietly veryquietly Custom(string) setmem(string) nopermanently ]

		version 11.0

		/*********************************

			Check user specifed version
			and apply it is valid

		*********************************/

		local stata_versions "11 11.0 11.1 11.2 12 12.0 12.1 13 13.0 13.1 14 14.0 14.1 14.2 15 15.0 15.1"

		if `:list versionnumber in stata_versions' == 0 {

			di as error "{phang}Only recent major releases are allowed. The releases currently allowed are:{break}`stata_versions'{p_end}"
			di ""
			error 198
			exit
		}

		*Return the value
		return local version "version `versionnumber'"

		*Set the version specfied in the command
		version `versionnumber'

		/*********************************

			Check settings related to older
			versions and Stata IC

		*********************************/

		*Test that maxvar is not set in combination when using Stata IC
		if 	!(c(MP) == 1 | c(SE) == 1) & "`maxvar'" != "" {

			di as error "{phang}In Stata IC the maximum number of variables allowed is fixed at 2,047 and maxvar() is therefore not allowed.{p_end}"
			error 198
		}

		if 	"`setmem'" != ""	{

			if `versionnumber' >= 12  {

				di as error "{phang}Option setmem() is only allowed when setting the version number to 11. Setmem() is only applicable in Stata 11 or earlier, but those versions wont be able to run this file as the version number is set to 12 or higher.{p_end}"
				error 198
			}

			*Split the memory setting string into value and byte unit
			local mem_strlen 	= strlen("`setmem'") -1
			local mem_number	= substr("`setmem'",  1 , `mem_strlen')
			local mem_bytetype	= substr("`setmem'", -1 , 1)

			*Test that the number part is a number and a integer
			cap confirm integer number `mem_number'
			if _rc {

				di as error "{phang}The value in setmem(`setmem') must be a number (followed by either b, k, m or g). See {help memory} for details.{p_end}"
				error _rc
			}

			*Test that the byte type is not anything else than b, k, m or g
			if !`=inlist(lower("`mem_bytetype'"), "b", "k", "m", "g")' {

				di as error "{phang}The last character in setmem(`setmem') must be either b, k, m or g. See {help memory} for details.{p_end}""
				error 7

			}
		}
		else {

			*Default value is 50M. This is probably too little, but
			*it will safely work. And users can manually increase this
			local setmem "50M"

		}

		/*********************************

			Check input for maxvar and matsize
			if specified, other wise set
			maximum value allowed.

		*********************************/

		*Setting maxvar requires a cleared memory. Therefore
		*maxvar() and noclear cannot be used at the same time.
		if "`maxvar'" != "" & "`clear'" != "" {

			di as error "{phang}It is not possible to set allowed maximum numbers of variables without clearing the data. noclear and maxvar() can therefore not be specified at the same time{p_end}"
			di ""
			error 198
			exit
		}

		foreach maxlocal in maxvar matsize {

			*Set locals with the max and min values fox maxvar and matsize
			if "`maxlocal'" == "maxvar" {

				*Stata 15 MP has a higher maximum number of maxvar
				if c(stata_version) >= 15 & c(MP) == 1 {
					local max 120000
				}
				else {
					*For Stata 15 SE and MP and SE for all lower versions
					local max 32767
				}
				local min 2048
			}
			if "`maxlocal'" == "matsize" {
				local max 11000
				local min 10
			}

			*Test if user set a value for this value
			if "``maxlocal''" != "" {

				*If user specified a value, test that it is between the min and the max
				if !(``maxlocal'' >= `min' & ``maxlocal'' <= `max') {

					di as error "{phang}`maxlocal' must be between `min' and `max' (inclusive) if you are using Stata SE or Stata MP. You entered ``maxlocal''.{p_end}"
					di ""
					*Throw appropriate error if below or above error
					if ``maxlocal'' < `min' error 910
					if ``maxlocal'' > `max' error 912
					exit
				}
			}

			else {
				*User did not specify value, use ieboilstart's defaults:

				if "`maxlocal'" == "maxvar" {
					*Set maxvar to max value allowed as this is often an issue when working with large survey data
					local `maxlocal' `max'
				}
				if "`maxlocal'" == "matsize" {
					*Set to the default as the maximum is rarely requered.
					local `maxlocal' 400
				}
			}
		}

		/*********************************

			Check other input

		*********************************/

		**Default is that these values are set as default values so that
		* these are the default values each time Stata starts.
		if "`nopermanently'" == "" {

			local permanently 		" , permanently"
			local permanently_col 	"{col 28}, permanently"

		}
		else {

			local permanently ""
		}

		/*********************************

			Set the settings

		*********************************/

		local setDispLocal "{col 5}{ul:Settings set by this command:}"

		*Set basic memory limits
		if "`clear'" == "" {

			*Setting
			clear all
			local setDispLocal "`setDispLocal'{break}{col 5}clear all"

			**Setting maxvar not allowed in Stata IC.
			if 	(c(MP) == 1 | c(SE) == 1) {

				*Setting
				set maxvar `maxvar' `permanently'
				local setDispLocal "`setDispLocal'{break}{col 5}set maxvar {col 22}`maxvar'`permanently_col'"
			}
		}

		*Setting
		set matsize `matsize' `permanently'
		local setDispLocal "`setDispLocal'{break}{col 5}set matsize {col 22}`matsize'`permanently_col'"

		****************
		*Memory	settings

		*For compatibility with Stata 11 the do file includes and if/else
		*condition testing version number. The memoroy settings introduced
		*in Stata 12 will be applied to Stata version more recent than
		*Stata 11, and set memory will be applied to Stata 11.
		if c(stata_version) >= 12 {

			*Setting
			set niceness 5  `permanently'
			local setDispLocal "`setDispLocal'{break}{col 5}set niceness{col 22}5`permanently_col'"

			*These settings cannot be modified with data in memory
			if "`clear'" == "" {

				*Settings
				set min_memory 0  `permanently'
				set max_memory .  `permanently'
				local setDispLocal "`setDispLocal'{break}{col 5}set min_memory {col 22}0`permanently_col'{break}{col 5}set max_memory {col 22}.`permanently_col'"

				*Set segment size to the largest value allowed by the operative system
				if c(bit) == 64 {
					*Setting
					set segmentsize	32m  `permanently'
					local setDispLocal "`setDispLocal'{break}{col 5}set segmentsize	{col 22}32m`permanently_col'"
				}
				else {
					*Setting
					set segmentsize	16m  `permanently'
					local setDispLocal "`setDispLocal'{break}{col 5}set segmentsize	{col 22}16m`permanently_col'"
				}
			}
		}
		else {

			*If this dofile is generated in Stata 11 then only the old
			*way of setting memory is included. This will be ignored by
			*more recent versios of Stata

			*Setting
			set memory `setmem'
			local setDispLocal "`setDispLocal'{break}{col 5}set memory {col 22}`setmem'"

		}

		*********************
		*Set default settings
		set more 		off `permanently'
		pause 			on
		set varabbrev 	off
		local setDispLocal "`setDispLocal'{break}{col 5}set more {col 22}off {col 28}, perm"
		local setDispLocal "`setDispLocal'{break}{col 5}pause {col 22}on"
		local setDispLocal "`setDispLocal'{break}{col 5}set varabbrev {col 22}off"

		/*********************************

			Add custom lines of code

		*********************************/

		if `"`custom'"' != "" {

			local setDispLocal `"`setDispLocal'{break} {break}{col 5}{ul:User specified settings:}"'

			*Create a local with the rowlabel input to be tokenized
			local custom_code_lines `custom'

			while `"`custom_code_lines'"' != "" {

				*Parsing name and label pair
				gettoken code_line custom_code_lines : custom_code_lines, parse("@")

				*Removing leadning or trailing spaces
				local code_line = trim(`"`code_line'"')

				*Set custom setting
				local setDispLocal `"`setDispLocal'{break}{col 5}`code_line'"'
				`code_line'

				*Parse char is not removed by gettoken
				local custom_code_lines = subinstr(`"`custom_code_lines'"' ,"@","",1)
			}
		}

		/*********************************

			Create return value and output message

		*********************************/

		if "`quietly'" == "" & "`veryquietly'" == "" {

			noi di ""
			noi di "{phang}{err:DISCLAIMER:} Due to how settings work in Stata, this command can only attempt to harmonize settings as much as possible across users, but no guarantee can be given that all commands will always behave identically unless the exact same version and type of Stata is used, with the same releases of user-contributed commands installed.{p_end}"
			noi di ""
			noi di `"`setDispLocal'"'
		}

		if  "`veryquietly'" == "" {

			noi di ""
			noi di "{phang}{err:IMPORTANT:} The most important setting of this command - the version - cannot be set inside the command due to technical reasons. The setting has been prepared by this command, and you only need to write \`r(version)' after this command (include the apostrophes).{p_end}"

		}

	}
	end

*! version 6.2 31JAN2019 DIME Analytics dimeanalytics@worldbank.org

	capture program drop ieboilstart
	program ieboilstart , rclass

	qui {

		syntax ,  Versionnumber(string) [noclear maxvar(numlist) matsize(numlist) Quietly veryquietly Custom(string) setmem(string) nopermanently ]

		version 11.0

		/*********************************

			Check user specifed version
			and apply it is valid

		*********************************/

		local stata_versions "11 11.0 11.1 11.2 12 12.0 12.1 13 13.0 13.1 14 14.0 14.1 14.2 15 15.0 15.1"

		if `:list versionnumber in stata_versions' == 0 {

			di as error "{phang}Only recent major releases are allowed. The releases currently allowed are:{break}`stata_versions'{p_end}"
			di ""
			error 198
			exit
		}

		*Return the value
		return local version "version `versionnumber'"

		*Set the version specfied in the command
		version `versionnumber'

		/*********************************

			Check settings related to older
			versions and Stata IC

		*********************************/

		*Test that maxvar is not set in combination when using Stata IC
		if 	!(c(MP) == 1 | c(SE) == 1) & "`maxvar'" != "" {

			di as error "{phang}In Stata IC the maximum number of variables allowed is fixed at 2,047 and maxvar() is therefore not allowed.{p_end}"
			error 198
		}

		if 	"`setmem'" != ""	{

			if `versionnumber' >= 12  {

				di as error "{phang}Option setmem() is only allowed when setting the version number to 11. Setmem() is only applicable in Stata 11 or earlier, but those versions wont be able to run this file as the version number is set to 12 or higher.{p_end}"
				error 198
			}

			*Split the memory setting string into value and byte unit
			local mem_strlen 	= strlen("`setmem'") -1
			local mem_number	= substr("`setmem'",  1 , `mem_strlen')
			local mem_bytetype	= substr("`setmem'", -1 , 1)

			*Test that the number part is a number and a integer
			cap confirm integer number `mem_number'
			if _rc {

				di as error "{phang}The value in setmem(`setmem') must be a number (followed by either b, k, m or g). See {help memory} for details.{p_end}"
				error _rc
			}

			*Test that the byte type is not anything else than b, k, m or g
			if !`=inlist(lower("`mem_bytetype'"), "b", "k", "m", "g")' {

				di as error "{phang}The last character in setmem(`setmem') must be either b, k, m or g. See {help memory} for details.{p_end}""
				error 7

			}
		}
		else {

			*Default value is 50M. This is probably too little, but
			*it will safely work. And users can manually increase this
			local setmem "50M"

		}

		/*********************************

			Check input for maxvar and matsize
			if specified, other wise set
			maximum value allowed.

		*********************************/

		*Setting maxvar requires a cleared memory. Therefore
		*maxvar() and noclear cannot be used at the same time.
		if "`maxvar'" != "" & "`clear'" != "" {

			di as error "{phang}It is not possible to set allowed maximum numbers of variables without clearing the data. noclear and maxvar() can therefore not be specified at the same time{p_end}"
			di ""
			error 198
			exit
		}

		foreach maxlocal in maxvar matsize {

			*Set locals with the max and min values fox maxvar and matsize
			if "`maxlocal'" == "maxvar" {

				*Stata 15 MP has a higher maximum number of maxvar
				if c(stata_version) >= 15 & c(MP) == 1 {
					local max 120000
				}
				else {
					*For Stata 15 SE and MP and SE for all lower versions
					local max 32767
				}
				local min 2048
			}
			if "`maxlocal'" == "matsize" {
				local max 11000
				local min 10
			}

			*Test if user set a value for this value
			if "``maxlocal''" != "" {

				*If user specified a value, test that it is between the min and the max
				if !(``maxlocal'' >= `min' & ``maxlocal'' <= `max') {

					di as error "{phang}`maxlocal' must be between `min' and `max' (inclusive) if you are using Stata SE or Stata MP. You entered ``maxlocal''.{p_end}"
					di ""
					*Throw appropriate error if below or above error
					if ``maxlocal'' < `min' error 910
					if ``maxlocal'' > `max' error 912
					exit
				}
			}

			else {
				*User did not specify value, use ieboilstart's defaults:

				if "`maxlocal'" == "maxvar" {
					*Set maxvar to max value allowed as this is often an issue when working with large survey data
					local `maxlocal' `max'
				}
				if "`maxlocal'" == "matsize" {
					*Set to the default as the maximum is rarely requered.
					local `maxlocal' 400
				}
			}
		}

		/*********************************

			Check other input

		*********************************/

		**Default is that these values are set as default values so that
		* these are the default values each time Stata starts.
		if "`nopermanently'" == "" {

			local permanently 		" , permanently"
			local permanently_col 	"{col 28}, permanently"

		}
		else {

			local permanently ""
		}

		/*********************************

			Set the settings

		*********************************/

		local setDispLocal "{col 5}{ul:Settings set by this command:}"

		*Set basic memory limits
		if "`clear'" == "" {

			*Setting
			clear all
			local setDispLocal "`setDispLocal'{break}{col 5}clear all"

			**Setting maxvar not allowed in Stata IC.
			if 	(c(MP) == 1 | c(SE) == 1) {

				*Setting
				set maxvar `maxvar' `permanently'
				local setDispLocal "`setDispLocal'{break}{col 5}set maxvar {col 22}`maxvar'`permanently_col'"
			}
		}

		*Setting
		set matsize `matsize' `permanently'
		local setDispLocal "`setDispLocal'{break}{col 5}set matsize {col 22}`matsize'`permanently_col'"

		****************
		*Memory	settings

		*For compatibility with Stata 11 the do file includes and if/else
		*condition testing version number. The memoroy settings introduced
		*in Stata 12 will be applied to Stata version more recent than
		*Stata 11, and set memory will be applied to Stata 11.
		if c(stata_version) >= 12 {

			*Setting
			set niceness 5  `permanently'
			local setDispLocal "`setDispLocal'{break}{col 5}set niceness{col 22}5`permanently_col'"

			*These settings cannot be modified with data in memory
			if "`clear'" == "" {

				*Settings
				set min_memory 0  `permanently'
				set max_memory .  `permanently'
				local setDispLocal "`setDispLocal'{break}{col 5}set min_memory {col 22}0`permanently_col'{break}{col 5}set max_memory {col 22}.`permanently_col'"

				*Set segment size to the largest value allowed by the operative system
				if c(bit) == 64 {
					*Setting
					set segmentsize	32m  `permanently'
					local setDispLocal "`setDispLocal'{break}{col 5}set segmentsize	{col 22}32m`permanently_col'"
				}
				else {
					*Setting
					set segmentsize	16m  `permanently'
					local setDispLocal "`setDispLocal'{break}{col 5}set segmentsize	{col 22}16m`permanently_col'"
				}
			}
		}
		else {

			*If this dofile is generated in Stata 11 then only the old
			*way of setting memory is included. This will be ignored by
			*more recent versios of Stata

			*Setting
			set memory `setmem'
			local setDispLocal "`setDispLocal'{break}{col 5}set memory {col 22}`setmem'"

		}

		*********************
		*Set default settings
		set more 		off `permanently'
		pause 			on
		set varabbrev 	off
		local setDispLocal "`setDispLocal'{break}{col 5}set more {col 22}off {col 28}, perm"
		local setDispLocal "`setDispLocal'{break}{col 5}pause {col 22}on"
		local setDispLocal "`setDispLocal'{break}{col 5}set varabbrev {col 22}off"

		/*********************************

			Add custom lines of code

		*********************************/

		if `"`custom'"' != "" {

			local setDispLocal `"`setDispLocal'{break} {break}{col 5}{ul:User specified settings:}"'

			*Create a local with the rowlabel input to be tokenized
			local custom_code_lines `custom'

			while `"`custom_code_lines'"' != "" {

				*Parsing name and label pair
				gettoken code_line custom_code_lines : custom_code_lines, parse("@")

				*Removing leadning or trailing spaces
				local code_line = trim(`"`code_line'"')

				*Set custom setting
				local setDispLocal `"`setDispLocal'{break}{col 5}`code_line'"'
				`code_line'

				*Parse char is not removed by gettoken
				local custom_code_lines = subinstr(`"`custom_code_lines'"' ,"@","",1)
			}
		}

		/*********************************

			Create return value and output message

		*********************************/

		if "`quietly'" == "" & "`veryquietly'" == "" {

			noi di ""
			noi di "{phang}{err:DISCLAIMER:} Due to how settings work in Stata, this command can only attempt to harmonize settings as much as possible across users, but no guarantee can be given that all commands will always behave identically unless the exact same version and type of Stata is used, with the same releases of user-contributed commands installed.{p_end}"
			noi di ""
			noi di `"`setDispLocal'"'
		}

		if  "`veryquietly'" == "" {

			noi di ""
			noi di "{phang}{err:IMPORTANT:} The most important setting of this command – the version – cannot be set inside the command due to technical reasons. The setting has been prepared by this command, and you only need to write \`r(version)' after this command (include the apostrophes).{p_end}"

		}

	}
	end


Update 02_hhchar.do

Update 02_hhchar.do

/*------------------------------------------------------------------------------
# Name:		02_hhchar
# Purpose:	Process household characteristics and education characteristics
# Author:	Patrick Gault
# Created:	2015/2/21
# License:	MIT License
# Ado(s):	labutil, labutil2 (ssc install labutil, labutil2)
# Dependencies: copylables, attachlabels, 00_SetupFoldersGlobals.do
#-------------------------------------------------------------------------------
*/

clear
capture log close
log using "$pathlog/01_hhchar.log", replace

use "$pathraw/GSEC2.dta", replace

* Merge education data to household roster using the force command
merge 1:1 HHID PID using "$pathraw/GSEC4.dta", force

/* Demographic list to calculate
1. Head of Household Sex
2. Relationship Status
*/

* Create head of household variable based on primary respondent and sex
g byte hoh = h2q4 == 1
la var hoh "Head of household"

g byte femhead = h2q3 == 2 & h2q4 == 1
la var femhead "Female head of household"

g agehead = h2q8 if hoh == 1
la var agehead "Age of head of household"

g ageheadsq = agehead^2
la var ageheadsq "Squared age of the head (for non-linear effects)"

* Relationship status variables
g byte marriedHohm = h2q10 == 1 & hoh==1
la var marriedHohm "married HOH monogamous"

g byte marriedHoh = hoh ==1 & inlist(h2q10, 1, 2)
la var marriedHoh "married head of household (any type)"

g byte marriedHohp = h2q10 == 2 & hoh==1
la var marriedHohp "married HOH polygamous"

g byte divorcedHead = (h2q10 == 3 & hoh==1)
la var divorcedHead "divorced HoH"

g byte divorcedFemhead = (h2q10 == 3 & femhead == 1)
la var divorcedFemhead "divorced Female head of household"

g byte widowHead = (h2q10 == 4 & hoh==1)
la var widowHead "widowed HoH"

g byte widowFemhead = (h2q10 == 4 & femhead == 1)
la var widowFemhead "Widowed Female head of household"

g byte singleHead = (h2q10==5 & hoh==1)
la var singleHead "single HoH"

g byte singleFemhead = (h2q10==5 & femhead)
la var singleFemhead "single HoH"

* Calculate household demographics (size, adult equivalent units, dep ratio, etc).

/* Household size - Household size refers to the number of usual members in a
household. Usual members are defined as those who have lived in the household
for at least 6 months in the past 12 months. However, it includes persons who
may have spent less than 6 months during the last 12 months in the household
but have joined the household with intention to live permanently or for an
extended period of time.
* http://www.ubos.org/UNHS0910/chapter2_householdcharacteristics.html */

*Create a flag variable for determining who is a usual member from above.
g byte hhmemb = (inlist(h2q7, 1, 2) & h2q5 >= 6)
egen hhsize = total(hhmemb), by(HHID)

* Create sex ratio for households
g byte male = h2q3 == 1 & hhmemb == 1
g byte female = h2q3 == 2 & hhmemb == 1
la var male "male hh members"
la var female "female hh members"

egen msize = total(male), by(HHID)
la var msize "number of males in hh"

egen fsize = total(female), by(HHID)
la var fsize "number of females in hh"

* Create a gender ratio variable
g sexRatio = msize/fsize
recode sexRatio (. = 0) if fsize==0
la var sexRatio "Ratio of males to females in HH"

* Calculate age demographics
g byte under15tmp = h2q8 <= 15 & hhmemb ==1
g byte under24tmp = h2q8 <= 24 &  hhmemb==1

egen under15 = total(under15tmp), by(HHID)
la var under15 "number of hh members under 15 years old"

egen under24 = total(under24tmp), by(HHID)
la var under15 "number of hh members under 24 years old"

egen under15male = total(under15tmp) if male==1, by(HHID)
la var under15male "number of male hh members under 15"

egen under15female = total(under15tmp) if female==1, by(HHID)
la var under15female "number of female hh members under 15"

egen under24male = total(under24tmp) if male==1, by(HHID)
la var under24male "number of male hh members under 24"

egen under24female = total(under24tmp) if female==1, by(HHID)
la var under24female "number of female hh members under 24"

/* Create intl. HH dependency ratio (age ranges appropriate for Bangladesh)
# HH Dependecy Ratio = [(# people 0-14 + those 65+) / # people aged 15-64 ] * 100 #
The dependency ratio is defined as the ratio of the number of members in the age groups
of 0–14 years and above 60 years to the number of members of working age (15–60 years).
The ratio is normally expressed as a percentage (data below are multiplied by 100 for pcts.*/
g byte numDepRatio = (h2q8 < 15 | h2q8 > 64) & hhmemb == 1
g byte demonDepRatio = numDepRatio != 1 & hhmemb == 1
egen totNumDepRatio = total(numDepRatio), by(HHID)
egen totDenomDepRatio = total(demonDepRatio), by(HHID)

* Check that numbers add to hhsize
assert hhsize == totNumDepRatio+totDenomDepRatio if hhmemb==1
g depRatio = (totNumDepRatio/totDenomDepRatio)*100 if totDenomDepRatio!=.
recode depRatio (. = 0) if totDenomDepRatio==0
la var depRatio "Dependency Ratio"

* Calculate household labor shares (ages 12 - 60)
/* Household Labor Shares */
g byte hhLabort = (h2q8>= 12 & h2q8<60) & hhmemb == 1
egen hhlabor = total(hhLabort), by(HHID)
la var hhlabor "hh labor age>11 & < 60"

g byte mlabort = (h2q8>= 12 & h2q8<60 & male == 1)
egen mlabor = total(mlabort), by(HHID)
la var mlabor "hh male labor age>11 & <60"

g byte flabort = (h2q8>= 12 & h2q8<60 & female == 2)
egen flabor = total(flabort), by(HHID)
la var flabor "hh female labor age>11 & <60"
drop hhLabort mlabort flabort

* Male/Female labor share in hh
g mlaborShare = mlabor/hhlabor
recode mlaborShare (. = 0) if hhlabor == 0
la var mlaborShare "share of working age males in hh"

g flaborShare = flabor/hhlabor
recode flaborShare (. = 0) if hhlabor == 0
la var flaborShare "share of working age females in hh"

* Generate adult equivalents in household
g male10 	= 1
g fem10_19 	= 0.84
g fem20		= 0.72
g child10	= 0.60

g ae = .
replace ae = male10 if (h2q8 >=10 ) & male == 1
replace ae = fem10_19 if (h2q8 >= 10 & h2q8 < 20) & female == 1
replace ae = fem20 if (h2q8 >= 20) & female == 1
replace ae = child10 if (h2q8) < 10 & hhmemb == 1
la var ae "Adult equivalents"

egen adultEquiv = total(ae), by(HHID)
la var adultEquiv "Total adult equivalent units"

**********************
* Education outcomes *
**********************
/* Literacy is defined as one’s ability to read with understanding and to
 write meaningfully in any language. */
g byte

* Save
save "$pathout/hhchar.dta", replace

* Keep a master file of only household id's for missing var checks
keep HHID PID
save "$pathout\hhid.dta", replace

* Create an html file of the log for internet sharability
log2html "$pathlog/02_hhchar", replace
log close

/*------------------------------------------------------------------------------
# Name:		02_hhchar
# Purpose:	Process household characteristics and education characteristics
# Author:	Patrick Gault
# Created:	2015/2/21
# License:	MIT License
# Ado(s):	labutil, labutil2 (ssc install labutil, labutil2)
# Dependencies: copylables, attachlabels, 00_SetupFoldersGlobals.do
#-------------------------------------------------------------------------------
*/

clear
capture log close
log using "$pathlog/01_hhchar.log", replace

use "$pathraw/GSEC2.dta", replace

* Merge education data to household roster using the force command
merge 1:1 HHID PID using "$pathraw/GSEC4.dta", force

/* Demographic list to calculate
1. Head of Household Sex
2. Relationship Status
*/

* Create head of household variable based on primary respondent and sex
g byte hoh = h2q4 == 1
la var hoh "Head of household"

g byte femhead = h2q3 == 2 & h2q4 == 1
la var femhead "Female head of household"

g agehead = h2q8 if hoh == 1
la var agehead "Age of head of household"

g ageheadsq = agehead^2
la var ageheadsq "Squared age of the head (for non-linear effects)"

* Relationship status variables
g byte marriedHohm = h2q10 == 1 & hoh==1
la var marriedHohm "married HOH monogamous"

g byte marriedHoh = hoh ==1 & inlist(h2q10, 1, 2)
la var marriedHoh "married head of household (any type)"

g byte marriedHohp = h2q10 == 2 & hoh==1
la var marriedHohp "married HOH polygamous"

g byte divorcedHead = (h2q10 == 3 & hoh==1)
la var divorcedHead "divorced HoH"

g byte divorcedFemhead = (h2q10 == 3 & femhead == 1)
la var divorcedFemhead "divorced Female head of household"

g byte widowHead = (h2q10 == 4 & hoh==1)
la var widowHead "widowed HoH"

g byte widowFemhead = (h2q10 == 4 & femhead == 1)
la var widowFemhead "Widowed Female head of household"

g byte singleHead = (h2q10==5 & hoh==1)
la var singleHead "single HoH"

g byte singleFemhead = (h2q10==5 & femhead)
la var singleFemhead "single HoH"

* Calculate household demographics (size, adult equivalent units, dep ratio, etc).

/* Household size - Household size refers to the number of usual members in a
household. Usual members are defined as those who have lived in the household
for at least 6 months in the past 12 months. However, it includes persons who
may have spent less than 6 months during the last 12 months in the household
but have joined the household with intention to live permanently or for an
extended period of time.
* http://www.ubos.org/UNHS0910/chapter2_householdcharacteristics.html */

*Create a flag variable for determining who is a usual member from above.
g byte hhmemb = (inlist(h2q7, 1, 2) & h2q5 >= 6)
egen hhsize = total(hhmemb), by(HHID)

* Create sex ratio for households
g byte male = h2q3 == 1 & hhmemb == 1
g byte female = h2q3 == 2 & hhmemb == 1
la var male "male hh members"
la var female "female hh members"

egen msize = total(male), by(HHID)
la var msize "number of males in hh"

egen fsize = total(female), by(HHID)
la var fsize "number of females in hh"

* Create a gender ratio variable
g sexRatio = msize/fsize
recode sexRatio (. = 0) if fsize==0
la var sexRatio "Ratio of males to females in HH"

* Calculate age demographics
g byte under15tmp = h2q8 <= 15 & hhmemb ==1
g byte under24tmp = h2q8 <= 24 &  hhmemb==1

egen under15 = total(under15tmp), by(HHID)
la var under15 "number of hh members under 15 years old"

egen under24 = total(under24tmp), by(HHID)
la var under15 "number of hh members under 24 years old"

egen under15male = total(under15tmp) if male==1, by(HHID)
la var under15male "number of male hh members under 15"

egen under15female = total(under15tmp) if female==1, by(HHID)
la var under15female "number of female hh members under 15"

egen under24male = total(under24tmp) if male==1, by(HHID)
la var under24male "number of male hh members under 24"

egen under24female = total(under24tmp) if female==1, by(HHID)
la var under24female "number of female hh members under 24"

/* Create intl. HH dependency ratio (age ranges appropriate for Bangladesh)
# HH Dependecy Ratio = [(# people 0-14 + those 65+) / # people aged 15-64 ] * 100 #
The dependency ratio is defined as the ratio of the number of members in the age groups
of 0–14 years and above 60 years to the number of members of working age (15–60 years).
The ratio is normally expressed as a percentage (data below are multiplied by 100 for pcts.*/
g byte numDepRatio = (h2q8 < 15 | h2q8 > 64) & hhmemb == 1
g byte demonDepRatio = numDepRatio != 1 & hhmemb == 1
egen totNumDepRatio = total(numDepRatio), by(HHID)
egen totDenomDepRatio = total(demonDepRatio), by(HHID)

* Check that numbers add to hhsize
assert hhsize == totNumDepRatio+totDenomDepRatio if hhmemb==1
g depRatio = (totNumDepRatio/totDenomDepRatio)*100 if totDenomDepRatio!=.
recode depRatio (. = 0) if totDenomDepRatio==0
la var depRatio "Dependency Ratio"

* Calculate household labor shares (ages 12 - 60)
/* Household Labor Shares */
g byte hhLabort = (h2q8>= 12 & h2q8<60) & hhmemb == 1
egen hhlabor = total(hhLabort), by(HHID)
la var hhlabor "hh labor age>11 & < 60"

g byte mlabort = (h2q8>= 12 & h2q8<60 & male == 1)
egen mlabor = total(mlabort), by(HHID)
la var mlabor "hh male labor age>11 & <60"

g byte flabort = (h2q8>= 12 & h2q8<60 & female == 2)
egen flabor = total(flabort), by(HHID)
la var flabor "hh female labor age>11 & <60"
drop hhLabort mlabort flabort

* Male/Female labor share in hh
g mlaborShare = mlabor/hhlabor
recode mlaborShare (. = 0) if hhlabor == 0
la var mlaborShare "share of working age males in hh"

g flaborShare = flabor/hhlabor
recode flaborShare (. = 0) if hhlabor == 0
la var flaborShare "share of working age females in hh"

* Generate adult equivalents in household
g male10 	= 1
g fem10_19 	= 0.84
g fem20		= 0.72
g child10	= 0.60

g ae = .
replace ae = male10 if (h2q8 >=10 ) & male == 1
replace ae = fem10_19 if (h2q8 >= 10 & h2q8 < 20) & female == 1
replace ae = fem20 if (h2q8 >= 20) & female == 1
replace ae = child10 if (h2q8) < 10 & hhmemb == 1
la var ae "Adult equivalents"

egen adultEquiv = total(ae), by(HHID)
la var adultEquiv "Total adult equivalent units"

**********************
* Education outcomes *
**********************

* Save
save "$pathout/hhchar.dta", replace

* Keep a master file of only household id's for missing var checks
keep HHID PID
save "$pathout\hhid.dta", replace

* Create an html file of the log for internet sharability
log2html "$pathlog/02_hhchar", replace
log close


redo: only dump .text section

redo: only dump .text section

#!/bin/bash
# automaton
# (c) 2013 Justin Gottula
# The source code of this project is distributed under the terms of the
# simplified BSD license. See the LICENSE file for details.

exec >&2

TARGET=$1
TARGET_BASE=$2
OUTPUT=$3

CC="ccache avr-gcc"
AS="avr-as"
LD="avr-ld"
NM="avr-nm"
OBJCOPY="avr-objcopy"
OBJDUMP="avr-objdump"

PROJ_NAME="automaton"

MCU="atmega324a"
CPU_KHZ="20000"

DEFINES=("F_CPU=${CPU_KHZ}000UL")
INC_DIRS=()
LIB_DIRS=()
LIBS=()

CFLAGS_CPU=("-mmcu=$MCU" -fpack-struct -ffreestanding)
CFLAGS_DEBUG=(-ggdb)
CFLAGS_OPT=(-Os -flto -fuse-linker-plugin -fwhole-program)
CFLAGS_WARN=(-Wall -Wextra -Wno-unused-function)
CFLAGS_ETC=(-fno-diagnostics-show-caret)

CFLAGS="-std=gnu11 ${CFLAGS_CPU[@]} ${CFLAGS_DEBUG[@]} ${CFLAGS_OPT[@]} \
${CFLAGS_WARN[@]} ${CFLAGS_ETC[@]}"
CFLAGS_COMPILE="$CFLAGS ${DEFINES[@]/#/-D} ${INC_DIRS[@]/#/-I}"
CFLAGS_LINK="$CFLAGS ${LIB_DIRS[@]/#/-L} ${LIBS[@]/#/-l}"

SOURCES=$(find src -type f -iname '*.c')
OBJECTS=${SOURCES/%.c/.o}

OUT_ELF="out/$PROJ_NAME.elf"
OUT_BIN="out/$PROJ_NAME.bin"
OUT_HEX="out/$PROJ_NAME.hex"
OUT_MAP="out/$PROJ_NAME.map"
OUT_DUMP="out/$PROJ_NAME.dump"

case "$TARGET" in
all)
	redo-ifchange $OUT_ELF $OUT_BIN $OUT_HEX $OUT_MAP $OUT_DUMP
	;;
$OUT_ELF)
	redo-ifchange $OBJECTS
	$CC $CFLAGS_LINK -o $OUTPUT $OBJECTS
	;;
$OUT_BIN)
	redo-ifchange $OUT_ELF
	$OBJCOPY -O binary $OUT_ELF $OUTPUT
	;;
$OUT_HEX)
	redo-ifchange $OUT_ELF
	$OBJCOPY -O ihex $OUT_ELF $OUTPUT
	;;
$OUT_MAP)
	redo-ifchange $OUT_ELF
	$NM -Cn $OUT_ELF >$OUTPUT
	;;
$OUT_DUMP)
	redo-ifchange $OUT_ELF
	$OBJDUMP -aCdfhpS $OUT_ELF >$OUTPUT
	;;
*.o)
	$CC $CFLAGS_COMPILE -o${TARGET/%.o/.dep} -MM -MG ${TARGET/%.o/.c}

	read DEPS <${TARGET/%.o/.dep}
	redo-ifchange ${DEPS#*:}

	$CC $CFLAGS_COMPILE -o$OUTPUT -c ${TARGET/%.o/.c}
	;;
clean)
	rm -rf $(find out -type f)
	rm -rf $(find src -type f -iname *.o)
	rm -rf $(find src -type f -iname *.dep)
	;;
load)
	redo-ifchange $OUT_HEX
	sudo avrdude -p ${MCU/atmega/m} -c usbasp -B 460800 -U flash:w:$OUT_HEX
	;;
*)
	echo "unknown target '$TARGET'"
	exit 1
	;;
esac

#!/bin/bash
# automaton
# (c) 2013 Justin Gottula
# The source code of this project is distributed under the terms of the
# simplified BSD license. See the LICENSE file for details.

exec >&2

TARGET=$1
TARGET_BASE=$2
OUTPUT=$3

CC="ccache avr-gcc"
AS="avr-as"
LD="avr-ld"
NM="avr-nm"
OBJCOPY="avr-objcopy"
OBJDUMP="avr-objdump"

PROJ_NAME="automaton"

MCU="atmega324a"
CPU_KHZ="20000"

DEFINES=("F_CPU=${CPU_KHZ}000UL")
INC_DIRS=()
LIB_DIRS=()
LIBS=()

CFLAGS_CPU=("-mmcu=$MCU" -fpack-struct -ffreestanding)
CFLAGS_DEBUG=(-ggdb)
CFLAGS_OPT=(-Os -flto -fuse-linker-plugin -fwhole-program)
CFLAGS_WARN=(-Wall -Wextra -Wno-unused-function)
CFLAGS_ETC=(-fno-diagnostics-show-caret)

CFLAGS="-std=gnu11 ${CFLAGS_CPU[@]} ${CFLAGS_DEBUG[@]} ${CFLAGS_OPT[@]} \
${CFLAGS_WARN[@]} ${CFLAGS_ETC[@]}"
CFLAGS_COMPILE="$CFLAGS ${DEFINES[@]/#/-D} ${INC_DIRS[@]/#/-I}"
CFLAGS_LINK="$CFLAGS ${LIB_DIRS[@]/#/-L} ${LIBS[@]/#/-l}"

SOURCES=$(find src -type f -iname '*.c')
OBJECTS=${SOURCES/%.c/.o}

OUT_ELF="out/$PROJ_NAME.elf"
OUT_BIN="out/$PROJ_NAME.bin"
OUT_HEX="out/$PROJ_NAME.hex"
OUT_MAP="out/$PROJ_NAME.map"
OUT_DUMP="out/$PROJ_NAME.dump"

case "$TARGET" in
all)
	redo-ifchange $OUT_ELF $OUT_BIN $OUT_HEX $OUT_MAP $OUT_DUMP
	;;
$OUT_ELF)
	redo-ifchange $OBJECTS
	$CC $CFLAGS_LINK -o $OUTPUT $OBJECTS
	;;
$OUT_BIN)
	redo-ifchange $OUT_ELF
	$OBJCOPY -O binary $OUT_ELF $OUTPUT
	;;
$OUT_HEX)
	redo-ifchange $OUT_ELF
	$OBJCOPY -O ihex $OUT_ELF $OUTPUT
	;;
$OUT_MAP)
	redo-ifchange $OUT_ELF
	$NM -Cn $OUT_ELF >$OUTPUT
	;;
$OUT_DUMP)
	redo-ifchange $OUT_ELF
	$OBJDUMP -aCDfhpS $OUT_ELF >$OUTPUT
	;;
*.o)
	$CC $CFLAGS_COMPILE -o${TARGET/%.o/.dep} -MM -MG ${TARGET/%.o/.c}

	read DEPS <${TARGET/%.o/.dep}
	redo-ifchange ${DEPS#*:}

	$CC $CFLAGS_COMPILE -o$OUTPUT -c ${TARGET/%.o/.c}
	;;
clean)
	rm -rf $(find out -type f)
	rm -rf $(find src -type f -iname *.o)
	rm -rf $(find src -type f -iname *.dep)
	;;
load)
	redo-ifchange $OUT_HEX
	sudo avrdude -p ${MCU/atmega/m} -c usbasp -B 460800 -U flash:w:$OUT_HEX
	;;
*)
	echo "unknown target '$TARGET'"
	exit 1
	;;
esac