text: /***************************************************************
   *
   * Copyright (C) 1990-2011, Condor Team, Computer Sciences Department,
   * University of Wisconsin-Madison, WI.
   * 
   * Licensed under the Apache License, Version 2.0 (the "License"); you
   * may not use this file except in compliance with the License.  You may
   * obtain a copy of the License at
   * 
   *    http://www.apache.org/licenses/LICENSE-2.0
   * 
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   *
   ***************************************************************/
  
  
  #include "condor_common.h"
  #include "subsystem_info.h"
  
  /*
   * Main routine and function for the startd.
   */
  #define _STARTD_NO_DECLARE_GLOBALS 1
  #include "startd.h"
  #include "vm_common.h"
  #include "VMManager.h"
  #include "VMRegister.h"
  #include "classadHistory.h"
  #include "misc_utils.h"
  #include "slot_builder.h"
  #include "history_queue.h"
  
  #if defined(WANT_CONTRIB) && defined(WITH_MANAGEMENT)
  #include "StartdPlugin.h"
  #if defined(WIN32)
  extern int load_startd_mgmt(void);
  #endif
  #endif
  
  // Define global variables
  
  // windows-specific: notifier for the condor "birdwatcher" (system tray icon)
  #ifdef WIN32
  CondorSystrayNotifier systray_notifier;
  #endif
  
  // Resource manager
  ResMgr*	resmgr;			// Pointer to the resource manager object
  
  // Polling variables
  int	polling_interval = 0;	// Interval for polling when there are resources in use
  int	update_interval = 0;	// Interval to update CM
  int	update_offset = 0;		// Interval offset to update CM
  
  // String Lists
  StringList *startd_job_attrs = NULL;
  StringList *startd_slot_attrs = NULL;
  static StringList *valid_cod_users = NULL; 
  
  // Hosts
  char*	accountant_host = NULL;
  
  // Others
  int		match_timeout;		// How long you're willing to be
  							// matched before claimed 
  int		killing_timeout;	// How long you're willing to be in
  							// preempting/killing before you drop the
  							// hammer on the starter
  int		vm_killing_timeout;	// How long you're willing to be
  							// in preempting/killing before you
  							// drop the hammer on the starter for VM universe jobs
  int		max_claim_alives_missed;  // how many keepalives can we miss
  								  // until we timeout the claim
  time_t	startd_startup;		// Time when the startd started up
  
  int		console_slots = 0;	// # of nodes in an SMP that care about
  int		keyboard_slots = 0;  //   console and keyboard activity
  int		disconnected_keyboard_boost;	// # of seconds before when we
  	// started up that we advertise as the last key press for
  	// resources that aren't connected to anything.  
  
  int		startd_noclaim_shutdown = 0;	
      // # of seconds we can go without being claimed before we "pull
      // the plug" and tell the master to shutdown.
  
  char* Name = NULL;
  
  #define DEFAULT_PID_SNAPSHOT_INTERVAL 15
  int		pid_snapshot_interval = DEFAULT_PID_SNAPSHOT_INTERVAL;
      // How often do we take snapshots of the pid families? 
  
  int main_reaper = 0;
  
  // Cron stuff
  StartdCronJobMgr	*cron_job_mgr;
  
  // Benchmark stuff
  StartdBenchJobMgr	*bench_job_mgr;
  
  // remote history queries
  HistoryHelperQueue *history_queue_mgr;
  
  // Cleanup reminders, for things we tried to cleanup but initially failed.
  // for instance, the execute directory on Windows when antivirus software is hold a file in it open.
  static int cleanup_reminder_timer_id = -1;
  int cleanup_reminder_timer_interval = 62; // default to doing at least some cleanup once a minute (ish)
  CleanupReminderMap cleanup_reminders;
  extern void register_cleanup_reminder_timer();
  
  /*
   * Prototypes of static functions.
   */
  
  void usage( char* );
  void main_init( int argc, char* argv[] );
  int init_params(int);
  void main_config();
  void finish_main_config();
  void main_shutdown_fast();
  void main_shutdown_graceful();
  extern "C" int do_cleanup(int,int,const char*);
  int reaper(int pid, int status);
  int	shutdown_reaper(int pid, int status ); 
  
  void
  usage( char* MyName)
  {
  	fprintf( stderr, "Usage: %s [option]\n", MyName );
  	fprintf( stderr, "  where [option] is one of:\n" );
  	fprintf( stderr, 
  			 "     [-skip-benchmarks]\t(now a no-op)\n" );
  	DC_Exit( 1 );
  }
  
  void
  main_init( int, char* argv[] )
  {
  	char**	ptr; 
  
  	// Reset the cron & benchmark managers to a known state
  	cron_job_mgr = NULL;
  	bench_job_mgr = NULL;
  
  		// Process command line args.
  	for(ptr = argv + 1; *ptr; ptr++) {
  		if(ptr[0][0] != '-') {
  			usage( argv[0] );
  		}
  		switch( ptr[0][1] ) {
  		case 's':
  			break;
  		case 'n':
  			ptr++;
  			if( !(ptr && *ptr) ) {
                  EXCEPT( "-n requires another arugment" );
              }
  			if (Name) {
  				free(Name);
  			}
              Name = build_valid_daemon_name( *ptr );
              dprintf( D_ALWAYS, "Using name: %s\n", Name );
              break;
  		default:
  			fprintf( stderr, "Error:  Unknown option %s\n", *ptr );
  			usage( argv[0] );
  		}
  	}
  
  		// Record the time we started up for use in determining
  		// keyboard idle time on SMP machines, etc.
  	startd_startup = time( 0 );
  
  		// Instantiate the Resource Manager object.
  	resmgr = new ResMgr;
  
  		// find all the starters we care about and get their classads. 
  	Starter::config();
  
  	ClassAd tmp_classad;
  	std::string starter_ability_list;
  	Starter::publish(&tmp_classad);
  	tmp_classad.LookupString(ATTR_STARTER_ABILITY_LIST, starter_ability_list);
  	if( starter_ability_list.find(ATTR_HAS_VM) != std::string::npos ) {
  		// Now starter has codes for vm universe.
  		resmgr->m_vmuniverse_mgr.setStarterAbility(true);
  		// check whether vm universe is available through vmgahp server
  		resmgr->m_vmuniverse_mgr.checkVMUniverse( false );
  	}
  
  		// Read in global parameters from the config file.
  		// We do this after we instantiate the resmgr, so we can know
  		// what num_cpus is, but before init_resources(), so we can
  		// use polling_interval to figure out how big to make each
  		// Resource's LoadQueue object.
  	init_params(1);		// The 1 indicates that this is the first time
  
  #if defined(WIN32)
  		// We do this on Win32 since Win32 uses last_x_event
  		// variable in a similar fasion to the X11 condor_kbdd, and
  		// thus it must be initialized.
  	command_x_event(0, 0);
  #endif
  
  		// create the class that tracks and limits reqmote history queries
  	history_queue_mgr = new HistoryHelperQueue();
  	history_queue_mgr->want_startd_history(true);
  	history_queue_mgr->setup(1000, param_integer("HISTORY_HELPER_MAX_CONCURRENCY", 50));
  
  		// Instantiate Resource objects in the ResMgr
  	resmgr->init_resources();
  
  		// Do a little sanity checking and cleanup
  	StringList execute_dirs;
  	resmgr->FillExecuteDirsList( &execute_dirs );
  	check_execute_dir_perms( execute_dirs );
  	cleanup_execute_dirs( execute_dirs );
  
  		// Compute all attributes
  	resmgr->compute_static();
  
  		// Startup Cron
  	cron_job_mgr = new StartdCronJobMgr( );
  	cron_job_mgr->Initialize( "startd" );
  
  		// Startup benchmarking
  	bench_job_mgr = new StartdBenchJobMgr( );
  	bench_job_mgr->Initialize( "benchmarks" );
  
  		// this does a bit of work that isn't valid yet 
  	resmgr->walk( &Resource::init_classad );
  
  	// Now that we have our classads, we can compute things that
  		// need to be evaluated
  	resmgr->compute_dynamic(true);
  
  		// If we EXCEPT, don't leave any starters lying around.
  	_EXCEPT_Cleanup = do_cleanup;
  
  		// register daemoncore stuff
  
  		//////////////////////////////////////////////////
  		// Commands
  		//////////////////////////////////////////////////
  
  		// These commands all read the ClaimId off the wire, find
  		// the resource with that ClaimId, and call appropriate
  		// action on that resource.  Plus, all of these commands only
  		// make sense when we're in the claimed state.  So, we can
  		// handle them all with a common handler.  For all of them,
  		// you need DAEMON permission.
  	daemonCore->Register_Command( ALIVE, "ALIVE", 
  								  command_handler,
  								  "command_handler", DAEMON,
  								  D_FULLDEBUG ); 
  	daemonCore->Register_Command( DEACTIVATE_CLAIM,
  								  "DEACTIVATE_CLAIM",  
  								  command_handler,
  								  "command_handler", DAEMON );
  	daemonCore->Register_Command( DEACTIVATE_CLAIM_FORCIBLY, 
  								  "DEACTIVATE_CLAIM_FORCIBLY", 
  								  command_handler,
  								  "command_handler", DAEMON );
  	daemonCore->Register_Command( PCKPT_FRGN_JOB, "PCKPT_FRGN_JOB", 
  								  command_handler,
  								  "command_handler", DAEMON );
  	daemonCore->Register_Command( REQ_NEW_PROC, "REQ_NEW_PROC", 
  								  command_handler,
  								  "command_handler", DAEMON );
  	if (param_boolean("ALLOW_SLOT_PAIRING", false)) {
  		daemonCore->Register_Command( SWAP_CLAIM_AND_ACTIVATION, "SWAP_CLAIM_AND_ACTIVATION",
  								  command_with_opts_handler,
  								  "command_handler", DAEMON );
  	}
  
  		// These commands are special and need their own handlers
  		// READ permission commands
  	daemonCore->Register_Command( GIVE_STATE,
  								  "GIVE_STATE",
  								  command_give_state,
  								  "command_give_state", READ );
  	daemonCore->Register_Command( GIVE_TOTALS_CLASSAD,
  								  "GIVE_TOTALS_CLASSAD",
  								  command_give_totals_classad,
  								  "command_give_totals_classad", READ );
  	daemonCore->Register_Command( QUERY_STARTD_ADS, "QUERY_STARTD_ADS",
  								  command_query_ads,
  								  "command_query_ads", READ );
  	if (history_queue_mgr) {
  		daemonCore->Register_CommandWithPayload(GET_HISTORY,
  			"GET_HISTORY",
  			(CommandHandlercpp)&HistoryHelperQueue::command_handler,
  			"command_get_history", history_queue_mgr, READ);
  	}
  
  		// DAEMON permission commands
  	daemonCore->Register_Command( ACTIVATE_CLAIM, "ACTIVATE_CLAIM",
  								  command_activate_claim,
  								  "command_activate_claim", DAEMON );
  	daemonCore->Register_Command( REQUEST_CLAIM, "REQUEST_CLAIM", 
  								  command_request_claim,
  								  "command_request_claim", DAEMON );
  	daemonCore->Register_Command( RELEASE_CLAIM, "RELEASE_CLAIM", 
  								  command_release_claim,
  								  "command_release_claim", DAEMON );
  	daemonCore->Register_Command( SUSPEND_CLAIM, "SUSPEND_CLAIM", 
  								  command_suspend_claim,
  								  "command_suspend_claim", DAEMON );
  	daemonCore->Register_Command( CONTINUE_CLAIM, "CONTINUE_CLAIM", 
  								  command_continue_claim,
  								  "command_continue_claim", DAEMON );	
  	
  	daemonCore->Register_Command( X_EVENT_NOTIFICATION,
  								  "X_EVENT_NOTIFICATION",
  								  command_x_event,
  								  "command_x_event", ALLOW,
  								  D_FULLDEBUG ); 
  	daemonCore->Register_Command( PCKPT_ALL_JOBS, "PCKPT_ALL_JOBS", 
  								  command_pckpt_all,
  								  "command_pckpt_all", DAEMON );
  	daemonCore->Register_Command( PCKPT_JOB, "PCKPT_JOB", 
  								  command_name_handler,
  								  "command_name_handler", DAEMON );
  #if !defined(WIN32)
  	daemonCore->Register_Command( DELEGATE_GSI_CRED_STARTD, "DELEGATE_GSI_CRED_STARTD",
  	                              command_delegate_gsi_cred,
  	                              "command_delegate_gsi_cred", DAEMON );
  #endif
  
  	daemonCore->Register_Command( COALESCE_SLOTS, "COALESCE_SLOTS",
  								  command_coalesce_slots,
  								  "command_coalesce_slots", DAEMON );
  
  		// ex-OWNER permission commands, now ADMINISTRATOR
  	daemonCore->Register_Command( VACATE_ALL_CLAIMS,
  								  "VACATE_ALL_CLAIMS",
  								  command_vacate_all,
  								  "command_vacate_all", ADMINISTRATOR );
  	daemonCore->Register_Command( VACATE_ALL_FAST,
  								  "VACATE_ALL_FAST",
  								  command_vacate_all,
  								  "command_vacate_all", ADMINISTRATOR );
  	daemonCore->Register_Command( VACATE_CLAIM,
  								  "VACATE_CLAIM",
  								  command_name_handler,
  								  "command_name_handler", ADMINISTRATOR );
  	daemonCore->Register_Command( VACATE_CLAIM_FAST,
  								  "VACATE_CLAIM_FAST",
  								  command_name_handler,
  								  "command_name_handler", ADMINISTRATOR );
  
  		// NEGOTIATOR permission commands
  	daemonCore->Register_Command( MATCH_INFO, "MATCH_INFO",
  								  command_match_info,
  								  "command_match_info", NEGOTIATOR );
  
  		// the ClassAd-only command
  	daemonCore->Register_Command( CA_AUTH_CMD, "CA_AUTH_CMD",
  								  command_classad_handler,
  								  "command_classad_handler", WRITE );
  	daemonCore->Register_Command( CA_CMD, "CA_CMD",
  								  command_classad_handler,
  								  "command_classad_handler", WRITE );
  
  	// Virtual Machine commands
  	if( vmapi_is_host_machine() == TRUE ) {
  		daemonCore->Register_Command( VM_REGISTER,
  				"VM_REGISTER",
  				command_vm_register,
  				"command_vm_register", DAEMON,
  				D_FULLDEBUG );
  	}
  
  		// Commands from starter for VM universe
  	daemonCore->Register_Command( VM_UNIV_GAHP_ERROR, 
  								"VM_UNIV_GAHP_ERROR",
  								command_vm_universe, 
  								"command_vm_universe", DAEMON, 
  								D_FULLDEBUG );
  	daemonCore->Register_Command( VM_UNIV_VMPID, 
  								"VM_UNIV_VMPID",
  								command_vm_universe, 
  								"command_vm_universe", DAEMON, 
  								D_FULLDEBUG );
  	daemonCore->Register_Command( VM_UNIV_GUEST_IP, 
  								"VM_UNIV_GUEST_IP",
  								command_vm_universe, 
  								"command_vm_universe", DAEMON, 
  								D_FULLDEBUG );
  	daemonCore->Register_Command( VM_UNIV_GUEST_MAC, 
  								"VM_UNIV_GUEST_MAC",
  								command_vm_universe, 
  								"command_vm_universe", DAEMON, 
  								D_FULLDEBUG );
  
  	daemonCore->Register_CommandWithPayload( DRAIN_JOBS,
  								  "DRAIN_JOBS",
  								  command_drain_jobs,
  											 "command_drain_jobs", ADMINISTRATOR);
  	daemonCore->Register_CommandWithPayload( CANCEL_DRAIN_JOBS,
  								  "CANCEL_DRAIN_JOBS",
  								  command_cancel_drain_jobs,
  								  "command_cancel_drain_jobs", ADMINISTRATOR);
  
  		//////////////////////////////////////////////////
  		// Reapers 
  		//////////////////////////////////////////////////
  	main_reaper = daemonCore->Register_Reaper( "reaper_starters", 
  		reaper, "reaper" );
  	ASSERT(main_reaper != FALSE);
  
  	daemonCore->Set_Default_Reaper( main_reaper );
  
  #if defined(WIN32)
  		// Pretend we just got an X event so we think our console idle
  		// is something, even if we haven't heard from the kbdd yet.
  		// We do this on Win32 since Win32 uses last_x_event
  		// variable in a similar fasion to the X11 condor_kbdd, and
  		// thus it must be initialized.
  	command_x_event(0, 0);
  #endif
  
  	resmgr->start_sweep_timer();
  	resmgr->start_update_timer();
  
  #if HAVE_HIBERNATION
  	resmgr->updateHibernateConfiguration();
  #endif /* HAVE_HIBERNATION */
  
  		// Evaluate the state of all resources and update CM 
  		// We don't just call eval_and_update_all() b/c we don't need
  		// to recompute anything.
  		// This is now called by a timer registered by start_update_timer()
  	//resmgr->update_all();
  
  #if defined(WANT_CONTRIB) && defined(WITH_MANAGEMENT)
  #if defined(UNIX)
     StartdPluginManager::Load();
  #elif defined(WIN32)
  	load_startd_mgmt();
  #endif
     StartdPluginManager::Initialize();
  #endif
  }
  
  
  void
  main_config()
  {
  	bool done_allocating;
  
  		// Reread config file for global settings.
  	init_params(0);
  		// Process any changes in the slot type specifications
  	done_allocating = resmgr->reconfig_resources();
  	if( done_allocating ) {
  		finish_main_config();
  	}
  }
  
  
  void
  finish_main_config( void ) 
  {
  #if defined(WIN32)
  	resmgr->reset_credd_test_throttle();
  #endif
  		// Recompute machine-wide attributes object.
  	resmgr->compute_static();
  		// Rebuild ads for each resource.  
  	resmgr->walk( &Resource::init_classad );  
  		// Reset various settings in the ResMgr.
  	resmgr->reset_timers();
  
  	dprintf( D_FULLDEBUG, "MainConfig finish\n" );
  	cron_job_mgr->Reconfig(  );
  	bench_job_mgr->Reconfig(  );
  	Starter::config();
  
  #if HAVE_HIBERNATION
  	resmgr->updateHibernateConfiguration();
  #endif /* HAVE_HIBERNATION */
  
  		// Re-evaluate and update the CM for each resource (again, we
  		// don't need to recompute, since we just did that, so we call
  		// the special case version).
  		// This is now called by a timer registered by reset_timers()
  	//resmgr->update_all();
  }
  
  
  int
  init_params( int first_time)
  {
  	static bool match_password_enabled = false;
  
  	if (first_time) {
  		std::string func_name("SlotEval");
  		classad::FunctionCall::RegisterFunction( func_name, OtherSlotEval );
  	}
  
  	resmgr->init_config_classad();
  
  	polling_interval = param_integer( "POLLING_INTERVAL", 5 );
  
  	update_interval = param_integer( "UPDATE_INTERVAL", 300, 1 );
  	update_offset = param_integer( "UPDATE_OFFSET", 0, 0 );
  
  	if( accountant_host ) {
  		free( accountant_host );
  	}
  	accountant_host = param("ACCOUNTANT_HOST");
  
  	match_timeout = param_integer( "MATCH_TIMEOUT", 120 );
  
  	killing_timeout = param_integer( "KILLING_TIMEOUT", 30 );
  	vm_killing_timeout = param_integer( "VM_KILLING_TIMEOUT", 60);
  	if (vm_killing_timeout < killing_timeout) { vm_killing_timeout = killing_timeout; } // use the larger of the two for VM universe
  
  	max_claim_alives_missed = param_integer( "MAX_CLAIM_ALIVES_MISSED", 6 );
  
  	sysapi_reconfig();
  
  	// Fill in *_JOB_ATTRS
  	//
  	if (startd_job_attrs) {
  		startd_job_attrs->clearAll();
  	} else {
  		startd_job_attrs = new StringList();
  	}
  	param_and_insert_unique_items("STARTD_JOB_ATTRS", *startd_job_attrs);
  	// merge in the deprecated _EXPRS config
  	param_and_insert_unique_items("STARTD_JOB_EXPRS", *startd_job_attrs);
  	// Now merge in the attrs required by HTCondor - this knob is a secret from users
  	param_and_insert_unique_items("SYSTEM_STARTD_JOB_ATTRS", *startd_job_attrs);
  	if (startd_job_attrs->isEmpty()) {
  		delete startd_job_attrs;
  		startd_job_attrs = NULL;
  	}
  
  	// Fill in *_SLOT_ATTRS
  	//
  	if (startd_slot_attrs) {
  		startd_slot_attrs->clearAll();
  	} else {
  		startd_slot_attrs = new StringList();
  	}
  	param_and_insert_unique_items("STARTD_SLOT_ATTRS", *startd_slot_attrs);
  	param_and_insert_unique_items("STARTD_SLOT_EXPRS", *startd_slot_attrs);
  
  	// now insert attributes needed by HTCondor
  	param_and_insert_unique_items("SYSTEM_STARTD_SLOT_ATTRS", *startd_slot_attrs);
  	if (startd_slot_attrs->isEmpty()) {
  		delete  startd_slot_attrs;
  		startd_slot_attrs = NULL;
  	}
  
  	console_slots = param_integer( "SLOTS_CONNECTED_TO_CONSOLE", -12345);
  	if (console_slots == -12345) {
  		// if no value set, try the old names...
  		console_slots = resmgr->m_attr->num_cpus();
  		console_slots = param_integer( "VIRTUAL_MACHINES_CONNECTED_TO_CONSOLE",
  		                param_integer( "CONSOLE_VMS",
  		                param_integer( "CONSOLE_CPUS",
  		                console_slots)));
  	}
  
  	keyboard_slots = param_integer( "SLOTS_CONNECTED_TO_KEYBOARD", -12345);
  	if (keyboard_slots == -12345) {
  		// if no value set, try the old names...
  		keyboard_slots = resmgr->m_attr->num_cpus();
  		keyboard_slots = param_integer( "VIRTUAL_MACHINES_CONNECTED_TO_KEYBOARD",
  		                 param_integer( "KEYBOARD_VMS",
  		                 param_integer( "KEYBOARD_CPUS", 1)));
  	}
  
  	disconnected_keyboard_boost = param_integer( "DISCONNECTED_KEYBOARD_IDLE_BOOST", 1200 );
  
  	startd_noclaim_shutdown = param_integer( "STARTD_NOCLAIM_SHUTDOWN", 0 );
  
  	// a 0 or negative value for the timer interval will disable cleanup reminders entirely
  	cleanup_reminder_timer_interval = param_integer( "STARTD_CLEANUP_REMINDER_TIMER_INTERVAL", 62 );
  
  	auto_free_ptr tmp(param("STARTD_NAME"));
  	if (tmp) {
  		if( Name ) {
  			free(Name);
  		}
  		Name = build_valid_daemon_name(tmp);
  		dprintf( D_FULLDEBUG, "Using %s for name\n", Name );
  	}
  
  	pid_snapshot_interval = param_integer( "PID_SNAPSHOT_INTERVAL", DEFAULT_PID_SNAPSHOT_INTERVAL );
  
  	if( valid_cod_users ) {
  		delete( valid_cod_users );
  		valid_cod_users = NULL;
  	}
  	tmp.set(param("VALID_COD_USERS"));
  	if (tmp) {
  		valid_cod_users = new StringList();
  		valid_cod_users->initializeFromString( tmp );
  	}
  
  	if( vmapi_is_virtual_machine() == TRUE ) {
  		vmapi_destroy_vmregister();
  	}
  	tmp.set(param("VMP_HOST_MACHINE"));
  	if (tmp) {
  		if (vmapi_is_my_machine(tmp.ptr())) {
  			dprintf( D_ALWAYS, "WARNING: VMP_HOST_MACHINE should be the hostname of host machine. In host machine, it doesn't need to be defined\n");
  		} else {
  			vmapi_create_vmregister(tmp.ptr());
  		}
  	}
  
  	if( vmapi_is_host_machine() == TRUE ) {
  		vmapi_destroy_vmmanager();
  	}
  	tmp.set(param("VMP_VM_LIST"));
  	if (tmp) {
  		if( vmapi_is_virtual_machine() == TRUE ) {
  			dprintf( D_ALWAYS, "WARNING: both VMP_HOST_MACHINE and VMP_VM_LIST are defined. Assuming this machine is a virtual machine\n");
  		}else {
  			vmapi_create_vmmanager(tmp.ptr());
  		}
  	}
  
  	InitJobHistoryFile( "STARTD_HISTORY" , "STARTD_PER_JOB_HISTORY_DIR");
  
  	bool new_match_password = param_boolean( "SEC_ENABLE_MATCH_PASSWORD_AUTHENTICATION", true );
  	if ( new_match_password != match_password_enabled ) {
  		IpVerify* ipv = daemonCore->getIpVerify();
  		if ( new_match_password ) {
  			ipv->PunchHole( DAEMON, SUBMIT_SIDE_MATCHSESSION_FQU );
  			ipv->PunchHole( CLIENT_PERM, SUBMIT_SIDE_MATCHSESSION_FQU );
  		} else {
  			ipv->FillHole( DAEMON, SUBMIT_SIDE_MATCHSESSION_FQU );
  			ipv->FillHole( CLIENT_PERM, SUBMIT_SIDE_MATCHSESSION_FQU );
  		}
  		match_password_enabled = new_match_password;
  	}
  
  
  	return TRUE;
  }
  
  // implement an exponential backoff by skipping some iterations
  // the algorithm is even powers of 2 until iteration 60, then once per 'hour' for 24 hours, then once per 'day' forever
  // The algorithm assumes that the iteration tick rate is once per minute.
  // if it is not then 'hour' or 'day' will be faster or slower to match the tick rate.
  //
  static bool retry_on_this_iter(int iter, CleanupReminder::category /*cat*/)
  {
  	// try once a day once we the we have been at it longer than a day.
  	if (iter > 60*24) return (iter % (60*24)) == 0;
  
  	// try once an hour once we the we have been at it longer than an hour
  	if (iter > 60) return (iter % 60) == 0;
  
  	// do exponential back off for the first hour
  	if ((iter & (iter-1)) == 0)
  		return true;
  	return false;
  }
  
  // process the cleanup reminders.
  void CleanupReminderTimerCallback()
  {
  	dprintf(D_FULLDEBUG, "In CleanupReminderTimerCallback() there are %d reminders\n", (int)cleanup_reminders.size());
  
  	for (auto jt = cleanup_reminders.begin(); jt != cleanup_reminders.end(); /* advance in the loop */) {
  		auto it = jt++; // so we can remove the current item if we manage to clean it up
  		it->second += 1; // record that we looked at this.
  		bool erase_it = false; // set this to true when we succeed (or don't need to try anymore)
  
  		const CleanupReminder & cr = it->first; // alias the CleanupReminder so that the code below is clearer
  
  		bool retry_now = retry_on_this_iter(it->second, cr.cat);
  		dprintf(D_FULLDEBUG, "cleanup_reminder %s, iter %d, retry_now = %d\n", cr.name.c_str(), it->second, retry_now);
  
  		// if our exponential backoff says we should retry this time, attempt the cleanup.
  		if (retry_now) {
  			int err=0;
  			switch (cr.cat) {
  			case CleanupReminder::category::exec_dir:
  				if (retry_cleanup_execute_dir(cr.name, cr.opt, err)) {
  					dprintf(D_ALWAYS, "Retry of directory delete '%s' succeeded. removing it from the retry list\n", cr.name.c_str());
  					erase_it = true;
  				} else {
  					dprintf(D_ALWAYS, "Retry of directory delete '%s' failed with error %d. will try again later\n", cr.name.c_str(), err);
  				}
  				break;
  			case CleanupReminder::category::account:
  				if (retry_cleanup_user_account(cr.name, cr.opt, err)) {
  					dprintf(D_ALWAYS, "Retry of account cleanup for '%s' succeeded. removing it from the retry list\n", cr.name.c_str());
  					erase_it = true;
  				} else {
  					dprintf(D_ALWAYS, "Retry of account cleanup '%s' failed with error %d. will try again later\n", cr.name.c_str(), err);
  				}
  				break;
  			}
  
  		}
  
  		// if we successfully cleaned up, or cleanup is now moot, remove the item from the list.
  		if (erase_it) {
  			cleanup_reminders.erase(it);
  		}
  	}
  
  	// if the collection of things to try and clean up is empty, turn off the timer
  	// it will get turned back on the next time an item is added to the collection
  	if (cleanup_reminders.empty() && cleanup_reminder_timer_id != -1) {
  		daemonCore->Cancel_Timer(cleanup_reminder_timer_id);
  		cleanup_reminder_timer_id = -1;
  		dprintf( D_ALWAYS, "Cancelling timer for cleanup reminders - collection is empty.\n" );
  	}
  }
  
  // register a timer to process the cleanup reminders, if the timer is not already registered
  void register_cleanup_reminder_timer()
  {
  	// a timer interval of 0 or negative will disable cleanup reminders
  	if (cleanup_reminder_timer_interval <= 0)
  		return;
  	if (cleanup_reminder_timer_id < 0) {
  		dprintf( D_ALWAYS, "Starting timer for cleanup reminders.\n" );
  		int id = daemonCore->Register_Timer(cleanup_reminder_timer_interval,
  								cleanup_reminder_timer_interval,
  								(TimerHandler)CleanupReminderTimerCallback,
  								"CleanupReminderTimerCallback");
  		if  (id < 0) {
  			EXCEPT( "Can't register DaemonCore timer for cleanup reminders" );
  		}
  		cleanup_reminder_timer_id = id;
  	}
  }
  
  void PREFAST_NORETURN
  startd_exit() 
  {
  	// print resources into log.  we need to do this before we free them
  	if(param_boolean("STARTD_PRINT_ADS_ON_SHUTDOWN", false)) {
  		dprintf(D_ALWAYS, "*** BEGIN AD DUMP ***\n");
  		resmgr->walk(&Resource::dropAdInLogFile);
  		dprintf(D_ALWAYS, "*** END AD DUMP ***\n");
  	}
  
  	// Shut down the cron logic
  	if( cron_job_mgr ) {
  		dprintf( D_ALWAYS, "Deleting cron job manager\n" );
  		cron_job_mgr->Shutdown( true );
  		delete cron_job_mgr;
  	}
  
  	// Shut down the benchmark job manager
  	if( bench_job_mgr ) {
  		dprintf( D_ALWAYS, "Deleting benchmark job mgr\n" );
  		bench_job_mgr->Shutdown( true );
  		delete bench_job_mgr;
  	}
  
  	// Cleanup the resource manager
  	if ( resmgr ) {
  #if HAVE_HIBERNATION
  		// don't want the final update, since it will overwrite 
  		// our off-line ad
  		if ( !resmgr->hibernating () ) {
  #else
  		if ( true ) {
  #endif /* HAVE_HIBERNATION */
  			dprintf( D_FULLDEBUG, "About to send final update to the central manager\n" );	
  			resmgr->final_update();
  			if ( resmgr->m_attr ) {
  				resmgr->m_attr->final_idle_dprintf();
  			}
  		}
  		
  			// clean-up stale claim-id files
  		int i;
  		char* filename;
  		for( i = 0; i <= resmgr->numSlots(); i++ ) { 
  			filename = startdClaimIdFile( i );
  			if (unlink(filename) < 0) {
  				dprintf( D_FULLDEBUG, "startd_exit: Failed to remove file '%s'\n", filename );
  			}
  			free( filename );
  			filename = NULL;
  		}
  
  		delete resmgr;
  		resmgr = NULL;
  	}
  
  #ifdef WIN32
  	systray_notifier.notifyCondorOff();
  #endif
  
  #if defined(WANT_CONTRIB) && defined(WITH_MANAGEMENT)
  	StartdPluginManager::Shutdown();
  #endif
  
  	dprintf( D_ALWAYS, "All resources are free, exiting.\n" );
  	DC_Exit(0);
  }
  
  void
  main_shutdown_fast()
  {
  	dprintf( D_ALWAYS, "shutdown fast\n" );
  
  		// Shut down the cron logic
  	if( cron_job_mgr ) {
  		cron_job_mgr->Shutdown( true );
  	}
  
  		// Shut down the benchmark logic
  	if( bench_job_mgr ) {
  		bench_job_mgr->Shutdown( true );
  	}
  
  		// If the machine is free, we can just exit right away.
  	startd_check_free();
  
  		// Remember that we're in shutdown-mode so we will refuse
  		// various commands. 
  	resmgr->markShutdown();
  
  	daemonCore->Reset_Reaper( main_reaper, "shutdown_reaper", 
  								 shutdown_reaper,
  								 "shutdown_reaper" );
  
  		// Quickly kill all the starters that are running
  	resmgr->walk( &Resource::killAllClaims );
  
  	daemonCore->Register_Timer( 0, 5, 
  								startd_check_free,
  								 "startd_check_free" );
  }
  
  
  void
  main_shutdown_graceful()
  {
  	dprintf( D_ALWAYS, "shutdown graceful\n" );
  
  		// Shut down the cron logic
  	if( cron_job_mgr ) {
  		cron_job_mgr->Shutdown( false );
  	}
  
  		// Shut down the benchmark logic
  	if( bench_job_mgr ) {
  		bench_job_mgr->Shutdown( false );
  	}
  
  		// If the machine is free, we can just exit right away.
  	startd_check_free();
  
  		// Remember that we're in shutdown-mode so we will refuse
  		// various commands. 
  	resmgr->markShutdown();
  
  	daemonCore->Reset_Reaper( main_reaper, "shutdown_reaper", 
  								 shutdown_reaper,
  								 "shutdown_reaper" );
  
  		// Release all claims, active or not
  	resmgr->walk( &Resource::releaseAllClaims );
  
  	daemonCore->Register_Timer( 0, 5, 
  								startd_check_free,
  								 "startd_check_free" );
  }
  
  
  int
  reaper(int pid, int status)
  {
  
  	if( WIFSIGNALED(status) ) {
  		dprintf(D_FAILURE|D_ALWAYS, "Starter pid %d died on signal %d (%s)\n",
  				pid, WTERMSIG(status), daemonCore->GetExceptionString(status));
  	} else {
  		int d_stat = status ? D_FAILURE : D_ALWAYS;
  		dprintf(d_stat|D_ALWAYS, "Starter pid %d exited with status %d\n",
  				pid, WEXITSTATUS(status));
  	}
  
  	// Adjust info for vm universe
  	resmgr->m_vmuniverse_mgr.freeVM(pid);
  
  	Starter * starter = findStarterByPid(pid);
  	Claim* claim = resmgr->getClaimByPid(pid);
  	if (claim) {
  		// this will call the starter->exited method also
  		claim->starterExited(starter, status);
  	} else if (starter) {
  		// claim is gone, we want to call the starter->exited method ourselves.
  		starter->exited(NULL, status);
  		delete starter;
  	} else {
  		dprintf(D_FAILURE|D_ALWAYS, "ERROR: Starter pid %d is not associated with a Starter object or a Claim.\n", pid);
  	}
  	return TRUE;
  }
  
  
  int
  shutdown_reaper(int pid, int status)
  {
  	reaper(pid,status);
  	startd_check_free();
  	return TRUE;
  }
  
  
  int
  do_cleanup(int,int,const char*)
  {
  	static int already_excepted = FALSE;
  
  	if ( already_excepted == FALSE ) {
  		already_excepted = TRUE;
  			// If the machine is already free, we can exit right away.
  		startd_check_free();		
  			// Otherwise, quickly kill all the active starters.
  		resmgr->walk( &Resource::void_kill_claim );
  		dprintf( D_FAILURE|D_ALWAYS, "startd exiting because of fatal exception.\n" );
  	}
  
  	return TRUE;
  }
  
  
  void
  startd_check_free()
  {	
  	if ( cron_job_mgr && ( ! cron_job_mgr->ShutdownOk() ) ) {
  		return;
  	}
  	if ( bench_job_mgr && ( ! bench_job_mgr->ShutdownOk() ) ) {
  		return;
  	}
  	if ( ! resmgr ) {
  		startd_exit();
  	}
  	if( ! resmgr->hasAnyClaim() ) {
  		startd_exit();
  	}
  	return;
  }
  
  
  int
  main( int argc, char **argv )
  {
  	set_mySubSystem( "STARTD", SUBSYSTEM_TYPE_STARTD );
  
  	dc_main_init = main_init;
  	dc_main_config = main_config;
  	dc_main_shutdown_fast = main_shutdown_fast;
  	dc_main_shutdown_graceful = main_shutdown_graceful;
  	return dc_main( argc, argv );
  }
  
  
  bool
  authorizedForCOD( const char* owner )
  {
  	if( ! valid_cod_users ) {
  		return false;
  	}
  	return valid_cod_users->contains( owner );
  }
  