text: #include "Facebook.h"
  #include "HttpServer.h"
  #include "sort.h"
  #include "Repair.h"
  
  // use this to get an access token for the event guru app
  // https://graph.facebook.com/oauth/access_token?
  // client_id=YOUR_APP_ID&client_secret=YOUR_APP_SECRET&
  // grant_type=client_credentials
  
  // use this to cache a facebook user's friends, and userid of themselves
  Facebookdb g_facebookdb;
  Likedb     g_likedb;
  
  static void queueSleepWrapper ( int fd, void *state );
  
  bool base64Decode ( char *dst, char *src, long dstSize ) ;
  
  ///////////////////////////
  //
  // FACEBOOKDB
  //
  ///////////////////////////
  
  void Facebookdb::reset() {
  	m_rdb.reset();
  }
  
  bool Facebookdb::init ( ) {
  
  	if ( ! g_conf.m_indexEventsOnly )  return true;
  
  	// load this here so calling saveQueryLoopState() does not overwrite it
  	loadQueryLoopState ();
  	// hit the queue
  	if ( ! g_loop.registerSleepCallback(500,NULL,queueSleepWrapper))
  		return false;
  	/*
  	char tmp[2048];
  	char *sr = "eyJhbGdvcml0aG0iOiJITUFDLVNIQTI1NiIsImV4cGlyZXMiOjEzMzA0NTkyMDAsImlzc3VlZF9hdCI6MTMzMDQ1Mjk3Miwib2F1dGhfdG9rZW4iOiJBQUFGRWczUUE1eWdCQUg1b3dJTEt6WkNaQ0FDNmNTUVRaQWVaQmE2WkFiT1J2dkllSzc2MktGWFg2RmxaQ29iZWFzcENzWE5BZGh1R2VMQm1VM1hnNmMyTm1JenhxRlZkQ3d1Z0ZLRzVHWkN6WXlWaUpwZkN4UlYiLCJ1c2VyIjp7ImNvdW50cnkiOiJ1cyIsImxvY2FsZSI6ImVuX1VTIiwiYWdlIjp7Im1pbiI6MjF9fSwidXNlcl9pZCI6IjEwMDAwMzUzMjQxMTAxMSJ9";
  	base64Decode ( tmp , sr , 2040 );
  	log("facebook: %s",tmp);
  	*/
  
  	// . what's max # of tree nodes?
  	// . assume avg facebookdb rec size of about 1000 bytes
  	// . NOTE: 32 bytes of the 1000 are overhead
  	long maxMem = 5000000;
  	long maxTreeNodes = maxMem / 1000;//82;
  	// each entry in the cache is usually just a single record, no lists,
  	// unless a hostname has multiple sites in it. has 24 bytes more 
  	// overhead in cache.
  	//long maxCacheNodes = g_conf.m_tagdbMaxCacheMem / 106;
  	// we now use a page cache for the banned turks table which
  	// gets hit all the time
  	//if(! m_pc.init ("facebookdb",RDB_TAGDB,10000000,GB_TFNDB_PAGE_SIZE))
  	//	return log("facebookdb: Tagdb init failed.");
  	// initialize our own internal rdb
  	if ( ! m_rdb.init ( g_hostdb.m_dir               ,
  			    "facebookdb"                     ,
  			    true                       , // dedup same keys?
  			    -1                         , // fixed record size
  			    2,//g_conf.m_tagdbMinFilesToMerge   ,
  			    maxMem, // 5MB g_conf.m_tagdbMaxTreeMem  ,
  			    maxTreeNodes               ,
  			    // now we balance so Sync.cpp can ordered huge list
  			    true                        , // balance tree?
  			    0 , //g_conf.m_tagdbMaxCacheMem ,
  			    0 , //maxCacheNodes              ,
  			    false                      , // half keys?
  			    false                      , //m_tagdbSaveCache
  			    NULL, // &m_pc                      ,
  			    false,  // is titledb
  			    false,  // preload disk page cache
  			    sizeof(key96_t),     // key size
  			    false , // bias disk page cache?
  			    true )) // iscollectionless? syncdb,facebookdb,...
  		return false;
  	// add the base since it is a collectionless rdb
  	return m_rdb.addColl ( NULL );
  }
  
  bool Facebookdb::addColl ( char *coll, bool doVerify ) {
  	if ( ! m_rdb.addColl ( coll ) ) return false;
  	return true;
  }
  
  ///////////////////////
  //
  // MSGFB
  //
  ///////////////////////
  
  Msgfb::Msgfb ( ) {
  	m_facebookReply = NULL;
  	//m_facebookReply2 = NULL;
  	m_msg7 = NULL;
  	m_fbId = 0LL;
  	m_inProgress = false;
  	reset();
  }
  
  #include "Process.h"
  
  void Msgfb::reset ( ) {
  	// this can happen if we try to save and exit while in progress
  	if ( g_process.m_mode != EXIT_MODE &&
  	     m_inProgress ) { char *xx=NULL;*xx=0; }
  	m_requests   = 0;
  	m_replies    = 0;
  	m_errno      = 0;
  	m_errorCount = 0;
  	m_fbId       = 0;
  	m_state      = NULL;
  	m_callback   = NULL;
  	m_collnum    = 0;
  	m_niceness   = MAX_NICENESS;
  	m_socket     = NULL;
  	m_retryCount = 0;
  	m_widgetId = 0;
  	m_userToUserWidgetId = 0LL;
  
  	//m_fbrec.reset();
  	if ( m_facebookReply ) {
  		mfree ( m_facebookReply , m_facebookAllocSize ,"msgfb");
  		m_facebookReply = NULL;
  	}
  	//if ( m_facebookReply2 ) {
  	//	mfree ( m_facebookReply2 , m_facebookAllocSize2 ,"msgfb");
  	//	m_facebookReply2 = NULL;
  	//}
  	m_rbuf.purge();
  
  	if ( m_msg7 ) {
  		mdelete ( m_msg7, sizeof(Msg7) , "FBInject" );
  		delete ( m_msg7);
  		m_msg7 = NULL;
  	}
  
  	m_list1.freeList();
  	m_list2.freeList();
  	m_list3.freeList();
  	m_hr.reset();// = NULL;
  	m_fbrecPtr = NULL;
  	m_likedbTable.reset();
  	m_evPtrBuf.purge();
  	m_evIdsBuf.purge();
  	m_fidBuf.purge();
  	m_eidBuf.purge();
  	m_dedupEidBuf.reset();
  	// free mem
  	m_fullReply.purge();
  }
  
  Msgfb::~Msgfb ( ) {
  	reset();
  }
  
  ///////////////////////
  //
  // MSGFB PIPELINE #1
  //
  ///////////////////////
  
  static void gotFBUserRecWrapper ( void *state ) {
  	Msgfb *mfb = (Msgfb *)state;
  	if ( ! mfb->gotFBUserRec ( ) ) return;
  	mfb->m_callback ( mfb->m_state );
  }	
  
  static void gotFBAccessTokenWrapper ( void *state , TcpSocket *s ) {
  	Msgfb *mfb = (Msgfb *)state;
  	if ( ! mfb->gotFBAccessToken( s ) ) return;
  	mfb->m_callback ( mfb->m_state );
  }
  
  // format like strncpy()
  bool base64Decode ( char *dst, char *src, long dstSize ) {
  
  	// make the map
  	static unsigned char s_bmap[256];
  	static bool s_init = false;
  	if ( ! s_init ) {
  		s_init = true;
  		memset ( s_bmap , 0 , 256 );
  		unsigned char val = 0;
  		for ( unsigned char c = 'A' ; c <= 'Z'; c++ ) 
  			s_bmap[c] = val++;
  		for ( unsigned char c = 'a' ; c <= 'z'; c++ ) 
  			s_bmap[c] = val++;
  		for ( unsigned char c = '0' ; c <= '9'; c++ ) 
  			s_bmap[c] = val++;
  		if ( val != 62 ) { char *xx=NULL;*xx=0; }
  		s_bmap['+'] = 62;
  		s_bmap['/'] = 63;
  	}
  		
  	// leave room for \0
  	char *dstEnd = dst + dstSize - 5;
  	unsigned char *p    = (unsigned char *)src;
  	unsigned char val;
  	for ( ; ; ) {
  		if ( *p ) {val = s_bmap[*p]; p++; } else val = 0;
  		// copy 6 bits
  		*dst <<= 6;
  		*dst |= val;
  		if ( *p ) {val = s_bmap[*p]; p++; } else val = 0;
  		// copy 2 bits
  		*dst <<= 2;
  		*dst |= (val>>4);
  		dst++;
  		// copy 4 bits
  		*dst = val & 0xf;
  		if ( *p ) {val = s_bmap[*p]; p++; } else val = 0;
  		// copy 4 bits
  		*dst <<= 4;
  		*dst |= (val>>2);
  		dst++;
  		// copy 2 bits
  		*dst = (val&0x3);
  		if ( *p ) {val = s_bmap[*p]; p++; } else val = 0;
  		// copy 6 bits
  		*dst <<= 6;
  		*dst |= val;
  		dst++;
  		// sanity
  		if ( dst >= dstEnd ) {
  			log("facebook: bas64decode breach");
  			//char *xx=NULL;*xx=0;
  			*dst = '\0';
  			return false;
  		}
  		if ( ! *p ) break;
  	}
  	// null term just in case
  	dst[1] = '\0';
  	return true;
  }
  
  
  bool Msgfb::getFacebookUserInfo ( HttpRequest *hr,
  				  TcpSocket *s,
  				  char *coll,
  				  void *state ,
  				  char *redirPath,
  				  void (* callback)(void *state) ,
  				  long niceness ) {
  
  	reset();
  	m_state      = state;
  	m_callback   = callback;
  	//m_hr         = hr;
  	m_oldFbrec   = NULL;
  	m_redirPath  = redirPath;
  	m_socket     = s;
  	m_collnum = g_collectiondb.getCollnum ( coll );
  	m_retryCount = 0;
  
  	// they just logged out
  	if ( hr->getLong("logout",0 ) ) return true;
  
  	// need to make a copy of it so we can access it
  	m_hr.copy ( hr );
  
  	// save this
  	m_widgetId = hr->getLongLongFromCookie("widgetid",0LL);
  
  	// make the facebook key url for tagdb lookup
  	char *fbidStr = m_hr.getStringFromCookie("fbid",NULL);
  	// check for "usefbid"
  	char *useFbid = m_hr.getString("usefbid",NULL);
  	// that overrides
  	if ( useFbid ) {
  		long long used = strtoull(useFbid,NULL,10);
  		long h32a = hash32 ( (char *)&used, 8 );
  		// this has to be a long long because fh is printed as
  		// an unsigned long and atol() will croak
  		long h32b = m_hr.getLongLong("fh",0);
  		if ( h32a != h32b ) {
  			useFbid = NULL;
  			log("facebook: bad usefbid=%lli h32=%lu fh=%lu",
  			    used,h32a,h32b);
  		}
  		else 
  			fbidStr = useFbid;
  	}
  
  	// they've logged in before if we got this cookie, otherwise,
  	// they got cookies off or they have not logged in ever yet, so
  	// we gotta ask facebook for the access token.
  	if ( ! fbidStr ) 
  		return downloadAccessToken ( );
  
  	// . make the facebook key url for facebook user rec
  	// . we are here because we are assuming they are already logged in
  	//   at some point in time and we should try to get their list
  	//   of friends from the facebookdb rec so if they click 
  	//   to show all the events their friends are going to we can do that
  	long long fbId = strtoull ( fbidStr , NULL, 10 );
  
  	// if they come into the canvas page we receive an encoded access
  	// token and fbid from facebook. so get that and do the lookup.
  	// if we do not have the username in the facebookdb rec or we do not
  	// even have a facebook id rec, then we will have to use cmd1 fql
  	// before they can be logged in.
  	char *sr = m_hr.getString("signed_request",NULL);
  	if ( sr ) sr = strchr(sr,'.');
  	if ( sr ) sr++;
  	// decode that
  	char dsr[2048];
  	if ( sr ) base64Decode ( dsr , sr , 2040 );
  	// parse that json for fb user id
  	char *jsonfbid = NULL;
  	if ( sr       ) jsonfbid = strstr ( dsr , "user_id\":\"" );
  	if ( jsonfbid ) jsonfbid += 10;
  	if ( jsonfbid ) fbId = atoll(jsonfbid);
  
  	// sometimes there's also an access token, if that is the case
  	// we can just right to the cmd1 fql call downloadFBUserInfo().
  	char *at = NULL;
  	if ( dsr ) at = strstr ( dsr , "oauth_token\":\"" );
  	if ( at  ) at += 14;
  	if ( at && fbId ) {
  		// copy into m_accessToken
  		char *p = at;
  		for ( ; *p && *p != '\"' && *p != ',' ; p++);
  		*p = '\0';
  		if ( p - at + 1 < MAX_TOKEN_LEN ) {
  			strcpy ( m_accessToken , at );
  			log("facebook: got access token from canvas app");
  			if ( ! downloadUserToUserRequestInfo() ) return false;
  			return downloadFBUserInfo();
  		}
  	}
  
  	// for the facebookdb lookup to be legit this must be non-zero
  	if ( fbId == 0LL )
  		return downloadAccessToken ( );
  
  
  	key96_t startKey;
  	key96_t endKey;
  	startKey.n1 = 0;
  	startKey.n0 = fbId;
  	endKey.n1 = 0;
  	endKey.n0 = fbId;
  	startKey.n0 <<= 1;
  	endKey.n0 <<= 1;
  	endKey.n0 |= 0x01;
  	if ( ! m_msg0.getList ( -1, // hostid
  				0 , // ip
  				0 , // port
  				0 , // maxcacheage
  				false, // addtocache
  				RDB_FACEBOOKDB,
  				"",//coll,
  				&m_list1,
  				(char *)&startKey,
  				(char *)&endKey,
  				10, // minrecsizes
  				this ,
  				gotFBUserRecWrapper,
  				niceness ) )
  		return false;
  	// i guess we got it without blocking
  	return gotFBUserRec();
  }
  
  bool Msgfb::gotFBUserRec ( ) {
  	// . it can be empty if never got saved!
  	// . or if they logged out then logged back in, fbid will be 0
  	if ( m_list1.getListSize() <= 0 ) return downloadAccessToken();
  	// cast the list
  	FBRec *fbrec = (FBRec *)m_list1.getList();
  	// save that ptr
  	m_fbrecPtr = fbrec;
  	// get timestamp on that
  	//long now     = getTimeGlobal();
  	//long elapsed = now - fbrec->m_accessTokenCreated;
  	// if stale, re-get it all from facebook.
  	//if ( elapsed >= 60*60 ) return downloadAccessToken ( );
  	// if still fresh, don't bother hitting facebook again, but we
  	// need to deserialize
  	deserializeMsg ( sizeof(FBRec) ,
  			 &fbrec->size_accessToken ,
  			 &fbrec->size_friendIds ,
  			 &fbrec->ptr_accessToken ,
  			 fbrec->m_buf );
  	// need to set this for printBlackBar()
  	m_fbId = fbrec->m_fbId;
  
  	log("facebook: loaded fbrec for fbid=%llu",fbrec->m_fbId);
  	log("facebook: loaded emailfreq=%li",(long)fbrec->m_emailFrequency);
  	log("facebook: loaded myradius=%li",(long)fbrec->m_myRadius);
  
  
  	// or if its an hour old i guess, get another token
  	//long expires = fbrec->m_accessTokenCreated + 3600;
  	// debug 
  	//expires = 0;
  	//if ( getTimeGlobal() > expires ) {
  	//	// use this to save again with new access token
  	//	m_oldFbrec = fbrec;
  	//	return downloadAccessToken();
  	//}
  	// use the same access token
  	return true;
  }
  
  bool Msgfb::downloadAccessToken ( ) {
  	//
  	// ok, no cookie, so see if user just pushed the login button
  	//
  	long  fbcodeLen = 0;
  	char *fbcode = m_hr.getString("code", &fbcodeLen, NULL);
  	// no they did not, no user info is available?
  	if ( ! fbcode ) return true;
  
  	// get current page url (minus the facebook code=)
  	SafeBuf cup;
  	m_hr.getCurrentUrl(cup);
  	//m_hr->getCurrentUrlPath(cup);
  	// but take out the &code=
  	char *fix = strstr(cup.getBufStart(),"&code=");
  	if ( fix ) *fix = 0;
  	cup.urlEncode();
  
  	// use code to get access token
  	// that calls https://graph.facebook.com/oauth/access_token?
  	// client_id=YOUR_APP_ID&redirect_uri=YOUR_URL&
  	// client_secret=YOUR_APP_SECRET&code=THE_CODE_FROM_ABOVE
  	char fbuf[1024];
  	SafeBuf fburl ( fbuf,1024 );
  	fburl.safePrintf ("https://graph.facebook.com/oauth/access_token?"
  			  "client_id=%s&"
  			  "redirect_uri=%s&"
  			  "client_secret=%s&"
  			  "code=%s"
  			  , APPID
  			  //, APPHOSTENCODEDNOSLASH
  			  //, m_redirPath
  			  // 
  			  // KEEP It simple because this redirect_uri
  			  // must EXACTLY match the one we used in
  			  // PageEvents.cpp. and it can't have cgi crap in it
  			  // because facebook strips them out...
  			  //
  			  //, "/" 
  			  , cup.getBufStart()
  			  , APPSECRET
  			  , fbcode );
  
  	// this fburl must match the previous url i think... i think the
  	// "page" in it changes up and mess wit hit
  	log("facebook: getting access token code=%s url=%s",fbcode,
  	    fburl.getBufStart());
  
  	// reset
  	g_errno = 0;
  	if ( ! g_httpServer.getDoc ( fburl.getBufStart() ,
  				     0 , // urlIp
  				     0                    , // offset
  				     -1                   ,
  				     0 , // ifModifiedSince ,
  				     this                 , // state
  				     gotFBAccessTokenWrapper , // callback
  				     10*1000      , // 10 sec timeout
  				     0 , // proxyip
  				     0 , // proxyport
  				     10000 , // maxTextDocLen   ,
  				     10000 , // maxOtherDocLen  ,
  				     g_conf.m_spiderUserAgent    ) )
  		// return false if blocked
  		return false;
  	// error?
  	if ( ! g_errno ) { char *xx=NULL;*xx=0; }
  	// let caller know we did not block
  	return gotFBAccessToken ( NULL );
  }
  
  bool Msgfb::gotFBAccessToken ( TcpSocket *s ) {
  
  	// some kind of error?
  	if ( g_errno ) {
  		log("facebook: error launching read of access token: %s", 
  		    mstrerror(g_errno));
  		m_errno = g_errno;
  		m_errorCount++;
  		return true;
  	}
  
  	// the access token should be in the reply
  	char *reply     = s->m_readBuf;
  	long  replySize = s->m_readOffset;
  
  	// mime error?
  	HttpMime mime;
  	// exclude the \0 i guess. use NULL for url.
  	mime.set ( reply, replySize - 1, NULL );
  	// not good?
  	long httpStatus = mime.getHttpStatus();
  	if ( httpStatus != 200 ) {
  		log("facebook: bad access request http status = %li. "
  		    "reply=%s",
  		    httpStatus ,
  		    reply );
  		g_errno = EBADREPLY;
  		m_errno = g_errno;
  		m_errorCount++;
  		return true;
  	}
  
  	// point to content
  	char *content = reply + mime.getMimeLen();
  
  	// assume no accesstoken provided
  	//long expires = 0;
  	m_accessToken[0] = '\0';
  
  	// look for access token
  	//sscanf(content,"access_token=%s&expires=%li",m_accessToken,&expires);
  	char *at = strstr(content,"access_token=");
  	if ( at ) {
  		char *p = at + 13;
  		char *start = p;
  		for ( ; *p && *p != '&' ;p++ );
  		long len = p - start;
  		if ( len > MAX_TOKEN_LEN ) { char *xx=NULL;*xx=0; }
  		memcpy ( m_accessToken , start , len );
  		m_accessToken [ len ] = '\0';
  	}
  
  	// error?
  	if ( ! m_accessToken[0] ) {
  		log("facebook: could not find access token");
  		g_errno = EBADREPLY;
  		m_errno = g_errno;
  		m_errorCount++;
  		return true;
  	}
  
  	// set this timestamp
  	//m_accessTokenCreated = getTimeGlobal();
  
  	// sanity
  	if ( gbstrlen(m_accessToken) > MAX_TOKEN_LEN ) { char *xx=NULL;*xx=0;}
  
  	if ( ! downloadUserToUserRequestInfo() ) return false;
  
  	return downloadFBUserInfo();
  }
  
  static void gotFQLUserInfoWrapper ( void *state , TcpSocket *s ) {
  	Msgfb *mfb = (Msgfb *)state;
  	// . returns false if it blocks
  	// . returns true with g_errno set on error
  	if ( ! mfb->gotFQLUserInfo ( s ) ) return;
  	// error?
  	if ( g_errno && mfb->m_retryCount++ < 5 ) {
  		// retry again. this returns false if blocks;
  		if ( ! mfb->downloadFBUserInfo() ) return;
  		// probably an error if it returns true!!
  	}
  	mfb->m_callback ( mfb->m_state );
  }
  
  
  
  
  
  ////////////////
  //
  // BEGIN SPECIAL FACEBOOK APPREQUEST parsing for m_userToUserWidgetId
  //
  ///////////////
  
  static void gotFBUserToUserRequestWrapper ( void *state , TcpSocket *s ) {
  	Msgfb *mfb = (Msgfb *)state;
  	if ( ! mfb->gotFBUserToUserRequest ( s ) ) return;
  	mfb->m_callback ( mfb->m_state );
  }
  
  // before calling downloadFBUserInfo() call this to set the m_widgetId
  // parameter correctly! so we can see which facebook user sent them
  // this user_to_user apprequest. we need to do this before downloading
  // the user info so we can store the m_originatingWidgetId into the
  // FBRec before we save it!
  bool Msgfb::downloadUserToUserRequestInfo ( ) {
  
  	// hmmm. hr is invalid here.
  	char *request_ids = m_hr.getString("request_ids",NULL);
  
  	if ( ! request_ids ) return true;
  
  	// if already got one, forget this...
  	if ( m_widgetId ) return true;
  
  	char fbuf[1024];
  	SafeBuf fburl ( fbuf,1024 );
  	fburl.safePrintf ("https://graph.facebook.com/me/apprequests/?"
  			  "request_ids=%s&access_token=%s"
  			  , request_ids
  			  , m_accessToken );
  
  	// this fburl must match the previous url i think... i think the
  	// "page" in it changes up and mess wit hit
  	log("facebook: getting referral fbid url=%s", fburl.getBufStart());
  
  	// reset
  	g_errno = 0;
  	if ( ! g_httpServer.getDoc ( fburl.getBufStart() ,
  				     0 , // urlIp
  				     0                    , // offset
  				     -1                   ,
  				     0 , // ifModifiedSince ,
  				     this                 , // state
  				     gotFBUserToUserRequestWrapper ,
  				     10*1000      , // 10 sec timeout
  				     0 , // proxyip
  				     0 , // proxyport
  				     10000 , // maxTextDocLen   ,
  				     10000 , // maxOtherDocLen  ,
  				     g_conf.m_spiderUserAgent    ) )
  		// return false if blocked
  		return false;
  	// error?
  	if ( ! g_errno ) { char *xx=NULL;*xx=0; }
  	// let caller know we did not block
  	return gotFBUserToUserRequest ( NULL );
  }
  
  bool Msgfb::gotFBUserToUserRequest ( TcpSocket *s ) {
  
  	// some kind of error?
  	if ( g_errno ) {
  		log("facebook: error launching read of fb request: %s", 
  		    mstrerror(g_errno));
  		m_errno = g_errno;
  		m_errorCount++;
  		return true;
  	}
  
  	// the access token should be in the reply
  	char *reply     = s->m_readBuf;
  	long  replySize = s->m_readOffset;
  
  	// mime error?
  	HttpMime mime;
  	// exclude the \0 i guess. use NULL for url.
  	mime.set ( reply, replySize - 1, NULL );
  	// not good?
  	long httpStatus = mime.getHttpStatus();
  	if ( httpStatus != 200 ) {
  		log("facebook: bad fb request reply http status = %li. "
  		    "reply=%s",
  		    httpStatus ,
  		    reply );
  		g_errno = EBADREPLY;
  		m_errno = g_errno;
  		m_errorCount++;
  		return true;
  	}
  
  	// point to content
  	char *content = reply + mime.getMimeLen();
  
  	// https://graph.facebook.com/me/apprequests/?request_ids=417422284951090&access_token=AAAFEg3QA5ygBALw0wqIMOZAD6zfbZBZBaH9mf7sw92kLtOqVtPjasQibZCo4P5R0HHztOnObBeoDoKbwM1ZChht04JJ7KrgkxcXwtNWcrngZDZD
  	//{
  	//	"data": [
  	//	{
  	//		"id": "417422284951090_100003381767946",
  	//		"application": {
  	//			"name": "Event Guru",
  	//			"namespace": "eventguru",
  	//			"canvas_name": "eventguru",
  	//			"id": "356806354331432"
  	//		},
  	//		"to": {
  	//			"name": "Jezebel Wells",
  	//			"id": "100003381767946"
  	//		},
  	//		"from": {
  	//			"name": "Matt Wells",
  	//			"id": "100003532411011"
  	//		},
  	//		"data": "100003532411011",
  	//		"message": "Hi my friend, I recommend EventGuru.com for discovering interesting local events. Plus, if you login to Event Guru I make a buck.",
  	//		"created_time": "2012-03-29T02:41:37+0000"
  	//	}
  	//	],
  	//	"paging": {
  	//		"previous": "https://graph.facebook.com/me/apprequests?request_ids=417422284951090&access_token=AAAFEg3QA5ygBALw0wqIMOZAD6zfbZBZBaH9mf7sw92kLtOqVtPjasQibZCo4P5R0HHztOnObBeoDoKbwM1ZChht04JJ7KrgkxcXwtNWcrngZDZD&limit=50&since=1332988897&__paging_token=417422284951090_100003381767946&__previous=1",
  	//		"next": "https://graph.facebook.com/me/apprequests?request_ids=417422284951090&access_token=AAAFEg3QA5ygBALw0wqIMOZAD6zfbZBZBaH9mf7sw92kLtOqVtPjasQibZCo4P5R0HHztOnObBeoDoKbwM1ZChht04JJ7KrgkxcXwtNWcrngZDZD&limit=50&until=1332988897&__paging_token=417422284951090_100003381767946"
  	//	}
  	//}
  
  	// mine out who sent it
  	char *from = strstr ( content , "\"from\":" );
  	long long id1 = 0LL;
  	if ( from ) {
  		char *ids = strstr ( from , "\"id\":" );
  		for ( ; ids && *ids && ! is_digit(*ids) ; ids++ );
  		if ( ids && *ids ) id1 = atoll(ids);
  	}
  
  
  	// mine out who sent it again... this is only present for paid invites
  	char *data = NULL;
  	// start mining AFTER "from" because there is a top "data" field!!
  	if ( from ) data = strstr ( from , "\"data\":" );
  	long long id2 = 0LL;
  	if ( data ) {
  		char *ids = strstr ( data , "\"id\":" );
  		for ( ; ids && *ids && ! is_digit(*ids) ; ids++ );
  		if ( ids && *ids ) id2 = atoll(ids);
  	}
  
  	// must match to be an official paid invite. we only include
  	// the "data" for the paid invites.
  	if ( id1 == id2 )
  		// set that to our widgetid then
  		m_widgetId = id1;
  
  	// download facebook info now
  	return downloadFBUserInfo();
  }
  
  ////////////////
  //
  // END SPECIAL FACEBOOK APPREQUEST parsing for user_to_user m_widgetId
  //
  ///////////////
  
  
  
  //long long mdw = 100003532411011LL; // my uid for Matt Wells
  // 100003316058818 // uid for flurbit
  // http://graph.facebook.com/502303355/picture shows pics for a uid
  bool Msgfb::downloadFBUserInfo ( ) {
  
  	log("facebook: downloading user info initial login");
  
  	SafeBuf cmd1;
  	// get your facebook id
  	cmd1.safePrintf ( "SELECT uid,username,first_name,last_name,name,pic_square,profile_update_time,timezone,religion,birthday,birthday_date,sex,hometown_location,current_location,activities,interests,is_app_user,music,tv,movies,books,about_me,status,online_presence,proxied_email,verified,website,is_blocked,contact_email,email,is_minor,work,education,sports,languages,likes_count,friend_count FROM user where uid=me()");
  
  	log("facebook: cmd1 = %s",cmd1.getBufStart());
  
  	cmd1.urlEncode();
  
  	// www.howtobe.pro/facebook-graph-api-graph-api-for-issuing-fql-queries
  	// make a url
  	SafeBuf ubuf;
  	ubuf.safePrintf("https://api.facebook.com/method/"
  			"fql.query?query=%s"
  			"&access_token=%s&format=xml"
  			, cmd1.getBufStart() 
  			, m_accessToken 
  			);
  
  	log("facebook: getting url = %s",ubuf.getBufStart());
  
  	// reset
  	g_errno = 0;
  	// get the results
  	if ( ! g_httpServer.getDoc ( ubuf.getBufStart() ,
  				     0 , // urlIp
  				     0                    , // offset
  				     -1                   ,
  				     0 , // ifModifiedSince ,
  				     this               , // state
  				     gotFQLUserInfoWrapper , // callback
  				     40*1000      , // 20 sec timeout
  				     0 , // proxyip
  				     0 , // proxyport
  				     30000000 , // maxTextDocLen   ,
  				     30000000 , // maxOtherDocLen  ,
  				     g_conf.m_spiderUserAgent    ) )
  		// return false if blocked
  		return false;
  	// otherwise, somehow got it without blocking... wtf?
  	//return gotFQLReply();
  	if ( ! g_errno ) { char *xx=NULL;*xx=0; }
  	log("fql: error getting doc: %s",mstrerror(g_errno));
  	return true;
  }
  
  static bool queueFBId ( long long fbId , collnum_t collnum );
  
  static void savedFBRecWrapper1 ( void *state ) {
  	Msgfb *mfb = (Msgfb *)state;
  	if ( ! g_errno ) queueFBId ( mfb->m_fbId , mfb->m_collnum );
  	mfb->m_callback ( mfb->m_state );
  }
  
  static void doneRecheckingWrapper ( void *state ) {
  	Msgfb *mfb = (Msgfb *)state;
  	if ( ! mfb->doneRechecking ( ) ) return;
  	mfb->m_callback ( mfb->m_state );
  }
  
  // returns true with g_errno set on error
  bool Msgfb::gotFQLUserInfo ( TcpSocket *s ) {
  
  	// bail on error
  	if ( g_errno ) {
  		log("fql: %s",mstrerror(g_errno));
  		m_errno = g_errno;
  		m_errorCount++;
  		return true;
  	}
  
  	// get reply
  	char *reply     = s->m_readBuf;
  	long  replySize = s->m_readOffset;
  
  	// we reference into this, so do not free it!!
  	m_facebookReply     = s->m_readBuf;
  	m_facebookReplySize = s->m_readOffset;
  	m_facebookAllocSize = s->m_readBufSize;
  	// do not allow tcpsocket to free it. we free it in destructor.
  	s->m_readBuf = NULL;
  
  
  	// mime error?
  	HttpMime mime;
  	// exclude the \0 i guess. use NULL for url.
  	mime.set ( reply, replySize - 1, NULL );
  	// not good?
  	long httpStatus = mime.getHttpStatus();
  	if ( httpStatus != 200 ) {
  		log("facebook: bad fql request http status = %li",
  		    httpStatus );
  		g_errno = EBADREPLY;
  		m_errno = g_errno;
  		m_errorCount++;
  		return true;
  	}
  
  	// point to content
  	char *content    = reply + mime.getMimeLen();
  	long  contentLen = reply + replySize - content;
  
  	// check for error
  	char *errMsg = strstr(content,"<error_msg>");
  	if ( errMsg ) {
  		log("facebook: error in fql reply: %s", content );
  		g_errno = EBADREPLY;
  		m_errno = g_errno;
  		m_errorCount++;
  		return true;
  	}
  
  	// set it on stack i guess
  	//FBRec fbrec;
  	// all the FBRec::ptr_* things reference into "reply"
  	// import friendids from existing rec if there, we just want to 
  	// save the new access token!!!!
  	if ( ! setFBRecFromFQLReply ( content,contentLen, &m_fbrecGen) ){
  		log("fql: error setting fb rec from fql");
  		g_errno = EBADREPLY;
  		return true;
  	}
  
  	// must be there!
  	if ( ! m_fbrecGen.m_fbId ) {
  		log("fql: failed to get facebook id from reply");
  		g_errno = EBADENGINEER;
  		m_errno = g_errno;
  		m_errorCount++;
  		return true;
  	}
  
  	// sanity
  	log("facebook: got initial facebook reply. fbid=%lli",m_fbId);
  
  	// point to it!
  	m_fbrecPtr = &m_fbrecGen;
  
  	// now that we got the fbid see if its in facebookdb again
  	long long fbId = m_fbrecGen.m_fbId;
  	key96_t startKey;
  	key96_t endKey;
  	startKey.n1 = 0;
  	startKey.n0 = fbId;
  	endKey.n1 = 0;
  	endKey.n0 = fbId;
  	startKey.n0 <<= 1;
  	endKey.n0 <<= 1;
  	endKey.n0 |= 0x01;
  	//char *coll = g_collectiondb.getColl(m_collnum);
  	if ( ! m_msg0.getList ( -1, // hostid
  				0 , // ip
  				0 , // port
  				0 , // maxcacheage
  				false, // addtocache
  				RDB_FACEBOOKDB,
  				"",//coll,
  				&m_list4,
  				(char *)&startKey,
  				(char *)&endKey,
  				12, // minrecsizes
  				this ,
  				doneRecheckingWrapper,
  				0 ) )//niceness ) )
  		return false;
  	// i guess we got it without blocking
  	return doneRechecking();
  }
  
  //
  // MERGE/UPDATE the old fbrec with the new fbrec
  //
  void mergeFBRec ( FBRec *dst , FBRec *src ) {
  	// if we had a pre-existing fbrec in facebookdb, do not just 
  	// save m_fbrecGen because it does not have ptr_friendIds set!
  	// so inherit or import the friendIds from the pre-existing 
  	// rec. otherwise we lose the friends until the cmd2 executes
  	// and that could be a while or that could fail!
  	if ( dst->size_friendIds <= 0 ) {
  		dst->ptr_friendIds        = src-> ptr_friendIds;
  		dst->size_friendIds       = src->size_friendIds;
  	}
  		// also this stuff too!
  	dst->m_emailFrequency     = src->m_emailFrequency;
  	dst->m_myRadius           = src->m_myRadius;
  	dst->ptr_mergedInterests  = src->ptr_mergedInterests;
  	dst->size_mergedInterests = src->size_mergedInterests;
  	dst->ptr_myLocation       = src->ptr_myLocation;
  	dst->size_myLocation      = src->size_myLocation;
  	// save this stuff now too for Emailer
  	dst->m_nextRetry        = src->m_nextRetry;
  	dst->m_timeToEmail      = src->m_timeToEmail;
  	dst->m_lastEmailAttempt = src->m_lastEmailAttempt;
  	// other stuff
  	dst->m_eventsDownloaded   = src->m_eventsDownloaded;
  	dst->m_accessTokenCreated = src->m_accessTokenCreated;
  	// new stuff
  	dst->m_flags = src->m_flags; // FB_INQUEUE
  	// for payment info:
  	dst->m_originatingWidgetId = src->m_originatingWidgetId;
  	// overwrite it if zero
  	if ( src->m_firstFacebookLogin )
  		dst->m_firstFacebookLogin = src->m_firstFacebookLogin;
  	// if it is zero in the old rec, overwrite it!
  	if ( src->m_lastLoginIP )
  		dst->m_lastLoginIP       = src->m_lastLoginIP;
  	//dst-> = src->;
  }
  
  // returns false if blocked, true otherwise
  bool Msgfb::saveFBRec ( FBRec *fbrec ) {
  
  	log("facebook: saving fbrec for fbid=%lli",m_fbId);
  
  	log("facebook: saving myradius=%li",fbrec->m_myRadius);
  	log("facebook: saving mylocation=%s",fbrec->ptr_myLocation);
  	log("facebook: saving emailfreq=%li",(long)fbrec->m_emailFrequency);
  
  	// . returns NULL and sets g_errno on error
  	// . "true" means we should make mr.ptr_* reference into the newly
  	//   serialized buffer
  	long replySize;
  	char *reply = serializeMsg ( sizeof(FBRec) ,
  				     &fbrec->size_accessToken,//1stsizeparm
  				     &fbrec->size_friendIds,// lastsizeparm
  				     &fbrec->ptr_accessToken , // firststrptr
  				     fbrec , // thisptr
  				     &replySize ,
  				     NULL , 
  				     0 ,
  				     false ); // true );
  	if ( ! reply ) {
  		log("facebook: could not save fbrec: %s",mstrerror(g_errno));
  		return true;
  	}
  		       
  	
  	// make the binary tag then for this facebook user
  	//char *rec     = (char *)&m_key;//fbId;
  	//long  recSize = (char *)&m_ids[m_numIds] - rec;
  
  	FBRec *serializedRec = (FBRec *)reply;
  
  	// set this after we know it
  	serializedRec->m_dataSize = replySize - sizeof(key96_t) - 4;
  
  
  	//char *coll = g_collectiondb.getColl ( m_collnum );
  	// use the list we got
  	key128_t startKey;
  	key128_t endKey;
  	startKey.setMin();
  	endKey.setMax();
  	// use m_list2 and not m_list since m_fbrecPtr is referencing
  	// m_list's content from our facebookdb lookup above
  	m_list2.set ( reply ,
  		      replySize ,
  		      NULL ,
  		      0 ,
  		      (char *)&startKey ,
  		      (char *)&endKey  ,
  		      -1 ,
  		      true , // own data? yeah, free it when done
  		      false ,
  		      sizeof(key96_t) );
  	// . just use TagRec::m_msg1 now
  	// . no, can't use that because tags are added using SafeBuf::addTag()
  	//   which first pushes the rdbid, so we gotta use msg4
  	// . if a host is down we have to fix msg1 (and msg4) so they both
  	//   just write to a file until that host comes back up.
  	if ( ! m_msg1.addList ( &m_list2 ,
  				RDB_FACEBOOKDB ,
  				"none",//coll ,
  				this ,
  				m_afterSaveCallback, // savedFBRecWrapper ,
  				false ,
  				0 ) ) // niceness
  		return false;
  	// no block?
  	//return downloadEvents();
  	return true;
  }
  
  // returns false if blocked, true otherwise
  bool Msgfb::doneRechecking ( ) {
  
  	if ( g_errno ) {
  		log("facebook: recheck lookup failed: %s",
  		    mstrerror(g_errno));
  		return true;
  	}
  
  	// try to set the old fbrec
  	if ( m_list4.getListSize() > 0 ) {
  		// cast the list
  		m_oldFbrec = (FBRec *)m_list4.getList();
  		deserializeMsg ( sizeof(FBRec) ,
  				 &m_oldFbrec->size_accessToken ,
  				 &m_oldFbrec->size_friendIds ,
  				 &m_oldFbrec->ptr_accessToken ,
  				 m_oldFbrec->m_buf );
  		// sanity
  		if ( m_fbId != m_oldFbrec->m_fbId ) 
  			log("facebook: fbid mismatch 3 %llu != %llu",
  			    m_fbId,m_oldFbrec->m_fbId);
  		// shortcuts
  		FBRec *src = m_oldFbrec;
  		FBRec *dst = &m_fbrecGen;
  		// merge src into dst. merge the old into the new.
  		mergeFBRec ( dst , src );
  	}
  	//
  	// if it did not exist, then this is the first time they logged
  	// in i guess...
  	//
  	else {
  		// . try to save this
  		// . PageEvents.cpp issues a Set-Cookie: widgetid=xxxxxx;
  		//   if si->m_widget is '1'. it uses si->m_widgetId which
  		//   is set from the cgi parms of the iframe.
  		// . use 1 if this is zero, they did not have one then...
  		// . it's important we save the widgetid here correctly
  		//   so we can payout the $1 reward.
  		// . if it was a facebook user_to_user app request we
  		//   should have called downloadUserToUserRequestInfo() to
  		//   set m_widgetId if it was not already set.
  		long long widgetId = m_widgetId;
  		if ( ! widgetId ) widgetId = 1;
  		m_fbrecGen.m_originatingWidgetId = widgetId;
  		m_fbrecGen.m_lastLoginIP = m_socket->m_ip;
  		// avoid a core
  		long now ;
  		if ( isClockInSync() ) now = getTimeGlobal();
  		else                   now = getTimeLocal();
  		m_fbrecGen.m_firstFacebookLogin = now;
  	}
  
  	// saveFBRec should queue it after it saves it. we want to make sure
  	// its only after a successful save because the facebookdb rec
  	// needs to have been saved so we can use its access token to get
  	// get event info of the user and his friends. but since that stuff
  	// is more involved we have a queue that handles the requests.
  	// that 2nd pipeline will set the FBRec::ptr_* things we did not set
  	// here and save them back to facebookdb.
  	m_afterSaveCallback = savedFBRecWrapper1;
  
  	log("facebook: saving rec for fbid=%lli",m_fbId);
  
  	// . queue it up. it returns true if added to the queue.
  	// . it won't add it if no room or its already in the queue.
  	if ( ! g_errno && queueFBId ( m_fbrecGen.m_fbId , m_collnum ) )
  		// . before saving it, mark it has being in the queue!! 
  		// . then PageEvents.cpp can tell you that it is still 
  		//   downloading you and your friends event info so its 
  		//   incomplete if you try to search for that stuff
  		// . once we process it we clear this flag and re-save this rec
  		m_fbrecGen.m_flags |= FB_INQUEUE;
  
  	// this calls serializeMsg() which mallocs a new reply to add
  	if ( ! saveFBRec( &m_fbrecGen ) ) return false;
  	// all done
  	return true;
  }
  
  class LikedbTableSlot {
  public:
  	long long m_uid;
  	long m_start_time;
  	long m_rsvp;
  };
  
  bool Msgfb::setFBRecFromFQLReply ( char *content,
  				   long contentLen,
  				   FBRec *fbrec ) {
  
  	// shortcut
  	char *reply = content;
  	// i've seen crappy facebook return essentially an empty
  	// reply here, so make sure we have all the fields!!
  	if ( ! strstr(reply,"<uid>") ||
  	     ! strstr(reply,"<name") ||
  	     ! strstr(reply,"<first_name") ||
  	     // might be <interests/>
  	     ! strstr(reply,"<interests") ) {
  		log("facebook: missing uid in facebook reply");
  		SafeBuf eb;
  		eb.safeMemcpy ( content,contentLen);
  		char fname[128];
  		sprintf(fname,"error.%llu",m_fbId);
  		eb.save("/tmp/",fname);
  		g_errno = EBADREPLY;
  		return false;
  	}
  	
  	Xml xml;
  	xml.set ( content,
  		  contentLen,
  		  false , // owndata
  		  0 , // allocsize
  		  true ,  // purexml
  		  TITLEREC_CURRENT_VERSION );
  
  	// shortcut
  	FBRec *f = fbrec;
  	// clear the entire rec
  	f->reset();
  
  	f->m_fbId = xml.getLongLong("uid",0LL);
  
  	// need this too i guess
  	m_fbId = f->m_fbId;
  
  
  	// scan for friend ids first since we add \0s below
  	char *p = strstr ( content , "<uid2>" );
  	for ( ; p ; p = strstr (p+1,"<uid2>") ) {
  		long long uid2 = strtoull(p+6,NULL,10);
  		if ( uid2 <= 0 ) continue;
  		// add it
  		if ( m_fidBuf.getAvail() < 8 && ! m_fidBuf.reserve(5000) ) 
  			return false;
  		// push it
  		m_fidBuf.pushLongLong ( uid2 );
  	}
  	// point to those. might have to re-set if we grow m_fidBuf more!
  	f->ptr_friendIds  = m_fidBuf.getBufStart();
  	f->size_friendIds = m_fidBuf.length();
  
  
  
  	f->ptr_firstName = xml.getString("first_name",&f->size_firstName);
  	f->ptr_lastName = xml.getString("last_name",&f->size_lastName);
  	// get the <name> tag right AFTER <last_name>, otherwise we
  	// get the multi query name tag "query1"
  	long n0 = xml.getNodeNum ("last_name");
  	if ( n0 >= 0 ) 
  		f->ptr_name = xml.getString(n0,n0+5,"name",&f->size_name);
  	f->ptr_pic_square = xml.getString("pic_square",&f->size_pic_square);
  	f->ptr_religion = xml.getString("religion",&f->size_religion);
  	f->ptr_birthday = xml.getString("birthday",&f->size_birthday);
  	f->ptr_birthday_date = xml.getString("birthday_date",
  					     &f->size_birthday_date);
  	f->ptr_sex = xml.getString("sex",&f->size_sex);
  	f->ptr_hometown_location =xml.getString("hometown_location",
  						&f->size_hometown_location);
  	f->ptr_current_location = xml.getString("current_location",
  						&f->size_current_location);
  	f->ptr_activities = xml.getString("activities",&f->size_activities);
  	f->ptr_interests = xml.getString("interests",&f->size_interests);
  	f->ptr_music = xml.getString("music",&f->size_music);
  	f->ptr_tv = xml.getString("tv",&f->size_tv);
  	f->ptr_movies = xml.getString("movies",&f->size_movies);
  	f->ptr_books = xml.getString("books",&f->size_books);
  	f->ptr_about_me = xml.getString("about_me",&f->size_about_me);
  	f->ptr_status = xml.getString("status",&f->size_status);
  	f->ptr_online_presence = xml.getString("online_presence",
  					       &f->size_online_presence);
  	f->ptr_proxied_email = xml.getString("proxied_email",
  					     &f->size_proxied_email);
  	f->ptr_website = xml.getString("website",&f->size_website);
  	f->ptr_contact_email = xml.getString("contact_email",
  					     &f->size_contact_email);
  	f->ptr_email = xml.getString("email",&f->size_email);
  	f->ptr_sports = xml.getString("sports",&f->size_sports);
  
  
  	// full nodes (lists)
  	f->ptr_work = xml.getNode("work",&f->size_work);
  	f->ptr_education = xml.getNode("education",&f->size_education);
  	f->ptr_languages = xml.getNode("languages",&f->size_languages);
  
  
  	f->m_timezone = xml.getLong("timezone",99);
  	f->m_is_app_user = xml.getLong("is_app_user",0);
  	f->m_is_blocked = xml.getLong("is_blocked",0);
  	f->m_verified = xml.getLong("verified",0);
  	f->m_is_minor = xml.getLong("is_minor",0);
  	f->m_likes_count = xml.getLong("likes_count",0);
  	f->m_friend_count = xml.getLong("friend_count",0);
  
  	//
  	// now that we got what we want, NULL term everything
  	//
  	if ( f->ptr_firstName) f->ptr_firstName[f->size_firstName++] = '\0';
  	if ( f->ptr_lastName) f->ptr_lastName[f->size_lastName++] = '\0';
  	if ( f->ptr_name) f->ptr_name[f->size_name++] = '\0';
  	if ( f->ptr_pic_square)f->ptr_pic_square[f->size_pic_square++] = '\0';
  	if ( f->ptr_religion) f->ptr_religion[f->size_religion++] = '\0';
  	if ( f->ptr_birthday)f->ptr_birthday[f->size_birthday++] = '\0';
  	if ( f->ptr_birthday_date) 
  		f->ptr_birthday_date[f->size_birthday_date++] = '\0';
  	if ( f->ptr_sex) f->ptr_sex[f->size_sex++] = '\0';
  	if ( f->ptr_hometown_location)
  		f->ptr_hometown_location[f->size_hometown_location++] = '\0';
  	if ( f->ptr_current_location)
  		f->ptr_current_location[f->size_current_location++] = '\0';
  	if ( f->ptr_activities) f->ptr_activities[f->size_activities++] = '\0';
  	if ( f->ptr_interests)f->ptr_interests[f->size_interests++] = '\0';
  	if ( f->ptr_music)f->ptr_music[f->size_music++] = '\0';
  	if ( f->ptr_tv)f->ptr_tv[f->size_tv++] = '\0';
  	if ( f->ptr_movies)f->ptr_movies[f->size_movies++] = '\0';
  	if ( f->ptr_books)f->ptr_books[f->size_books++] = '\0';
  	if ( f->ptr_about_me)f->ptr_about_me[f->size_about_me++] = '\0';
  	if ( f->ptr_status)f->ptr_status[f->size_status++] = '\0';
  	if ( f->ptr_online_presence)
  		f->ptr_online_presence[f->size_online_presence++] = '\0';
  	if ( f->ptr_proxied_email)
  		f->ptr_proxied_email[f->size_proxied_email++] = '\0';
  	if ( f->ptr_website)
  		f->ptr_website[f->size_website++] = '\0';
  	if ( f->ptr_contact_email)
  		f->ptr_contact_email[f->size_contact_email++] = '\0';
  	if ( f->ptr_email) f->ptr_email[f->size_email++] = '\0';
  	if ( f->ptr_sports)f->ptr_sports[f->size_sports++] = '\0';
  	if ( f->ptr_work) f->ptr_work[f->size_work++] = '\0';
  	if ( f->ptr_education)f->ptr_education[f->size_education++] = '\0';
  	if ( f->ptr_languages)f->ptr_languages[f->size_languages++] = '\0';
  
  	// allow dups! one eid can have many likes
  	m_likedbTable.set(8,sizeof(LikedbTableSlot),0,NULL,0,true,0,"lkdbtab");
  
  	//
  	// . add the eids facebook provided into m_eidBuf
  	// . create a likedb record for each one to add to likedb
  	//
  	bool hadError = false;
  	char *line = NULL;
  	// scan the xmlnodes, looking for event_member
  	long i; for ( i = 0 ; i < xml.m_numNodes ; i++ ) {
  
  		// get an <event_member> tag
  		XmlNode *node = &xml.m_nodes[i];
  		char *s = node->m_node;
  		if ( strncmp(s,"<event_member",13) ) continue;
  
  		line = s;
  
  		hadError = true;
  
  		// skip til end of tag
  		s += 13;
  		for ( ; *s && *s != '>' ; s++ ) ;
  		// eof?
  		if ( ! *s ) break;
  		// skip >
  		s++;
  
  		// next tag
  		for ( ; *s && *s != '<' ; s++ ) ;
  		// must be <uid> afterwards
  		if ( strncmp ( s, "<uid>" , 5 ) ) break;
  		// skip that
  		s += 5;
  		// long long
  		long long uid = strtoull ( s , NULL, 10 );
  		// skip til next tag
  		for ( ; *s && *s != '<' ; s++ ) ;		
  		// must be /uid
  		if ( strncmp ( s, "</uid>", 6 ) ) break;
  		// skip that
  		s += 6;
  
  		// next tag
  		for ( ; *s && *s != '<' ; s++ ) ;
  		// must be <eid>
  		if ( strncmp ( s, "<eid>" , 5 ) ) break;
  		// skip that
  		s += 5;
  		// long long
  		long long eid = strtoull ( s , NULL, 10 );
  		// skip til next tag
  		for ( ; *s && *s != '<' ; s++ ) ;		
  		// must be /uid
  		if ( strncmp ( s, "</eid>", 6 ) ) break;
  		// skip that
  		s += 6;
  
  		// next tag
  		for ( ; *s && *s != '<' ; s++ ) ;
  		// must be <eid>
  		if ( strncmp ( s, "<rsvp_status>" , 13 ) ) break;
  		// skip that
  		s += 13;
  		// skip whitespace
  		for ( ; *s && is_wspace_a(*s) ; s++ );
  		// long long
  		char *rsvp_status = s;
  		// skip til next tag
  		for ( ; *s && *s != '<' ; s++ ) ;
  		// must be /uid
  		if ( strncmp ( s, "</rsvp_status>", 14 ) ) break;
  		// null term so "status" is null terminated
  		//*s = '\0';
  		// skip that
  		s += 14;
  
  		
  		// next tag
  		for ( ; *s && *s != '<' ; s++ ) ;
  		// must be <eid>
  		if ( strncmp ( s, "<start_time>" , 12 ) ) break;
  		// skip that
  		s += 12;
  		// long long
  		long start_time = (unsigned long)(atoll ( s ));
  		// skip til next tag
  		for ( ; *s && *s != '<' ; s++ ) ;		
  		// must be /uid
  		if ( strncmp ( s, "</start_time>", 13 ) ) break;
  		// skip that
  		s += 13;
  
  		// next tag
  		hadError = false;
  
  		if ( g_conf.m_logDebugFacebook ) {
  			char c = rsvp_status[6];
  			rsvp_status[6] = '\0';
  			log("facebook: got event "
  			    "eid=%llu "
  			    "uid=%llu "
  			    "rsvp_status=%s "
  			    "start_time=%lu"
  			    , eid
  			    , uid
  			    , rsvp_status
  			    , start_time 
  			    );
  			rsvp_status[6] = c;
  		}
  		
  		// ensure enough room
  		if ( m_eidBuf.getAvail() < 8 && ! m_eidBuf.reserve(5000) ) 
  			return false;
  
  		// dedup eidbuf
  		if ( ! m_dedupEidBuf.isInTable ( &eid ) ) {
  			// push it in
  			m_eidBuf.pushLongLong (eid );
  			// add it now
  			m_dedupEidBuf.addKey ( &eid );
  		}
  
  		// TODO: what about uid/rsvp_status/start_time ???
  		// we need that info for adding to likedb so make a likedb
  		// key here and we'll add that if the inject goes through
  		//long recSize;
  		//char *rec = g_likedb.makeRec ( uid         ,
  		//			       docId       ,
  		//			       gbeventId   ,
  		//			       rsvp_status ,
  		//			       start_time  ,
  		//			       &recSize    );
  		// a problem?
  		//if ( ! rec ) 
  		//	continue;
  
  		// convert rsvpstatus to a value
  		long rsvp = 0;
  		if      ( ! strncmp(rsvp_status,"not_replied",11) ) 
  			rsvp = LF_INVITED;
  		else if ( ! strncmp(rsvp_status,"attending",9  ) ) 
  			rsvp = LF_GOING;
  		else if ( ! strncmp(rsvp_status,"declined",8   ) ) 
  			rsvp = LF_DECLINED;
  		else if ( ! strncmp(rsvp_status,"unsure" ,6    ) ) 
  			rsvp = LF_MAYBE_FB;
  		else 
  			log("facebook: unknown rsvp_status=%s",rsvp_status);
  
  		// the data
  		LikedbTableSlot lts;
  		lts.m_uid         = uid;
  		lts.m_start_time  = start_time;
  		lts.m_rsvp        = rsvp;
  
  		// store that in table so if we end up injecting this
  		// eid successfully, we'll scan the table and add all these
  		// likedb recs for it. return false with g_errno set on error.
  		if ( ! m_likedbTable.addKey ( &eid , &lts ) )
  			return false;
  	}
  
  	// if we broke out when we should not have...
  	if ( hadError ) {
  		log("facebook: had error processing event_members reply %s",
  		    line);
  		g_errno = EBADREPLY;
  		return false;
  	}
  
  
  	// point to the access token from the old facebookdb rec so the
  	// new one keeps it going
  	fbrec->ptr_accessToken = m_accessToken;
  	fbrec->size_accessToken = gbstrlen(m_accessToken)+1;
  	fbrec->m_accessTokenCreated = getTimeGlobal();
  
  	// set key, just the fbid
  	fbrec->m_key.n1 = 0;
  	fbrec->m_key.n0 = fbrec->m_fbId;
  	fbrec->m_key.n0 <<= 1;
  	fbrec->m_key.n0 |= 0x01;
  
  	return true;
  }
  
  /////////////////////////////////////
  //
  // MSGFB PIPELINE #2
  //
  /////////////////////////////////////
  
  // high priority queue for ppl that login
  long long g_fbq1   [100];
  collnum_t g_colls1 [100];
  long      g_n1 = 0;
  // low priority queue for passive facebookdb scanning
  //long long g_fbq2   [100];
  //collnum_t g_colls2 [100];
  //long      g_n2 = 0;
  // used for queue
  Msgfb g_msgfb;
  
  bool isInQueue ( long long fbId , collnum_t collnum ) {
  	for ( long i = 0 ; i < g_n1 ; i++ ) 
  		if ( g_fbq1 [ i ] == fbId ) return true;
  	return false;
  }
  
  bool queueFBId ( long long fbId , collnum_t collnum ) {
  	// skip matt wells for now
  	//if ( fbId == 100003532411011LL ) {
  	//	log("facebook: skipping matt wells in queue");
  	//	return true;
  	//}
  	if ( g_n1 >= 100 )
  		return log("facebook: could not add fbid=%lli to queue",fbId);
  
  	// make sure not already in
  	if ( isInQueue ( fbId , collnum ) ) return false;
  	log("facebook: queueing fbid=%lli",fbId);
  	g_fbq1   [ g_n1 ] = fbId;
  	g_colls1 [ g_n1 ] = collnum;
  	g_n1++;
  	return true;
  }
  
  /*
  static void doneProcessingWrapper ( void *state ) {
  	// shortcut
  	long err = g_msgfb.m_errno;
  	// or inherit this. we might have forgotten to set m_errno
  	if ( ! err && g_errno ) err = g_errno;
  	// no longer in progress
  	g_msgfb.m_inProgress = false;
  	g_msgfb.reset();
  	// note it
  	log("facebook: done with queue for fbid=%llu. error=%s",
  	    g_fbq1[0],mstrerror(err));
  	// save it for potential re-add
  	//long long fsaved = g_fbq1  [0];
  	//collnum_t csaved = g_colls1[0];
  	// shift queue down
  	for ( long i = 1 ; i < g_n1 ; i++ ) {
  		g_fbq1  [i-1] = g_fbq1  [i];
  		g_colls1[i-1] = g_colls1[i];
  	}
  	// one less in queue
  	g_n1--;
  	// . on error, re-add to the end of the queue
  	// . leave this out for now. we should have some kinda download
  	//   loop perhaps to download it. at least leave this out until we
  	//   have a backoff scheme in place.
  	//if ( err ) queueFBId ( fsaved , csaved );
  }
  */
  
  #define NUM_MSGFBS 3
  Msgfb g_msgfbs[NUM_MSGFBS];
  long  g_numOut = 0;
  
  // evaluate events associated with the fbuserids in the queue
  void queueSleepWrapper ( int fd, void *state ) {
  	// skip for now
  	//return;
  	// return if empty
  	if ( g_n1 == 0 ) return;
  	// sanity
  	if ( g_n1 >= 100 ) { char *xx=NULL;*xx=0; }
  	// wait for clock to be in sync
  	if ( ! isClockInSync() ) 
  		return;
  	// wait until done repairing... so we do not inject events!
  	if ( g_repair.isRepairActive() ) 
  		return;
  	// get an fbid
  	if ( g_numOut >= (long)NUM_MSGFBS ) return;
  	// return if all out and no more to put out
  	if ( g_n1 <= g_numOut ) return;
  	// get the next fbid
  	long long fbId = g_fbq1[g_numOut];
  	// get one not in use
  	Msgfb *mfb = NULL;
  	for ( long i = 0 ; i < NUM_MSGFBS ; i++ ) {
  		if ( g_msgfbs[i].m_inProgress ) continue;
  		mfb = &g_msgfbs[i];
  		break;
  	}
  	// return if all in progress. how can this be?
  	if ( ! mfb ) return;
  	// get the fbid
  	//long long fbId    = g_fbq1[0];
  	//collnum_t collnum = g_colls1[0];
  	// inc this now!
  	g_numOut++;
  	// set it up
  	mfb->m_fbId = fbId;
  	mfb->m_phase = 0;
  	// and launch it. will not re-launch since it sets m_inProgress = true
  	mfb->queueLoop();
  	// set it up
  	//if( ! g_msgfb.processFBId (fbid,collnum,NULL,doneProcessingWrapper) )
  	//	return;
  	// error?
  	//log("fbqueue: error of some sort = %s",mstrerror(g_errno));
  	// wtf?
  	//g_msgfb.m_inProgress = false;
  }
  
  ////////////
  //
  // the queue loop
  //
  ////////////
  
  static bool s_init = false;
  static long s_flip = 0;
  static SafeBuf s_tbuf1;
  static SafeBuf s_tbuf2;
  static long s_ptr1 = 0; // facebook dictionary query cursor
  static long s_ptr2 = 0; // facebook location query cursor
  static long s_ptr3 = 0; // stubhub cursor
  static long long s_ptr4 = 0; // eventbrite cursor
  static long long s_ptr5 = 0; // local facebookdb scanner cursor
  static long      s_holdOffStubHubTill = 0;
  static long long s_lastEventBriteEventId = 0;
  static long      s_eventBriteWaitUntil = 0;
  static long      s_localWaitUntil = 0;
  
  static char *getNextQuery ();
  
  static void queueLoopWrapper ( void *state ) {
  	Msgfb *msgfb = (Msgfb *)state;
  	msgfb->queueLoop();
  }
  
  static void queueLoopWrapper2 ( void *state , TcpSocket *s ) {
  	Msgfb *msgfb = (Msgfb *)state;
  	msgfb->m_socket = s;
  	msgfb->queueLoop();
  }
  
  static void queueLoopWrapper5 ( void *state , RdbList *list, Msg5 *msg5 ) {
  	Msgfb *msgfb = (Msgfb *)state;
  	msgfb->queueLoop();
  }
  
  void Msgfb::queueLoop ( ) {
  
  	// skip for now
  	//return;
  
  	// assume in use
  	//m_inProgress = true;
  
  	if ( m_phase == 0 ) {
  		// do not reset this!
  		long long saved = m_fbId;
  		// make sure this is empty and ready to go
  		reset();
  		// re-establish this
  		m_fbId = saved;
  		// set this
  		m_inProgress = true;
  	}
  
  	// if an fbid of -3 means to get scan facebookdb locally
  	if ( m_phase == 0 && m_fbId == -3LL ) {
  		// use msg5 then i guess
  		key96_t startKey;
  		key96_t endKey;
  		startKey.n1 = 0;
  		startKey.n0 = s_ptr5;
  		endKey.n1 = 0;
  		endKey.n0 = 0xffffffffffffffffLL;
  		startKey.n0 <<= 1;
  		endKey.n0 <<= 1;
  		endKey.n0 |= 0x01;
  		// advance phase to a special phase
  		m_phase = 50;
  		if ( ! m_msg5.getList ( RDB_FACEBOOKDB,
  					"", // coll
  					&m_list33,
  					&startKey,
  					&endKey,
  					13, // minrecsizes
  					true , // includetree?
  					false, // addtocache?
  					0, // maxcacheage
  					0, // startfilenum
  					-1, // numfiles
  					this ,
  					queueLoopWrapper5,
  					m_niceness ,
  					true // doErrorCorrection?
  					) )
  			return;
  	}
  
  	// handle special phase from msg5 call above
  	if ( m_phase == 50 ) {
  		// grab the rec
  		if ( g_errno ) {
  			log("fbqueue: error getting local facebookdb rec: %s", 
  			    mstrerror(g_errno));
  			goto error;
  		}
  		// empty is means done
  		if ( m_list33.getListSize() <= 0 ) {
  			// set this
  			long now = getTimeGlobal();
  			// wait for a day then
  			s_localWaitUntil = now + 86400;
  			// reset the scan for the scan tomorrow
  			s_ptr5 = 0;
  			// all done
  			m_phase = 16;
  			// remove from queue
  			goto skipdown;
  		}
  		// get the facebook rec... why?
  		m_oldFbrec = (FBRec *)m_list33.getList();
  		// get is fbid and queue that
  		if ( m_oldFbrec->m_fbId < 0 ) {
  			log("fbqueue: bad fbid in local scan: %lli",
  			    m_oldFbrec->m_fbId);
  			g_errno = EBADREPLY;
  			goto error;
  		}
  		// ok, queue that fbid now
  		queueFBId ( m_oldFbrec->m_fbId , m_collnum );
  		// update our scan cursor
  		s_ptr5 = m_oldFbrec->m_fbId + 1;
  		// and remove ourselves from the queue
  		m_phase = 16;
  		goto skipdown;
  	}
  
  
  	// if an fbid of -2 means to get some eventbrite
  	if ( m_fbId == -2LL && m_phase == 0 ) {
  		// . loop over all events sorted by id
  		// . use &since_id=<s_ptr4> to avoid duppage
  		// . we assume they only increment that id, if that is
  		//   not the case we'll have to restart from 0 each
  		//   time, and maybe two a roll twice per day
  		// . they say they limit to 1000 requests per day!
  		SafeBuf cmd10;
  		cmd10.safePrintf(
  				 "http://www.eventbrite.com/xml/event_search?"
  				 "app_key=QSMWWLSZUJT3D5ZGPF&"
  				 "page=1&"
  				 // ask for 100 results
  				 "max=100&"
  				 "sort_by=id&"
  				 "since_id=%lli"
  				 // no need to add 1 to s_ptr4, just make
  				 // sure that it exactly equals the LAST
  				 // id we injected because it is EXCLUDED
  				 // from these results and eventbrite starts
  				 // with s_ptr4+1
  				 ,s_ptr4 
  				);
  		log("fbspider: getting eventbrite results: cmd10 = %s",
  		    cmd10.getBufStart() );
  		// advance to phase to parse the events out of the reply
  		m_phase = 11;
  		// this will be set in queueLoopWrapper2()
  		m_socket = NULL;
  		// use defaults i guess
  		char *ua = g_conf.m_spiderUserAgent;
  		char *proto = "HTTP/1.0";
  		// pillage from eventbrite
  		if ( ! g_httpServer.getDoc ( cmd10.getBufStart() ,
  					     0 , // urlIp
  					     0                    , // offset
  					     -1                   ,
  					     0 , // ifModifiedSince ,
  					     this               , // state
  					     queueLoopWrapper2,
  					     80*1000      , // 20 sec timeout
  					     0 , // proxyip
  					     0 , // proxyport
  					     100000000 , // maxTextDocLen   ,
  					     100000000 , // maxOtherDocLen  ,
  					     ua ,
  					     proto ))
  			// return false if blocked
  			return;
  		// this is not possible, must be error!
  		log("fbspider: got error downloading eventbrite results = %s",
  		    mstrerror(g_errno));
  		goto error;
  	}
  
  	// if an fbid of -1 means to close our eyes and suck down some stubhub
  	if ( m_fbId == -1LL && m_phase == 0 ) {
  
  		long now = getTimeGlobal();
  		// now subtract 12 so that gmtime() call below will
  		// return current time in hawaii or right before that...
  		now -= 12*3600;
  
  		SafeBuf cmd9;
  		cmd9.safePrintf("http://www.stubhub.com/"
  				"listingCatalog/select/?q="
  				"event_date%%3A["
  				);
  
  		if ( s_ptr3 == 0 ) s_ptr3 = now;
  
  		// . convert into canonical time format
  		// . this should be basically close to the time in hawaii
  		//   which lags pretty much everyone else, that way
  		//   we will get events about to happen right now in hawaii
  		struct tm *tt = gmtime(&s_ptr3);
  
  		// . now print it out for the stubhub query
  		// . it uses the SOLR format that lucene uses for queries
  		//   like "event_data:[X TO Y]" for doing ranges
  		// http://www.stubhub.com/listingCatalog/select/?q=event_date%3A[2012-05-04T00:00:00.001Z%20TO%202012-05-04T13:59:00.001Z]
  		// &start=0&rows=10
  		char tbuf[512];
  		strftime ( tbuf , 512, "%Y-%m-%dT%H:00:00Z" , tt );
  		cmd9.urlEncode ( tbuf );
  		cmd9.safePrintf("+TO+");
  
  		// add the half day here
  		long delta = 12*3600;
  
  		// 3+ months out use a full day
  		if ( s_ptr3 - now > 90*86400 ) delta = 24*3600;
  
  		// subtract 1 to avoid dups for next call in case stubhub
  		// uses a closed interval, and not half open
  		s_ptr3 += delta - 1;
  
  		// get the endpoint then
  		tt = gmtime(&s_ptr3);
  		strftime ( tbuf , 512, "%Y-%m-%dT%H:00:00Z" , tt );
  		cmd9.urlEncode ( tbuf );
  		cmd9.safePrintf("]");
  		cmd9.safePrintf("&fl=event_id,"
  				"act_primary,event_date_time_local,"
  				"keywords,lat_lon,"
  				"urlpath,venueDetailUrlPath,description,"
  				"ancestorGenreDescriptions"
  				);
  
  
  		// add the second back
  		s_ptr3++;
  
  		log("fbspider: getting stubhub results: cmd9 = %s",
  		    cmd9.getBufStart() );
  
  		// advance to phase to parse the events out of the reply
  		m_phase = 11;
  
  		// this will be set in queueLoopWrapper2()
  		m_socket = NULL;
  
  		char *ua = g_conf.m_spiderUserAgent;
  		char *proto = "HTTP/1.0";
  		//if ( m_fbId < 0 ) {
  		//	ua = "Mozilla/5.0 (X11; U; Linux i686; "
  		//		"en-US; rv:1.9.2.7) Gecko/20100715 "
  		//			"Ubuntu/10.04 (lucid) Firefox/3.6.7";
  		//	proto = "HTTP/1.1";
  		//}
  
  		// pillage from stubhub
  		if ( ! g_httpServer.getDoc ( cmd9.getBufStart() ,
  					     0 , // urlIp
  					     0                    , // offset
  					     -1                   ,
  					     0 , // ifModifiedSince ,
  					     this               , // state
  					     queueLoopWrapper2,
  					     80*1000      , // 20 sec timeout
  					     0 , // proxyip
  					     0 , // proxyport
  					     100000000 , // maxTextDocLen   ,
  					     100000000 , // maxOtherDocLen  ,
  					     ua ,
  					     proto ))
  			// return false if blocked
  			return;
  		// this is not possible, must be error!
  		log("fbspider: got error downloading stubhub results = %s",
  		    mstrerror(g_errno));
  		goto error;
  	}
  
  
  	// if an fbid of 0 is used that means to do a query against
  	// facebook's massively lame event search engine
  	if ( m_fbId == 0LL && m_phase == 0 ) {
  		// . ok, pick a query to send to facebook
  		// . pick the next most popular word, 
  		//   or the next most popular location name
  		// . interleave between words and location names
  		char *fbq = getNextQuery ( );
  		// don't core!
  		if ( ! fbq ) {
  			log("fbspider: getNextQuery() returned null. "
  			    "possibly a state without an official name "
  			    "and getStateName() returned NULL?");
  			g_errno = EBADENGINEER;
  			goto error;
  		}
  		// . make the query
  		// . &format=xml is pointless and will not work!
  		//   they made it 100% json now
  		SafeBuf cmd8;
  		cmd8.safePrintf("https://graph.facebook.com/"
  				"search?"
  				"fields=id,privacy,picture,name,location,"
  				"venue,description,"
  				"start_time,end_time&"
  				"q="
  				);
  		// encode the query
  		cmd8.urlEncode ( fbq );
  		// and put in the rest
  		cmd8.safePrintf("&"
  				"type=event&"
  				//"access_token=AAAAAAITEghMBADXfRdrOzCBZAV7"
  				//"2K1kQurpnnyPDBIYHCSoPpRLOvdJFBZBESUZA0BgZ"
  				//"AWr8dGCpNCOZBMh58opiYAnAZAavKPGaMnoKq32v9"
  				//"CiYxy2WtD&"
  				"limit=600&"
  				"offset=0"
  				);
  
  		log("fbspider: getting fb results: cmd8 = %s",
  		    cmd8.getBufStart() );
  
  		// advance to phase to parse the events out of the reply
  		m_phase = 11;
  
  		// this will be set in queueLoopWrapper2()
  		m_socket = NULL;
  
  		// pillage from facebook!
  		if ( ! g_httpServer.getDoc ( cmd8.getBufStart() ,
  					     0 , // urlIp
  					     0                    , // offset
  					     -1                   ,
  					     0 , // ifModifiedSince ,
  					     this               , // state
  					     queueLoopWrapper2,
  					     40*1000      , // 20 sec timeout
  					     0 , // proxyip
  					     0 , // proxyport
  					     100000000 , // maxTextDocLen   ,
  					     100000000 , // maxOtherDocLen  ,
  					     g_conf.m_spiderUserAgent    ) )
  			// return false if blocked
  			return;
  		// this is not possible, must be error!
  		log("fbspider: got error downloading fb results = %s",
  		    mstrerror(g_errno));
  		goto error;
  	}
  
  
  	// first phase is loading the facebook rec
  	if ( m_phase == 0 ) {
  		key96_t startKey;
  		key96_t endKey;
  		startKey.n1 = 0;
  		startKey.n0 = m_fbId;
  		endKey.n1 = 0;
  		endKey.n0 = m_fbId;
  		startKey.n0 <<= 1;
  		endKey.n0 <<= 1;
  		endKey.n0 |= 0x01;
  		// now we use this
  		m_dedupEidBuf.set (8,0,0,NULL,0,false,m_niceness,"deibuf");
  		// advance phase
  		m_phase = 1;
  		if ( ! m_msg0.getList ( -1, // hostid
  					0 , // ip
  					0 , // port
  					0 , // maxcacheage
  					false, // addtocache
  					RDB_FACEBOOKDB,
  					"",//coll,
  					&m_list1,
  					(char *)&startKey,
  					(char *)&endKey,
  					13, // minrecsizes
  					this ,
  					queueLoopWrapper,
  					m_niceness ) )
  			return;
  	}
  	// check for errors
  	if ( m_phase == 1 ) {
  		if ( g_errno ) {
  			log("fbqueue: error getting facebookdb rec: %s", 
  			    mstrerror(g_errno));
  			goto error;
  		}
  		// empty is bad
  		if ( m_list1.getListSize() <= 0 ) {
  			log("fbqueue: facebookdb rec is empty. wtf? fbid=%lli",
  			    m_fbId);
  			g_errno = EBADREPLY;
  			goto error;
  		}
  		// get the facebook rec... why?
  		m_oldFbrec = (FBRec *)m_list1.getList();
  		// sanity
  		if ( m_oldFbrec->m_fbId != m_fbId ) {
  			log("fbqueue: fbid mismatch. fbid=%lli", m_fbId);
  			g_errno = EBADREPLY;
  			goto error;
  		}
  		// deserialize...
  		deserializeMsg ( sizeof(FBRec) ,
  				 &m_oldFbrec->size_accessToken ,
  				 &m_oldFbrec->size_friendIds ,
  				 &m_oldFbrec->ptr_accessToken ,
  				 m_oldFbrec->m_buf );
  		// copy for calling fql
  		strcpy ( m_accessToken , m_oldFbrec->ptr_accessToken );
  		
  	retryDownload:
  
  		// note it
  		log("facebook: downloading user info again");
  		// get your facebook user info again in case it changed
  		SafeBuf cmd;
  		cmd.safePrintf ( "SELECT uid,username,first_name,last_name,name,pic_square,profile_update_time,timezone,religion,birthday,birthday_date,sex,hometown_location,current_location,activities,interests,is_app_user,music,tv,movies,books,about_me,status,online_presence,proxied_email,verified,website,is_blocked,contact_email,email,is_minor,work,education,sports,languages,likes_count,friend_count FROM user where uid=me()");
  		cmd.urlEncode();
  		// www.howtobe.pro/facebook-graph-api-graph-api-for-issuing-
  		// fql-queries make a url
  		SafeBuf ubuf;
  		ubuf.safePrintf("https://api.facebook.com/method/"
  				"fql.query?query=%s"
  				"&access_token=%s&format=xml"
  				, cmd.getBufStart() 
  				, m_accessToken 
  				);
  		log("facebook: cmd = %s",ubuf.getBufStart());
  		// advance phase
  		m_phase = 2;
  		// reset
  		g_errno = 0;
  		// get the results
  		if ( ! g_httpServer.getDoc ( ubuf.getBufStart() ,
  					     0 , // urlIp
  					     0                    , // offset
  					     -1                   ,
  					     0 , // ifModifiedSince ,
  					     this               , // state
  					     queueLoopWrapper2 ,
  					     40*1000      , // 20 sec timeout
  					     0 , // proxyip
  					     0 , // proxyport
  					     30000000 , // maxTextDocLen   ,
  					     30000000 , // maxOtherDocLen  ,
  					     g_conf.m_spiderUserAgent    ) )
  			// return false if blocked
  			return;
  		// otherwise, somehow got it without blocking... wtf?
  		//return gotFQLReply();
  		if ( ! g_errno ) { char *xx=NULL;*xx=0; }
  		log("fql: error getting doc: %s",mstrerror(g_errno));
  		goto error;
  	}
  	// got reply
  	if ( m_phase == 2 ) {
  		// bail on http reply error
  		if ( g_errno ) {
  			log("fbqueue: error getting fb user rec: %s", 
  			    mstrerror(g_errno));
  			goto error;
  		}
  		m_phase = 3;
  		// a quick sanity check
  		char *reply = m_socket->m_readBuf;
  		long  replyLen = m_socket->m_readOffset;
  		// i've seen crappy facebook return essentially an empty
  		// reply here, so make sure we have all the fields!!
  		if ( ! strstr(reply,"<uid>") ||
  		     ! strstr(reply,"<name") ||
  		     ! strstr(reply,"<first_name") ||
  		     // might be <interests/>
  		     ! strstr(reply,"<interests") ) {
  			log("facebook: missing uid in facebook reply #1");
  			SafeBuf eb;
  			eb.safeMemcpy ( reply,replyLen);
  			char fname[128];
  			sprintf(fname,"error1.%llu",m_fbId);
  			eb.save("/tmp/",fname);
  			// do a retry
  			if ( m_retryCount++ < 5 ) {
  				log("facebook: retrying %li.",m_retryCount);
  				goto retryDownload;
  			}
  			// otherwise abandon ship
  			g_errno = EBADREPLY;
  			goto error;
  		}
  		// copy reply into our final SafeBuf buffer
  		m_fullReply.safeMemcpy ( m_socket->m_readBuf,
  					 m_socket->m_readOffset );
  	}
  	// download friend list
  	if ( m_phase == 3 ) {
  		// note it
  		log("facebook: downloading friend list fbid=%lli",m_fbId);
  		// get your facebook user info again in case it changed
  		SafeBuf cmd;
  		cmd.safePrintf ( "SELECT uid2 from friend WHERE uid1=me()");
  		cmd.urlEncode();
  		SafeBuf ubuf;
  		ubuf.safePrintf("https://api.facebook.com/method/"
  				"fql.query?query=%s"
  				"&access_token=%s&format=xml"
  				, cmd.getBufStart() 
  				, m_accessToken 
  				);
  		log("facebook: cmd = %s",ubuf.getBufStart());
  		// advance phase
  		m_phase = 4;
  		// reset start for phase 3
  		m_chunkStart = 0;
  		// reset chunksize for phase 3
  		m_chunkSize = 300;
  		// reset
  		g_errno = 0;
  		// get the results
  		if ( ! g_httpServer.getDoc ( ubuf.getBufStart() ,
  					     0 , // urlIp
  					     0                    , // offset
  					     -1                   ,
  					     0 , // ifModifiedSince ,
  					     this               , // state
  					     queueLoopWrapper2 ,
  					     40*1000      , // 20 sec timeout
  					     0 , // proxyip
  					     0 , // proxyport
  					     30000000 , // maxTextDocLen   ,
  					     30000000 , // maxOtherDocLen  ,
  					     g_conf.m_spiderUserAgent    ) )
  			// return false if blocked
  			return;
  		// otherwise, somehow got it without blocking... wtf?
  		if ( ! g_errno ) { char *xx=NULL;*xx=0; }
  		log("fql: error getting doc: %s",mstrerror(g_errno));
  		goto error;
  	}
  	// got reply
  	if ( m_phase == 4 ) {
  		// mime error?
  		HttpMime mime;
  		// exclude the \0 i guess. use NULL for url.
  		mime.set ( m_socket->m_readBuf, m_socket->m_readOffset-1,NULL);
  		// not good?
  		long httpStatus = mime.getHttpStatus();
  		if ( httpStatus != 200 ) g_errno = EBADREPLY;
  		// bail on http reply error
  		if ( g_errno ) {
  			log("fbqueue: error getting friend list fbid=%lli: "
  			    "%s", m_fbId,   mstrerror(g_errno));
  			goto error;
  		}
  		m_phase = 5;
  		// copy reply into our final SafeBuf buffer
  		m_fullReply.safeMemcpy ( m_socket->m_readBuf,
  					 m_socket->m_readOffset );
  	}
  
  
  	// download some event member rsvp statuses
  	if ( m_phase == 5 ) {
  	recallMembers:
  		// note it
  		log("facebook: downloading event members #%li-#%li "
  		    "fbid=%lli",
  		    m_chunkStart,m_chunkStart+m_chunkSize,m_fbId);
  		// get status of each friend attending an event & what eventid
  		SafeBuf cmd;
  		// if this fails then chunk down to 50 instead of 300
  		cmd.safePrintf ( "SELECT uid, eid, rsvp_status, start_time "
  				 "FROM event_member where uid IN "
  				 "( SELECT uid2 from friend WHERE uid1=me() "
  				 "LIMIT %li,%li) "
  				 , m_chunkStart , m_chunkSize 
  				 // this start_time from the event_member table
  				 // is not accurate. it is often in the past! 
  				 // wtf?
  				 //"AND start_time > now()"
  				 );
  		cmd.urlEncode();
  		SafeBuf ubuf;
  		ubuf.safePrintf("https://api.facebook.com/method/"
  				"fql.query?query=%s"
  				"&access_token=%s&format=xml"
  				, cmd.getBufStart() 
  				, m_accessToken 
  				);
  		log("facebook: cmd = %s",ubuf.getBufStart());
  		// advance chunkstart
  		m_chunkStart += m_chunkSize;
  		// advance phase
  		m_phase = 6;
  		// reset
  		g_errno = 0;
  		// get the results
  		if ( ! g_httpServer.getDoc ( ubuf.getBufStart() ,
  					     0 , // urlIp
  					     0                    , // offset
  					     -1                   ,
  					     0 , // ifModifiedSince ,
  					     this               , // state
  					     queueLoopWrapper2 ,
  					     40*1000      , // 20 sec timeout
  					     0 , // proxyip
  					     0 , // proxyport
  					     30000000 , // maxTextDocLen   ,
  					     30000000 , // maxOtherDocLen  ,
  					     g_conf.m_spiderUserAgent    ) )
  			// return false if blocked
  			return;
  		// otherwise, somehow got it without blocking... wtf?
  		if ( ! g_errno ) { char *xx=NULL;*xx=0; }
  		log("fql: error getting doc: %s",mstrerror(g_errno));
  		goto error;
  	}
  	// got a reply
  	if ( m_phase == 6 ) {
  		// mime error?
  		HttpMime mime;
  		// exclude the \0 i guess. use NULL for url.
  		mime.set ( m_socket->m_readBuf, m_socket->m_readOffset-1,NULL);
  		// not good?
  		long httpStatus = mime.getHttpStatus();
  		if ( httpStatus != 200 ) g_errno = EBADREPLY;
  		// bail on http reply error
  		if ( g_errno ) {
  			log("fbqueue: error getting event members fbid=%lli: "
  			    "%s", m_fbId,   mstrerror(g_errno));
  			// count it
  			m_errorCount++;
  			// retry?
  			if ( m_errorCount < 10 ) goto recallMembers;
  			// final phase. wrap it up
  			goto error;
  		}
  		// advance
  		m_phase = 7;
  		// copy reply into our final SafeBuf buffer
  		m_fullReply.safeMemcpy ( m_socket->m_readBuf,
  					 m_socket->m_readOffset );
  		// if we had some repeat!
  		if ( strstr ( m_socket->m_readBuf , "<eid>" ) ) {
  			log("facebook: recalling event members");
  			goto recallMembers;
  		}
  		// reset for my events
  		m_myChunkStart = 0;
  		m_myChunkSize = 300;
  	}
  
  	if ( m_phase == 7 ) {
  		// get MY events
  	recallMyEvents:
  		// note it
  		log("facebook: downloading my event ids #%li-#%li "
  		    "fbid=%lli",
  		    m_myChunkStart,m_myChunkStart+m_myChunkSize,m_fbId);
  		// get status of each friend attending an event & what eventid
  		SafeBuf cmd;
  		// if this fails then chunk down to 50 instead of 300
  		cmd.safePrintf ( "SELECT uid, eid, rsvp_status, start_time "
  				 "FROM event_member where uid=me() "
  				 "LIMIT %li,%li"
  				 , m_myChunkStart , m_myChunkSize 
  				 // this start_time from the event_member table
  				 // is not accurate. it is often in the past! 
  				 // wtf?
  				 //"AND start_time > now()"
  				 );
  		cmd.urlEncode();
  		SafeBuf ubuf;
  		ubuf.safePrintf("https://api.facebook.com/method/"
  				"fql.query?query=%s"
  				"&access_token=%s&format=xml"
  				, cmd.getBufStart() 
  				, m_accessToken 
  				);
  		log("facebook: cmd = %s",ubuf.getBufStart());
  		// advance chunkstart
  		m_myChunkStart += m_myChunkSize;
  		// advance phase
  		m_phase = 8;
  		// reset
  		g_errno = 0;
  		// get the results
  		if ( ! g_httpServer.getDoc ( ubuf.getBufStart() ,
  					     0 , // urlIp
  					     0                    , // offset
  					     -1                   ,
  					     0 , // ifModifiedSince ,
  					     this               , // state
  					     queueLoopWrapper2 ,
  					     40*1000      , // 20 sec timeout
  					     0 , // proxyip
  					     0 , // proxyport
  					     30000000 , // maxTextDocLen   ,
  					     30000000 , // maxOtherDocLen  ,
  					     g_conf.m_spiderUserAgent    ) )
  			// return false if blocked
  			return;
  		// otherwise, somehow got it without blocking... wtf?
  		if ( ! g_errno ) { char *xx=NULL;*xx=0; }
  		log("fql: error getting doc: %s",mstrerror(g_errno));
  		goto error;
  	}
  	// got a reply
  	if ( m_phase == 8 ) {
  		// mime error?
  		HttpMime mime;
  		// exclude the \0 i guess. use NULL for url.
  		mime.set ( m_socket->m_readBuf, m_socket->m_readOffset-1,NULL);
  		// not good?
  		long httpStatus = mime.getHttpStatus();
  		if ( httpStatus != 200 ) g_errno = EBADREPLY;
  		// bail on http reply error
  		if ( g_errno ) {
  			log("fbqueue: error getting my events fbid=%lli: "
  			    "%s", m_fbId,   mstrerror(g_errno));
  			// count it
  			m_errorCount++;
  			// retry?
  			if ( m_errorCount < 10 ) goto recallMembers;
  			// final phase. wrap it up
  			goto error;
  		}
  		// advance
  		m_phase = 9;
  		// copy reply into our final SafeBuf buffer
  		m_fullReply.safeMemcpy ( m_socket->m_readBuf,
  					 m_socket->m_readOffset );
  		// if we had some repeat!
  		if ( strstr ( m_socket->m_readBuf , "<eid>" ) ) {
  			log("facebook: recalling my events");
  			goto recallMyEvents;
  		}
  		// init this for downloading events
  		m_eventStartNum = 0;
  		// start off high, reduce to 10 on error
  		m_eventStep = 100;
  	}
  
  	// save the fbrec from m_fullReply
  	if ( m_phase == 9 ) {
  		// save to disk for debug
  		char fname[64];
  		sprintf(fname,"%lli.txt",m_fbId);
  		m_fullReply.save("/tmp/",fname);
  		// compile into a new facebook rec
  		char *content    = m_fullReply.getBufStart();
  		long  contentLen = m_fullReply.length();
  		// sanity
  		long long origFbId = m_fbId;
  		// is this messing up our Msgfb::m_fbId???
  		if ( ! setFBRecFromFQLReply(content,contentLen,&m_fbrecGen)) {
  			log("fql: error setting fb rec from fql");
  			g_errno = EBADREPLY;
  			goto error;
  		}
  		// must be there!
  		if ( ! m_fbrecGen.m_fbId ) {
  			log("fql: failed to get facebook id from reply");
  			g_errno = EBADENGINEER;
  			goto error;
  		}
  		// sanity
  		if ( m_fbId != origFbId ) { char *xx=NULL;*xx=0; }
  		// we loaded this in phase 0
  		FBRec *src = m_oldFbrec;
  		// we just made this from facebook replies
  		FBRec *dst = &m_fbrecGen;
  		// merge src into dst. merge the old into the new.
  		mergeFBRec ( dst , src );
  		// advance to downloading events now
  		m_phase = 10;
  		// come here when done saving
  		m_afterSaveCallback = queueLoopWrapper;
  		// save the merged rec now
  		if ( ! saveFBRec( &m_fbrecGen ) ) return;
  	}
  
  	// get events now
  	if ( m_phase == 10 ) {
  	recallEvents:
  		// reset these so we do not re-inject the same events
  		m_evPtrBuf.reset();
  		m_evIdsBuf.reset();
  		// skip matt wells for now
  		if ( m_fbId == MATTWELLS ) { // 100003532411011LL ) {
  			log("facebook: skipping matt wells event members "
  			    "download");
  			// save it so we can turn off the in queue bit
  			//m_phase = 15;
  			//goto skipdown;
  		}
  		// note it
  		log("facebook: downloading events #%li-#%li fbid=%lli",
  		    m_eventStartNum,m_eventStartNum+m_eventStep,m_fbId);
  		SafeBuf cmd;
  		cmd.safePrintf ( "SELECT eid, "
  				 "name, "
  				 // deprecated as of july 5th 2012
  				 //"tagline, "
  				 //"nid, "
  				 "pic_small, "
  				 "pic_big, "
  				 "pic_square, "
  				 "pic, "
  				 "host, "
  				 "description, "
  				 // deprecated as of july 5th 2012
  				 //"event_type, "
  				 //"event_subtype, "
  				 "start_time, "
  				 "end_time, "
  				 "creator, "
  				 "update_time, "
  				 "location, "
  				 "venue, "
  				 "privacy, "
  				 "hide_guest_list, "
  				 "can_invite_friends "
  				 "FROM event WHERE "
  				 // this does not seem to work either!
  				 //"start_time > now() AND "
  				 "eid IN (" );
  		// from buf
  		long max = m_eidBuf.length() / 8;
  		// did we add any?
  		bool printed = false;
  		// shortcut
  		long long *eids = (long long *)m_eidBuf.getBufStart();
  		// list some eids here
  		for ( long i = m_eventStartNum ; 
  		      i < max &&
  		      i < m_eventStartNum + m_eventStep ; i++ ) {
  			if ( printed ) cmd.pushChar(',');
  			printed = true;
  			cmd.safePrintf("%lli",eids[i]);
  		}
  		// end it
  		cmd.safePrintf(")");
  
  		cmd.urlEncode();
  		SafeBuf ubuf;
  		ubuf.safePrintf("https://api.facebook.com/method/"
  				"fql.query?query=%s"
  				"&access_token=%s&format=xml"
  				, cmd.getBufStart() 
  				, m_accessToken 
  				);
  		log("facebook: cmd = %s",ubuf.getBufStart());
  		// advance chunkstart
  		m_eventStartNum += m_eventStep;
  		// advance phase
  		m_phase = 11;
  		// reset
  		g_errno = 0;
  		// get the results
  		if ( ! g_httpServer.getDoc ( ubuf.getBufStart() ,
  					     0 , // urlIp
  					     0                    , // offset
  					     -1                   ,
  					     0 , // ifModifiedSince ,
  					     this               , // state
  					     queueLoopWrapper2 ,
  					     40*1000      , // 20 sec timeout
  					     0 , // proxyip
  					     0 , // proxyport
  					     30000000 , // maxTextDocLen   ,
  					     30000000 , // maxOtherDocLen  ,
  					     g_conf.m_spiderUserAgent    ) )
  			// return false if blocked
  			return;
  		// otherwise, somehow got it without blocking... wtf?
  		if ( ! g_errno ) { char *xx=NULL;*xx=0; }
  		log("fql: error getting doc: %s",mstrerror(g_errno));
  		goto error;
  	}
  	// got events reply
  	if ( m_phase == 11 ) {
  		// mime error?
  		HttpMime mime;
  		// exclude the \0 i guess. use NULL for url.
  		if ( m_socket && ! g_errno ) {
  			mime.set ( m_socket->m_readBuf, 
  				   m_socket->m_readOffset-1,NULL);
  			// not good?
  			long httpStatus = mime.getHttpStatus();
  			if ( httpStatus != 200 ) g_errno = EBADREPLY;
  		}
  		// bail on http reply error
  		if ( g_errno ) {
  			char *reply = "";
  			if ( m_socket && m_socket->m_readBuf )
  				reply = m_socket->m_readBuf;
  			log("fbqueue: error getting events fbid=%lli: "
  			    "%s : %s", m_fbId,   mstrerror(g_errno),reply);
  			// bail right away if doing spider
  			if ( m_fbId <= 0 ) goto error;
  			// final phase. wrap it up
  			if ( ++m_errorCount >= 10 ) goto error;
  			// first time?
  			if ( m_eventStep > 10 ) m_eventStartNum -= m_eventStep;
  			// reduce step
  			m_eventStep = 10;
  			// retry
  			goto recallEvents;
  		}
  
  		m_rbuf.purge();
  
  		// convert json to xml
  		m_rbuf.safeMemcpy ( m_socket->m_readBuf ,
  				    m_socket->m_readOffset );
  
  		// nuke the read buf then from the http server
  		mfree ( m_socket->m_readBuf , m_socket->m_readBufSize,"sss" );
  		// do not allow tcpsocket to free it again
  		m_socket->m_readBuf = NULL;
  
  		char *type = NULL;
  		if ( m_fbId ==  0LL ) type = "facebook";
  		if ( m_fbId == -1LL ) type = "stubhub";
  		if ( m_fbId == -2LL ) type = "eventbrite";
  
  		// show it
  		if ( type )
  			log("facebook: got %s reply (%li bytes)"
  			    , type , m_rbuf.length() );
  
  		//fprintf(stderr,"full raw reply=%s",m_rbuf.getBufStart());
  
  		// do not do jsontoxml on the mime header!
  		char *rb = m_rbuf.getBufStart();
  		char *rc = strstr(rb,"\r\n\r\n");
  		long  pos = 0;
  		if ( rc ) pos = (rc+4) - rb;
  
  		// convert json to xml if it is in json. only for facebook
  		m_rbuf.convertJSONtoXML ( m_niceness , pos );
  
  		//fprintf(stderr,
  		//"full converted reply=%s",m_rbuf.getBufStart());
  
  		// show it
  		//if ( m_fbId == 0LL )
  		//	log("facebook: got fb reply xml: %s",
  		//	    m_rbuf.getBufStart() );
  
  
  		// did it have some events?
  		//char *hadSome = strstr ( m_socket->m_readBuf , "<eid>" ) ;
  		// save reply
  		//m_facebookReply2     = m_socket->m_readBuf;
  		//m_facebookReplySize2 = m_socket->m_readOffset;
  		//m_facebookAllocSize2 = m_socket->m_readBufSize;
  
  		// sanity. must be \0. httpserver should add this if not
  		// already there!
  		//if ( m_facebookReply2 [ m_facebookReplySize2 ] ) {
  		//	log("fbqueue: facebookreply2 does not end in \\0");
  		//	g_errno = EBADREPLY;
  		//	goto error;
  		//}
  
  		// . set m_numEvents and m_eventPtrs from m_facebookReply
  		// . they reference into the reply, so do not free 
  		//   m_facebookReply until destructor is called
  		m_numEvents = 0;
  
  		// . get the event delimeter in the xml
  		// . both facebook and eventbrite use this
  		char *delim = "<event>";
  		char *dend  = "</event>";
  		// stubhub uses "<doc>" to delineate events
  		if ( m_fbId == -1 ) {
  			delim = "<doc>";
  			dend  = "</doc>";
  		}
  
  		// parse the events into an array of ptrs for easier injection
  		// scan for <event> tags and set m_evptrs safebuf to 
  		// each ptr to those
  		char *p = m_rbuf.getBufStart(); // facebookReply2;
  		for ( ; *p ; ) {
  			// scan to first <event> tag
  			p = strstr ( p , delim );
  			if ( ! p ) break;
  			// store start for ptr
  			char *start = p + gbstrlen(delim);
  			// find end
  			char *end = strstr ( p , dend );
  			if ( ! end ) break;
  			// null term it
  			*end = '\0';
  			// for next round
  			p = end + gbstrlen(dend);
  			// try to get event id
  			char *ep = NULL;
  			if ( m_fbId >= 0 ) {
  				ep = strstr ( start, "<eid>");
  				if ( ep ) ep += 5;
  			}
  			// is it a stubhub event? 
  			// <str name="event_id">2659223</str>
  			if ( m_fbId == -1 ) {
  				ep = strstr ( start, "=\"event_id\">");
  				if ( ep ) ep += 12;
  			}
  			// . is it an eventbrite event?
  			// . the first <id> after the <event> is the event id
  			if ( m_fbId == -2 ) {
  				ep = strstr ( start, "<id>");
  				if ( ep ) ep += 4;
  			}
  
  			if ( ! ep ) continue;
  			long long eid = strtoull ( ep , NULL , 10 );
  			if ( eid == 0 ) continue;
  			if ( eid < 0 ) log("facebook: wtf? eid is 0");
  
  			// for eventbrite record the last eventid because
  			// we use that for paging through
  			if ( m_fbId == -2 )
  				s_lastEventBriteEventId = eid;
  
  			// store it
  			if ( ! m_evPtrBuf.pushLong ( (long)start)) goto error;
  			if ( ! m_evIdsBuf.pushLongLong ( eid)) goto error;
  			// count them
  			m_numEvents++;
  		}
  		log("facebook: got %li events from %s",m_numEvents,type);
  
  		// if we got 0 events from an eventbrite request, then
  		// wait for an hour before retrying
  		if ( m_numEvents == 0 && m_fbId == -2 ) {
  			log("facebook: pausing eventbrite feed for 1 hour");
  			long now = getTimeGlobal();
  			s_eventBriteWaitUntil = now + 3600;
  		}
  
  		// reset loop
  		m_i = 0;
  		// make a new state
  		if ( m_numEvents > 0 ) {
  			Msg7 *msg7;
  			try { msg7 = new (Msg7); }
  			catch ( ... ) { 
  				g_errno = ENOMEM;
  				log("facebook: inject msg7 new(%i): %s", 
  				    sizeof(Msg7),mstrerror(g_errno));
  				goto error;
  			}
  			mnew ( msg7, sizeof(Msg7) , "PageInjct7" );
  			// save it for freeing in destructor
  			m_msg7 = msg7;
  		}
  		// advance
  		m_phase = 12;
  		// if no events, skip injection and adding likes i guess...
  		if ( m_numEvents == 0 ) m_phase = 15;
  	}
  
  	// injection loop
  	if ( m_phase == 12 ) {
  	recallInject:
  		char *coll = g_collectiondb.getColl ( m_collnum );
  		char      **eventPtrs = (char **)m_evPtrBuf.getBufStart();
  		long long  *eventIds  = (long long *)m_evIdsBuf.getBufStart();
  		// get ptr to it
  		char *content    = eventPtrs[m_i];
  		long  contentLen = gbstrlen(content);
  		// debug thing
  		//if ( eventIds[m_i] != 314901535212815LL ) {m_i++; continue;}
  		// make a fake url
  		char url[128];
  		sprintf(url,"http://www.facebook.com/events/%llu",
  			eventIds[m_i]);
  		// is it a stubhub event? <str name="event_id">2659223</str>
  		if ( m_fbId == -1 )
  			sprintf(url,"http://www.stubhub.com/%llu",
  				eventIds[m_i]);
  		if ( m_fbId == -2 )
  			sprintf(url,"http://www.eventbrite.com/%llu",
  				eventIds[m_i]);
  
  		// test debug (on for eventbrite now)
  		if ( g_conf.m_logDebugFacebook )
  			log("facebook: %s",content);
  		//
  		// set m_privacy for event being injected
  		//
  		char *s = strstr(content,"<privacy");
  		// skip that
  		if ( s ) s += 8;
  		// skip til '>'
  		for ( ; s && *s && *s !='>' ; s++ );
  		// skip actual >
  		if ( s && *s == '>' ) s++;
  		// skip whitespace
  		for ( ; s && *s && is_wspace_a(*s) ; s++ );
  		// compare
  		m_privacy = 0;
  		if ( s && ! strncasecmp(s,"secret",6) ) 
  			m_privacy = LF_PRIVATE;
  		if ( s && ! strncasecmp(s,"closed",6) ) 
  			m_privacy = LF_PRIVATE;
  
  		// event brite privacy. 0 means private event.
  		if ( m_fbId == -2 && s && ! strncasecmp(s,"0",1) )
  			m_privacy = LF_PRIVATE;
  		
  		//if ( eventIds[m_i] == 273883416016761LL ) 
  		//	m_privacy = LF_PRIVATE;
  		// test
  		//m_privacy = LF_PRIVATE;
  		
  		// use a forced ip for speed! otherwise it takes 
  		// forever lookup up www.facebook.com for some reason!
  		long forcedIp = atoip("69.171.224.39");
  		// advance up here
  		m_i++;
  		// and the phase too
  		m_phase = 13;
  		// inject just that
  		if ( ! m_msg7->inject ( url ,
  					forcedIp,
  					content ,
  					contentLen ,
  					false, // recyclecontent
  					CT_XML, // contentType,
  					coll ,
  					false ,
  					NULL, // username
  					NULL , // pwd
  					m_niceness,
  					this ,
  					queueLoopWrapper ) )
  			return;
  		// bail on error
  		if ( g_errno ) goto error;
  		// how did this happen? it needs to block... otherwise
  		// we have to add to likedb here.
  		char *xx=NULL;*xx=0;
  	}
  
  	if ( m_phase == 13 ) {
  		// error?
  		if ( g_errno ) 
  			log("facebook: injection had error: %s",
  			    mstrerror(g_errno));
  		// add the likes
  		XmlDoc *xd = &m_msg7->m_xd;
  		/*
  		// if event was not found or added for some reason...
  		if ( xd->m_numHashableEvents <= 0 ) {
  			// try to do the next one
  			if ( m_i < m_numEvents ) goto recallInject;
  			// or another round?
  			if ( m_numEvents > 0 ) goto recall
  		}
  		if ( xd->m_indexCode && xd->m_indexCodeValid ) {
  			// note it
  			log("facebook: could not index doc: %s",
  			    mstrerror(xd->m_indexCode));
  			// try to do the next one
  			if ( m_i < m_numEvents ) goto recallInject;
  		}
  		*/
  		if ( xd->m_indexCode && xd->m_indexCodeValid )
  			// note it
  			log("facebook: could not index doc: %s",
  			    mstrerror(xd->m_indexCode));
  		// advancwe
  		m_phase = 14;
  		//
  		// ADD LIKES
  		//
  		// scan the event_members reply we got and cross-reference
  		// those facebook eventids with our eventhash/evid/docid guys
  		// we got in the injection reply to see if we added the 
  		// facebook event to our db. in that case, we also want to add
  		// the  maybe/goingto/invitedto/notgoing flags.
  		// uses the eventhash64, eventid, docid of event added!
  		// returns false with g_errno set on error.
  		if ( ! makeLikedbKeyList ( m_msg7 , &m_list3 ) )
  			goto error;
  		// if nothing to add, we are done
  		//if ( m_list3.getListSize() == 0 && m_i < m_numEvents ) 
  		//	goto recallInject;
  		char *coll = g_collectiondb.getColl ( m_collnum );
  		// add that
  		if ( ! m_msg1.addList ( &m_list3 ,
  					RDB_LIKEDB ,
  					coll ,
  					this ,
  					queueLoopWrapper,
  					false ,
  					m_niceness ) ) // niceness
  			return;
  	}
  
  	if ( m_phase == 14 ) {
  		// we saved the likes...
  		// try to do the next one
  		if ( m_i < m_numEvents ) goto recallInject;
  		// . all done, but need to load more events?
  		// . skip if fbid is zero, which means its facebook spider
  		if ( m_fbId > 0 && m_numEvents > 0 ) goto recallEvents;
  		// ok, i guess, we are done
  		m_phase = 15;
  	}
  
   skipdown:
  	if ( m_phase == 15 ) {
  		// we can nuke this now
  		m_rbuf.purge();
  		// advance
  		m_phase = 16;
  		// a 0 fbid is spidering facebook
  		if ( m_fbId > 0 ) {
  			// final save of rec to clear the FB_INQUEUE bit
  			m_afterSaveCallback = queueLoopWrapper;
  			log("facebook: saving final rec for fbid=%lli",m_fbId);
  			m_fbrecGen.m_flags &= ~FB_INQUEUE;
  			m_fbrecGen.m_eventsDownloaded = getTimeGlobal();
  			// this calls serializeMsg() which mallocs a 
  			// new reply to add
  			if ( ! saveFBRec( &m_fbrecGen ) ) return;
  		}
  	}
  
  	// . remove from queue
  	// . all done!
  	if ( m_phase == 16 ) {
  		// shortcut
  		long err = m_errno;
  		// or inherit this. we might have forgotten to set m_errno
  		if ( ! err && g_errno ) err = g_errno;
  		// no longer in progress
  		m_inProgress = false;
  		// this will purge fullreply
  		reset();
  		// note it
  		log("facebook: done with queue for fbid=%lli. error=%s",
  		    g_fbq1[0],mstrerror(err));
  
  		// if we are eventbrite, update s_ptr4 to the last eventid
  		// that we had on that page
  		s_ptr4 = s_lastEventBriteEventId;
  
  		// save it for potential re-add
  		//long long fsaved = g_fbq1  [0];
  		//collnum_t csaved = g_colls1[0];
  		// shift queue down
  		for ( long i = 1 ; i < g_n1 ; i++ ) {
  			g_fbq1  [i-1] = g_fbq1  [i];
  			g_colls1[i-1] = g_colls1[i];
  		}
  		// one less in queue
  		g_n1--;
  		g_numOut--;
  		return;
  	}
  
   error:
  	log("facebook: queue fbid %lli had error: %s",
  	    m_fbId,mstrerror(g_errno));
  	// no longer in progress
  	m_inProgress = false;
  	// this will purge fullreply
  	reset();
  	// save it for potential re-add
  	long long fsaved = g_fbq1  [0];
  	collnum_t csaved = g_colls1[0];
  	// shift queue down
  	for ( long i = 1 ; i < g_n1 ; i++ ) {
  		g_fbq1  [i-1] = g_fbq1  [i];
  		g_colls1[i-1] = g_colls1[i];
  	}
  	// one less in queue
  	g_n1--;
  	g_numOut--;
  	// re-add on certain errors
  	if ( g_errno == ETIMEDOUT || g_errno == ESOCKETCLOSED ) {
  		log("facebook: re-queue fbid %lli from error: %s",
  		    m_fbId,mstrerror(g_errno));
  		queueFBId ( fsaved , csaved );
  	}
  }
  
  // . these are ptrs to likedb records
  // . these first long long is the least significant
  // . the 2nd long long is more
  int likedbCmp ( const void *a , const void *b ) {
  	const key192_t *k1 = (key192_t *)a;
  	const key192_t *k2 = (key192_t *)b;
  	if ( k1->n2 < k2->n2 ) return -1;
  	if ( k1->n2 > k2->n2 ) return  1;
  	if ( k1->n1 < k2->n1 ) return -1;
  	if ( k1->n1 > k2->n1 ) return  1;
  	if ( k1->n0 < k2->n0 ) return -1;
  	if ( k1->n0 > k2->n0 ) return  1;
  	return 0;
  }
  
  // scan the event_members reply we got and cross-reference
  // those facebook eventids with our eventhash/evid/docid guys
  // we got in the injection reply to see if we added the facebook
  // event to our db. in that case, we also want to add the 
  // maybe/goingto/invitedto/notgoing flags
  // returns false with g_errno set on error.
  bool Msgfb::makeLikedbKeyList ( Msg7 *msg7 , RdbList *list ) {
  	// reset
  	list->reset();
  	// sanity
  	if ( m_i-1 < 0 ) { char *xx=NULL;*xx=0; }
  	// shortcuts
  	XmlDoc *xd = &msg7->m_xd;
  	long long docId = xd->m_docId;
  	// none if no events!
  	if ( ! xd->size_eventData ) return true;
  	if ( ! xd->m_eventDataValid ) return true;
  	// . get first event and use that one
  	// . really, being facebook, there should only be one eventid!
  	//   but maybe later it may be different and we'll have to "like"
  	//   each eventid/docid combo
  	EventDisplay *ed = (EventDisplay *)xd->ptr_eventData;
  	if ( ! ed ) {
  		log("facebook: eventdisplay is null! wtf?");
  		return true;
  	}
  	long gbeventId = ed->m_indexedEventId;
  	//long gbeventHash32 = (long)((unsigned long)ed->m_eventHash64);
  	unsigned long long evh64 = ed->m_eventHash64;
  	// shortcuts
  	//char      **eventPtrs = (char **)m_evPtrBuf.getBufStart();
  	long long  *eventIds  = (long long *)m_evIdsBuf.getBufStart();
  	// what facebook eventid did we just inject?
  	long long eid = eventIds[m_i-1];
  	long count = 0;
  	SafeBuf tmpBuf;
  	if ( ! tmpBuf.reserve ( 50 ) ) return false;
  
  	// but don't use lts->m_start_time, it is not UTC really
  	// so use our own...
  	Interval *ii = (Interval *)ed->m_int;
  	// must be there
  	if ( ed->m_intSize <= 0 ) {
  		log("facebook: wtf? no intervals!");
  		return true;
  	}
  	long start_time = ii->m_a;
  		
  	// scan the table to see what users had an rsvp_status for this event
  	for ( long i = 0 ; i < m_likedbTable.m_numSlots ; i++ ) {
  		// skip empties
  		if ( ! m_likedbTable.m_flags[i] ) continue;
  		// get the record in there
  		long long *eid2 = (long long *)m_likedbTable.getKeyFromSlot(i);
  		// skip if not a match
  		if ( *eid2 != eid ) continue;
  		// the data is a key that we made above
  		LikedbTableSlot *lts;
  		lts = (LikedbTableSlot *)m_likedbTable.getDataFromSlot(i);
  		// debug note
  		if ( g_conf.m_logDebugFacebook )
  			log("facebook: makerec uid=%lli",lts->m_uid);
  		// assume they "like" it
  		long long value = 1LL;
  		// unless it is "negative"
  		//if ( negative ) value = 0LL;
  		// make the flags
  		long flag = lts->m_rsvp;
  		// indicate from facebook so we do not upload it
  		flag |= LF_FROMFACEBOOK;
  		// . this makes two recs to add to likedb
  		// . the first one starts with uid in the key so it will be 
  		//   used to find events that your friends are 
  		// . the second one starts with docid/evid so it can be used
  		//   to lookup who likes an event when we generate the summary
  		//   for the event because its in the search results
  		char *recs = g_likedb.makeRecs ( lts->m_uid        ,
  						 docId             ,
  						 gbeventId         ,
  						 start_time        ,
  						 //lts->m_start_time ,
  						 //lts->m_rsvp       ,
  						 flag              ,
  						 evh64 ,
  						 value             );
  		//continue;
  		// add to list otherwise
  		tmpBuf.safeMemcpy ( recs , (long)LIKEDB_RECSIZE*2 );
  		count++;
  	}
  	// add a separate rec for LF_PRIVATE if we need to because we can't
  	// set more than one bit in a likedb key otherwise you can't UNDO
  	// or delete key bits easily... i guess the event could change 
  	// privacy status the next time we index it, so this needs to
  	// override... and since the uid is 0 (the system uid) let's ignore
  	// the first record and only add the second.
  	// and because i only want to add these keys when the event is
  	// private to avoid bloat, let's just add a negative key when its
  	// public in order to nuke any possible private key.
  	char *recs = g_likedb.makeRecs ( 0LL               , // uid
  					 docId             ,
  					 gbeventId         ,
  					 start_time        ,
  					 LF_FROMFACEBOOK|LF_PRIVATE,
  					 evh64             ,
  					 1LL               );
  	// point to the 2nd key
  	char *rec2 = recs + LIKEDB_RECSIZE;
  	long recSize = (long)LIKEDB_RECSIZE;
  	// make it a del/negative key by clearing the lowest bit
  	if ( ! m_privacy ) { 
  		rec2[0] &= 0xfe;
  		// negative keys are keys only - no data!!!
  		recSize = (long)LIKEDB_KEYSIZE;
  	}
  	// add to list otherwise
  	tmpBuf.safeMemcpy ( rec2 , recSize );
  	count++;
  
  	// debug
  	//key192_t *k2 = (key192_t *)rec2;
  	//log("key: 0x%llx 0x%llx 0x%llx",k2->n2,k2->n1,k2->n0);
  	// all done if nothing
  	if ( count == 0 ) return true;
  	// sort the records in tmp now
  	char *buf = tmpBuf.getBufStart();
  	// sort for rdblist
  	gbqsort ( buf , count , (long)LIKEDB_RECSIZE, likedbCmp );
  	// use the list we got
  	key192_t startKey;
  	key192_t endKey;
  	startKey.setMin();
  	endKey.setMax();
  	// that is our list
  	list->set ( buf ,
  		    tmpBuf.length() ,
  		    buf, // alloc
  		    tmpBuf.getCapacity() , // allocSize
  		    (char *)&startKey ,
  		    (char *)&endKey  ,
  		    LIKEDB_DATASIZE , // fixed datasize
  		    true , // own data? yeah, free it when done
  		    false , // use half keys? no.
  		    sizeof(key192_t) );
  	// steal it from safebuf so it doesn't free it
  	tmpBuf.detachBuf();
  	return true;
  }
  		
  
  /*
  
  YE OLD PIPELINE
  
  
  static void gotRecWrapper ( void *state ) {
  	Msgfb *mfb = (Msgfb *)state;
  	if ( ! mfb->hitFacebook ( ) ) return;
  	mfb->m_callback ( mfb->m_state );
  }
  
  // get it from facebookdb
  bool Msgfb::processFBId ( long long fbId , 
  			  collnum_t collnum,
  			  void *state , 
  			  void (* callback) (void *) ) {
  
  	reset();
  
  	m_inProgress = true;
  	m_fbId = fbId;
  	m_collnum = collnum;
  	m_state = state;
  	m_callback = callback;
  
  	// sanity
  	if  ( ! m_fbId ) { char *xx=NULL;*xx=0; }
  
  	long niceness = 0;
  	key96_t startKey;
  	key96_t endKey;
  	startKey.n1 = 0;
  	startKey.n0 = m_fbId;
  	endKey.n1 = 0;
  	endKey.n0 = m_fbId;
  	startKey.n0 <<= 1;
  	endKey.n0 <<= 1;
  	endKey.n0 |= 0x01;
  	//char *coll = g_collectiondb.getColl ( m_collnum );
  	if ( ! m_msg0.getList ( -1, // hostid
  				0 , // ip
  				0 , // port
  				0 , // maxcacheage
  				false, // addtocache
  				RDB_FACEBOOKDB,
  				"none",//coll,
  				&m_list1,
  				(char *)&startKey,
  				(char *)&endKey,
  				10, // minrecsizes
  				this ,
  				gotRecWrapper,
  				niceness ) )
  		return false;
  	// i guess we got it without blocking
  	return hitFacebook();
  }
  
  static void gotFQLReplyWrapper ( void *state , TcpSocket *s ) {
  	Msgfb *mfb = (Msgfb *)state;
  	if ( ! mfb->gotFQLReply( s ) ) return;
  	mfb->m_callback ( mfb->m_state );
  }
  
  // once we get the facebook access token, we can get the user info
  // https://graph.facebook.com/me?access_token=ACCESS_TOKEN 
  // NO... i would use one fql call at this point...
  // if it says "Requires valid signature" you need the access token
  // fql console: http://developers.facebook.com/docs/reference/rest/fql.query/
  bool Msgfb::hitFacebook ( ) {
  
  	log("facebook: downloading event_members for %llu",m_fbId);
  
  	if ( g_errno ) {
  		log("fbqueue: error getting facebookdb rec: %s", 
  		    mstrerror(g_errno));
  		return true;
  	}
  
  	// empty is bad
  	if ( m_list1.getListSize() <= 0 ) {
  		log("fbqueue: facebookdb rec is empty. wtf? fbid=%lli",
  		    m_fbId);
  		g_errno = EBADREPLY;
  		return true;
  	}
  
  	// get the facebook rec... why?
  	m_fbrecPtr = (FBRec *)m_list1.getList();
  
  	// sanity
  	if ( m_fbrecPtr->m_fbId != m_fbId ) {
  		log("fbqueue: fbid mismatch. fbid=%lli", m_fbId);
  		g_errno = EBADREPLY;
  		return true;
  	}
  
  	// deserialize...
  	deserializeMsg ( sizeof(FBRec) ,
  			 &m_fbrecPtr->size_accessToken ,
  			 &m_fbrecPtr->size_friendIds ,
  			 &m_fbrecPtr->ptr_accessToken ,
  			 m_fbrecPtr->m_buf );
  
  	// copy for calling fql
  	strcpy ( m_accessToken , m_fbrecPtr->ptr_accessToken );
  
  	// get your facebook user info again in case it changed
  	SafeBuf cmd1;
  	cmd1.safePrintf ( "SELECT uid,username,first_name,last_name,name,pic_square,profile_update_time,timezone,religion,birthday,birthday_date,sex,hometown_location,current_location,activities,interests,is_app_user,music,tv,movies,books,about_me,status,online_presence,proxied_email,verified,website,is_blocked,contact_email,email,is_minor,work,education,sports,languages,likes_count,friend_count FROM user where uid=me()");
  
  
  	// get all friends for saving into likedb
  	SafeBuf cmd2;
  	cmd2.safePrintf ( "SELECT uid2 from friend WHERE uid1=me()");
  
  	// get status of each friend attending an event and what eventid
  	SafeBuf cmd3;
  	cmd3.safePrintf ( "SELECT uid, eid, rsvp_status, start_time "
  			  "FROM event_member where uid IN "
  			  "( SELECT uid2 from friend WHERE uid1=me()) "
  			  // this start_time from the event_member table
  			  // is not accurate. it is often in the past! wtf?
  			  //"AND start_time > now()"
  			  );
  
  	// composite
  	SafeBuf json;
  	json.safePrintf("{"
  			"\"query1\":\"%s\""
  			","
  			"\"query2\":\"%s\""
  			","
  			"\"query3\":\"%s\""
  			, cmd1.getBufStart()
  			, cmd2.getBufStart()
  			, cmd3.getBufStart()
  			);
  	json.safePrintf("}");
  	json.urlEncode();
  			
  	// www.howtobe.pro/facebook-graph-api-graph-api-for-issuing-fql-queries
  	// make a url
  	SafeBuf ubuf;
  	ubuf.safePrintf("https://api.facebook.com/method/"
  			//"fql.query?query="
  			"fql.multiquery?queries=%s"
  			"&access_token=%s&format=xml"
  			, json.getBufStart() 
  			, m_accessToken 
  			);
  
  	log("facebook: queryurl = %s", ubuf.getBufStart() );
  
  	// reset
  	g_errno = 0;
  	// get the results
  	if ( ! g_httpServer.getDoc ( ubuf.getBufStart() ,
  				     0 , // urlIp
  				     0                    , // offset
  				     -1                   ,
  				     0 , // ifModifiedSince ,
  				     this               , // state
  				     gotFQLReplyWrapper , // callback
  				     40*1000      , // 20 sec timeout
  				     0 , // proxyip
  				     0 , // proxyport
  				     30000000 , // maxTextDocLen   ,
  				     30000000 , // maxOtherDocLen  ,
  				     g_conf.m_spiderUserAgent    ) )
  		// return false if blocked
  		return false;
  	// otherwise, somehow got it without blocking... wtf?
  	//return gotFQLReply();
  	if ( ! g_errno ) { char *xx=NULL;*xx=0; }
  	log("fql: error getting doc: %s",mstrerror(g_errno));
  	return true;
  }
  
  static void savedFBRecWrapper2 ( void *state ) {
  	Msgfb *mfb = (Msgfb *)state;
  	if ( ! mfb->downloadEvents ( ) ) return;
  	// final save of rec to clear the FB_INQUEUE bit
  	if ( ! mfb->doFinalFBRecSave ( ) ) return;
  	mfb->m_callback ( mfb->m_state );
  }
  
  bool Msgfb::gotFQLReply ( TcpSocket *s ) {
  
  	// bail on error
  	if ( g_errno ) {
  		log("fql: %s",mstrerror(g_errno));
  		m_errno = g_errno;
  		m_errorCount++;
  		return true;
  	}
  
  	// get reply
  	char *reply     = s->m_readBuf;
  	long  replySize = s->m_readOffset;
  
  	// we reference into this, so do not free it!!
  	m_facebookReply     = s->m_readBuf;
  	m_facebookReplySize = s->m_readOffset;
  	m_facebookAllocSize = s->m_readBufSize;
  	// do not allow tcpsocket to free it. we free it in destructor.
  	s->m_readBuf = NULL;
  
  
  	// mime error?
  	HttpMime mime;
  	// exclude the \0 i guess. use NULL for url.
  	mime.set ( reply, replySize - 1, NULL );
  	// not good?
  	long httpStatus = mime.getHttpStatus();
  	if ( httpStatus != 200 ) {
  		log("facebook: bad fql request 2 http status = %li. reply=%s"
  		    , httpStatus 
  		    , reply
  		    );
  		log("facebook: resuming despite error to download friends "
  		    "for fbid=%lli",m_fbId);
  		//g_errno = EBADREPLY;
  		m_errno = EBADREPLY;
  		//m_errorCount++;
  		//return true;
  	}
  
  	// point to content
  	char *content    = reply + mime.getMimeLen();
  	long  contentLen = reply + replySize - content;
  
  	// check for error
  	char *errMsg = strstr(content,"<error_msg>");
  	if ( errMsg ) {
  		log("facebook: error in fql reply: %s", content );
  		log("facebook: resuming despite error to download friends "
  		    "for fbid=%lli",m_fbId);
  		//g_errno = EBADREPLY;
  		m_errno = EBADREPLY;
  		//m_errorCount++;
  		//return true;
  	}
  
  	m_fbrecGen.reset();
  
  	//
  	// . set m_fbrecGen now !!
  	// . compare to m_fbrecPtr to see what eventids are new
  	//
  	if ( ! m_errno &&
  	     ! setFBRecFromFQLReply ( content     ,
  				      contentLen  ,
  				      &m_fbrecGen ) ) {
  		log("fql: error setting fb rec from fql. pipeline 2.");
  		g_errno = EBADREPLY;
  		return true;
  	}
  
  	// merge fbrecPtr into m_fbrecGen
  	if ( ! m_errno ) mergeFBRec ( &m_fbrecGen , m_fbrecPtr );
  
  	// must match
  	if ( ! m_errno && m_fbId != m_fbrecGen.m_fbId ) {
  		log("fql: fbid mismatch in fql reply.");
  		g_errno = EBADENGINEER;
  		m_errno = g_errno;
  		m_errorCount++;
  		return true;
  	}
  
  	// just make sure
  	m_afterSaveCallback = savedFBRecWrapper2;
  
  	// start at event #0
  	m_eventStartNum = 0;
  	// 100 events at a time
  	m_eventStep     = 100;
  
  	// save that before we start downloading the events
  	if ( ! m_errno && ! saveFBRec ( &m_fbrecGen ) ) return false;
  	// try to download new events
  	return downloadEvents ( );
  }
  
  
  static void injectFBEventsWrapper ( void *state, TcpSocket *s ) {
  	Msgfb *mfb = (Msgfb *)state;
  	if ( ! mfb->injectFBEvents ( s ) ) return;
  	// error? try advancing!
  	if ( mfb->m_errno && mfb->m_eventStartNum < 500 ) {
  		if ( mfb->m_eventStep == 100 ) mfb->m_eventStep = 10;
  		else mfb->m_eventStartNum += mfb->m_eventStep;
  		if ( ! mfb->downloadEvents() ) return;
  	}
  	// final save of rec to clear the FB_INQUEUE bit
  	if ( ! mfb->doFinalFBRecSave ( ) ) return;
  	if ( mfb->m_callback ) mfb->m_callback ( mfb->m_state );
  }
  	
  // download the new events and then inject them (ptr_newEvents)
  bool Msgfb::downloadEvents ( ) {
  
  	// reset this since we check for it in injectFBEventsWrapper
  	m_errno = 0;
  
  	log("facebook: downloading events for %llu (#%li-#%li)",
  	    m_fbId,m_eventStartNum,m_eventStartNum+m_eventStep);
  
  	// skip matt wells for now
  	if ( m_fbId == MATTWELLS ) { // 100003532411011LL ) {
  		log("facebook: skipping matt wells event download");
  		// final save of rec to clear the FB_INQUEUE bit
  		return doFinalFBRecSave ( );
  	}
  
  	// save some mem
  	//m_list1.freeList();
  
  	//long now = getTimeGlobal();
  
  	// sanity checks
  	if ( m_eventStartNum <  0 ) { char *xx=NULL; *xx=0; }
  	if ( m_eventStep     <= 0 ) { char *xx=NULL; *xx=0; }
  
  
  	if ( m_eventStartNum >= 100 && 
  	     m_eventStartNum < 130 &&
  	     m_errorCount >= 10 ) {
  		log("facebook: too many errors in event downloads. "
  		    "fbid=%lli . giving up. start=%li errcount=%li",
  		    m_fbId,m_eventStartNum,m_errorCount);
  		return doFinalFBRecSave();
  	}
  
  	// get the events your friends are related to
  	SafeBuf cmd4;
  	cmd4.safePrintf ( "SELECT eid, "
  			  "name, "
  			  "tagline, "
  			  "nid, "
  			  "pic_small, "
  			  "pic_big, "
  			  "pic_square, "
  			  "pic, "
  			  "host, "
  			  "description, "
  			  "event_type, "
  			  "event_subtype, "
  			  "start_time, "
  			  "end_time, "
  			  "creator, "
  			  "update_time, "
  			  "location, "
  			  "venue, "
  			  "privacy, "
  			  "hide_guest_list, "
  			  "can_invite_friends "
  			  "FROM event WHERE "
  			  "start_time > now() AND "
  			  "eid IN (" 
  			  // how do i include events i am assoc. with too?
  			  " SELECT eid FROM event_member WHERE "
  			  "uid IN "
  			  "( SELECT uid2 from friend WHERE uid1=me()) ) "
  			  "LIMIT %li,%li"
  			  , m_eventStartNum
  			  , m_eventStep
  			  );
  
  	// list all the new event ids here
  	//long long *newIds = (long long *)m_eidBuf.getBufStart();
  	//long       n      = m_eidBuf.length() / 8;
  	//bool firstOne = true;
  	//for ( long i = 0 ; i < n ; i++ ) {
  	//	if ( ! firstOne ) cmd4.pushChar(',');
  	//	firstOne = false;
  	//	cmd4.safePrintf("%llu",newIds[i]);
  	//}
  	//cmd4.safePrintf ( ")" // ORDER by start_time ASC "
  	//		  // LIMIT 1001,100 etc. 
  	//		  // LIMIT x,y (x is offset, y is # results)
  	//		  //"LIMIT 0,100"
  	//		  //, fbId 
  	//		  );
  
  	// composite
  	cmd4.urlEncode();
  			
  	// www.howtobe.pro/facebook-graph-api-graph-api-for-issuing-fql-queries
  	// make a url
  	SafeBuf ubuf;
  	ubuf.safePrintf("https://api.facebook.com/method/"
  			"fql.query?query=%s"
  			"&access_token=%s&format=xml"
  			, cmd4.getBufStart() 
  			, m_accessToken 
  			);
  
  	log("facebook: cmd4url = %s", ubuf.getBufStart() );
  
  	// reset
  	g_errno = 0;
  	// get the results
  	if ( ! g_httpServer.getDoc ( ubuf.getBufStart() ,
  				     0 , // urlIp
  				     0                    , // offset
  				     -1                   ,
  				     0 , // ifModifiedSince ,
  				     this               , // state
  				     injectFBEventsWrapper , // callback
  				     40*1000      , // 20 sec timeout
  				     0 , // proxyip
  				     0 , // proxyport
  				     30000000 , // maxTextDocLen   ,
  				     30000000 , // maxOtherDocLen  ,
  				     g_conf.m_spiderUserAgent    ) )
  		// return false if blocked
  		return false;
  	// otherwise, somehow got it without blocking... wtf?
  	//return gotFQLReply();
  	log("fql: http get did not block!");
  	if ( ! g_errno ) { char *xx=NULL;*xx=0; }
  	log("fql: error getting doc: %s",mstrerror(g_errno));
  	// final save of rec to clear the FB_INQUEUE bit
  	return doFinalFBRecSave ( );
  }
  
  static void addLikesWrapper ( void *state ) {
  	Msgfb *mfb = (Msgfb *)state;
  	if ( ! mfb->addLikes ( ) ) return;
  	if ( ! mfb->doInjectionLoop ( ) ) return;
  	// try to read more!!
  	if ( mfb->m_numEvents > 0 ) {
  		mfb->m_eventStartNum += mfb->m_eventStep;
  		if ( ! mfb->downloadEvents() ) return;
  	}
  	// final save of rec to clear the FB_INQUEUE bit
  	if ( ! mfb->doFinalFBRecSave ( ) ) return;
  	if ( mfb->m_callback ) mfb->m_callback ( mfb->m_state );
  }
  
  #define MAXEVENTPTRS 1000
  
  bool Msgfb::injectFBEvents ( TcpSocket *s ) {
  
  	// bail on error
  	if ( g_errno ) {
  		log("fql: %s",mstrerror(g_errno));
  		m_errno = g_errno;
  		m_errorCount++;
  		return true;
  	}
  
  	// get reply
  	char *reply     = s->m_readBuf;
  	long  replySize = s->m_readOffset;
  
  	// mime error?
  	HttpMime mime;
  	// exclude the \0 i guess. use NULL for url.
  	mime.set ( reply, replySize - 1, NULL );
  	// not good?
  	long httpStatus = mime.getHttpStatus();
  	if ( httpStatus != 200 ) {
  		log("facebook: bad fql request 3 http status = %li. reply=%s",
  		    httpStatus ,reply );
  		g_errno = EBADREPLY;
  		m_errno = g_errno;
  		m_errorCount++;
  		return true;
  	}
  
  	// point to content
  	char *content = reply + mime.getMimeLen();
  
  	// check for error
  	char *errMsg = strstr(content,"<error_msg>");
  	if ( errMsg ) {
  		log("facebook: error in fql reply2: %s", content );
  		g_errno = EBADREPLY;
  		m_errno = g_errno;
  		m_errorCount++;
  		return true;
  	}
  
  	// if we are re-using this class. need to reset some things.
  	m_evPtrBuf.reset();
  	m_evIdsBuf.reset();
  
  	// save reply
  	m_facebookReply2     = s->m_readBuf;
  	m_facebookReplySize2 = s->m_readOffset;
  	m_facebookAllocSize2 = s->m_readBufSize;
  	// do not allow tcpsocket to free it. we free it in destructor.
  	s->m_readBuf = NULL;
  
  	// . set m_numEvents and m_eventPtrs from m_facebookReply
  	// . they reference into the reply, so do not free m_facebookReply
  	//   until destructor is called
  
  	m_numEvents = 0;
  
  	// scan for <event> tags and set m_evptrs safebuf to each ptr to those
  	char *p = content;
  	for ( ; *p ; ) {
  		// scan to first <event> tag
  		p = strstr ( p , "<event>" );
  		if ( ! p ) break;
  		// store start for ptr
  		char *start = p + 7;
  		// find end
  		char *end = strstr ( p , "</event>" );
  		if ( ! end ) break;
  		// null term it
  		*end = '\0';
  		// for next round
  		p = end + 8;
  		// try to get event id
  		char *ep = strstr ( start, "<eid>");
  		if ( ! ep ) continue;
  		long long eid = strtoull ( ep + 5 , NULL , 10 );
  		if ( eid == 0 ) continue;
  		if ( eid < 0 ) log("facebook: wtf? eid is 0");
  		// store it
  		if ( ! m_evPtrBuf.pushLong     ( (long)start ) ) return false;
  		if ( ! m_evIdsBuf.pushLongLong ( eid         ) ) return false;
  		// count them
  		m_numEvents++;
  	}
  
  	long askedFor = m_eidBuf.length() / 8;
  	if ( askedFor != m_numEvents )
  		log("facebook: asked for %li events but got %li",
  		    askedFor,m_numEvents );
  
  	// bail if none!
  	if ( m_numEvents <= 0 ) return true;
  
  	// make a new state
  	Msg7 *msg7;
  	try { msg7 = new (Msg7); }
  	catch ( ... ) { 
  		g_errno = ENOMEM;
  		log("facebook: inject msg7 new(%i): %s", 
  		    sizeof(Msg7),mstrerror(g_errno));
  		return true;
  	}
  	mnew ( msg7, sizeof(Msg7) , "PageInject" );
  	// save it for freeing in destructor
  	m_msg7 = msg7;
  
  	m_i = 0;
  
  	return doInjectionLoop ( );
  }
  
  bool Msgfb::doInjectionLoop ( ) {
  
  	char *coll = g_collectiondb.getColl ( m_collnum );
  
  	char      **eventPtrs = (char **)m_evPtrBuf.getBufStart();
  	long long  *eventIds  = (long long *)m_evIdsBuf.getBufStart();
  
  	for ( ; m_i < m_numEvents ;) {
  		// get ptr to it
  		char *content    = eventPtrs[m_i];
  		long  contentLen = gbstrlen(content);
  		// debug thing
  		//if ( eventIds[m_i] != 314901535212815LL ) {m_i++; continue; }
  		// yoyo tuesdays:
  		//if ( eventIds[m_i] != 371365776212884LL ) {m_i++; continue; }
  		// latin dance night
  		//if ( eventIds[m_i] != 111680095620647LL ) {m_i++; continue; }
  		//if (eventIds[m_i] != 273883416016761LL )  {m_i++; continue; }
  		// temp thing
  		//m_c = content[contentLen];
  		//content[contentLen] = '\0';
  		// make a fake url
  		char url[128];
  		sprintf(url,"http://www.facebook.com/events/%llu",
  			eventIds[m_i]);
  		// test debug
  		if ( g_conf.m_logDebugFacebook )
  			log("facebook: %s",content);
  
  		//
  		// set m_privacy for event being injected
  		//
  		char *s = strstr(content,"<privacy");
  		// skip that
  		if ( s ) s += 8;
  		// skip til '>'
  		for ( ; s && *s && *s !='>' ; s++ );
  		// skip actual >
  		if ( s && *s == '>' ) s++;
  		// skip whitespace
  		for ( ; s && *s && is_wspace_a(*s) ; s++ );
  		// compare
  		m_privacy = 0;
  		if ( ! strncasecmp(s,"secret",6) ) 
  			m_privacy = LF_PRIVATE;
  		if ( ! strncasecmp(s,"closed",6) ) 
  			m_privacy = LF_PRIVATE;
  
  		//if ( eventIds[m_i] == 273883416016761LL ) 
  		//	m_privacy = LF_PRIVATE;
  		// test
  		//m_privacy = LF_PRIVATE;
  
  		// use a forced ip for speed! otherwise it takes forever
  		// lookup up www.facebook.com for some reason!!!
  		long forcedIp = atoip("69.171.224.39");
  
  		// advance to next event to inject
  		m_i++;
  		// inject just that
  		if ( ! m_msg7->inject ( url ,
  					forcedIp,
  					content ,
  					contentLen ,
  					false, // recyclecontent
  					CT_XML, // contentType,
  					coll ,
  					false ,
  					NULL, // username
  					NULL , // pwd
  					MAX_NICENESS, 
  					this ,
  					addLikesWrapper ) )
  			return false;
  		// bail on error
  		if ( g_errno ) return true;
  		// how did this happen? it needs to block... otherwise
  		// we have to add to likedb here.
  		char *xx=NULL;*xx=0;
  	}
  	// it did not block, i gues we are done
  	//if ( ! doneInjecting ( ) ) return false;
  	return true;
  }
  
  void doneAddingLikesWrapper ( void *state ) {
  	Msgfb *mfb = (Msgfb *)state;
  	if ( ! mfb->doInjectionLoop() ) return;
  	// try to read more!!
  	if ( mfb->m_numEvents > 0 ) {
  		mfb->m_eventStartNum += mfb->m_eventStep;
  		if ( ! mfb->downloadEvents() ) return;
  	}
  	// final save of rec to clear the FB_INQUEUE bit
  	if ( ! mfb->doFinalFBRecSave ( ) ) return;
  	mfb->m_callback ( mfb->m_state );
  }
  
  
  bool Msgfb::addLikes ( ) { // doneInjecting ( ) {
  
  	// get ptr to it so we can revert the character
  	//char *content    = m_eventPtrs[m_i-1];
  	//long  contentLen = m_eventLens[m_i-1];
  	//content[contentLen] = m_c;
  
  	XmlDoc *xd = &m_msg7->m_xd;
  
  	// if event was not found or added for some reason...
  	if ( xd->m_numHashableEvents <= 0 ) {
  		// try to do the next one
  		if ( m_i < m_numEvents && ! doInjectionLoop ( ) ) return false;
  		// all done! no need to loop back up for more, they're done
  		return true;
  	}
  
  	if ( xd->m_indexCode && xd->m_indexCodeValid ) {
  		// note it
  		log("facebook: could not index doc: %s",
  		    mstrerror(xd->m_indexCode));
  		// try to do the next one
  		if ( m_i < m_numEvents && ! doInjectionLoop ( ) ) return false;
  		// all done! no need to loop back up for more, they're done
  		return true;
  	}
  
  	// get msg7 reply. it needs to have 
  
  	//
  	// ADD LIKES
  	//
  	// scan the event_members reply we got and cross-reference
  	// those facebook eventids with our eventhash/evid/docid guys
  	// we got in the injection reply to see if we added the 
  	// facebook event to our db. in that case, we also want to add
  	// the  maybe/goingto/invitedto/notgoing flags.
  	// uses the eventhash64, eventid, docid of event added!
  	// returns false with g_errno set on error.
  	if ( ! makeLikedbKeyList ( m_msg7 , &m_list3 ) )
  		return true;
  
  	// if nothing to add, we are done
  	if ( m_list3.getListSize() == 0 ) {
  		// try to do the next one
  		if ( m_i < m_numEvents && ! doInjectionLoop ( ) ) return false;
  		// all done! no need to loop back up for more, they're done
  		return true;
  	}
  
  
  	// extract info from state
  	//TcpSocket *s = m_msg7->m_socket;
  	//long long docId  = xd->m_docId;
  	//long      hostId = 0;//msg7->m_msg7.m_hostId;
  
  	char *coll = g_collectiondb.getColl ( m_collnum );
  
  	// add that
  	if ( ! m_msg1.addList ( &m_list3 ,
  				RDB_LIKEDB ,
  				coll ,
  				this ,
  				doneAddingLikesWrapper ,
  				false ,
  				0 ) ) // niceness
  		return false;
  
  	// this might just add to tree in a single server setup so it
  	// will not block...
  	//if ( ! g_errno ) { char *xx=NULL;*xx=0; }
  	// error must be set!
  	return true;
  	//return doInjectionLoop ( );
  }
  
  // all done with everything!
  static void savedFinalFBRecWrapper3 ( void *state ) {
  	Msgfb *mfb = (Msgfb *)state;
  	mfb->m_callback ( mfb->m_state );
  }
  
  
  // final save of rec to clear the FB_INQUEUE bit
  bool Msgfb::doFinalFBRecSave ( ) {
  	m_afterSaveCallback = savedFinalFBRecWrapper3;
  	log("facebook: saving final rec for fbid=%lli",m_fbId);
  	m_fbrecGen.m_flags &= ~FB_INQUEUE;
  	m_fbrecGen.m_eventsDownloaded = getTimeGlobal();
  	// this calls serializeMsg() which mallocs a new reply to add
  	return saveFBRec( &m_fbrecGen );
  }
  
  */
  
  /////////////////////
  //
  // LIKEDB
  //
  //////////////////////
  
  
  void Likedb::reset() {
  	m_rdb.reset();
  }
  
  bool Likedb::init ( ) {
  
  	long long uid = 123456789123LL;
  	long long docId = 999888777666LL;
  	long eventId = 12345;
  	long rsvp_status = LF_GOING;//|LF_HIDE;
  	long start_time = 6543210;
  	unsigned long long eventHash64 = 9999997398453LL;
  	unsigned long eventHash32 = (unsigned long)eventHash64;
  	long value = 999888;
  	char *recs = g_likedb.makeRecs ( uid         ,
  					 docId       ,
  					 eventId   ,
  					 start_time  ,
  					 rsvp_status ,
  					 eventHash64 ,
  					 value );
  	char *p = recs;
  	long long uid2 = g_likedb.getUserIdFromRec ( p );
  	if ( uid2 != uid ) { char *xx=NULL;*xx=0; }
  	long flags = g_likedb.getFlagsFromRec ( p );
  	if ( flags != rsvp_status ) { char *xx=NULL;*xx=0; }
  	unsigned long eh = g_likedb.getEventHash32FromRec ( p );
  	if ( eh != eventHash32 ) { char *xx=NULL;*xx=0; }
  	if ( g_likedb.getValueFromRec ( p ) != value ) { char *xx=NULL;*xx=0;}
  
  	p += LIKEDB_RECSIZE;
  	uid2 = g_likedb.getUserIdFromRec ( p );
  	if ( uid2 != uid ) { char *xx=NULL;*xx=0; }
  	flags = g_likedb.getFlagsFromRec ( p );
  	eh = g_likedb.getEventHash32FromRec ( p );
  	if ( eh != eventHash32 ) { char *xx=NULL;*xx=0; }
  	if ( flags != rsvp_status ) { char *xx=NULL;*xx=0; }
  
  
  	// . what's max # of tree nodes?
  	// . NOTE: 32 bytes of the 82 are overhead
  	long maxMem = 50000000;
  	long maxTreeNodes = maxMem / 48;
  	// initialize our own internal rdb
  	return m_rdb.init ( g_hostdb.m_dir               ,
  			    "likedb"                     ,
  			    true                       , // dedup same keys?
  			    LIKEDB_DATASIZE , // fixed record size
  			    2,//g_conf.m_tagdbMinFilesToMerge   ,
  			    maxMem, // 5MB g_conf.m_tagdbMaxTreeMem  ,
  			    maxTreeNodes               ,
  			    // now we balance so Sync.cpp can ordered huge list
  			    true                        , // balance tree?
  			    0 , //g_conf.m_tagdbMaxCacheMem ,
  			    0 , //maxCacheNodes              ,
  			    false                      , // half keys?
  			    false                      , //m_tagdbSaveCache
  			    NULL , // pagecache - tree only!
  			    false,  // is titledb
  			    false,  // preload disk page cache
  			    sizeof(key192_t),     // key size
  			    false ); // bias disk page cache?
  }
  
  bool Likedb::addColl ( char *coll, bool doVerify ) {
  	if ( ! m_rdb.addColl ( coll ) ) return false;
  	return true;
  }
  
  // FIRST REC KEY:
  //
  // key192_t:
  // uuuuuuuu uuuuuuuu uuuuuuuu uuuuuuuu 
  // uuuuuuuu uuuuuuuu uuuuuuuu uuuuuuuu u=uid (fb userid)
  // dddddddd dddddddd dddddddd dddddddd d=docid
  // dddddd00 00000000 eeeeeeee eeeeeeee e=gbeventid D=delbit
  // ssssssss ssssssss ssssssss ssssssss start_time
  // ffffffff ffffffff ffffffff ffffff0D flags (LF_HIDE,etc.)
  
  // SECOND REC KEY:
  //
  // key192_t:
  // dddddddd dddddddd dddddddd dddddddd d=docid
  // dddddd00 00000000 eeeeeeee eeeeeeee e=gbeventid
  // uuuuuuuu uuuuuuuu uuuuuuuu uuuuuuuu u=uid (fb userid)
  // uuuuuuuu uuuuuuuu uuuuuuuu uuuuuuuu
  // ssssssss ssssssss ssssssss ssssssss start_time
  // ffffffff ffffffff ffffffff ffffff1D flags (LF_HIDE,etc.)
  
  
  // data:
  // hhhhhhhh hhhhhhhh hhhhhhhh hhhhhhhh eventhash32 
  // vvvvvvvv vvvvvvvv vvvvvvvv vvvvvvvv v=value of flag or facebook event id
  // vvvvvvvv vvvvvvvv vvvvvvvv vvvvvvvv
  //
  // NOTE: we set "v" to facebook event id for the LF_ADDEDTOFACEBOOK flag
  //
  
  
  long Likedb::getEventIdFromRec ( void *rec ) {
  	key192_t *k = (key192_t *)rec;
  	if ( k->n0 & LF_TYPEBIT )
  		return k->n2 & 0xffff;
  	return k->n1 & 0xffff;
  }
  
  void Likedb::setEventId ( char *rec , long eventId ) {
  	key192_t *k = (key192_t *)rec;
  	// sanbity
  	if ( eventId & 0xffff0000 ) { char *xx=NULL;*xx=0;}
  	// is it a SECOND REC?
  	if ( k->n0 & LF_TYPEBIT ) {
  		// clear out old event id bits
  		k->n2 &= 0xffffffffffff0000LL;
  		// or in new, shifted up 1 for delbit
  		k->n2 |= eventId;
  	}
  	else {
  		// clear out old event id bits
  		k->n1 &= 0xffffffffffff0000LL;
  		// or in new, shifted up 1 for delbit
  		k->n1 |= eventId;
  	}
  }
  
  // use our docid/eventid because that is what we use in datedb when
  // doing a search. the docid and eventid should be returned by the msg7
  // inject reply.
  char *Likedb::makeRecs ( long long  uid         ,
  			 long long  docId       ,
  			 long       eventId     ,
  			 long       start_time  ,
  			 long       rsvp_status ,
  			 unsigned long long eventHash64 ,
  			 long long  value       ) {
  	// sanity
  	if ( rsvp_status & LF_TYPEBIT ) { char *xx=NULL;*xx=0; }
  	if ( rsvp_status & LF_DELBIT  ) { char *xx=NULL;*xx=0; }
  	// only one flag can be set!!!
  	long ignore = 0;
  	ignore |= LF_DELBIT;
  	ignore |= LF_TYPEBIT;
  	ignore |= LF_ISEVENTGURUID;
  	ignore |= LF_FROMFACEBOOK;
  	if ( getNumBitsOn32(rsvp_status & ~ignore)!=1) { char *xx=NULL;*xx=0;}
  
  	if ( docId   < 0      ) { char *xx=NULL;*xx=0; }
  	if ( eventId > 0xffff ) { char *xx=NULL;*xx=0; }
  	if ( eventId < 0      ) { char *xx=NULL;*xx=0; }
  	// the record
  	static char s_buf[2*LIKEDB_RECSIZE];
  	// store a 16 byte key first
  	key192_t k;
  	// the destination ptr
  	char *p = s_buf;
  
  	unsigned long eventHash32 = (unsigned long)eventHash64;
  
  	//log("facebook: adding eventhash64 = %llu",eventHash64 );
  	//log("facebook: adding eventhash32 = %lu",eventHash32 );
  
  	//
  	// make the first type of rec
  	//
  
  	// . this is the facebook userid OR the eventGuruId
  	// . can also be the userid we assign someone i guess on our end
  	k.n2 = uid;
  	// reset
  	k.n1 = 0;
  	// then docid
  	k.n1 |= docId;
  	// then 10 zero bits
  	k.n1 <<= 10;
  	// make room for event id
  	k.n1 <<= 16;
  	k.n1 |= eventId;
  	// starttime
  	k.n0 = start_time;
  	k.n0 <<= 32;
  	k.n0 |= rsvp_status;
  	// we are a positive key!
  	k.n0 |= LF_DELBIT;
  	// store that
  	*(key192_t *)p = k;
  	// skip key
  	p += sizeof(key192_t);
  	// then 4 byte data
  	*(long *)p = eventHash32;
  	p += 4;
  	// then the value
  	*(long long *)p = value;
  	p += 8;
  
  
  	//
  	// now make the 2nd rec
  	//
  	k.n2 = k.n1;
  	k.n1 = uid;
  	// this is the 2nd type of rec, so set this bit
  	k.n0 |= LF_TYPEBIT;
  	// store second key
  	*(key192_t *)p = k;
  	// skip second key
  	p += sizeof(key192_t);
  	// now this
  	*(long *)p = eventHash32;
  	p += 4;
  	// then the value
  	*(long long *)p = value;
  	p += 8;
  	
  	return s_buf;
  }
  
  // make a "type 2" key (docid leads)
  key192_t Likedb::makeStartKey ( long long docId, long eventId ) {
  	key192_t k;
  	// reset
  	k.n2 = docId;
  	// then 10 zero bits
  	k.n2 <<= 10;
  	// make room for event id
  	k.n2 <<= 16;
  	k.n2 |= eventId;
  	// any user id
  	k.n1 = 0LL;
  	// any starttime or flags
  	k.n0 = 0LL;
  	return k;
  }
  
  // make a "type 2" key (docid leads)
  key192_t Likedb::makeEndKey ( long long docId, long eventId ) {
  	key192_t k;
  	// reset
  	k.n2 = docId;
  	// then 10 zero bits
  	k.n2 <<= 10;
  	// make room for event id
  	k.n2 <<= 16;
  	k.n2 |= eventId;
  	// max user id
  	k.n1 = 0xffffffffffffffffLL;
  	// max starttime and flags
  	k.n0 = 0xffffffffffffffffLL;
  	return k;
  }
  
  long long Likedb::getUserIdFromRec ( void *rec ) {
  	key192_t *k = (key192_t *)rec;
  	if ( k->n0 & LF_TYPEBIT ) return k->n1;
  	return k->n2;
  }
  
  long long Likedb::getDocIdFromRec ( char *rec ) {
  	key192_t *k = (key192_t *)rec;
  	if ( k->n0 & LF_TYPEBIT ) return k->n2 >> 26;
  	return k->n1 >> 26;
  }
  
  key192_t Likedb::makeStartKey2 ( long long uid ) {
  	key192_t k;
  	k.n2 = uid;
  	k.n1 = 0;
  	k.n0 = 0;
  	return k;
  }
  
  // . a facebook user uses likedb to tag an event rsvp_status
  // . similar to facebook's event_members table, but uses our technology
  // . use Rdb, but prohibit from dumping to disk! must always be in tree.
  // . add this to likedb (or remove if "neg" is true.
  bool Msgfc::addLikedbTag ( long long userId ,
  			   long long docId,
  			   long      gbeventId,
  			   unsigned long long eventHash64 ,
  			   long start_time,
  			   long rsvp , // LF_* #define's above
  			   bool negative , // turn off that flag?
  			   char *coll,
  			   void *state ,
  			   void (* callback)(void *) ) {
  
  	if ( userId < 0 ) { char *xx=NULL;*xx=0; }
  
  	
  
  	char *p = m_recs;
  	long size = (long)LIKEDB_RECSIZE*2;
  	long count = 0;
  	//long eventHash32 = (long)((unsigned long long)eventHash64);
  	long long value = 1LL;
  	if ( negative ) value = 0LL;
  
  	//if ( eventHash == 0 || eventHash32 == -1 ) { char *xx=NULL;*xx=0;}
  
  	// sanity
  	long ignore;
  	ignore |= LF_DELBIT;
  	ignore |= LF_TYPEBIT;
  	ignore |= LF_ISEVENTGURUID;
  	ignore |= LF_FROMFACEBOOK;
  	if ( getNumBitsOn32 ( rsvp & ~ignore ) != 1 ) { char *xx=NULL;*xx=0; }
  
  	// only use start_time for "going". everything else tags all
  	// occurences of the event: like,hide,accept,reject,invited.
  	// otherwise if you like an event you have to unlike the exact
  	// same occurence to turn the flags off!
  	if ( ! ( rsvp & LF_GOING ) ) start_time = 0;
  
  	// . this makes two recs to add to likedb
  	// . the first one starts with uid in the key so it will be 
  	//   used to find events that your friends are 
  	// . the second one starts with docid/evid so it can be used
  	//   to lookup who likes an event when we generate the summary
  	//   for the event because its in the search results
  	char *recs = g_likedb.makeRecs ( userId ,
  					 docId             ,
  					 gbeventId         ,
  					 start_time ,
  					 rsvp       ,
  					 eventHash64     ,
  					 value );
  	// add to list otherwise
  	memcpy ( p , recs , size );
  	p += size;
  	count += 2;
  	
  	// sort for rdblist
  	gbqsort ( m_recs , count , (long)LIKEDB_RECSIZE, likedbCmp );
  
  	// use the list we got
  	key192_t startKey;
  	key192_t endKey;
  	startKey.setMin();
  	endKey.setMax();
  	// that is our list
  	m_list6.set ( m_recs ,
  		      p - m_recs , // listsize
  		      NULL, // alloc
  		      0 , // allocSize
  		      (char *)&startKey ,
  		      (char *)&endKey  ,
  		      LIKEDB_DATASIZE , // fixed datasize
  		      true , // own data? yes, free it when done
  		      false , // use half keys? no.
  		      LIKEDB_KEYSIZE );
  	// add that
  	if ( ! m_msg1.addList ( &m_list6 ,
  				RDB_LIKEDB ,
  				coll ,
  				state, // this ,
  				callback, //doneAddingLikedbTagWrapper ,
  				false ,
  				0 ) ) // niceness
  		return false;
  
  	return true;
  }
  
  long Likedb::getUserFlags ( long long userId , 
  			    long start_time ,
  			    char *list, 
  			    long listSize ) {
  	// bail if not valid user id
  	if ( userId == 0LL ) return 0;
  	// scan
  	char *p     = list;
  	char *pend  = list + listSize;
  	long  flags = 0;
  	for ( ; p < pend ; p += LIKEDB_RECSIZE ) {
  		// check for matching userid
  		long long uid = g_likedb.getUserIdFromRec ( p );
  		if ( uid != userId ) continue;
  		// got it
  		long ff = g_likedb.getFlagsFromRec ( p );
  		// get value
  		long long val = g_likedb.getValueFromRec ( p );
  		// skip if 0, that means unset!
  		if ( ! val ) continue;
  		// restrict to just this instance of its a "GOING" flag
  		if ( ff & LF_GOING ) {
  			long start = g_likedb.getStartTimeFromRec ( p );
  			if ( start&&start_time && start!=start_time) continue;
  		}
  		// keep tabs
  		flags |= ff;
  	}
  	return flags;
  }
  
  long Likedb::getPositiveFlagsFromRec  ( char *rec ) {
  	if ( ! g_likedb.getValueFromRec ( rec ) ) return 0;
  	long flags = (*(long *)rec) & ~(LF_DELBIT|LF_TYPEBIT);
  	return flags;
  }
  
  char *Likedb::getRecFromLikedbList ( long long userId ,
  				     long start_time ,
  				     long flags , 
  				     char *list ,
  				     long  listSize ) {
  	// bail if not valid user id
  	if ( userId == 0LL ) return NULL;
  	// scan
  	char *p     = list;
  	char *pend  = list + listSize;
  	for ( ; p < pend ; p += LIKEDB_RECSIZE ) {
  		// check for matching userid
  		long long uid = g_likedb.getUserIdFromRec ( p );
  		if ( uid != userId ) continue;
  		// got it
  		long ff = g_likedb.getFlagsFromRec ( p );
  		// must match
  		if ( ! ( ff & flags) ) continue;
  		// get value
  		long long val = g_likedb.getValueFromRec ( p );
  		// skip if 0, that means unset!
  		if ( ! val ) continue;
  		// restrict to just this instance of its a "GOING" flag
  		if ( flags & LF_GOING ) {
  			long start = g_likedb.getStartTimeFromRec ( p );
  			if ( start&&start_time && start!=start_time) continue;
  		}
  		return p;
  	}
  	return NULL;
  }
  
  
  // http://developers.facebook.com/docs/reference/rest/events.create/
  // http://developers.facebook.com/docs/reference/api/event/
  
  // scan over all events that have someone going to them or maybe going
  // and make sure facebook has that status. 
  // therefore maybe add LF_EGMAYBE and LF_FBMAYBE as separate flags
  // so we know if the bit can from facebook or not. once we update facebook
  // with the event then we can set LF_FBMAYBE or LF_FBGOING for that user
  // assuming they have a facebook id. also, if we initially use their 
  // eventguruid and they tag an event then later they log in and we get
  // their facebookid, we have to make sure to update the event on facebook
  // to reflect they are unsure/goingtoit.
  
  // also, if an event has LF_ACCEPTED we should upload it to facebook under
  // our appid.
  
  // so, let's be perpetually scanning likedb to do this...
  // maybe just host #0 should do it to avoid slamming facebook?
  
  /*
  //////////////////////////
  //
  //
  // ADD EVENT TO FACEBOOK
  //
  //
  //////////////////////////
  
  bool Msgfb::addEventToFacebook ( char *title ,
  				 char *desc  ,
  				 long  start_time ,
  				 long  end_time ,
  				 void *state ,
  				 void (* callback)(void *state) ,
  				 long niceness ) {
  
  	// how do we get the accesstoken for the app? must have to pass
  	// in our secret somehow.
  	if ( ! getAppAccessToken ( ) ) return false;
  
  	return gotAppAccessToken();
  }
  
  static void addedFBEventWrapper ( void *state , TcpSocket *s ) {
  	Msgfb *mfb = (Msgfb *)state;
  	if ( ! mfb->addedFBEvent ( s ) ) return;
  	mfb->m_callback ( mfb->m_state );
  }
  
  bool Msgfb::gotAppAccessToken ( ) {
  
  	SafeBuf args;
  	args.safePrintf("name=");
  	args.urlEncode( title );
  	args.safePrintf("&description=");
  	args.urlEncode( description );
  	args.safePrintf("&start_time=%lu"
  			"&end_time=%lu"
  			"&latitude=%.07f"
  			"&longitude=%.07f"
  			, start_time
  			, end_time 
  			, latitude
  			, longitude 
  			);
  
  	SafeBuf purl;
  	purl.safePrintf("https://graph.facebook.com/%s/events?"
  			"access_token=%s&"
  			"%s"
  			, APPID
  			, m_appAccessToken
  			, args.getBufStart() );
  
  	// reset
  	g_errno = 0;
  	// . get the results
  	// . TODO: make sure post puts the args in the post section
  	if ( ! g_httpServer.getDoc ( purl.getBufStart() ,
  				     0 , // urlIp
  				     0                    , // offset
  				     -1                   ,
  				     0 , // ifModifiedSince ,
  				     this               , // state
  				     addedFBEventWrapper , // callback
  				     40*1000      , // 40 sec timeout
  				     0 , // proxyip
  				     0 , // proxyport
  				     30000000 , // maxTextDocLen   ,
  				     30000000 , // maxOtherDocLen  ,
  				     g_conf.m_spiderUserAgent    ,
  				     "HTTP/1.0",
  				     true ) ) // doPost?
  		// return false if blocked
  		return false;
  	
  	// otherwise, somehow got it without blocking... wtf?
  	if ( ! g_errno ) { char *xx=NULL;*xx=0; }
  	log("facebook: error adding event: %s",mstrerror(g_errno));
  	return true;
  }
  
  bool Msgfb::addedFBEvent ( TcpSocket *s ) {
  
  	// get the event id from reply
  	char *reply     = s->m_readBuf;
  	long  replySize = s->m_readOffset;
  
  	char *s = strstr ( reply , "id" );
  	if ( ! s ) {
  		log("facebook: add event reply had no eid");
  		g_errno = EBADREPLY;
  		return true;
  	}
  
  	// get it otherwise
  	char *p = s;
  	for ( ; *p && !is_digit(*p) ; p++ );
  	if ( ! is_digit ( *p ) ) {
  		log("facebook: add event reply had no eid 2");
  		g_errno = EBADREPLY;
  		return true;
  	}
  
  	long long eid = strtoull(p,NULL,10);
  
  	// add it to likedb as being in facebook now!
  	char flags = LF_ADDEDTOFACEBOOK;
  
  	if ( ! m_msgfc.addLikedbTag ( 0 , // eventGuruId,
  				      APPID , // facebookId,
  				      m_docIdToAdd,
  				      m_eventIdToAdd,
  				      m_eventHash64ToAdd,
  				      m_start_timeToAdd
  				      flags , // LF_* #define's above
  				      false, // negative - turn off that flag?
  				      m_coll ,
  				      this ,
  				      addedLikedbTag2 ) )
  		return false;
  
  	
  	
  
  
  
  
  }
  
  ///////////////////////////
  //
  //
  // GET APP ACCESS TOKEN
  //
  //
  ///////////////////////////
  
  static char s_appAccessToken[256];
  static bool s_appAccessTokenValid = false;
  static bool s_inProgress = false;
  
  static void gotAppAccessTokenWrapper ( void *state , TcpSocket *s ) {
  	Msgfb *mfb = (Msgfb *)state;
  	if ( ! mfb->gotAppAccessToken ( s ) ) return;
  	mfb->m_callback ( mfb->m_state );
  }
  
  bool Msgfb::getAppAccessToken ( ) {
  
  	if ( s_appAccessTokenValid ) {
  		strcpy ( m_appAccessToken , s_appAccessToken );
  		return true;
  	}
  
  	// must not be in progress
  	if ( s_inProgress ) { char *xx=NULL;*xx=0; }
  
  	s_inProgress = true;
  
  	// use code to get access token
  	// that calls https://graph.facebook.com/oauth/access_token?
  	// client_id=YOUR_APP_ID&redirect_uri=YOUR_URL&
  	// client_secret=YOUR_APP_SECRET
  	char fburl[1024];
  	sprintf(fburl,
  		"https://graph.facebook.com/oauth/access_token?"
  		"client_id=%s&"
  		"client_secret=%s&"
  		"grant_type=client_credentials"
  		, APPID
  		, APPSECRET
  		);
  	// reset
  	g_errno = 0;
  	if ( ! g_httpServer.getDoc ( fburl ,
  				     0 , // urlIp
  				     0                    , // offset
  				     -1                   ,
  				     0 , // ifModifiedSince ,
  				     this                 , // state
  				     gotAppAccessTokenWrapper , // callback
  				     10*1000      , // 10 sec timeout
  				     0 , // proxyip
  				     0 , // proxyport
  				     10000 , // maxTextDocLen   ,
  				     10000 , // maxOtherDocLen  ,
  				     g_conf.m_spiderUserAgent    ) )
  		// return false if blocked
  		return false;
  	// error?
  	if ( ! g_errno ) { char *xx=NULL;*xx=0; }
  	// all done
  	s_inProgress = false;
  	// let caller know we did not block
  	return gotAppAccessToken ( NULL );
  }
  
  bool Msgfb::gotAppAccessToken ( TcpSocket *s ) {
  
  	// all done
  	s_inProgress = false;
  
  	// some kind of error?
  	if ( g_errno ) {
  		log("facebook: error launching read of app access token: %s", 
  		    mstrerror(g_errno));
  		return;
  	}
  
  	// the access token should be in the reply
  	char *reply     = s->m_readBuf;
  	long  replySize = s->m_readOffset;
  
  	// mime error?
  	HttpMime mime;
  	// exclude the \0 i guess. use NULL for url.
  	mime.set ( reply, replySize - 1, NULL );
  	// not good?
  	long httpStatus = mime.getHttpStatus();
  	if ( httpStatus != 200 ) {
  		log("facebook: bad app access request http status = %li",
  		    httpStatus );
  		g_errno = EBADREPLY;
  		return;
  	}
  
  	// point to content
  	char *content = reply + mime.getMimeLen();
  
  	// assume no accesstoken provided
  	s_appAccessToken[0] = '\0';
  
  	// look for access token
  	//sscanf(content,"access_token=%s&expires=%li",m_accessToken,&expires);
  	char *at = strstr(content,"access_token=");
  	if ( at ) {
  		char *p = at + 13;
  		char *start = p;
  		for ( ; *p && *p != '&' ;p++ );
  		long len = p - start;
  		if ( len > MAX_TOKEN_LEN ) { char *xx=NULL;*xx=0; }
  		memcpy ( s_appAccessToken , start , len );
  		s_appAccessToken [ len ] = '\0';
  	}
  
  	// error?
  	if ( ! s_appAccessToken[0] ) {
  		log("facebook: could not find app access token");
  		g_errno = EBADREPLY;
  		return;
  	}
  
  	// sanity
  	if ( gbstrlen(m_accessToken) > MAX_TOKEN_LEN ) { char *xx=NULL;*xx=0;}
  
  	// set this timestamp
  	//m_accessTokenCreated = getTimeGlobal();
  
  	s_appAccessTokenValid = true;
  
  	strcpy ( m_appAccessToken , s_appAccessToken );
  
  	return true;
  }
  
  
  
  
  
  // we should also have a spiderloop to continually search for and find
  // events on facebook and add them to our db
  */
  
  
  
  ///////////////////////
  //
  //
  // THE EMAILER
  //
  // . g_emailer is in Process.cpp and so is its 60 second sleep callback
  //
  ///////////////////////
  
  // . some sleepwrapper should call this once every 10 seconds or so
  bool Emailer::emailEntryLoop ( ) {
  
  	// temporarily disable
  	return true;
  
  	// skip if in progress already
  	if ( m_emailInProgress ) return true;
  
  	// wait for clock to be in sync
  	if ( ! isClockInSync() ) return true;
  
  	long now = getTimeGlobal();
  	if ( m_lastEmailLoop && now - m_lastEmailLoop < 60 ) return true;
  
  	// just use first event collection
  	CollectionRec *cr = NULL;
  	for ( long i = 0 ; i < g_collectiondb.m_numRecs ; i++ ) {
  		cr = g_collectiondb.m_recs[i];
  		if ( ! cr ) continue;
  		if ( ! cr->m_indexEventsOnly ) { cr = NULL; continue; }
  		break;
  	}
  	if ( ! cr ) return true;
  
  	// lock it out
  	m_emailInProgress = true;
  
  	// we are doing a scan loop, not sending a single email
  	//m_sendSingleEmail = false;
  
  	// save this
  	m_coll = cr->m_coll;
  	m_collnum = g_collectiondb.getCollnum ( m_coll );
  
  	// . make sure m_emailTree is full
  	// . this returns false if blocked, true otherwise
  	// . it should call emailScan() when done if it blocks
  	if ( ! populateEmailTree ( ) ) return false;
  
  	// do the scan loop
  	return emailScan( NULL );
  }
  
  // . ok, now m_emailTree should be fully populated
  // . returns false if blocked, true otherwise
  bool Emailer::emailScan ( EmailState *es ) {
  
  	if ( es && es->m_sendSingleEmail ) {
  		es->m_singleCallback ( es->m_singleState );
  		return true;
  	}
  
  	log("emailer: scanning fbids");
  
   loop:
  	long now = getTimeGlobal();
  	// scan for the fbids in the email tree
  	long n = m_emailTree.getFirstNode();
  	// get the key if n is good
  	key96_t *kp = NULL;
  	if ( n >= 0 ) kp = (key96_t *)m_emailTree.getKey(n);
  	// check time. stop scanning if in the future!
  	if ( kp && kp->n1 > (unsigned long)now ) n = -1;
  	// none remain?
  	if ( n < 0 ) {
  		// return false if waiting for email replies
  		if ( m_emailRequests > m_emailReplies ) return false;
  		// clear this so we can run again later
  		m_emailInProgress = false;
  		// i guess update this to the completion time
  		long now = getTimeGlobal();
  		m_lastEmailLoop = now;
  		return true;
  	}
  	// save id
  	long long fbId = kp->n0;
  	// nuke that node
  	m_emailTree.deleteNode ( n , true );
  	// . ok launch an email. pass in the facebook id
  	// . returns false if blocked, true otherwise
  	launchEmail ( fbId );
  	// . if we hit the outstanding limit, "block"
  	// . when an email reply comes it it should re-call
  	//   emailScan()... and it should remove itself from
  	//   m_emailTree so we do not re-do it!
  	if ( m_emailRequests - m_emailReplies >= MAX_OUTSTANDING_EMAILS )
  		return false;
  	// do another one!
  	goto loop;
  }
  
  //static void generateEventsEmailWrapper ( void *state ) {
  //	Emailer *em = (Emailer *)state;
  //	if ( ! em->generateEventsEmail( ) ) return;
  //}
  
  // from PageEvents.cpp:
  extern bool sendPageEvents2 ( TcpSocket *s ,
  			      HttpRequest *hr ,
  			      SafeBuf *resultsBuf,
  			      SafeBuf *emailLikedbListBuf,
  			      void *state  ,
  			      void (* emailCallback)(void *state) ,
  			      SafeBuf *providedBuf ,
  			      void *providedState  ,
  			      void (* providedCallback)(void *state) );
  
  static void gotPageToEmailWrapper ( void *state ) {
  	EmailState *es = (EmailState *)state;
  	Emailer *em = es->m_emailer;
  	if ( ! em->getMailServerIP( es ) ) return;
  	// scan next
  	em->emailScan( es );
  }
  
  bool Emailer::launchEmail ( long long fbId ) {
  
  	// we need an email state now!
  	EmailState *es = NULL;
  	for ( long i = 0 ; i < MAX_OUTSTANDING_EMAILS ; i++ ) {
  		if ( m_emailStates[i].m_inUse ) continue;
  		es = &m_emailStates[i];
  		break;
  	}
  	// how can this happen?
  	if ( ! es ) { char *xx=NULL;*xx=0; }
  
  	// make a fake http request
  	//SafeBuf hrsb;
  
  	//return true;
  	
  	// must remain on stack since the copied HttpRequest will point into
  	// this or the SearchInput will point into this
  	es->m_hrsb.safePrintf("GET /?"
  			      "c=%s&"
  			      "showpersonal=1&"
  			      //"where=anywhere&"
  			      // this should override the fbid in the cookie
  			      "usefbid=%lli&"
  			      "fh=%lu&"
  			      "usecookie=0&"
  			      "map=0&"
  			      "n=25&"
  			      "emailformat=1"
  			      " HTTP/1.0\r\n\r\n"
  			      , m_coll
  			      , fbId
  			      , hash32((char *)&fbId,8)
  			      );
  
  	HttpRequest hr;
  	hr.set (es->m_hrsb.getBufStart(), 
  		es->m_hrsb.length() , 
  		(TcpSocket *)NULL );
  
  	TcpSocket *s = NULL;
  
  	// two counts
  	m_emailRequests++;
  
  	// claim it
  	es->m_inUse = true;
  	// point to emailer
  	es->m_emailer = this;
  	// who are we sending to?
  	es->m_fbId = fbId;
  	// container class
  	es->m_emailer = this;
  	// reset this
  	es->m_errno = 0;
  	// our collection
  	es->m_coll = m_coll;
  	// we are doing a loop! so return to emailScan() function
  	es->m_sendSingleEmail = false;
  	// clear these
  	es->m_emailResultsBuf   .purge();
  	es->m_emailLikedbListBuf.purge();
  
  	// . use that to generate the search results
  	// . returns false if blocked, true otherwise
  	if ( ! sendPageEvents2 ( s , 
  				 &hr ,  // this is copied right away!
  				 // our special parms:
  				 &es->m_emailResultsBuf,
  				 &es->m_emailLikedbListBuf,
  				 es ,
  				 gotPageToEmailWrapper ,
  				 NULL ,
  				 NULL ,
  				 NULL ) ) 
  		return false;
  	// we got it
  	return getMailServerIP ( es );
  }
  
  static void gotMXIpWrapper ( void *state , long ip ) {
  	EmailState *es = (EmailState *)state;
  	Emailer *em = es->m_emailer;
  	if ( ! em->gotMXIp ( es ) ) return;
  	// scan next
  	em->emailScan( es );
  }
  
  
  // . PageEvents.cpp should have stored the html content into m_emailResultsBuf
  // . so email that to the recipient
  bool Emailer::getMailServerIP ( EmailState *es ) {
  
  	//
  	// if no results, skip it the next couple functions
  	//
  	if ( es->m_emailResultsBuf.length() == 0 ) {
  		log("emailer: no results for user %llu",es->m_fbId);
  		return gotEmailReply ( es , NULL );
  	}
  
  	// get email address from the msg
  	char *rb = es->m_emailResultsBuf.getBufStart();
  
  	char *emailTo = NULL;
  	if ( rb ) emailTo = strstr(rb,"RCPT To:<");
  	// bail on error
  	if ( ! emailTo ) {
  		log("emailer: no email address for %llu",es->m_fbId);
  		es->m_emailResultsBuf.purge();
  		es->m_emailLikedbListBuf.purge();
  		es->m_inUse = false;
  		m_emailReplies++;
  		return true;
  	}
  
  	// get domain from that
  	char *p = emailTo;
  	char *pend = p + 256;
  	for ( ; *p && *p != '@' && p < pend ; p++ ) ;
  	if ( p >= pend || ! *p ) {
  		log("emailer: no at sign in email address "
  		    "for %llu",es->m_fbId);
  		es->m_emailResultsBuf.purge();
  		es->m_emailLikedbListBuf.purge();
  		es->m_inUse = false;
  		m_emailReplies++;
  		return true;
  	}
  
  	// skip over '@' sign
  	p++;
  	// set domain
  	char *dom = p;
  	// scan domain length
  	for ( ; *p && *p != '>' && p < pend ; p++ ) ;
  	long domLen = p - dom;
  	if ( p >= pend || ! *p || domLen > 80 ) {
  		log("emailer: no valid subdomain in email address "
  		    "for %llu",es->m_fbId);
  		es->m_emailResultsBuf.purge();
  		es->m_emailLikedbListBuf.purge();
  		es->m_inUse = false;
  		m_emailReplies++;
  		return true;
  	}
  	
  	
  	// get the ip. use kinda a fake hostname to pass into MsgC
  	// so that it understands its a special MX record lookup
  	char *dst = es->m_emailSubdomain;
  	memcpy ( dst , "gbmxrec-" , 8 );
  	dst += 8;
  	memcpy ( dst , dom , domLen );
  	dst += domLen;
  	*dst = '\0';
  
  	// . now get the ip for that. get the MX record IP!!!
  	// . it will recognize the gbmxrec- prepension and ask for the
  	//   MX record
  	if ( ! es->m_msgc.getIp ( es->m_emailSubdomain ,
  				  dst - es->m_emailSubdomain ,
  				  &es->m_ip ,
  				  es ,
  				  gotMXIpWrapper ) )
  		return false;
  
  	return gotMXIp ( es );
  }
  
  static void gotEmailReplyWrapper ( void *state , TcpSocket *s ) {
  	EmailState *es = (EmailState *)state;
  	Emailer *em = es->m_emailer;
  	if ( ! em->gotEmailReply ( es , s ) ) return;
  	// scan next
  	em->emailScan( es );
  }
  
  bool Emailer::gotMXIp ( EmailState *es ) {
  
  	log("facebook: got mx ip of %s for %s",
  	    iptoa(es->m_ip), es->m_emailSubdomain );
  
  	// our problem? like ENOME?
  	if ( g_errno ) {
  		log("emailer: had server side error getting ip: %s",
  		    mstrerror(g_errno));
  		es->m_errno = g_errno;
  		return gotEmailReply ( es , NULL );
  	}
  
  	// shortcut
  	long ip = es->m_ip;//msgc.getIp();
  	// problem?
  	if ( ip == 0 || ip == -1 ) {
  		log("emailer: bad ip of %li for %s for %llu",
  		    ip,
  		    es->m_emailSubdomain,
  		    es->m_fbId);
  		es->m_errno = EBADIP;
  		g_errno     = EBADIP;
  		return gotEmailReply ( es , NULL );
  	}
  
  	// send the message
  	TcpServer *ts = g_httpServer.getTcp();
  	// log it
  	log ( LOG_WARN, "emailer: Sending email to %llu size=%li", 
  	      es->m_fbId , es->m_emailResultsBuf.length());
  
  	/*
  	//
  	// THIS ONE WORKS so work backwards from here if you have issues
  	//
  	SafeBuf *eb = &es->m_emailResultsBuf;
  	eb->reset();
  	eb->safePrintf(
  		       "EHLO gigablast.com\r\n"
  		       "Mail From:<mwells2@gigablast.com>\r\n"
  		       "RCPT To:<app+a4gdq01pp8.2qufhgd443.c6277eee23cba81f44f0decbcb1a4d03@proxymail.facebook.com>\r\n"
  		       "DATA\r\n"
  		       "From: mwells <mwells2@gigablast.com>\r\n"
  		       "MIME-Version: 1.0\r\n"
  		       "To: app+a4gdq01pp8.2qufhgd443.c6277eee23cba81f44f0decbcb1a4d03@proxymail.facebook.com\r\n"
  		       "Subject: testing\r\n"
  		       "Content-Type: text/html; charset=UTF-8; format=flowed\r\n"
  		       "Content-Transfer-Encoding: 8bit\r\n"
  		       "\r\n"
  		       "\r\n"
  		       "<table cellpadding=3 cellspacing=0></table>\r\n"
  		       ".\r\n"
  		       "QUIT\r\n"
  		       );
  	*/
  
  	//
  	// debug by dumping to file!!!
  	//
  	char filename[512];
  	long now = getTimeLocal();
  	sprintf ( filename,"html/email/email.%llu.%lu"
  		  , es->m_fbId
  		  , now
  		  );
  	es->m_emailResultsBuf.save(g_hostdb.m_dir,filename);
  	log("facebook: saving email %s", filename);
  	SafeBuf embuf;
  	embuf.load(g_hostdb.m_dir,"html/email/email.html");
  	embuf.safePrintf("<a href=/email/email.%llu.%lu>email.%llu.%lu</a><br>"
  			 , es->m_fbId
  			 , now
  			 , es->m_fbId
  			 , now
  			 );
  	embuf.save(g_hostdb.m_dir,"html/email/email.html");
  
  	log("facebook: emailing %li bytes",
  	    es->m_emailResultsBuf.length() );
  
  
  	//
  	// skip actual email for now!
  	//
  	//gotEmailReply( es , NULL );
  	//if ( ! es->m_sendSingleEmail ) return true;
  
  	if ( ! ts->sendMsg ( ip,
  			     25, // smtp (send mail transfer protocol) port
  			     es->m_emailResultsBuf.getBufStart(),
  			     es->m_emailResultsBuf.length(),
  			     es->m_emailResultsBuf.length(),
  			     es->m_emailResultsBuf.length(),
  			     es,
  			     gotEmailReplyWrapper,
  			     60*1000,
  			     1000*1024,
  			     1000*1024 ) )
  		return false;
  	// we did not block, so update facebook rec with timestamps
  	gotEmailReply( es , NULL );
  	// we did not block
  	return true;
  }
  
  static void gotRecWrapper3 ( void *state ) {
  	EmailState *es = (EmailState *)state;
  	Emailer *em = es->m_emailer;
  	if ( ! em->gotRec3 ( es ) ) return;
  	// scan next
  	em->emailScan( es );
  }
  	
  
  bool Emailer::gotEmailReply ( EmailState *es , TcpSocket *s ) {
  
  	// don't free it that's our job!
  	if ( s ) s->m_sendBuf = NULL;
  
  	// free allocated memory
  	es->m_emailResultsBuf.purge();
  
  	if ( g_errno ) { 
  		log("emailer: got error sending to fbid=%lli: %s",es->m_fbId,
  		    mstrerror(g_errno));
  		es->m_errno = g_errno;
  		// reset these errors just in case
  		g_errno = 0;
  	}
  	// . show the reply
  	// . seems to crash if we log the read buffer... no \0?
  	if ( s && s->m_readBuf )
  		log("emailer: got email server reply: %s", s->m_readBuf );
  	else
  		log("emailer: missing email server reply!");
  
  
  	log("emailer: getting fbrec for fbid=%lli",es->m_fbId);
  
  	// load the facebookdb rec so we can update it and save it then
  	key96_t startKey;
  	key96_t endKey;
  	startKey.n1 = 0;
  	startKey.n0 = es->m_fbId;
  	endKey.n1 = 0;
  	endKey.n0 = es->m_fbId;
  	startKey.n0 <<= 1;
  	endKey.n0 <<= 1;
  	endKey.n0 |= 0x01;
  	if ( ! m_msg0.getList ( -1, // hostid
  				0 , // ip
  				0 , // port
  				0 , // maxcacheage
  				false, // addtocache
  				RDB_FACEBOOKDB,
  				"",//m_coll,
  				&es->m_list9,
  				(char *)&startKey,
  				(char *)&endKey,
  				11, // minrecsizes
  				es, // this ,
  				gotRecWrapper3,
  				MAX_NICENESS ) )
  		return false;
  	// i guess we got it without blocking
  	return gotRec3 ( es );
  }
  
  static void savedUpdatedRecWrapper ( void *state ) {
  	EmailState *es = (EmailState *)state;
  	Emailer *em = es->m_emailer;
  	if ( ! em->savedUpdatedRec ( es ) ) return;
  	// scan next
  	em->emailScan( es );
  }
  
  
  bool Emailer::gotRec3 ( EmailState *es ) {
  
  	// error loading?
  	if ( g_errno ) {
  		log("emailer: error loading facebookdb rec for %llu",
  		    es->m_fbId );
  		es->m_errno = g_errno;
  	}
  
  
  	// empty is bad
  	if ( es->m_list9.getListSize() <= 0 ) {
  		log("emailer: facebookdb rec is empty. wtf? fbid=%lli",
  		    es->m_fbId);
  		es->m_errno = EBADREPLY;
  	}
  
  	// get the facebook rec... why?
  	FBRec *rec = (FBRec *)es->m_list9.getList();
  
  	// assume no error
  	if ( ! es->m_errno ) rec->m_nextRetry = 0;
  
  	long now = getTimeGlobal();
  
  	// on error...
  	if ( es->m_errno ) {
  		// did we have a previous attempt?
  		long elapsed = now - rec->m_lastEmailAttempt ;
  		// our first time? set to 6 hours retry then.
  		if ( rec->m_nextRetry == 0 ) elapsed = 0;
  		// ok, add 3 hours and double that
  		long wait = 2 * (elapsed + 3*3600);
  		// store that then
  		rec->m_nextRetry = now + wait;
  	}
  
  	// update the last send time
  	rec->m_lastEmailAttempt = now;
  
  	// . add the facebookdb rec back now with updated times
  	// . just use TagRec::m_msg1 now
  	// . no, can't use that because tags are added using SafeBuf::addTag()
  	//   which first pushes the rdbid, so we gotta use msg4
  	// . if a host is down we have to fix msg1 (and msg4) so they both
  	//   just write to a file until that host comes back up.
  	if ( ! es->m_msg1.addList ( &es->m_list9 ,
  				    RDB_FACEBOOKDB ,
  				    "none",//m_coll ,
  				    es ,
  				    savedUpdatedRecWrapper,
  				    false ,
  				    0 ) ) // niceness
  		return false;
  	// this does not block if only one host and in memory
  	return savedUpdatedRec ( es );
  }
  
  static void doneAddingEmailedLikesWrapper ( void *state ) {
  	EmailState *es = (EmailState *)state;
  	Emailer *em = es->m_emailer;
  	if ( ! em->doneAddingEmailedLikes ( es ) ) return;
  	// scan next
  	em->emailScan(es);
  }
  
  bool Emailer::savedUpdatedRec ( EmailState *es ) {
  	// now add to likedb if no error so we do not re-email these
  	// same events. start_time is non-zero so it is just the single
  	// instances of each event in the case of recurring events.
  	SafeBuf *eb = &es->m_emailLikedbListBuf;
  	// sort the records in tmp now
  	char *buf     = eb->getBufStart();
  	long  bufSize = eb->length();
  	// how many?
  	long count = bufSize / (long)LIKEDB_RECSIZE;
  	// sort for rdblist
  	gbqsort ( buf , count , (long)LIKEDB_RECSIZE, likedbCmp );
  	// use the list we got
  	key192_t startKey;
  	key192_t endKey;
  	startKey.setMin();
  	endKey.setMax();
  	// that is our list
  	es->m_list5.set ( buf ,
  			  bufSize ,
  			  buf, // alloc
  			  eb->getCapacity() , // allocSize
  			  (char *)&startKey ,
  			  (char *)&endKey  ,
  			  LIKEDB_DATASIZE , // fixed datasize
  			  true , // own data? yeah, free it when done
  			  false , // use half keys? no.
  			  sizeof(key192_t) );
  	// steal it from safebuf so it doesn't free it
  	eb->detachBuf();
  	// note it
  	log("facebook: adding events to likedb to prevent re-emailing. "
  	    "listsize=%li",es->m_list5.getListSize());
  	// add that
  	if ( ! es->m_msg1.addList ( &es->m_list5 ,
  				    RDB_LIKEDB ,
  				    es->m_coll ,
  				    es , // this ,
  				    doneAddingEmailedLikesWrapper,
  				    false ,
  				    0 ) ) // niceness
  		return false;
  	// it did not block
  	return doneAddingEmailedLikes ( es );
  }
  
  
  bool Emailer::doneAddingEmailedLikes ( EmailState *es ) {
  	m_emailReplies++;
  	es->m_emailLikedbListBuf.purge();
  	es->m_inUse = false;
  	return true;
  }
  
  /////////////
  //
  // code to send an individual email
  //
  ////////////
  
  // need to set EmailState::m_fbId, m_emailResultsBuf
  bool Emailer::sendSingleEmail (  EmailState *es , long long fbId ) {
  
  	es->m_sendSingleEmail = true;
  	// claim it
  	es->m_inUse = true;
  	// point to emailer
  	es->m_emailer = this;
  	// who are we sending to?
  	es->m_fbId = fbId;
  	// container class
  	es->m_emailer = this;
  	// reset this
  	es->m_errno = 0;
  	// send it off
  	if ( ! getMailServerIP( es ) ) return false;
  	return true;
  }
  
  
  
  
  /////////////
  //
  // code to populate the m_emailTree
  //
  /////////////
  
  
  // returns false if blocked, true otherwise
  bool Emailer::populateEmailTree ( ) {
  
  	// not if already in progress
  	if ( m_populateInProgress ) return true;
  	// stop if emailing now, it needs the tree
  	//if ( m_emailInProgress ) return true;
  	// re-scan only once per hour
  	long now = getTimeGlobal();
  	if ( m_lastScan && now - m_lastScan < 3600 ) return true;
  	// update that
  	m_lastScan = now;
  	// lock it up just in case...
  	m_populateInProgress = true;
  
  	// init the tree the first tim eonly
  	if ( ! m_init ) {
  		// . what's max # of tree nodes?
  		// . assume avg facebookdb rec size of about 1000 bytes
  		// . NOTE: 32 bytes overhead?
  		long maxMem = 10000000;
  		long maxTreeNodes = maxMem / 32;
  		if ( ! m_emailTree.set ( 0 ,
  					 maxTreeNodes ,
  					 true , // balance?
  					 maxMem,
  					 true , // owndata?
  					 "emailtree", // allocname
  					 false , // datainptrs
  					 NULL , // dbname
  					 sizeof(key96_t) )) { // keysize
  			log("email: failed to init email tree");
  			return true;
  		}
  		// only do once
  		m_init = true;
  	}
  
  	// clear out all nodes
  	m_emailTree.clear();
  	// reset start key for scan
  	m_startKey.setMin();
  
  	// returns false if blocked, true otherwise
  	return scanLoop();
  }
  
  
  static void gotScanListWrapper ( void *state, RdbList *list , Msg5 *msg5 ) {
  	// use this
  	Emailer *em = (Emailer *)state;
  	// this never blocks
  	em->gotScanList ();
  	// and resume the loop. return if it blocked.
  	if ( ! em->scanLoop () ) return;
  	// it did not block, it must be done...
  	// we were spawned from emailEntryLoop(), so go back there
  	em->emailScan( NULL );
  }
  
  // . scan facebookdb and get every facebookid, and couple it with the
  //   time we gotta send the email
  // . sort by that in emailTree
  // . re-scan facebookdb every few hours in case of new entries or if
  //   someone updates their email
  // . i would also call addToEmailTree if a new facebookdb rec comes in.
  //   perhaps do that from Rdb.cpp?
  // . returns false if blocked true otherwise
  bool Emailer::scanLoop ( ) {
  
  	key96_t endKey   ;
  	endKey.setMax();
  	// get a meg at a time
  	long minRecSizes = 1024*1024;
  	key96_t oldk; oldk.setMin();
  
   loop:
  	// use msg5 to get the list, should ALWAYS block since no threads
  	if ( ! m_msg5.getList ( RDB_FACEBOOKDB    ,
  				m_coll          ,
  				&m_list7         ,
  				&m_startKey      ,
  				&endKey        ,
  				minRecSizes   ,
  				true          , // includeTree
  				false         , // add to cache?
  				0             , // max cache age
  				0             , // startFileNum  ,
  				-1            , // numFiles      ,
  				this          , // state
  				gotScanListWrapper , // callback
  				MAX_NICENESS  , // niceness
  				false         )) // err correction?
  		// return false if we blocked
  		return false;
  	// stuff the m_emailTree with some data based on m_list
  	gotScanList( );
  	// if something, get more
  	if ( ! m_list7.isEmpty() ) goto loop;
  	// stop
  	m_populateInProgress = false;
  	// i guess we did not block?
  	return true;
  }
  
  void Emailer::gotScanList ( ) {
  
  	long now = getTimeGlobal();
  	long dayStart = now  - ( now % 86400 );
  
  	if ( m_list7.isEmpty() ) return;
  
  	// loop over entries in list
  	for ( m_list7.resetListPtr() ; ! m_list7.isExhausted() ;
  	      m_list7.skipCurrentRecord() ) {
  		// get it
  		char *drec = m_list7.getCurrentRec();
  		// sanity check. delete key?
  		if ( (drec[0] & 0x01) == 0x00 ) continue;
  
  		FBRec *fr = (FBRec *)drec;
  
  		char ef = fr->m_emailFrequency;
  		// 0 means none provided, so let's default it to weekly
  		//if ( ef == 0 ) continue;
  		if ( ef == 0 ) ef = 2;
  		// 3 means never
  		// 1 is daily, 2 is weekly
  		if ( ef == 3 ) continue;
  		// strange?
  		if ( ef != 1 && ef != 2 ) {
  			log("email: strange freq = %li",(long)ef);
  			continue;
  		}
  		// shortcut
  		unsigned long long fbId = fr->m_fbId;
  
  		// is assigned to us for emailing?
  		Host *group = g_hostdb.getMyGroup();
  		long hpg = g_hostdb.getNumHostsPerShard();
  		long i = fbId % hpg;
  		Host *h = &group[i];
  		// skip if not assigned to us
  		if ( h->m_hostId != g_hostdb.m_hostId ) continue;
  
  		// add him to our list. sorted by next email time and
  		// fbid. so its a key96_t
  		key96_t k;
  		k.n0 = fr->m_fbId;
  		k.n1 = fr->m_nextRetry;
  		// at what time of day to email ( in minutes)? UTC
  		long tte = dayStart + fr->m_timeToEmail * 60;
  		// when was the last attempt to email?
  		long success = fr->m_lastEmailAttempt;
  
  		// reset this for debug
  		//success = 0;
  
  		// . "success" is non-zero if we had at least one successful
  		//   emailing to this person
  		// . for daily frequency we must wait at least a day after
  		//   the last successful email
  		// . we have minus 4 hours in case the email got off to
  		//   a late start
  		if ( ef == 1 && success && now - success < 20*3600 )
  			tte += 24*3600;
  		// same goes for weekly emails
  		if ( ef == 2 && success && now - success < 7*24*3600-4*3600 )
  			tte += 7*24*3600;
  		// assume that's the unix timestamp then (UTC)
  		k.n1 = tte;
  		// if non-zero, this overrides. this is non-zero if we
  		// had our last email fail
  		if ( fr->m_nextRetry ) k.n1 = fr->m_nextRetry;
  		// HACK TIME!
  		//k.n1 = 0;
  		// add to the tree now
  		if ( m_emailTree.addNode(m_collnum,(char *)&k,NULL,0) >= 0 ) 
  			continue;
  		// error!
  		log("email: email tree add error: %s",mstrerror(g_errno));
  	}
  	m_startKey = *(key96_t *)m_list7.getLastKey();
  	m_startKey += (unsigned long) 1;
  	// watch out for wrap around
  	//if ( startKey < *(key96_t *)list.getLastKey() ) return;
  }
  
  
  
  ///////////////////////
  //
  // FACEBOOK SPIDER
  //
  ///////////////////////
  
  // https://graph.facebook.com/search?fields=id,privacy,picture,name,location,venue,description,"start_time,end_time&type=event&q=china&limit=10&offset=0
  
  // . call this every second
  // . https://developers.facebook.com/docs/reference/api/#searching
  void facebookSpiderSleepWrapper ( int fd , void *state ) {
  	// only for host #0
  	//if ( g_hostdb.m_hostId != 0 ) return;
  
  	// all spiders off?
  	if ( ! g_conf.m_spideringEnabled ) return;
  
  	// if nothing on queue, push a 0 fbid on there to initiate
  	// the query spider algo on facebook
  	//if ( g_n1 >= 2 ) return;
  	// for now
  	//if ( g_n1 >= 1 ) return;
  	// flag
  	bool gotIt = false;
  	collnum_t collnum;
  	// get event collection
  	for ( long i = 0 ; i < g_collectiondb.m_numRecs ; i++ ) {
  		// get it
  		CollectionRec *cr = g_collectiondb.m_recs[i];
  		// skip if empty
  		if ( ! cr ) continue;
  		// or not events
  		if ( ! cr->m_indexEventsOnly ) continue;
  		// ok, use that
  		collnum = cr->m_collnum;
  		// flag it
  		gotIt = true;
  	}
  	// return if no such collection
  	if ( ! gotIt ) return;
  
  	// do we have an stubhubs already queued?
  	bool hasLocalFBId  = false;
  	bool hasEventBrite = false;
  	bool hasStubHub    = false;
  	bool hasFacebook   = false;
  	for ( long i = 0 ; i < g_n1 ; i++ ) {
  		if ( g_fbq1[i] == -3 ) hasLocalFBId  = true;
  		if ( g_fbq1[i] == -2 ) hasEventBrite = true;
  		if ( g_fbq1[i] == -1 ) hasStubHub    = true;
  		if ( g_fbq1[i] >=  0 ) hasFacebook   = true;
  	}
  
  	// need this
  	long now = getTimeGlobal();
  
  	// ok, it's empty! 0 fbid has special meaning. it means to
  	// do a spider round on facebook
  	if ( ! hasFacebook && g_conf.m_facebookSpideringEnabled &&
  	     // only for host #0
  	     g_hostdb.m_hostId == 0 )
  		queueFBId ( 0 , collnum );
  
  	// . s_ptr3 is now used for stub hub and is a time_t!
  	// . reset it if it's over a year out
  	if ( ! hasStubHub && g_conf.m_stubHubSpideringEnabled &&
  	     // only for host #0
  	     g_hostdb.m_hostId == 0 ) {
  		// if ptr is over a year into the future then reset to
  		// 0 and wait for 12 hours!!! stubhub.com spiders through
  		// in like an hour!
  		if ( s_ptr3 - now > 365*86400 )	{
  			// give it a delay too!
  			s_holdOffStubHubTill = now + 12*3600;
  			// log it
  			log("stubhub: stubhub spider completed. "
  			    "waiting for 12 hours before "
  			    "hitting stubhub again." );
  			// and reset our timer thing
  			s_ptr3 = 0;
  		}
  		// are we done waiting?
  		if ( now > s_holdOffStubHubTill )
  			// queue the stubhub
  			queueFBId ( -1 , collnum );
  	}
  
  	// . s_ptr3 is on the day we need to download events from eventbrite
  	//   that were create on that day
  	// . if its 0 then play catch up until we hit today, then just
  	//   hit it like once per hour for events created today
  	if ( ! hasEventBrite && 
  	     g_conf.m_eventBriteSpideringEnabled &&
  	     // only for host #0
  	     g_hostdb.m_hostId == 0 &&
  	     // if we got no results we delay like an hour until we
  	     // try again, in hopes someone added some new events to
  	     // eventbrite's index
  	     now > s_eventBriteWaitUntil )
  		// queue the eventbrite
  		queueFBId ( -2 , collnum );
  
  	// this is for all hosts in stripe #0, not just host #0
  	if ( ! hasLocalFBId &&
  	     // skip if not in bottom part
  	     g_hostdb.m_myHost->m_stripe == 0 &&
  	     // we do this once per day
  	     now > s_localWaitUntil ) 
  		// queue the local facebookdb scan
  		queueFBId ( -3 , collnum );
  
  }
  
  #include "Speller.h"
  
  // use unified dictionary
  char *getNextQuery ( ) {
  
  	if ( g_hostdb.m_hostId != 0 ) { 
  		log("qloop: wtf! not host #0");
  		return NULL;
  	}
  
  	if ( ! s_init ) {
  		// load query loop state
  		//loadQueryLoopState();
  		// try to load from disk
  		//if ( loadSortByPopTable() ) s_init = true;
  		bool s1 = false;
  		bool s2 = false;
  		s1 = s_tbuf1.load ( g_hostdb.m_dir,"/popsortwords.dat" );
  		s2 = s_tbuf2.load ( g_hostdb.m_dir,"/popsortplaces.dat" );
  		// if both loaded we are done
  		if ( s1 && s2 ) s_init = true;
  		// clear if one loaded but the other did not
  		else { s_tbuf1.reset(); s_tbuf2.reset(); }
  	}
  
  	// if load was unsuccessful, then create
  	if ( ! s_init ) {
  		// ok, create it and save it
  		HashTableX *ud = &g_speller.m_unifiedDict;
  		// init trees
  		RdbTree tree1;
  		RdbTree tree2;
  		tree1.set ( 4,  // fixeddatasize
  			    3000000, // maxnodes
  			    true, // do balancing
  			    -1 , // maxmem
  			    false, // owndata?
  			    "tree1", // allocname
  			    false, // datainptrs?
  			    NULL, // dbname
  			    12 ); // keysize
  		tree2.set ( 4,  // fixeddatasize
  			    5000000, // maxnodes
  			    true, // do balancing
  			    -1 , // maxmem
  			    false, // owndata?
  			    "tree2", // allocname
  			    false , // datainptrs?
  			    NULL, // dbname
  			    12 ); // keysize
  		// for keeping keys unique
  		long count = 0;
  		// scan the unified dictionary 
  		long n1 = ud->m_numSlots;
  		for ( long i = 0 ; i < n1 ; i++ ) {
  			// skip if empty
  			if ( ! ud->m_flags[i] ) continue;
  			// . get the ptr into m_unifiedBuf
  			// . word/phrase\tlangid\tpop\tlangid\tpop....
  			char *p = *(char **)ud->getValueFromSlot(i);
  			// point to \0 ending the word or the phonetic
  			char *w = p - 1;
  			// back up again until we are at the beginning of
  			// that word or phonetic
  			for ( ; w[-1] ; w-- );
  			// is it a phonetic?
  			if ( is_upper_a(w[0]) || w[0]=='*' ) {
  				// point to the \0 before it
  				w--;
  				// and back up to the start of the word/phrase
  				// that the phonetic represents
  				for ( ;w > g_speller.m_unifiedBuf &&w[-1];w--);
  			}
  			// scan word or phrase, we only want words not
  			// phrases for this... phrases are too spammy!
  			bool hadSpace = false;
  			for ( char *x = w; *x ; x++ ) {
  				if ( ! is_wspace_a ( *x ) ) continue;
  				hadSpace = true;
  				break;
  			}
  			// skip if its a phrase
  			if ( hadSpace ) continue;
  			// get the max pop from all the language/pop tuples
  			long maxPop = -2;
  			long pop;
  		subloop:
  			// skip over langid
  			for ( ; *p && *p !='\t';p++ );
  			// crazy? a pop should follow it!
  			if ( ! *p ) goto done;
  			// skip that
  			p++;
  			// get the pop
  			pop = atol(p);
  			// if negative make 0
  			if ( pop < 0 ) pop = 0;
  			// get max pop
  			if ( pop > maxPop ) maxPop = pop;
  			// skip over next tab, should be langid or \0
  			for ( ; *p && *p !='\t';p++ );
  			// if no more, get next word line
  			if ( ! *p ) goto done;
  			// skip that
  			if ( *p ) p++;
  			// get more if they are there
  			goto subloop;
  			// store it in tree
  		done:
  			// how is this possible?
  			if ( maxPop < 0 ) continue;
  			// make the key
  			key_t k;
  			k.n1 = ~((unsigned long)maxPop);
  			k.n0 = count++;
  			// store offset
  			long woff = w - g_speller.m_unifiedBuf;
  			// . add to b-tree to sort by pop
  			// . data is the word/phrase ptr
  			tree1.addNode(0,k,(char *)woff,4);
  		}
  
  		// . now add the cities in there too!
  		// . scan the cities
  		// . g_nameTable is from Address.cpp
  		long n2 = g_nameTable.m_numSlots;
  		for ( long i = 0 ; i < n2 ; i++ ) {
  			// skip if empty
  			if ( ! g_nameTable.m_flags[i] ) continue;
  			// get it
  			long offset = *(long *)g_nameTable.getValueFromSlot(i);
  			// get the ptr into g_pbuf for it
  			PlaceDesc *pd = (PlaceDesc *)(g_pbuf+offset);
  			// get the pop
  			unsigned long pop = pd->m_population;
  			// make the key
  			key_t k;
  			k.n1 = ~pop;
  			// "China" has many spellings and each one has an
  			// entry in the g_nameTable BUT they hash to the
  			// same PlaceDesc ptr, so use that as part of they
  			// key for making sure we have no dups in tree2.
  			k.n0 = (unsigned long long)pd;
  			//note it
  			//log("adding pop=%lu",pop);
  			// add to b-tree to sort by pop
  			tree2.addNode(0,k,(char *)offset,4);
  		}
  		// serialize tree1
  		for (long n=tree1.getLowestNode();n>=0;n=tree1.getNextNode(n)){
  			// get data. this one is a slot # in m_unifiedDict
  			long woff = (long)tree1.getData(n);
  			// store it
  			s_tbuf1.pushLong(woff);
  		}
  		long reps = 0;
  		// serialize tree2
  		for (long n=tree2.getLowestNode();n>=0;n=tree2.getNextNode(n)){
  			// get data. this one is an offset into g_pbuf
  			long i = (long)tree2.getData(n);
  			// sample
  			PlaceDesc *pd = (PlaceDesc *)(g_pbuf+i);
  			// print it
  			if ( reps++ == 0 && pd->m_population < 1000000 ) {
  				char *xx=NULL;*xx=0; }
  			// store it
  			s_tbuf2.pushLong(i);
  		}
  		// save both
  		s_tbuf1.save ( g_hostdb.m_dir,"/popsortwords.dat" );
  		s_tbuf2.save ( g_hostdb.m_dir,"/popsortplaces.dat" );
  		// do not re-do
  		s_init = true;
  		// init the ptrs then
  		//s_ptr1 = 0;
  		//s_ptr2 = 0;
  		// save state
  		saveQueryLoopState();
  	}
  
  	if ( s_flip == 0 ) s_flip = 1;
  	else               s_flip = 0;
  
  	if ( s_flip == 0 && ! g_speller.m_unifiedBuf ) {
  		log("facebook: unifiedDict not loaded! skipping pop words "
  		    "facebook spidering.");
  		s_flip = 1;
  	}
  
  	// get the next word or location
  	if ( s_flip == 0 ) {
  		long woff = ((long *)(s_tbuf1.getBufStart())) [s_ptr1];
  		s_ptr1++;
  		if ( s_ptr1 * 4 > s_tbuf1.length() ) s_ptr1 = 0;
  
  		// just to keep things somewhat fresh, let's cycle
  		// once we hit 15,000 words which is about 5 days
  		// at the current rate. i am planning on increasing
  		// the fb spider rate though a little if i can since it
  		// seems to not be rate-limited so far
  		if ( s_ptr1 > 15000 ) s_ptr1 = 0;
  		
  		char *v = g_speller.m_unifiedBuf + woff;
  		return v;
  	}
  
  	if ( s_flip == 1 ) {
  		long poff = s_ptr2 * 4;
  		s_ptr2++;
  		if ( s_ptr2 * 4 > s_tbuf2.length() ) s_ptr2 = 0;
  
  		// just to keep things somewhat fresh, let's cycle
  		// once we hit 15,000 words which is about 5 days
  		// at the current rate. i am planning on increasing
  		// the fb spider rate though a little if i can since it
  		// seems to not be rate-limited so far
  		if ( s_ptr2 > 15000 ) s_ptr2 = 0;
  
  		long offset = *(long *)(s_tbuf2.getBufStart() + poff);
  		// get the ptr into g_pbuf for it
  		PlaceDesc *pd = (PlaceDesc *)(g_pbuf+offset);
  
  		if ( pd->m_flags & PDF_STATE )
  			return (char *)pd->getStateName();
  		if ( pd->m_flags & PDF_COUNTRY )
  			return (char *)pd->getCountryName();
  		// crap, official names has many dups because we have one
  		// place desc for every slang name of a place.
  		return pd->getOfficialName();
  	}
  
  	// shouldn't be here!
  	char *xx=NULL;*xx=0;
  	return NULL;
  }
  
  
  
  // . save the state of getNextQuery() 
  // . save the queue g_fbq1[100],g_colls1[100],g_n1
  // . called from Process.cpp
  bool saveQueryLoopState ( ) {
  	SafeBuf ss;
  	ss.pushLong(s_flip);
  	ss.pushLong(s_ptr1);
  	ss.pushLong(s_ptr2);
  	// the queue of fbids
  	ss.pushLong(g_n1);
  	ss.safeMemcpy((char *)g_fbq1,g_n1*4);
  	ss.safeMemcpy((char *)g_colls1,g_n1*4);
  	ss.pushLong(s_ptr3);
  	ss.pushLong(s_holdOffStubHubTill);
  	ss.pushLongLong(s_ptr4);
  	ss.pushLong(s_eventBriteWaitUntil);
  	// local scan
  	ss.pushLongLong(s_ptr5);
  	ss.pushLong    (s_localWaitUntil);
  	log("facebook: saving fbloop.dat. "
  	    "s_ptr1=%li "
  	    "s_ptr2=%li "
  	    "s_ptr3=%li "
  	    "s_ptr4=%lli "
  	    "s_ptr5=%lli "
  	    "s_holdOffStubHubTill=%lu "
  	    "s_eventBriteWaitUntil=%lu "
  	    "s_localWaitUntil=%lu "
  	    "g_n1=%li",
  	    s_ptr1,
  	    s_ptr2,
  	    s_ptr3,
  	    s_ptr4,
  	    s_ptr5,
  	    s_holdOffStubHubTill,
  	    s_eventBriteWaitUntil,
  	    s_localWaitUntil,
  	    g_n1);
  	return ss.save(g_hostdb.m_dir,"fbloop.dat");
  }
  
  bool loadQueryLoopState ( ) {
  	SafeBuf ss;
  	if ( ! ss.load(g_hostdb.m_dir,"fbloop.dat") ) return false;
  	// assign
  	char *p = ss.getBufStart();
  	char *pend = p + ss.length();
  	s_flip = *(long *)p; p += 4;
  	s_ptr1 = *(long *)p; p += 4;
  	s_ptr2 = *(long *)p; p += 4;
  	g_n1   = *(long *)p; p += 4;
  	memcpy ( g_fbq1   , p , g_n1 * 4 ); p += g_n1 * 4;
  	memcpy ( g_colls1 , p , g_n1 * 4 ); p += g_n1 * 4;
  	if ( p >= pend ) goto done;
  	s_ptr3 = *(long *)p; p += 4;
  	if ( p >= pend ) goto done;
  	s_holdOffStubHubTill = *(long *)p; p += 4;
  	if ( p >= pend ) goto done;
  	s_ptr4 = *(long long *)p; p += 8;
  	s_eventBriteWaitUntil = *(long *)p; p += 4;
  	// local scan
  	if ( p >= pend ) goto done;
  	s_ptr5           = *(long long *)p; p += 8;
  	s_localWaitUntil = *(long      *)p; p += 4;
   done:
  	log("facebook: loaded fbloop.dat. "
  	    "s_ptr1=%li "
  	    "s_ptr2=%li "
  	    "s_ptr3=%li "
  	    "s_ptr4=%lli "
  	    "s_ptr5=%lli "
  	    "s_holdOffStubHubTill=%lu "
  	    "s_eventBriteWaitUntil=%lu "
  	    "s_localWaitUntil=%lu "
  	    "g_n1=%li",
  	    s_ptr1,
  	    s_ptr2,
  	    s_ptr3,
  	    s_ptr4,
  	    s_ptr5,
  	    s_holdOffStubHubTill,
  	    s_eventBriteWaitUntil,
  	    s_localWaitUntil,
  	    g_n1);
  	return true;
  }
  