text: #include "Collectiondb.h"
  #include "HttpServer.h"
  #include "Stats.h"
  #include "Statistics.h"
  #include "Query.h"
  #include "Speller.h"
  #include "Msg40.h"
  #include "Pages.h"
  #include "Highlight.h"
  #include "SearchInput.h"
  #include <math.h>
  #include "SafeBuf.h"
  #include "Pos.h"
  #include "Bits.h"
  #include "sort.h"
  #include "CountryCode.h"
  #include "Posdb.h"
  #include "PosdbTable.h"
  #include "PageResults.h"
  #include "PageRoot.h"
  #include "Proxy.h"
  #include "Json.h"
  #include "Images.h" //Thumbnail*
  #include "HttpMime.h"
  #include "Process.h"
  #include "Linkdb.h"
  #include "XmlDoc.h"
  #include "ip.h"
  #include "GbUtil.h"
  #include "Conf.h"
  #include "Mem.h"
  #include "RobotsBlockedResultOverride.h"
  #include "QueryLanguage.h"
  #include "FxLanguage.h"
  #include "Errno.h"
  #ifdef _VALGRIND_
  #include <valgrind/memcheck.h>
  #endif
  
  
  static bool printSearchResultsHeader(State0 *st);
  static bool printResult(State0 *st, int32_t ix, int32_t *numPrintedSoFar);
  static bool printSearchResultsTail(State0 *st);
  static bool printSearchFiltersBar ( SafeBuf *sb , HttpRequest *hr ) ;
  static bool printMenu ( SafeBuf *sb , int32_t menuNum , HttpRequest *hr ) ;
  
  static void gotQueryLanguageWrapper(void *state, const std::vector<std::pair<lang_t, double>> &languages);
  static bool gotQueryLanguage(State0 *st, const std::vector<std::pair<lang_t, double>> &languages);
  
  static void gotResultsWrapper  ( void *state ) ;
  static bool gotResults         ( void *state ) ;
  
  static bool replaceParm ( const char *cgi , SafeBuf *newUrl , HttpRequest *hr ) ;
  static bool replaceParm2 ( const char *cgi , SafeBuf *newUrl ,
  			   const char *oldUrl , int32_t oldUrlLen ) ;
  
  
  static bool printPairScore(SafeBuf *sb, const SearchInput *si, const PairScore *ps, Msg20Reply *mr);
  
  static bool printScoresHeader ( SafeBuf *sb ) ;
  
  static bool printMetaContent ( Msg40 *msg40 , int32_t i ,State0 *st, SafeBuf *sb );
  
  static bool printSingleScore(SafeBuf *sb, const SearchInput *si, const SingleScore *ss, Msg20Reply *mr);
  
  static bool printSingleTerm(SafeBuf *sb, const Query *q, const SingleScore *ss);
  static bool printTermPairs(SafeBuf *sb, const Query *q, const PairScore *ps);
  
  static bool printLogoAndSearchBox (SafeBuf *sb , class HttpRequest *hr, const SearchInput *si );
  
  State0::State0()
  	: m_sb()
  	, m_header(true)
  	, m_collnum(-1)
  	, m_si()
  	, m_msg40()
  	, m_socket(nullptr)
  	, m_startTime(0)
  	, m_gotResults(false)
  	, m_errno(0)
  	, m_numDocIds(0)
  	, m_took(0)
  	, m_hr()
  	, m_qesb()
  	, m_xd(nullptr)
  	, m_socketStartTimeHack(0)
  	, m_primaryQueryLanguage(langUnknown) {
  }
  
  static bool sendReply(State0 *st, const char *reply, int32_t rlen) {
  
  	int32_t savedErr = g_errno;
  
  	TcpSocket *sock = st->m_socket;
  	if ( ! sock ) { 
  		log("results: not sending back results on an empty socket."
  		    "socket must have closed on us abruptly.");
  		//g_process.shutdownAbort(true); }
  	}
  	SearchInput *si = &st->m_si;
  	const char *ct = "text/html";
  	if ( si->m_format == FORMAT_XML ) ct = "text/xml"; 
  	if ( si->m_format == FORMAT_JSON ) ct = "application/json";
  
  	const char *charset = "utf-8";
  	char format = si->m_format;
  
  	// . filter anything < 0x20 to 0x20 to keep XML legal
  	// . except \t, \n and \r, they're ok
  	// . gotta set "f" down here in case it realloc'd the buf
  	if ( format == FORMAT_XML && reply ) {
  		unsigned char *f = (unsigned char *)reply;
  		for ( ; *f ; f++ ) 
  			if ( *f < 0x20 && *f!='\t' && *f!='\n' && *f!='\r' ) 
  				*f = 0x20;
  	}
  
  
  	logf(LOG_DEBUG,"gb: sending back %" PRId32" bytes",rlen);
  
  	Statistics::register_query_time(si->m_q.m_numWords, si->m_queryLangId, savedErr, (gettimeofdayInMilliseconds() - st->m_startTime));
  
  	// . log the time
  	// . do not do this if g_errno is set lest m_sbuf1 be bogus b/c
  	//   it failed to allocate its buf to hold terminating \0 in
  	//   SearchInput::setQueryBuffers()
  	if ( ! g_errno && st->m_took >= g_conf.m_logQueryTimeThreshold ) {
  		logf(LOG_WARN,"Query took %" PRId64" ms for %s. results=%" PRId32,
  		     st->m_took,
  		     si->m_sbuf1.getBufStart(),
  		     st->m_msg40.getNumResults());
  	}
  
  	if ( ! savedErr ) {
  		// . one hour cache time... no 1000 hours, basically infinite
  		// . no because if we redo the query the results are cached
  		//int32_t cacheTime = 3600;//*1000;
  		// no... do not use cache
  		int32_t cacheTime = -1;
  		// the "Check it" link on add url uses &usecache=0 to tell
  		// the browser not to use its cache...
  		//if ( hr->getLong("usecache",-1) == 0 ) cacheTime = 0;
  		//
  		// send back the actual search results
  		//
  		if ( sock )
  		g_httpServer.sendDynamicPage(sock,
  					     reply,
  					     rlen,//strlen(reply),
  					     // don't let the ajax re-gen
  					     // if they hit the back button!
  					     // so make this 1 hour, not 0
  					     cacheTime, // cachetime in secs
  					     false, // POSTReply?
  					     ct,
  					     -1, // httpstatus -1 -> 200
  					     NULL, // cookieptr
  					     charset );
  
  		// free st after sending reply since "st->m_sb" = "reply"
  		mdelete(st, sizeof(State0), "PageResults2");
  		delete st;
  		return true;
  	}
  
  	mdelete(st, sizeof(State0), "PageResults2");
  	delete st;
  
  	// if we had a broken pipe from the browser while sending
  	// them the search results, then we end up closing the socket fd
  	// in TcpServer::sendChunk() > sendMsg() > destroySocket()
  	if ( sock && sock->m_numDestroys ) {
  		log("results: not sending back error on destroyed socket "
  		    "sd=%" PRId32,sock->m_sd);
  		return true;
  	}
  
  	int32_t status = 500;
  	if (savedErr == EBADREQUEST ||
  	    savedErr == ENOPERM ||
  	    savedErr == ENOCOLLREC) 
  		status = 400;
  
  	if ( sock )
  	g_httpServer.sendQueryErrorReply(sock,
  					 status,
  					 mstrerror(savedErr),
  					 format,//xml,
  					 savedErr, 
  					 "There was an error!");
  	return true;
  }
  
  static bool printCSSHead(SafeBuf *sb) {
  	sb->safePrintf(
  			      "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML "
  			      "4.01 Transitional//EN\">\n"
  			      //"<meta http-equiv=\"Content-Type\" "
  			      //"content=\"text/html; charset=utf-8\">\n"
  			      "<html>\n"
  			      "<head>\n"
  			      "<title>Gigablast Search Results</title>\n"
  			      "<style><!--"
  			      "body {"
  			      "font-family:Arial, Helvetica, sans-serif;"
  			      );
  
  	sb->safePrintf(	      "color: #000000;"
  			      "font-size: 12px;"
  			      //"margin: 20px 5px;"
  			      "}"
  			      "a:link {color:#00c}"
  			      "a:visited {color:#551a8b}"
  			      "a:active {color:#f00}"
  			      ".bold {font-weight: bold;}"
  			      ".bluetable {background:#d1e1ff;"
  			      "margin-bottom:15px;font-size:12px;}"
  			      ".url {color:#008000;}"
  			      ".cached, .cached a {font-size: 10px;"
  			      "color: #666666;"
  			      "}"
  			      "table {"
  			      "font-family:Arial, Helvetica, sans-serif;"
  			      "color: #000000;"
  			      "font-size: 12px;"
  			      "}"
  			      ".directory {font-size: 16px;}"
  			      "-->\n"
  			      "</style>\n"
  			      "</head>\n"
  			      );
  	return true;
  }
  
  // . returns false if blocked, true otherwise
  // . sets g_errno on error
  // . "msg" will be inserted into the access log for this request
  bool sendPageResults ( TcpSocket *s , HttpRequest *hr ) {
  	// make a new state
  	State0 *st;
  	try {
  		st = new State0();
  	} catch(std::bad_alloc&) {
  		g_errno = ENOMEM;
  		log(LOG_ERROR, "query: Query failed. "
  		    "Could not allocate %" PRId32" bytes for query. "
  		    "Returning HTTP status of 500.",(int32_t)sizeof(State0));
  		Statistics::register_query_time(0, langUnknown, g_errno, 0);
  
  		return g_httpServer.sendQueryErrorReply(s, 500, mstrerror(g_errno), hr->getReplyFormat(), g_errno,
  		                                        "Query failed. Could not allocate memory to execute a search. Please try later.");
  	}
  
  	mnew(st, sizeof(State0), "PageResults2");
  
  	// copy yhits
  	if (!st->m_hr.copy(hr)) {
  		return sendReply(st,nullptr,0);
  	}
  
  	// set this in case SearchInput::set fails!
  	st->m_socket = s;
  
  	// record timestamp so we know if we got our socket closed and swapped
  	st->m_socketStartTimeHack = s->m_startTime;
  
  	// you have to say "&header=1" to get back the header for json now.
  	// later on maybe it will default to on.
  	st->m_header = hr->getLong("header", 1);
  
  	// get query parameters
  	int32_t queryLen = 0;
  	const char *query = hr->getString("q", &queryLen, "");
  
  	const char *fx_qlang = hr->getString("fx_qlang", nullptr, "");
  	const char *fx_blang = hr->getString("fx_blang", nullptr, "");
  	const char *fx_country = hr->getString("fx_country", nullptr, "");
  	const char *fx_fetld = hr->getString("fx_fetld", nullptr, "");
  
  	
  	// First try our built-in langauge detection which uses CLD2 and the top-quality accept-language item
  	std::string contentLanguage;
  	if (strlen(fx_qlang) && getLangIdFromAbbr(fx_qlang) != langUnknown) {
  		contentLanguage = fx_qlang;
  	} else if (strlen(fx_blang)) {
  		// en-GB,en;q=0.8,fr;q=0.6,en-US;q=0.4
  		std::string blang_hint(fx_blang);
  
  		std::vector<std::pair<std::string, double>> blang_quality_pairs;
  		// split between entries
  		auto items = split(blang_hint, ',');
  		for (const auto &item : items) {
  			// en-US;q=0.4
  			auto pairs = split(item, ';');
  			double q = 1;
  			if (pairs.size() == 2) {
  				// has language quality factor
  				auto nvp = split(pairs[1], '=');
  				if (nvp.size() == 2) {
  					q = strtod(nvp[1].c_str(), nullptr);
  				}
  			}
  
  			// en-US
  			auto tokens = split(pairs[0], '-');
  
  			blang_quality_pairs.emplace_back(pairs[0], q);
  		}
  
  		std::sort(blang_quality_pairs.begin(), blang_quality_pairs.end(),
  		          [](const std::pair<std::string, double> &a, const std::pair<std::string, double> &b) {
  			          return a.second > b.second;
  		          });
  
  		auto it = std::unique(blang_quality_pairs.begin(), blang_quality_pairs.end(),
  		                      [](const std::pair<std::string, double> &a, const std::pair<std::string, double> &b) {
  			                      return a.first == b.first;
  		                      });
  		blang_quality_pairs.erase(it, blang_quality_pairs.end());
  
  		if (blang_quality_pairs.size()) {
  			double max_weight = blang_quality_pairs[0].second;
  			for (auto blang_quality_pair : blang_quality_pairs) {
  				if (!contentLanguage.empty()) {
  					contentLanguage.append(",");
  				}
  
  				// append all blang with max weight
  				if (blang_quality_pair.second == max_weight) {
  					contentLanguage.append(blang_quality_pair.first);
  					continue;
  				}
  
  				break;
  			}
  		}
  	}
  
  	const char *tld_hint = fx_fetld;
  	if (!tld_hint || strlen(tld_hint) == 0 || strlen(tld_hint) != 2) {
  		tld_hint = fx_country;
  	}
  
  	st->m_primaryQueryLanguage = FxLanguage::getLangIdCLD2(true, query, queryLen, contentLanguage.c_str(), contentLanguage.size(),
  	                                                       tld_hint, strlen(tld_hint), true);
  
  
  	//We can't use st->m_si.m_baseScoringParameters.allLanguageWeightsAreTheSame() because the cgi parameters have not set the members in SearchInput yet.
  	//So we have to check if any CGI parameter using the "lw_" prefix has been specified.
  	bool any_specific_language_weight_specified = false;
  	for(int32_t i = 0; i < hr->getNumFields(); i++) {
  		if(!hr->getValue(i))
  			continue;
  		const char *full_field_name = hr->getField(i);
  		if(strncmp(full_field_name,"lw_",3)==0) {
  			any_specific_language_weight_specified = true;
  			break;
  		}
  	}
  	
  	if(!any_specific_language_weight_specified) {
  		// Then try the external language detection server
  		if (g_queryLanguage.getLanguage(st, gotQueryLanguageWrapper, fx_qlang, fx_blang, fx_country, fx_fetld, query)) {
  			// blocked
  			return false;
  		}
  	} else
  		log(LOG_DEBUG,"Explicit language weight specified. Not contacting external language detection server");
  	return gotQueryLanguage(st, {});
  }
  
  static void gotQueryLanguageWrapper(void *state, const std::vector<std::pair<lang_t, double>> &language_weights) {
  	State0 *st = reinterpret_cast<State0 *>(state);
  	if (!language_weights.empty()) {
  		st->m_primaryQueryLanguage = language_weights.front().first;
  	}
  
  	gotQueryLanguage(st,language_weights);
  }
  
  static bool gotQueryLanguage(State0 *st, const std::vector<std::pair<lang_t, double>> &language_weights) {
  	// . parse it up
  	// . this returns false and sets g_errno and, maybe, g_msg on error
  	SearchInput *si = &st->m_si;
  
  	// si just copies the ptr into the httprequest
  	// into stuff like SearchInput::m_defaultSortLanguage
  	// so do not use the "hr" on the stack. SearchInput::
  	// m_hr points to the hr we pass into
  	// SearchInput::set
  	if (!si->set(st->m_socket, &st->m_hr, st->m_primaryQueryLanguage, language_weights)) {
  		log("query: set search input: %s",mstrerror(g_errno));
  		if ( ! g_errno ) g_errno = EBADENGINEER;
  		return sendReply(st,NULL,0);
  	}
  
  	// save collnum now
  	st->m_collnum = si->m_cr ? si->m_cr->m_collnum : -1;
  
  	st->m_numDocIds = si->m_docsWanted;
  
  	// add query stat
  	st->m_startTime = gettimeofdayInMilliseconds();
  
  	// debug msg
  	log(LOG_DEBUG, "query: Getting search results for q=%s", st->m_si.m_displayQuery);
  
  	// for now disable queries
  	if (!g_conf.m_queryingEnabled) {
  		g_errno = EQUERYINGDISABLED;
  		return sendReply(st, nullptr,0);
  	}
  
  	// LAUNCH RESULTS
  
  	// . get some results from it
  	// . this returns false if blocked, true otherwise
  	// . it also sets g_errno on error
  	// . use a niceness of 0 for all queries so they take precedence
  	//   over the indexing process
  	// . this will copy our passed "query" and "coll" to it's own buffer
  	// . we print out matching docIds to int32_t if m_isDebug is true
  	// . no longer forward this, since proxy will take care of evenly
  	//   distributing its msg 0xfd "forward" requests now
  	st->m_gotResults = st->m_msg40.getResults(si, false, st, gotResultsWrapper);
  
  	// save error
  	st->m_errno = g_errno;
  
  	// wait for results?
  	if (!st->m_gotResults) {
  		return false;
  	}
  
  	// otherwise call gotResults which returns false if blocked, true else
  	// and sets g_errno on error
  	return gotResults(st);
  }
  
  static void gotResultsWrapper ( void *state ) {
  	// cast our State0 class from this
  	State0 *st = (State0 *) state;
  
  	// save error
  	st->m_errno = g_errno;
  
  	// mark as gotten
  	st->m_gotResults = true;
  
  	gotResults(st);
  }
  
  // . make a web page from results stored in msg40
  // . send it on TcpSocket "s" when done
  // . returns false if blocked, true otherwise
  // . sets g_errno on error
  static bool gotResults ( void *state ) {
  	// cast our State0 class from this
  	State0 *st = (State0 *) state;
  
  	int64_t nowMS = gettimeofdayInMilliseconds();
  	// log the time
  	int64_t took = nowMS - st->m_startTime;
  	// record that
  	st->m_took = took;
  
  	//log("results: debug: in gotResults state=%" PTRFMT,(PTRTYPE)st);
  
  	// grab the query
  	Msg40 *msg40 = &(st->m_msg40);
  	//char  *q    = msg40->getQuery();
  	//int32_t   qlen = msg40->getQueryLen();
  
  	SearchInput *si = &st->m_si;
  
  	// if we lost the socket because we were streaming and it
  	// got closed from a broken pipe or something, then Msg40.cpp
  	// will set st->m_socket to NULL if the fd ends up ending closed
  	// because someone else might be using it and we do not want to
  	// mess with their TcpSocket settings.
  	if ( ! st->m_socket ) {
  		log("results: socket is NULL. sending failed.");
  		return sendReply(st,NULL,0);
  	}
  
  	// if we skipped a shard because it was dead, usually we provide
  	// the results anyway, but if this switch is true then return an
  	// error code instead. this is the 'all or nothing' switch.
  	if ( msg40->m_msg3a.m_skippedShards > 0 &&
  	     ! g_conf.m_returnResultsAnyway ) {
  	       char reply[256];
  	       sprintf ( reply , 
  			 "%" PRId32" shard(s) out of %" PRId32" did not "
  			 "respond to query."
  			 , msg40->m_msg3a.m_skippedShards
  			 , g_hostdb.m_numShards );
  	       g_errno = ESHARDDOWN;
  	       return sendReply(st,reply,strlen(reply));
  	}
  
  
  	// collection rec must still be there since SearchInput references 
  	// into it, and it must be the SAME ptr too!
  	CollectionRec *cr = si->m_cr;//g_collectiondb.getRec ( collnum );
  	if ( ! cr ) { // || cr != si->m_cr ) {
  		g_errno = ENOCOLLREC;
  		return sendReply(st,NULL,0);
  	}
  
  	// this causes ooms everywhere, not a good fix
  	if ( ! msg40->m_msg20 && ! si->m_docIdsOnly && msg40->m_errno ) {
  		log("msg40: failed to get results q=%s",si->m_q.originalQuery());
  	 	//g_errno = ENOMEM;
  		g_errno = msg40->m_errno;
  		return sendReply(st,NULL,0);
  	}
  
  
   	int32_t numResults = msg40->getNumResults();
  
  	SafeBuf *sb = &st->m_sb;
  
  	// print logo, search box, results x-y, ... into st->m_sb
  	printSearchResultsHeader ( st );
  
  	// then print each result
  	// don't display more than docsWanted results
  	int32_t count = msg40->getDocsWanted();
  	bool hadPrintError = false;
  	int32_t numPrintedSoFar = 0;
  
  	for ( int32_t i = 0 ; count > 0 && i < numResults ; i++ ) {
  		//////////
  		//
  		// prints in xml or html
  		//
  		//////////
  		if ( ! printResult ( st , i , &numPrintedSoFar ) ) {
  			hadPrintError = true;
  			break;
  		}
  
  		// limit it
  		count--;
  	}
  
  
  	if ( hadPrintError ) {
  		if ( ! g_errno ) g_errno = EBADENGINEER;
  		log("query: had error: %s",mstrerror(g_errno));
  	}
  
  	// wrap it up with Next 10 etc.
  	printSearchResultsTail ( st );
  
  	// END SERP DIV
  	if ( si->m_format == FORMAT_WIDGET_IFRAME ||
  	     si->m_format == FORMAT_WIDGET_AJAX )
  		sb->safePrintf("</div>");
  
  	// send it off
  	sendReply(st, st->m_sb.getBufStart(), st->m_sb.length());
  
  	return true;
  }
  
  
  static bool printLeftNavColumn ( SafeBuf &sb, State0 *st ) {
  
  	SearchInput *si = &st->m_si;
  	CollectionRec *cr = si->m_cr;
  
  	char format = si->m_format;
  
  	if ( format == FORMAT_HTML ) {
  		const char *title = "Search Results";
  		sb.safePrintf("<title>Gigablast - %s</title>\n",title);
  		sb.safePrintf("<style><!--\n");
  		sb.safePrintf("body {\n");
  		sb.safePrintf("font-family:Arial, Helvetica, sans-serif;\n");
  		sb.safePrintf("color: #000000;\n");
  		sb.safePrintf("font-size: 12px;\n");
  		sb.safePrintf("margin: 0px 0px;\n");
  		sb.safePrintf("letter-spacing: 0.04em;\n");
  		sb.safePrintf("}\n");
  		sb.safePrintf("a {text-decoration:none;}\n");
  		sb.safePrintf(".bold {font-weight: bold;}\n");
  		sb.safePrintf(".bluetable {background:#d1e1ff;"
  			      "margin-bottom:15px;font-size:12px;}\n");
  		sb.safePrintf(".url {color:#008000;}\n");
  		sb.safePrintf(".cached, .cached a {font-size: 10px;"
  			      "color: #666666;\n");
  		sb.safePrintf("}\n");
  		sb.safePrintf("table {\n");
  		sb.safePrintf("font-family:Arial, Helvetica, sans-serif;\n");
  		sb.safePrintf("color: #000000;\n");
  		sb.safePrintf("font-size: 12px;\n");
  		sb.safePrintf("}\n");
  		sb.safePrintf(".directory {font-size: 16px;}\n"
  			      ".nav {font-size:20px;align:right;}\n"
  			      );
  		sb.safePrintf("-->\n");
  		sb.safePrintf("</style>\n");
  		sb.safePrintf("\n");
  		sb.safePrintf("</head>\n");
  		sb.safePrintf("<script>\n");
  		sb.safePrintf("<!--\n");
  		sb.safePrintf("var openmenu=''; var inmenuclick=0;");
  		sb.safePrintf("function x(){document.f.q.focus();}\n");
  		sb.safePrintf("// --></script>\n");
  		sb.safePrintf("<body "
  
  			      "onmousedown=\""
  
  			      "if (openmenu != '' && inmenuclick==0) {"
  			      "document.getElementById(openmenu)."
  			      "style.display='none'; openmenu='';"
  			      "}"
  
  			      "inmenuclick=0;"
  			      "\" "
  
  			      "onload=\"x()\">\n");
  
  		//
  		// DIVIDE INTO TWO PANES, LEFT COLUMN and MAIN COLUMN
  		//
  		sb.safePrintf("<TABLE border=0 height=100%% cellpadding=0 "
  			      "cellspacing=0>"
  			      "\n<TR>\n");
  
  		//
  		// first the nav column
  		//
  
  		// . also prints <TD>...</TD>. true=isSearchresults
  		// . tabName = "search"
  		printLeftColumnRocketAndTabs ( &sb , true , cr , "search" );
  
  		//
  		// now the MAIN column
  		//
  		sb.safePrintf("\n</TD>"
  			      "<TD valign=top style=padding-left:30px;>\n");
  	}
  
  	return true;
  }
  
  static bool printIgnoredWords ( SafeBuf *sb , const SearchInput *si ) {
  	// mention ignored query terms
  	// we need to set another Query with "keepAllSingles" set to false
  	const Query *qq2 = &si->m_q;
  	bool firstIgnored = true;
  	for ( int32_t i = 0 ; i < qq2->m_numWords ; i++ ) {
  		//if ( si->m_xml ) break;
  		const QueryWord *qw = &qq2->m_qwords[i];
  		// only print out words ignored cuz they were stop words
  		if ( qw->m_ignoreWord != IGNORE_QSTOP ) continue;
  		// print header -- we got one
  		if ( firstIgnored ) {
  			if ( si->m_format == FORMAT_XML )
  				sb->safePrintf ("\t<ignoredWords><![CDATA[");
  			else if ( si->m_format == FORMAT_JSON )
  				sb->safePrintf ("\t\"ignoredWords\":\"");
  			else if ( si->m_format == FORMAT_HTML )
  				sb->safePrintf ("<br><font "
  					       "color=\"#707070\">The "
  					       "following query words "
  					       "were ignored: "
  					       "<b>");
  			firstIgnored = false;
  		}
  		// print the word
  		sb->utf8Encode2(qw->m_word, qw->m_wordLen);
  		sb->safePrintf (" ");
  	}
  	// print tail if we had ignored terms
  	if ( ! firstIgnored ) {
  		sb->incrementLength(-1);
  		if ( si->m_format == FORMAT_XML )
  			sb->safePrintf("]]></ignoredWords>\n");
  		else if ( si->m_format == FORMAT_JSON )
  			sb->safePrintf("\",\n");
  		else if ( si->m_format == FORMAT_HTML )
  			sb->safePrintf ("</b>. Preceed each with a '+' or "
  				       "wrap in "
  				       "quotes to not ignore.</font>");
  	}
  	return true;
  }
  
  static bool printSearchResultsHeader(State0 *st) {
  
  	const SearchInput *si = &st->m_si;
  
  	// grab the query
  	Msg40 *msg40 = &(st->m_msg40);
  	const char *q    = msg40->getQuery();
  	int32_t     qlen = msg40->getQueryLen();
  
    	//char  local[ 128000 ];
  	//SafeBuf sb(local, 128000);
  	SafeBuf *sb = &st->m_sb;
  	// reserve 1.5MB now!
  	if ( ! sb->reserve(1500000 ,"pgresbuf" ) ) // 128000) )
  		return false;
  	// just in case it is empty, make it null terminated
  	sb->nullTerm();
  
  	// print first [ for json
  	if ( si->m_format == FORMAT_JSON ) {
  		if ( st->m_header ) sb->safePrintf("{\n");
  		// this is just for diffbot really...
  		else                sb->safePrintf("[\n");
  	}
  
  	CollectionRec *cr = si->m_cr;
  	HttpRequest *hr = &st->m_hr;
  
  	// if there's a ton of sites use the post method otherwise
  	// they won't fit into the http request, the browser will reject
  	// sending such a large request with "GET"
  	const char *method = "GET";
  	if ( si->m_sites && strlen(si->m_sites)>800 ) method = "POST";
  
  
  	if ( si->m_format == FORMAT_HTML &&
  	     cr->m_htmlHead.length() ) {
  		return expandHtml ( *sb ,
  				    cr->m_htmlHead.getBufStart(),
  				    cr->m_htmlHead.length(),
  				    q,
  				    qlen,
  				    hr,
  				    si,
  				    method,
  				    cr);
  	}
  				 
  
  	if ( si->m_format == FORMAT_HTML ) {
  		printCSSHead (sb);
  		sb->safePrintf("<body>");
  	}
  
  	if ( si->m_format==FORMAT_WIDGET_IFRAME ) {
  		printCSSHead(sb);
  		sb->safePrintf("<body style=padding:0px;margin:0px;>");
  
  		int32_t refresh = hr->getLong("refresh",0);
  		if ( refresh )
  			sb->safePrintf("<meta http-equiv=\"refresh\" "
  				       "content=%" PRId32">",refresh);
  	}
  
  	// lead with user's widget header which usually has custom style tags
  	if ( si->m_format == FORMAT_WIDGET_IFRAME ||
  	     si->m_format == FORMAT_WIDGET_AJAX ) {
  		const char *header = hr->getString("header",NULL);
  		if ( header ) sb->safeStrcpy ( header );
  	}
  
  	// if we are xml/json we call this below otherwise we lose
  	// the header of <?xml...> or whatever
  	if ( si->m_format == FORMAT_HTML ) {
  		printLeftNavColumn ( *sb,st );
  
  		printLogoAndSearchBox ( sb, &st->m_hr, si );
  	}
  
  	// the calling function checked this so it should be non-null
  	const char *coll = cr->m_coll;
  	int32_t collLen = strlen(coll);
  
  	if ( si->m_format == FORMAT_WIDGET_IFRAME ||
  	     si->m_format == FORMAT_WIDGET_AJAX ) {
  		int32_t widgetwidth = hr->getLong("widgetwidth",150);
  		int32_t widgetHeight = hr->getLong("widgetheight",400);
  		//int32_t iconWidth = 25;
  
  		// put image in this div which will have top:0px JUST like
  		// the div holding the search results we print out below
  		// so that the image does not scroll when you use the
  		// scrollbar. holds the magifying glass img and searchbox.
  		sb->safePrintf("<div class=magglassdiv "
  			       "style=\"position:absolute;"
  			       "right:15px;"
  			       "z-index:10;"
  			       "top:0px;\">");
  
  		//int32_t refresh = hr->getLong("refresh",15);
  		const char *oq = hr->getString("q",NULL);
  		if ( ! oq ) oq = "";
  		const char *prepend = hr->getString("prepend");
  		if ( ! prepend ) prepend = "";
  		const char *displayStr = "none";
  		if ( prepend[0] ) displayStr = "";
  		// to do a search we need to re-call the ajax,
  		// just call reload like the one that is called every 15s or so
  		sb->safePrintf("<form "//method=get action=/search "
  			       // use "1" as arg to force reload
  			       "onsubmit=\"widget123_reload(1);"
  
  			       // let user know we are loading
  			       "var w=document.getElementById("
  			       "'widget123_scrolldiv');"
  			       // just set the widget content to the reply
  			       "if (w) "
  			       "w.innerHTML='<br><br><b>Loading Results..."
  			       "</b>';"
  
  			       // prevent it from actually submitting
  			       "return false;\">");
  
  		sb->safePrintf("<img "
  			       "style=\""
  			       //"position:absolute;" // absolute or relative?
  			       // put it on TOP of the other stuff
  			       "z-index:10;"
  			       "margin-top:3px;"
  			       //"right:10px;"
  			       //"right:2px;"
  			       //"width:%" PRId32"px;"
  			       // so we are to the right of the searchbox
  			       "float:right;"
  			       "\" "
  			       "onclick=\""
  			       "var e=document.getElementById('sbox');"
  			       "if(e.style.display == 'none') {"
  			       "e.style.display = '';"
  			       // give it focus
  			       "var qb=document.getElementById('qbox');"
  			       "qb.focus();"
  			       "} else {"
  			       "e.style.display = 'none';"
  			       "}"
  			       "\" " // end function
  			       " "
  			       "width=35 "
  			       "height=31 "
  			       "src=\"/magglass.png\">"
  			       );
  
  		//char *origq = hr->getString("q");
  		// we sort all results by spider date now so PREPEND
  		// the actual user query 
  		const char *origq = hr->getString("prepend");
  		if ( ! origq ) origq = "";
  		sb->safePrintf("<div id=sbox style=\"float:left;"
  			       "display:%s;"
  			       "opacity:0.83;"
  			       //"background-color:gray;"
  			       //"padding:5px;"
  			       "\">"
  			       // the box that holds the query
  			       "<input type=text id=qbox name=qbox "
  			       "size=%" PRId32" " //name=prepend "
  			       "value=\"%s\"  "
  			       "style=\"z-index:10;"
  			       "font-weight:bold;"
  			       "font-size:18px;"
  			       "border:4px solid black;"
  			       "margin:3px;"
  			       "\">"
  			       , displayStr
  			       , widgetwidth / 23 
  			       , origq
  			       );
  		sb->safePrintf("</div>"
  			       "</form>\n"
  			       );
  
  		// . BEGIN SERP DIV
  		// . div to hold the search results
  		// . this will have the scrollbar to just scroll the serps
  		//   and not the magnifying glass
  		sb->safePrintf("</div>"
  			       "<div id=widget123_scrolldiv "
  			       "onscroll=widget123_append(); "
  			       "style=\"position:absolute;"
  			       "top:0px;"
  			       "overflow-y:auto;"
  			       "overflow-x:hidden;"
  			       "width:%" PRId32"px;"
  			       "height:%" PRId32"px;\">"
  			       , widgetwidth
  			       , widgetHeight);
  	}
  
  	// xml
  	if ( si->m_format == FORMAT_XML )
  		sb->safePrintf("<?xml version=\"1.0\" "
  			      "encoding=\"UTF-8\" ?>\n"
  			      "<response>\n" );
  
  	int64_t nowMS = gettimeofdayInMilliseconds();
  
  	// show current time
  	if ( si->m_format == FORMAT_XML ) {
  		sb->safePrintf("\t<currentTimeUTC>%" PRIu32"</currentTimeUTC>\n",
  			      (uint32_t)(nowMS/1000));
  	} 
  	else if ( st->m_header && si->m_format == FORMAT_JSON ) {
  	    sb->safePrintf("\"currentTimeUTC\":%" PRIu32",\n",
  			   (uint32_t)(nowMS/1000));
  	}
  
  	// show result validity time
  	if ( si->m_format == FORMAT_XML ) {
  		int32_t expireTimeUTC = nowMS/1000 + g_conf.m_defaultQueryResultsValidityTime;
  		sb->safePrintf("\t<expireTimeUTC>%" PRIu32"</expireTimeUTC>\n",
  			      expireTimeUTC);
  	} else if ( st->m_header && si->m_format == FORMAT_JSON ) {
  		int32_t expireTimeUTC = nowMS/1000 + g_conf.m_defaultQueryResultsValidityTime;
  		sb->safePrintf("\"expireTimeUTC\":%" PRIu32",\n",
  			   expireTimeUTC);
  	}
  
  	// show response time if not doing Quality Assurance
  	if ( si->m_format == FORMAT_XML )
  		sb->safePrintf("\t<responseTimeMS>%" PRId64"</responseTimeMS>\n",
  			      st->m_took);
  	else if ( st->m_header && si->m_format == FORMAT_JSON )
  	    sb->safePrintf("\"responseTimeMS\":%" PRId64",\n", st->m_took);
  
  	// out of memory allocating msg20s?
  	if ( st->m_errno ) {
  		log("query: Query failed. Had error processing query: %s",
  		    mstrerror(st->m_errno));
  		g_errno = st->m_errno;
  		return false;
  	}
  
  
  	if ( si->m_format == FORMAT_XML ) {
  		sb->safePrintf("\t<numResultsOmitted>%" PRId32
  			       "</numResultsOmitted>\n",
  			       msg40->m_omitCount);
  		sb->safePrintf("\t<numShardsSkipped>%" PRId32"</numShardsSkipped>\n",
  			       msg40->m_msg3a.m_skippedShards);
  		sb->safePrintf("\t<totalShards>%" PRId32"</totalShards>\n",
  			       g_hostdb.m_numShards );
  		sb->safePrintf("\t<pctSearched>%f</pctSearched>\n",
  			       msg40->m_msg3a.m_pctSearched);
  	}
  
  	if ( st->m_header && si->m_format == FORMAT_JSON ) {
  		sb->safePrintf("\"numResultsOmitted\":%" PRId32",\n",
  			       msg40->m_omitCount);
  		sb->safePrintf("\"numShardsSkipped\":%" PRId32",\n",
  			       msg40->m_msg3a.m_skippedShards);
  		sb->safePrintf("\"totalShards\":%" PRId32",\n",
  			       g_hostdb.m_numShards );
  		sb->safePrintf("\"pctSearched\":%f,\n",
  			       msg40->m_msg3a.m_pctSearched);
  	}
  
  	// save how many docs are in this collection
  	int64_t docsInColl = -1;
  	RdbBase *base = getRdbBase ( RDB_CLUSTERDB , st->m_collnum );
  
  	// estimate it
  	if ( base ) {
  		docsInColl = base->estimateNumGlobalRecs();
  	}
  
  	// include number of docs in the collection corpus
  	if ( docsInColl >= 0LL ) {
  		if ( si->m_format == FORMAT_XML) {
  			sb->safePrintf ( "\t<docsInCollection>%" PRId64"</docsInCollection>\n", docsInColl );
  		} else if ( st->m_header && si->m_format == FORMAT_JSON) {
  			sb->safePrintf("\"docsInCollection\":%" PRId64",\n", docsInColl);
  		}
  	}
  
   	int32_t numResults = msg40->getNumResults();
  	bool moreFollow = msg40->moreResultsFollow();
  	// an estimate of the # of total hits
  	int64_t totalHits = msg40->getNumTotalHits();
  	// only adjust upwards for first page now so it doesn't keep chaning
  	if ( totalHits < numResults ) totalHits = numResults;
  
  	if ( si->m_format == FORMAT_XML )
  		sb->safePrintf("\t<hits>%" PRId64"</hits>\n",(int64_t)totalHits);
  	else if ( st->m_header && si->m_format == FORMAT_JSON )
  		sb->safePrintf("\"hits\":%" PRId64",\n", (int64_t)totalHits);
  
  	if ( si->m_format == FORMAT_XML )
  		sb->safePrintf("\t<moreResultsFollow>%" PRId32
  				"</moreResultsFollow>\n"
  				,(int32_t)moreFollow);
  	else if ( st->m_header && si->m_format == FORMAT_JSON )
  		sb->safePrintf("\"moreResultsFollow\":%" PRId32",\n",
  				(int32_t)moreFollow);
  
  	// print individual query term info
  	if ( si->m_format == FORMAT_XML ) {
  		const Query *q = &si->m_q;
  		sb->safePrintf("\t<queryInfo>\n");
  		sb->safePrintf("\t\t<fullQuery><![CDATA[");
  		cdataEncode(sb, q->originalQuery());
  		sb->safePrintf("]]></fullQuery>\n");
  		sb->safePrintf("\t\t<queryLanguageAbbr>"
  			       "<![CDATA[%s]]>"
  			       "</queryLanguageAbbr>\n"
  			       , getLanguageAbbr(si->m_queryLangId) );
  		sb->safePrintf("\t\t<queryLanguage>"
  			       "<![CDATA[%s]]>"
  			       "</queryLanguage>\n"
  			       , getLanguageString(si->m_queryLangId) );
  		// print query words we ignored, like stop words
  		printIgnoredWords ( sb , si );
  
  		sb->safePrintf("\t\t<queryNumTermsTotal>"
  			       "%" PRId32
  			       "</queryNumTermsTotal>\n"
  			       , q->m_numTermsUntruncated );
  		sb->safePrintf("\t\t<queryNumTermsUsed>"
  			       "%" PRId32
  			       "</queryNumTermsUsed>\n"
  			       , q->m_numTerms );
  		int32_t tval = 0;
  		if ( q->m_numTerms < q->m_numTermsUntruncated ) tval = 1;
  		sb->safePrintf("\t\t<queryWasTruncated>"
  			       "%" PRId32
  			       "</queryWasTruncated>\n"
  			       , tval );
  
  		for ( int i = 0 ; i < q->m_numTerms ; i++ ) {
  			sb->safePrintf("\t\t<term>\n");
  			const QueryTerm *qt = &q->m_qterms[i];
  			sb->safePrintf("\t\t\t<termNum>%i</termNum>\n",i);
  			const char *term = qt->m_term;
  			sb->safePrintf("\t\t\t<termStr><![CDATA[");
  			const char *printTerm = qt->m_term;
  			if ( is_wspace_a(term[0])) printTerm++;
  			cdataEncode(sb, printTerm,qt->m_termLen);
  			sb->safePrintf("]]>"
  				       "</termStr>\n");
  			// syn?
  			const QueryTerm *sq = qt->m_synonymOf;
  			// what language did synonym come from?
  			if ( sq ) {
  				// language map from wiktionary
  				sb->safePrintf("\t\t\t<termLang>"
  					       "<![CDATA[");
  				bool first = true;
  				for ( int i = 0 ; i < langLast ; i++ ) {
  					uint64_t bit = (uint64_t)1 << i;
  					if ( ! (qt->m_langIdBits&bit))continue;
  					const char *str = getLanguageAbbr(i);
  					if ( ! first ) sb->pushChar(',');
  					first = false;
  					sb->safeStrcpy ( str );
  				}
  				sb->safePrintf("]]></termLang>\n");
  			}
  
  			if ( sq ) {
  				const char *term = sq->m_term;
  				const char *printTerm = term;
  				if ( is_wspace_a(term[0])) printTerm++;
  				sb->safePrintf("\t\t\t<synonymOf>"
  					       "<![CDATA[%*.*s]]>"
  					       "</synonymOf>\n"
  					       ,(int)sq->m_termLen,(int)sq->m_termLen,printTerm);
  			}				
  			sb->safePrintf("\t\t\t<termFreq>%" PRId64"</termFreq>\n"
  				       ,qt->m_termFreq);
  			sb->safePrintf("\t\t\t<termFreqWeight>%.2f</termFreqWeight>\n"
  				       ,qt->m_termFreqWeight);
  			sb->safePrintf("\t\t\t<termHash48>%" PRId64"</termHash48>\n"
  				       ,qt->m_termId);
  			sb->safePrintf("\t\t\t<termHash64>%" PRIu64"</termHash64>\n"
  				       ,qt->m_rawTermId);
  			const QueryWord *qw = qt->m_qword;
  			sb->safePrintf("\t\t\t<prefixHash64>%" PRIu64"</prefixHash64>\n"
  				       ,qw->m_prefixHash);
  			sb->safePrintf("\t\t</term>\n");
  		}
  		sb->safePrintf("\t</queryInfo>\n");
  	}			
  
  	// print individual query term info
  	if ( si->m_format == FORMAT_JSON && st->m_header ) {
  		const Query *q = &si->m_q;
  		sb->safePrintf("\"queryInfo\":{\n");
  		sb->safePrintf("\t\"fullQuery\":\"");
  		sb->jsonEncode(q->originalQuery());
  		sb->safePrintf("\",\n");
  		sb->safePrintf("\t\"queryLanguageAbbr\":\"");
  		sb->jsonEncode ( getLanguageAbbr(si->m_queryLangId) );
  		sb->safePrintf("\",\n");
  		sb->safePrintf("\t\"queryLanguage\":\"");
  		sb->jsonEncode ( getLanguageString(si->m_queryLangId) );
  		sb->safePrintf("\",\n");
  		
  		//If the language weights are dissimilar then print the top 5
  		//make a temporari sorted-by-weight list of languages
  		struct langweight_t {
  			lang_t lang;
  			double weight;
  		} langweights[MAX_LANGUAGES];
  		for(int i=0; i<MAX_LANGUAGES; i++) {
  			langweights[i].lang = (lang_t)i;
  			langweights[i].weight = si->m_baseScoringParameters.m_languageWeights[i];
  		}
  		std::sort(langweights, langweights+MAX_LANGUAGES, [](const langweight_t& a, const langweight_t& b) {
  			return a.weight>b.weight;
  		});
  		//if the weigths are not all identical then print the top 5
  		if(langweights[0].weight!=langweights[MAX_LANGUAGES-1].weight) {
  			sb->safePrintf("\t\"languageWeights\": [ ");
  			for(int i=0; i<7; i++) {
  				const char *l = getLanguageAbbr(langweights[i].lang);
  				sb->safePrintf("{\"lang\":\"%s\", \"weight\":%.3f}", l, langweights[i].weight);
  				if(i+1<7)
  					sb->safePrintf(", ");
  			}
  			sb->safePrintf(" ],\n");
  		}
  		
  		// print query words we ignored, like stop words
  		printIgnoredWords ( sb , si );
  
  		sb->safePrintf("\t\"queryNumTermsTotal\":"
  			       "%" PRId32",\n"
  			       , q->m_numTermsUntruncated );
  		sb->safePrintf("\t\"queryNumTermsUsed\":"
  			       "%" PRId32",\n"
  			       , q->m_numTerms );
  		int32_t tval = 0;
  		if ( q->m_numTerms < q->m_numTermsUntruncated ) tval = 1;
  		sb->safePrintf("\t\"queryWasTruncated\":"
  			       "%" PRId32",\n"
  			       , tval );
  			
  		sb->safePrintf("\t\"terms\":[\n");
  		for ( int i = 0 ; i < q->m_numTerms ; i++ ) {
  			const QueryTerm &qt = q->m_qterms[i];
  			sb->safePrintf("\t\t{\n");
  			sb->safePrintf("\t\t\t\"termNum\":%i,\n",i);
  			sb->safePrintf("\t\t\t\"termStr\":\"");
  			sb->jsonEncode(qt.m_term, qt.m_termLen);
  			sb->safePrintf("\",\n");
  			sb->safePrintf("\t\t\t\"isPhrase\":%s,\n", qt.m_isPhrase?"true":"false");
  			sb->safePrintf("\t\t\t\"termHash48\":%" PRId64",\n",   qt.m_termId);
  			sb->safePrintf("\t\t\t\"termHash64\":%" PRIu64",\n",    qt.m_rawTermId);
  			//m_termSign?
  			sb->safePrintf("\t\t\t\"termFreq\":%" PRId64",\n",      qt.m_termFreq);
  			sb->safePrintf("\t\t\t\"termFreqWeight\":%.2f,\n",      qt.m_termFreqWeight);
  			sb->safePrintf("\t\t\t\"queryStopWord\":%s,\n",         qt.m_isQueryStopWord?"true":"false");
  			sb->safePrintf("\t\t\t\"termWeight\":%.2f,\n",          qt.m_termWeight);
  			sb->safePrintf("\t\t\t\"userWeight\":%.2f,\n",          qt.m_userWeight);
  			sb->safePrintf("\t\t\t\"userNotRequired\":%s,\n",       qt.m_userNotRequired?"true":"false");
  			sb->safePrintf("\t\t\t\"ignored\":%s,\n",               qt.m_ignored?"true":"false");
  			const QueryTerm *synqt = qt.m_synonymOf;
  			if(synqt) {
  				if(qt.m_langIdBitsValid) {
  					// what language did synonym come from?
  					// language map from wiktionary
  					sb->safePrintf("\t\t\t\"termLang\":\"");
  					bool first = true;
  					for(int i = 0; i < langLast; i++) {
  						uint64_t bit = (uint64_t)1 << i;
  						if(!(qt.m_langIdBits&bit))
  							continue;
  						const char *str = getLanguageAbbr(i);
  						if(!first)
  							sb->pushChar(',');
  						first = false;
  						sb->jsonEncode(str);
  					}
  					sb->safePrintf("\",\n");
  				}
  				
  				sb->safePrintf("\t\t\t\"synonymOf\":\"");
  				sb->jsonEncode(synqt->m_term, synqt->m_termLen);
  				sb->safePrintf("\",\n");
  			}
  			const char *fieldCodeName = getFieldCodeName(qt.m_fieldCode);
  			if(fieldCodeName)
  				sb->safePrintf("\t\t\t\"fieldCodeName\":\"%s\",\n", fieldCodeName);
  			sb->safePrintf("\t\t\t\"required\":%s,\n",              qt.m_isRequired?"true":"false");
  
  			sb->safePrintf("\t\t\t\"prefixHash64\":%" PRIu64"\n", qt.m_qword->m_prefixHash);
  
  			sb->safePrintf("\t\t}");
  			// don't end last query term attr on a omma
  			if( i + 1 < q->m_numTerms)
  				sb->pushChar(',');
  			sb->pushChar('\n');
  		}
  		sb->safePrintf("\t]\n"); // end "terms":[]
  		sb->safePrintf("},\n"); //end "queryInfo":{}
  	}
  
  	if ( si->m_format == FORMAT_JSON && st->m_header ) {
  		sb->safePrintf("\"results\":[\n");
  		return true;
  	}
  
  	// debug
  	if ( si->m_debug ) {
  		logf(LOG_DEBUG,"query: Displaying up to %" PRId32" results.", numResults);
  	}
  
  	// get some result info from msg40
  	int32_t firstNum   = msg40->getFirstResultNum() ;
  
  	// numResults may be more than we requested now!
  	int32_t n = msg40->getDocsWanted();
  	if ( n > numResults )  n = numResults;
  
  	// . make the query class here for highlighting
  	// . keepAllSingles means to convert all individual words into
  	//   QueryTerms even if they're in quotes or in a connection (cd-rom).
  	//   we use this for highlighting purposes
  	Query qq;
  	qq.set(si->m_displayQuery, langUnknown, 1.0, 1.0, &si->m_word_variations_config, false,
  	       si->m_allowHighFrequencyTermCache, ABS_MAX_QUERY_TERMS);
  	//syn-todo: in the call above si->m_queryExpansion was used for both 'queryExpansion' and 'useQueryStopWords'. Why?
  
  	if ( g_errno ) return false;
  
  	DocIdScore *dpx = NULL;
  	if ( numResults > 0 ) dpx = msg40->getScoreInfo(0);
  
  	if ( si->m_format == FORMAT_XML && dpx ) {
  		float max = 0.0;
  
  		// max pairwise
  		float lw = getHashGroupWeight(HASHGROUP_INLINKTEXT);
  
  		// square that location weight
  		lw *= lw;
  
  		// assume its an inlinker's text, who has rank 15!!!
  		lw *= getLinkerWeight(MAXSITERANK);
  
  		// single weights
  		float maxtfw1 = 0.0;
  		int32_t maxi1;
  
  		// now we can have multiple SingleScores for the same term!
  		// because we take the top MAX_TOP now and add them to
  		// get the term's final score.
  		for ( int32_t i = 0 ; i< dpx->m_numSingles ; i++ ) {
  			SingleScore *ssi = &dpx->m_singleScores[i];
  			float tfwi = ssi->m_tfWeight;
  			if ( tfwi <= maxtfw1 ) continue;
  			maxtfw1 = tfwi;
  			maxi1 = i;
  		}
  		float maxtfw2 = 0.0;
  		for ( int32_t i = 0 ; i< dpx->m_numSingles ; i++ ) {
  			if ( i == maxi1 ) continue;
  			SingleScore *ssi = &dpx->m_singleScores[i];
  			float tfwi = ssi->m_tfWeight;
  			if ( tfwi <= maxtfw2 ) continue;
  			maxtfw2 = tfwi;
  		}
  		// only 1 term?
  		if ( almostEqualFloat(maxtfw2, 0.0) ) maxtfw2 = maxtfw1;
  		// best term freqs
  		max *= maxtfw1 * maxtfw2;
  		// site rank effect
  		max *= MAXSITERANK*si->m_baseScoringParameters.m_siteRankMultiplier + 1;
  		sb->safePrintf ("\t\t<theoreticalMaxFinalScore>%f"
  			       "</theoreticalMaxFinalScore>\n",
  			       max );
  	}
  	
  
  
  	// debug msg
  	log ( LOG_TIMING ,
  	     "query: Got %" PRId32" search results in %" PRId64" ms for q=%s",
  	      numResults,gettimeofdayInMilliseconds()-st->m_startTime,
  	      qq.getQuery());
  
  	st->m_qesb.nullTerm();
  
  	// encode query buf
  	const char *dq    = si->m_displayQuery;
  	if ( dq ) {
  		urlEncode(&st->m_qesb,dq);
  	}
  
  	// print it with commas into "thbuf" and null terminate it
  	char thbuf[64];
  	ulltoa ( thbuf , totalHits );
  
  	char inbuf[128];
  	ulltoa ( inbuf , docsInColl );
  
  	// otherwise, we had no error
  	if ( numResults == 0 && si->m_format == FORMAT_HTML ) {
  		sb->safePrintf ( "No results found in <b>%s</b> collection.",
  				cr->m_coll);
  	}
  	// the token is currently in the collection name so do not show that
  	else if ( numResults == 0 && 
  		  ( si->m_format == FORMAT_WIDGET_IFRAME ||
  		    si->m_format == FORMAT_WIDGET_AJAX ) ) {
  		sb->safePrintf ( "No results found. Wait for spider to "
  				 "kick in.");
  	}
  	else if ( moreFollow && si->m_format == FORMAT_HTML ) {
  		sb->safePrintf ("Results <b>%" PRId32"</b> to <b>%" PRId32"</b> of "
  			       "exactly <b>%s</b> from an index "
  			       "of about %s pages" , 
  			       firstNum + 1          ,
  			       firstNum + n          ,
  			       thbuf                 ,
  			       inbuf
  			       );
  	}
  	// otherwise, we didn't get enough results to show this page
  	else if ( si->m_format == FORMAT_HTML ) {
  		sb->safePrintf ("Results <b>%" PRId32"</b> to <b>%" PRId32"</b> of "
  			       "exactly <b>%s</b> from an index "
  			       "of about %s pages" , 
  			       firstNum + 1          ,
  			       firstNum + n          ,
  			       thbuf                 ,
  			       inbuf
  			       );
  	}
  
  	if ( si->m_format == FORMAT_HTML )
  		sb->safePrintf(" in %.02f seconds",((float)st->m_took)/1000.0);
  
  
  	//
  	// if query was a url print add url msg
  	//
  	const char *url = NULL;
  	if ( !strncmp(q,"url:"    ,4) && qlen > 4 ) url = q+4;
  	if ( !strncmp(q,"http://" ,7) && qlen > 7 ) url = q;
  	if ( !strncmp(q,"https://",8) && qlen > 8 ) url = q;
  	if ( !strncmp(q,"www."    ,4) && qlen > 4 ) url = q;
  	// find end of url
  	const char *ue = url;
  	for ( ; ue && *ue && ! is_wspace_a(*ue) ; ue++ ) ;
  	if ( numResults == 0 && si->m_format == FORMAT_HTML && url ) {
  		sb->safePrintf("<br><br>"
  			      "Could not find that url in the "
  			      "index. Try <a href=\"/addurl?u=");
  		urlEncode(sb,url,ue-url,false,false);
  		sb->safePrintf("\">Adding it.</a>");
  	}
  
  	// sometimes ppl search for "www.whatever.com" so ask them if they
  	// want to search for url:www.whatever.com
  	if ( numResults > 0  && si->m_format == FORMAT_HTML && url && url == q){
  		sb->safePrintf("<br><br>"
  			      "Did you mean to "
  			      "search for the url "
  			      "<a href=\"/search?q=url%%3A");
  		urlEncode(sb,url,ue-url,false,false);
  		sb->safePrintf("\">");
  		sb->safeMemcpy(url,ue-url);
  		sb->safePrintf("</a> itself?");
  	}
  
  
  	// is it the main collection?
  	bool isMain = false;
  	if ( collLen == 4 && strncmp ( coll, "main", 4) == 0 ) isMain = true;
  
  	// print "in collection ***" if we had a collection
  	if (collLen>0 && numResults>0 && !isMain && si->m_format==FORMAT_HTML )
  		sb->safePrintf (" in collection <b>%s</b>",coll);
  
  
  	printIgnoredWords ( sb , si );
  
  
  	if ( si->m_format == FORMAT_HTML ) {
  		sb->safePrintf("<br><br>");
  
  		sb->safePrintf("<table cellpadding=0 cellspacing=0>"
  			      "<tr><td valign=top>");
  	}
  
  	// debug
  	if ( si->m_debug )
  		logf(LOG_DEBUG,"query: Printing up to %" PRId32" results. "
  		     "bufStart=%p",
  		     numResults,
  		     sb->getBufPtr());
  
  	return true;
  }
  
  
  bool printSearchResultsTail(State0 *st) {
  	SafeBuf *sb = &st->m_sb;
  
  	SearchInput *si = &st->m_si;	
  
  	Msg40 *msg40 = &(st->m_msg40);
  
  	CollectionRec *cr = si->m_cr;
  	const char *coll = cr->m_coll;
  
  	if ( si->m_format == FORMAT_JSON ) {	
  		// remove last },\n if there and replace with just \n
  		const char *e = sb->getBufPtr() - 2;
  		if ( sb->length()>=2 &&
  		     e[0]==',' && e[1]=='\n') {
  			sb->m_length -= 2;
  			sb->safePrintf("\n");
  		}
  		// print ending ] for json search results
  		sb->safePrintf("]\n");
  
  		if ( st->m_header ) sb->safePrintf("}\n");
  
  		//////////////////////
  		// for some reason if we take too long to write out this
  		// tail we get a SIGPIPE on a firefox browser.
  		//////////////////////
  
  		// all done for json
  		return true;
  	}
  
  	// grab the query
  	const char  *q    = msg40->getQuery();
  	int32_t   qlen = msg40->getQueryLen();
  
  	HttpRequest *hr = &st->m_hr;
  
  	// get some result info from msg40
  	int32_t firstNum   = msg40->getFirstResultNum() ;
  
  	// end the two-pane table
  	if ( si->m_format == FORMAT_HTML) sb->safePrintf("</td></tr></table>");
  
  	//
  	// PRINT PREV 10 NEXT 10 links!
  	// 
  
  	// center everything below here
  	if ( si->m_format == FORMAT_HTML ) sb->safePrintf ( "<br><center>" );
  
  	int32_t remember = sb->length();
  
  	// now print "Prev X Results" if we need to
  	if ( firstNum < 0 ) firstNum = 0;
  
  	StackBuf<300> args;
  	// show banned?
  	if ( si->m_showBanned && ! si->m_isMasterAdmin )
  		args.safePrintf("&sb=1");
  	if ( ! si->m_showBanned && si->m_isMasterAdmin )
  		args.safePrintf("&sb=0");
  
  	// collection
  	args.safePrintf("&c=%s",coll);
  	// formatting info
  	if ( si->m_format == FORMAT_WIDGET_IFRAME ||
  	     si->m_format == FORMAT_WIDGET_AJAX ) {
  		args.safePrintf("&format=widget");
  		int32_t widgetwidth = hr->getLong("widgetwidth",250);
  		args.safePrintf("&widgetwidth=%" PRId32,widgetwidth);
  	}
  
  	// carry over the sites we are restricting the search results to
  	if ( si->m_sites )
  		//whiteListBuf.getBufStart());
  		args.safePrintf("&sites=%s",si->m_sites);
  
  
  	if ( si->m_format == FORMAT_HTML &&
  	     msg40->m_omitCount ) { // && firstNum == 0 ) { 
  		// . add our cgi to the original url
  		// . so if it has &qlang=de and they select &qlang=en
  		//   we have to replace it... etc.
  		StackBuf<> newUrl;
  		// show banned results
  		replaceParm2 ("sb=1",
  			      &newUrl,
  			      hr->getOrigUrlRequest(),
  			      hr->getOrigUrlRequestLen() );
  		// no deduping by summary or content hash etc.
  		StackBuf<> newUrl2;
  		replaceParm2("dr=0",&newUrl2,newUrl.getBufStart(),
  			     newUrl.length());
  		// and no site clustering
  		StackBuf<> newUrl3;
  		replaceParm2 ( "sc=0", &newUrl3 , newUrl2.getBufStart(),
  			     newUrl2.length());
  		// start at results #0 again
  		StackBuf<> newUrl4;
  		replaceParm2 ( "s=0", &newUrl4 , newUrl3.getBufStart(),
  			     newUrl3.length());
  		// show errors
  		StackBuf<> newUrl5;
  		replaceParm2 ( "showerrors=1", 
  			       &newUrl5 , 
  			       newUrl4.getBufStart(),
  			       newUrl4.length());
  		
  		
  		sb->safePrintf("<center>"
  			       "<i>"
  			       "%" PRId32" results were omitted because they "
  			       "were considered duplicates, banned, errors "
  			       "<br>"
  			       "or "
  			       "from the same site as other results. "
  			       "<a href=%s>Click here to show all results</a>."
  			       "</i>"
  			       "</center>"
  			       "<br><br>"
  			       , msg40->m_omitCount
  			       , newUrl5.getBufStart() );
  	}
  
  
  	if ( firstNum > 0 && 
  	     (si->m_format == FORMAT_HTML || 
  	      si->m_format == FORMAT_WIDGET_IFRAME //||
  	      //si->m_format == FORMAT_WIDGET_AJAX
  	      ) ) {
  		int32_t ss = firstNum - msg40->getDocsWanted();
  		
  		//sb->safePrintf("<a href=\"/search?s=%" PRId32"&q=",ss);
  		// our current query parameters
  		//sb->safeStrcpy ( st->m_qe );
  		// print other args if not zero
  		//sb->safeMemcpy ( &args );
  
  		// make the cgi parm to add to the original url
  		char nsbuf[128];
  		sprintf(nsbuf,"s=%" PRId32,ss);
  		// get the original url and add/replace in &s=xxx
  		StackBuf<> newUrl;
  		replaceParm ( nsbuf , &newUrl , hr );
  
  
  		// close it up
  		sb->safePrintf ("<a href=\"%s\"><b>"
  			       "<font size=+0>Prev %" PRId32" Results</font>"
  			       "</b></a>"
  				, newUrl.getBufStart()
  				, msg40->getDocsWanted() );
  	}
  
  	// now print "Next X Results"
  	if ( msg40->moreResultsFollow() && 
  	     (si->m_format == FORMAT_HTML || 
  	      si->m_format == FORMAT_WIDGET_IFRAME 
  	      //si->m_format == FORMAT_WIDGET_AJAX 
  	      )) {
  		int32_t ss = firstNum + msg40->getDocsWanted();
  		// print a separator first if we had a prev results before us
  		if ( sb->length() > remember ) sb->safePrintf ( " &nbsp; " );
  		// add the query
  		//sb->safePrintf ("<a href=\"/search?s=%" PRId32"&q=",ss);
  		// our current query parameters
  		//sb->safeStrcpy ( st->m_qe );
  		// print other args if not zero
  		//sb->safeMemcpy ( &args );
  
  		// make the cgi parm to add to the original url
  		char nsbuf[128];
  		sprintf(nsbuf,"s=%" PRId32,ss);
  		// get the original url and add/replace in &s=xxx
  		StackBuf<> newUrl;
  		replaceParm ( nsbuf , &newUrl , hr );
  
  		// close it up
  		sb->safePrintf("<a href=\"%s\"><b>"
  			      "<font size=+0>Next %" PRId32" Results</font>"
  			      "</b></a>"
  			       , newUrl.getBufStart()
  			       , msg40->getDocsWanted() );
  	}
  
  
  	// print try this search on...
  	// an additional <br> if we had a Next or Prev results link
  	if ( sb->length() > remember &&
  	     si->m_format == FORMAT_HTML ) 
  		sb->safeMemcpy ("<br>" , 4 ); 
  
  	//
  	// END PRINT PREV 10 NEXT 10 links!
  	// 
  
  	if ( si->m_format == FORMAT_HTML ) {
  		sb->safePrintf("<input name=c type=hidden value=\"%s\">",coll);
  	}
  
  	bool isAdmin = (si->m_isMasterAdmin || si->m_isCollAdmin);
  	if ( si->m_format != FORMAT_HTML ) isAdmin = false;
  
  	// TODO: print cache line in light gray here
  	// TODO: "these results were cached X minutes ago"
  	if ( msg40->getCachedTime() > 0 && si->m_format == FORMAT_HTML ) {
  		sb->safePrintf("<br><br><font size=1 color=707070>"
  			       "<b><center>");
  		sb->safePrintf ( " These results were cached " );
  		// this cached time is this local cpu's time
  		int32_t diff = getTime() - msg40->getCachedTime();
  		if      ( diff < 60   ) sb->safePrintf ("%" PRId32" seconds", diff );
  		else if ( diff < 2*60 ) sb->safePrintf ("1 minute");
  		else                    sb->safePrintf ("%" PRId32" minutes",diff/60);
  		sb->safePrintf ( " ago. [<a href=\"/pageCache.html\">"
  				"<font color=707070>Info</font></a>]");
  		sb->safePrintf ( "</center></font>");
  	}
  
  	
  
  	if ( si->m_format == FORMAT_XML ) {
  		sb->safePrintf("</response>\n");
  	}
  
  	if ( si->m_format == FORMAT_HTML && 
  	     cr->m_htmlTail.length() == 0 ) {
  		sb->safePrintf ( "<br>"
  				 "<center>"
  				 "<font color=gray>"
  				 "Copyright &copy; 2014. All Rights "
  				 "Reserved.<br/>"
  				 "Powered by the <a href=\"http://www."
  				 "gigablast.com/\">GigaBlast</a> open source "
  				 "search engine."
  				 "</font>"
  				 "</center>\n"
  				 "<br>\n"
  				 );
  	}
  
  
  	// if we did not use ajax, print this tail here now
  	if ( si->m_format == FORMAT_HTML ) {
  		sb->safePrintf( "</body>\n"
  				"</html>\n"
  				);
  	}
  
  	// ajax widgets will have this outside the downloaded content
  	if ( si->m_format == FORMAT_WIDGET_IFRAME ) {
  		sb->safePrintf ( "<br>"
  				"<center>"
  				"<font color=gray>"
  				 // link to edit the list of widget sites
  				 // or various other widget content properties
  				 // because we can't edit the width/height
  				 // of the widget like this.
  				 "<a href=\"/widget?inlineedit=1\">edit</a> "
  				 "&bull; "
  				 //"Copyright &copy; 2014. All Rights "
  				 //"Reserved.<br/>"
  				"Powered by <a href=http://www.diffbot.com>"
  				 "Diffbot</a>."
  				"</font>"
  				"</center>\n"
  				"</body>\n"
  				"</html>\n"
  				 );
  	}
  
  	if ( sb->length() == 0 && si->m_format == FORMAT_JSON )
  		sb->safePrintf("[]\n");
  
  	if ( sb->length() == 0 ) {
  		sb->pushChar('\n');
  		sb->nullTerm();
  	}
  
  	if ( si->m_format == FORMAT_HTML &&
  	     cr->m_htmlTail.length() &&
  	     ! expandHtml ( *sb ,
  			    cr->m_htmlTail.getBufStart(),
  			    cr->m_htmlTail.length(),
  			    q,
  			    qlen,
  			    hr,
  			    si,
  			    NULL, // method,
  			    cr) )
  			return false;
  
  	return true;
  }
  
  
  static int linkSiteRankCmp (const void *v1, const void *v2) {
  	Inlink *i1 = *(Inlink **)v1;
  	Inlink *i2 = *(Inlink **)v2;
  	return i2->m_siteRank - i1->m_siteRank;
  }
  
  static bool printInlinkText ( SafeBuf *sb , Msg20Reply *mr , SearchInput *si ,
  		       int32_t *numPrinted ) {
  	*numPrinted = 0;
  	// . show the "LinkInfo"
  	// . Msg20.cpp will have "computed" the LinkInfo if we set
  	//   Msg20Request::m_computeLinkInfo to true, but if we set
  	//   Msg20Request::m_getLinkInfo to true it will just get it
  	//   from the TitleRec, which is much faster but more stale.
  	// . "&inlinks=1" is slow and fresh, "&inlinks=2" is fast
  	//   and stale. Both are really only for BuzzLogic.
  	LinkInfo *info = (LinkInfo *)mr->ptr_linkInfo;//inlinks;
  	// Corrupted linkinfo has been seen. In that case just log an error and do not try to print any link texts
  	if(info && mr->size_linkInfo!=info->m_lisize) {
  		log(LOG_ERROR, "results: mr->size_linkInfo(%d) != info->m_lisize (%d)", mr->size_linkInfo, info->m_lisize);
  		return false;
  	}
  
  	// NULLify if empty
  	if ( mr->size_linkInfo <= 0 ) info = NULL;
  	// do not both if none
  	if ( info && ! info->m_numStoredInlinks ) info = NULL;
  	// bail?
  	if ( ! info ) return true;
  	// now sort them up
  	Inlink *ptrs[3000]; //could use #define from Msg25/linkdb.h but we don't really have to match
  	int32_t numLinks = 0;
  	for(Inlink *k = info->getNextInlink(NULL); k; k = info->getNextInlink(k)) {
  		ptrs[numLinks++] = k;
  		if ( numLinks >= 3000 ) break;
  	}
  	// sort them
  	gbsort ( ptrs , numLinks , sizeof(Inlink *) , linkSiteRankCmp );
  	// print xml starter
  	if ( si->m_format == FORMAT_XML ) sb->safePrintf("\t\t<inlinks>\n");
  	// loop through the inlinks
  	bool printedInlinkText = false;
  	bool firstTime = true;
  	int32_t inlinkId = 0;
  	int64_t  starttime = gettimeofdayInMilliseconds();
  
  	for ( int32_t i = 0 ; i < numLinks ; i++ ) {
  		Inlink *k = ptrs[i];
  		if ( ! k->getLinkText() ) continue;
  		if ( ! si->m_doQueryHighlighting && 
  		     si->m_format == FORMAT_HTML ) 
  			continue;
  		const char *str   = k->getLinkText();//ptr_linkText;
  		int32_t strLen = strnlen(k->getLinkText(),k->size_linkText);
  
  		const char *frontTag =
  		     "<font style=\"color:black;background-color:yellow\">" ;
  		const char *backTag = "</font>";
  		if ( si->m_format == FORMAT_XML ) {
  			frontTag = "<b>";
  			backTag  = "</b>";
  		}
  		if ( si->m_format == FORMAT_WIDGET_IFRAME ||
  		     si->m_format == FORMAT_WIDGET_AJAX ) {
  			frontTag = "<font style=\"background-color:yellow\">" ;
  		}
  
  		Highlight hi;
  		SafeBuf hb;
  		int32_t hlen = hi.set ( &hb, str, strLen , &si->m_hqq, frontTag, backTag );
  		if ( hlen <= 0 ) {
  			continue;
  		}
  
  		// skip it if nothing highlighted
  		if ( hi.getNumMatches() == 0 ) continue;
  
  		if ( si->m_format == FORMAT_XML ) {
  			sb->safePrintf("\t\t\t<inlink "
  				      "docId=\"%" PRId64"\" "
  				      "url=\"",
  				      k->m_docId );
  			// encode it for xml
  			sb->htmlEncode ( k->getUrl(),//ptr_urlBuf,
  					k->size_urlBuf - 1 , false );
  			sb->safePrintf("\" "
  				      //"hostId=\"%" PRIu32"\" "
  				      "firstindexed=\"%" PRIu32"\" "
  				      // not accurate!
  				      //"lastspidered=\"%" PRIu32"\" "
  				      "wordposstart=\"%" PRId32"\" "
  				      "id=\"%" PRId32"\" "
  				      "siterank=\"%" PRId32"\" "
  				      "text=\"",
  				      //hh ,
  				      //(int32_t)k->m_datedbDate,
  				      (uint32_t)k->m_firstIndexedDate,
  				      //(uint32_t)k->m_lastSpidered,
  				      (int32_t)k->m_wordPosStart,
  				      inlinkId,
  				      //linkScore);
  				      (int32_t)k->m_siteRank
  				      );
  			// HACK!!!
  			k->m_siteHash = inlinkId;
  			// inc it
  			inlinkId++;
  			// encode it for xml
  			if ( !sb->htmlEncode ( hb.getBufStart(),
  					      hb.length(),
  					      false)) 
  				return false;
  			sb->safePrintf("\"/>\n");
  			continue;
  		}
  
  
  		if ( firstTime ) {
  			sb->safePrintf("<font size=-1>");
  			sb->safePrintf("<table border=1>"
  				      "<tr><td colspan=10>"
  				      "<center>"
  				      "<b>Inlinks with Query Terms</b>"
  				      "</center>"
  				      "</td></tr>"
  				      "<tr>"
  				      "<td>Inlink Text</td>"
  				      "<td>From Site</td>"
  				      "<td>Site IP</td>"
  				      "<td>Site Rank</td>"
  				      "</tr>"
  				      );
  		}
  		firstTime = false;
  		sb->safePrintf("<tr><td>"
  			      "<a href=\"/get?c=%s&d=%" PRId64"&cnsp=0\">"
  			      //"<a href=\"/print?"
  			      //"page=7&"
  			      //"c=%s&"
  			      //"d=%" PRId64"\">"
  			      //k->getUrl());
  			      ,si->m_cr->m_coll
  			      ,k->m_docId);
  		if ( ! sb->safeMemcpy(&hb) ) return false;
  		int32_t hostLen = 0;
  		const char *host = getHostFast(k->getUrl(),&hostLen,NULL);
  		sb->safePrintf("</td><td>");
  		if ( host ) sb->safeMemcpy(host,hostLen);
  		sb->safePrintf("</td><td>");
  		char ipbuf[16];
  		sb->safePrintf("<a href=\"/search?c=%s&q=ip%%3A%s"
  			       "&n=100\">"
  			       ,si->m_cr->m_coll,iptoa(k->m_ip,ipbuf));
  		sb->safePrintf("%s</a>",iptoa(k->m_ip,ipbuf));
  		sb->safePrintf("</td><td>%" PRId32"</td></tr>"
  			       ,(int32_t)k->m_siteRank);
  		//sb->safePrintf("<br>");
  		printedInlinkText = true;
  		*numPrinted = *numPrinted + 1;
  	}
  
  	int64_t took = gettimeofdayInMilliseconds() - starttime;
          if ( took > 2 )
                  log("timing: took %" PRId64" ms to highlight %" PRId32" links."
                      ,took,numLinks);
  
  
  	// closer for xml
  	if ( si->m_format == FORMAT_XML ) sb->safePrintf("\t\t</inlinks>\n");
  	//if ( printedInlinkText ) sb->safePrintf("<br>\n");
  	if ( printedInlinkText )
  		sb->safePrintf("</font>"
  			      "</table>"
  			      "<br>");
  	return true;
  }
  
  // use this for xml as well as html
  static bool printResult(State0 *st, int32_t ix, int32_t *numPrintedSoFar) {
  	SafeBuf *sb = &st->m_sb;
  
  	HttpRequest *hr = &st->m_hr;
  
  	CollectionRec *cr = g_collectiondb.getRec ( st->m_collnum );
  	if ( ! cr ) {
  		log("query: printResult: collnum %" PRId32" gone",
  		    (int32_t)st->m_collnum);
  		return true;
  	}
  
  
  	// shortcuts
  	SearchInput *si    = &st->m_si;
  	Msg40       *msg40 = &st->m_msg40;
  
  	// ensure not all cluster levels are invisible
  	if ( si->m_debug )
  		logf(LOG_DEBUG,"query: result #%" PRId32" clusterlevel=%" PRId32,
  		     ix, (int32_t)msg40->getClusterLevel(ix));
  
  	int64_t d = msg40->getDocId(ix);
  	// this is normally a double, but cast to float
  	float docScore = (float)msg40->getScore(ix);
  	unsigned docFlags = msg40->getFlags(ix);
  
  	if ( si->m_docIdsOnly ) {
  		if ( si->m_format == FORMAT_XML ) {
  			sb->safePrintf("\t<result>\n"
  				       "\t\t<docId>%" PRId64"</docId>\n"
  				       "\t</result>\n", 
  				       d );
  		}
  		else
  		if ( si->m_format == FORMAT_JSON ) {
  			sb->safePrintf("\t{\n"
  				       "\t\t\"docId\":%" PRId64"\n"
  				       "\t},\n",
  				       d );
  		}
  		else {
  			sb->safePrintf("%" PRId64"<br/>\n", d );
  		}
  
  		// inc it
  		*numPrintedSoFar = *numPrintedSoFar + 1;
  		return true;
  	}
  
  	Msg20 *m20 = msg40->m_msg20[ix];
  
  	// get the reply
  	Msg20Reply *mr = m20->m_r;
  		
  
  	// . sometimes the msg20reply is NULL so prevent it coring
  	// . i think this happens if all hosts in a shard are down or timeout
  	//   or something
  	if ( ! mr ) {
  		if(si->m_format == FORMAT_HTML)
  			sb->safePrintf("<i>getting summary for docid %" PRId64" had error: %s</i><br>\n", d, mstrerror(m20->m_errno));
  		return true;
  	}
  
  
  	int32_t cursor = -1;
  	if ( si->m_format == FORMAT_XML  ) cursor = sb->length();
  	if ( si->m_format == FORMAT_JSON ) cursor = sb->length();
  
  	if ( si->m_format == FORMAT_XML ) 
  		sb->safePrintf("\t<result>\n" );
  
  	if ( si->m_format == FORMAT_JSON ) {
  		if ( *numPrintedSoFar != 0 ) {
  			sb->safePrintf(",\n");
  		}
  		sb->safePrintf("\t{\n" );
  	}
  
  
  	if ( mr->ptr_content && si->m_format == FORMAT_XML ) {
  		sb->safePrintf("\t\t<content><![CDATA[");
  		cdataEncode(sb, mr->ptr_content);
  		sb->safePrintf("]]></content>\n");
  	}
  		
  	if ( mr->ptr_content && si->m_format == FORMAT_JSON ) {
  		sb->safePrintf("\t\t\"content\":\"");
  		sb->jsonEncode(mr->ptr_content);
  		sb->safePrintf("\",\n");
  	}
  
  	Highlight hi;
  
  	// get the url
  	const char *url = mr->ptr_ubuf;
  	int32_t urlLen = mr->size_ubuf - 1;
  	int32_t err = mr->m_errno;
  
  	// . remove any session ids from the url
  	// . for speed reasons, only check if its a cgi url
  	Url uu;
  	uu.set( url, urlLen, false, true );
  	url    = uu.getUrl();
  	urlLen = uu.getUrlLen();
  
  	bool isAdmin = (si->m_isMasterAdmin || si->m_isCollAdmin);
  	if ( si->m_format == FORMAT_XML ) isAdmin = false;
  
  	if ( si->m_format == FORMAT_HTML ) {
  		sb->safePrintf("<table><tr><td>");
  	}
  
  	if ( si->m_showBanned ) {
  		if ( err == EDOCBANNED   ) err = 0;
  		if ( err == EDOCFILTERED ) err = 0;
  	}
  
  	// if this msg20 had an error print "had error"
  	if ( err || urlLen <= 0 || ! url ) {
  		// revert back so we do not break the json/xml
  		if ( cursor >= 0 ) sb->m_length = cursor;
  		// it's unprofessional to display this in browser
  		// so just let admin see it
  		if ( isAdmin && si->m_format == FORMAT_HTML ) {
  			sb->safePrintf("<i>docId %" PRId64" had error: %s</i><br><br>", mr->m_docId, mstrerror(err));
  		}
  		log("query: docId %" PRId64" had error: %s.", mr->m_docId,mstrerror(err));
  		// wrap it up if clustered
  		// DO NOT inc it otherwise puts a comma in there and
  		// screws up the json
  		//*numPrintedSoFar = *numPrintedSoFar + 1;
  		return true;
  	}
  
  	// if we have a thumbnail show it next to the search result,
  	// base64 encoded. do NOT do this for the WIDGET, only for search
  	// results in html/xml.
  	if ( (si->m_format == FORMAT_HTML || si->m_format == FORMAT_XML ) && si->m_showImages && mr->ptr_imgData ) {
  		ThumbnailArray *ta = (ThumbnailArray *)mr->ptr_imgData;
  		ThumbnailInfo *ti = ta->getThumbnailInfo(0);
  		if ( si->m_format == FORMAT_XML )
  			sb->safePrintf("\t\t");
  		ti->printThumbnailInHtml ( sb , 
  					   100 ,  // max width
  					   100 ,  // max height
  					   true ,  // add <a href>
  					   NULL ,
  					   " style=\"margin:10px;\" ",
  					   si->m_format );
  		if ( si->m_format == FORMAT_XML ) {
  			sb->safePrintf("\t\t<imageHeight>%" PRId32"</imageHeight>\n",
  				       ti->m_dy);
  			sb->safePrintf("\t\t<imageWidth>%" PRId32"</imageWidth>\n",
  				       ti->m_dx);
  			sb->safePrintf("\t\t<origImageHeight>%" PRId32
  				       "</origImageHeight>\n",
  				       ti->m_origDY);
  			sb->safePrintf("\t\t<origImageWidth>%" PRId32
  				       "</origImageWidth>\n",
  				       ti->m_origDX);
  			sb->safePrintf("\t\t<imageUrl><![CDATA[");
  			cdataEncode(sb, ti->getUrl());
  			sb->safePrintf("]]></imageUrl>\n");
  		}
  		if ( si->m_format == FORMAT_JSON ) {
  			sb->safePrintf("\t\t\"imageHeight\":%" PRId32",\n",
  				       ti->m_dy);
  			sb->safePrintf("\t\t\"imageWidth\":%" PRId32",\n",
  				       ti->m_dx);
  			sb->safePrintf("\t\t\"origImageHeight\":%" PRId32",\n",
  				       ti->m_origDY);
  			sb->safePrintf("\t\t\"origImageWidth\":%" PRId32",\n",
  				       ti->m_origDX);
  			sb->safePrintf("\t\t\"imageUrl\":\"");
  			sb->jsonEncode(ti->getUrl());
  			sb->safePrintf("\",\n");
  		}
  	}
  
  	bool isWide = false;
  	int32_t newdx = 0;
  
  	// print image for widget
  	if ( ( si->m_format == FORMAT_WIDGET_IFRAME ||
  	       si->m_format == FORMAT_WIDGET_AJAX ||
  	       si->m_format == FORMAT_WIDGET_APPEND ) ) {
  
  		int32_t widgetWidth = hr->getLong("widgetwidth",200);
  
  		// prevent coring
  		if ( widgetWidth < 1 ) widgetWidth = 1;
  
  		// each search result in widget has a div around it
  		sb->safePrintf("<div "
  			       "class=result "
  			       // we need the docid and score of last result
  			       // when we append new results to the end
  			       // of the widget for infinite scrolling
  			       // using the scripts in PageBasic.cpp
  			       "docid=%" PRId64" "
  			       "score=%f " // double
  			       "style=\""
  			       "width:%" PRId32"px;"
  			       "min-height:%" PRId32"px;"
  			       "height:%" PRId32"px;"
  			       "padding:%" PRId32"px;"
  			       "position:relative;"
  			       // summary overflows w/o this!
  			       "overflow-y:hidden;"
  			       "overflow-x:hidden;"
  			       "\""
  			       ">"
  			       , mr->m_docId
  			       // this is a double now. this won't work
  			       // for streaming...
  			       , msg40->m_msg3a.getScores()[ix]
  			       // subtract 8 for scrollbar on right
  			       , widgetWidth - 2*8 - 8 // padding is 8px
  			       , (int32_t)RESULT_HEIGHT
  			       , (int32_t)RESULT_HEIGHT
  			       , (int32_t)PADDING
  			       );
  		if ( mr->ptr_imgData ) {
  			ThumbnailArray *ta = (ThumbnailArray *)mr->ptr_imgData;
  			ThumbnailInfo *ti = ta->getThumbnailInfo(0);
  			// account for scrollbar on the right
  			int32_t maxWidth = widgetWidth - (int32_t)SCROLLBAR_WIDTH;
  			int32_t maxHeight = (int32_t)RESULT_HEIGHT;
  			// false = do not print <a href> link on image
  			ti->printThumbnailInHtml ( sb , 
  						   maxWidth ,
  						   maxHeight , 
  						   false , // add <a href>
  						   &newdx );
  		}
  
  		sb->safePrintf ( "<a "
  				 "target=_blank "
  				 "style=\"text-decoration:none;"
  				 // don't let scroll bar obscure text
  				 "margin-right:%" PRId32"px;"
  				 ,(int32_t)SCROLLBAR_WIDTH
  				 );
  
  		// if thumbnail is wide enough put text on top of it, otherwise
  		// image is to the left and text is to the right of image
  		if ( newdx > .5 * widgetWidth ) {
  			isWide = true;
  			sb->safePrintf("position:absolute;"
  				       "bottom:%" PRId32";"
  				       "left:%" PRId32";"
  				       , (int32_t) PADDING 
  				       , (int32_t) PADDING 
  				       );
  		}
  		// to align the text verticall we gotta make a textbox div
  		// otherwise it wraps below image! mdw
  		//else
  		//	sb->safePrintf("vertical-align:middle;");
  		else
  			sb->safePrintf("position:absolute;"
  				       "bottom:%" PRId32";"
  				       "left:%" PRId32";"
  				       , (int32_t) PADDING
  				       , (int32_t) PADDING + newdx + 10 );
  
  		// close the style and begin the url
  		sb->safePrintf( "\" "
  				"href=\"" 
  				 );
  
  		// truncate off -diffbotxyz%" PRId32"
  		int32_t newLen = urlLen;
  		// print the url in the href tag
  		sb->safeMemcpy ( url , newLen ); 
  		// then finish the a href tag and start a bold for title
  		sb->safePrintf ( "\">");
  		
  		sb->safePrintf("<b style=\""
  			       "text-decoration:none;"
  			       "font-size: 15px;"
  			       "font-weight:bold;"
  			       "background-color:rgba(0,0,0,.5);"
  			       "color:white;"
  			       "font-family:arial;"
  			       "text-shadow: 2px 2px 0 #000 "
  			       ",-2px -2px 0 #000 "
  			       ",-2px  2px 0 #000 "
  			       ", 2px -2px 0 #000 "
  			       ", 2px -2px 0 #000 "
  			       ", 0px -2px 0 #000 "
  			       ",  0px 2px 0 #000 "
  			       ", -2px 0px 0 #000 "
  			       ",  2px 0px 0 #000 "
  			       ";"
  			       "\">");
  		// then title over image
  	}
  
  	// only do link here if we have no thumbnail so no bg image
  	if ( (si->m_format == FORMAT_WIDGET_IFRAME ||
  	      si->m_format == FORMAT_WIDGET_APPEND ||
  	      si->m_format == FORMAT_WIDGET_AJAX   ) &&
  	     ! mr->ptr_imgData ) {
  		sb->safePrintf ( "<a style=text-decoration:none;"
  				 "color:white; "
  				 "href=" );
  		// truncate off -diffbotxyz%" PRId32"
  		int32_t newLen = urlLen;
  		// print the url in the href tag
  		sb->safeMemcpy ( url , newLen ); 
  		// then finish the a href tag and start a bold for title
  		sb->safePrintf ( ">");
  	}
  
  
  	// the a href tag
  	if ( si->m_format == FORMAT_HTML ) sb->safePrintf ( "\n\n" );
  
  	// then if it is banned 
  	if ( mr->m_isBanned && si->m_format == FORMAT_HTML )
  		sb->safePrintf("<font color=red><b>BANNED</b></font> ");
  
  
  	///////
  	//
  	// PRINT THE TITLE
  	//
  	///////
  
  	// the a href tag
  	if ( si->m_format == FORMAT_HTML ) {
  		sb->safePrintf ( "<a href=\"" );
  		// truncate off -diffbotxyz%" PRId32"
  		int32_t newLen = urlLen;
  		// print the url in the href tag
  		sb->safeMemcpy ( url , newLen ); 
  		// then finish the a href tag and start a bold for title
  		sb->safePrintf ( "\">");
  	}
  
  	const char  *str = nullptr;
  	int32_t strLen = 0;
  
  	// . then the title  (should be NULL terminated)
  	// . the title can be NULL
  	// . highlight it first
  	// . the title itself should not have any tags in it!
  	str = mr->ptr_tbuf;
  	strLen = mr->size_tbuf - 1;
  	if (!str || strLen < 0) {
  		strLen = 0;
  	}
  
  	std::string preferredResultLang = si->getPreferredResultLanguage();
  	std::string overriddenTitle;
  	// override title
  	if (strLen == 0 && mr->m_indexCode == EDOCDISALLOWEDROOT) {
  		overriddenTitle = g_robotsBlockedResultOverride.getTitle(preferredResultLang, uu);
  		str = overriddenTitle.c_str();
  		strLen = overriddenTitle.length();
  	}
  
  	const char *frontTag = "<font style=\"color:black;background-color:yellow\">" ;
  	const char *backTag = "</font>";
  	if ( si->m_format == FORMAT_XML ) {
  		frontTag = "<b>";
  		backTag  = "</b>";
  	}
  	if ( si->m_format == FORMAT_WIDGET_IFRAME || 
  	     si->m_format == FORMAT_WIDGET_APPEND ||
  	     si->m_format == FORMAT_WIDGET_AJAX ) {
  		frontTag = "<font style=\"background-color:yellow\">" ;
  	}
  	int32_t cols = si->m_summaryMaxWidth;
  
  	// url encode title
  	StackBuf<> tmpTitle;
  	if ( str && strLen ) {
  		tmpTitle.htmlEncode(str, strLen, false);
  	}
  
  	StackBuf<> hb;
  	if ( str && strLen && si->m_doQueryHighlighting ) {
  		hi.set ( &hb, tmpTitle.getBufStart(), tmpTitle.length(), &si->m_hqq, frontTag, backTag);
  
  		// reassign!
  		str = hb.getBufStart();
  		strLen = hb.length();
  	}
  
  	// . use "UNTITLED" if no title
  	// . msg20 should supply the dmoz title if it can
  	if ( strLen == 0 &&  si->m_format != FORMAT_XML &&  si->m_format != FORMAT_JSON ) {
  		str = "<i>UNTITLED</i>";
  		strLen = strlen(str);
  	}
  
  	if ( str && strLen &&
  	     ( si->m_format == FORMAT_HTML ||
  	       si->m_format == FORMAT_WIDGET_IFRAME ||
  	       si->m_format == FORMAT_WIDGET_APPEND ||
  	       si->m_format == FORMAT_WIDGET_AJAX ) 
  	     ) {
  		// determine if TiTle wraps, if it does add a <br> count for each wrap
  		if ( !sb->brify( str, strLen, cols ) ) {
  			return false;
  		}
  	}
  
  	// close up the title tag
  	if ( si->m_format == FORMAT_XML ) {
  		sb->safePrintf("\t\t<title><![CDATA[");
  		if ( str ) {
  			cdataEncode(sb, str);
  		}
  		sb->safePrintf("]]></title>\n");
  	}
  
  	if ( si->m_format == FORMAT_JSON ) {
  		sb->safePrintf("\t\t\"title\":\"");
  		if ( str ) {
  			sb->jsonEncode(str);
  		}
  		sb->safePrintf("\",\n");
  	}
  
  
  	if ( si->m_format == FORMAT_HTML ) 
  		sb->safePrintf ("</a><br>\n" ) ;
  
  
  	// close the title tag stuf
  	if ( si->m_format == FORMAT_WIDGET_IFRAME ||
  	     si->m_format == FORMAT_WIDGET_APPEND ||
  	     si->m_format == FORMAT_WIDGET_AJAX ) 
  		sb->safePrintf("</b></a>\n");
  
  	//
  	// print <h1> tag contents. hack for client.
  	//
  	char *hp = mr->ptr_htag;
  	char *hpend = hp + mr->size_htag;
  	for ( ; hp && hp < hpend ; ) {
  		if ( si->m_format == FORMAT_XML ) {
  			sb->safePrintf("\t\t<h1Tag><![CDATA[");
  			cdataEncode(sb, hp);
  			sb->safePrintf("]]></h1Tag>\n");
  		}
  		if ( si->m_format == FORMAT_JSON ) {
  			sb->safePrintf("\t\t\"h1Tag\":\"");
  			sb->jsonEncode(hp);
  			sb->safePrintf("\",\n");
  		}
  		// it is a \0 separated list of headers generated from XmlDoc::getHeaderTagBuf()
  		hp += strlen(hp) + 1;
  	}
  
  	// print the [cached] link?
  	bool printCached;
  	if ( mr->m_contentLen <= 0 )
  		printCached = false; //nothing to show
  	else if ( isAdmin )
  		printCached = true; //admin can bypass noarchive tag
  	else if( mr->m_noArchive )
  		printCached = false; //page doesn't want to be archived. honour that.
  	else
  		printCached = true;
  
  	/////
  	//
  	// print content type after title
  	//
  	/////
  	unsigned char ctype = mr->m_contentType;
  	const char *cs = g_contentTypeStrings[ctype];
  
  	if ( si->m_format == FORMAT_XML )
  		sb->safePrintf("\t\t<contentType><![CDATA[%s]]></contentType>\n", cs);
  
  	if ( si->m_format == FORMAT_JSON )
  		sb->safePrintf("\t\t\"contentType\":\"%s\",\n",cs);
  
  	if ( si->m_format == FORMAT_HTML && ctype != CT_HTML && ctype != CT_UNKNOWN ) {
  		sb->safePrintf(" <b><font style=color:white;background-color:maroon;>");
  		const char *p = cs;
  		for ( ; *p ; p++ ) {
  			char c = to_upper_a(*p);
  			sb->pushChar(c);
  		}
  		sb->safePrintf("</font></b> &nbsp;");
  	}
  	
  
  	////////////
  	//
  	// print the summary
  	//
  	////////////
  
  	// do the normal summary
  	str = mr->ptr_displaySum;
  
  	// sometimes the summary is longer than requested because for
  	// summary deduping purposes (see "pss" parm in Parms.cpp) we do not
  	// get it as short as requested. so use mr->m_sumPrintSize here
  	// not mr->size_sum
  	strLen = mr->size_displaySum - 1;
  
  	// this includes the terminating \0 or \0\0 so back up
  	if ( strLen < 0 ) {
  		strLen  = 0;
  	}
  
  	// override summary
  	std::string overriddenSummary;
  	if (strLen == 0 && mr->m_indexCode == EDOCDISALLOWEDROOT) {
  		overriddenSummary = g_robotsBlockedResultOverride.getSummary(preferredResultLang, uu);
  		str = overriddenSummary.c_str();
  		strLen = overriddenSummary.length();
  	}
  
  	bool printSummary = true;
  
  	// do not print summaries for widgets by default unless overridden with &summary=1
  	int32_t defSum = 0;
  
  	// if no image then default the summary to on
  	if ( ! mr->ptr_imgData ) {
  		defSum = 1;
  	}
  
  	if ( (si->m_format == FORMAT_WIDGET_IFRAME ||
  	      si->m_format == FORMAT_WIDGET_APPEND ||
  	      si->m_format == FORMAT_WIDGET_AJAX ) && 
  	     hr->getLong("summaries",defSum) == 0 ) {
  		printSummary = false;
  	}
  
  	if ( printSummary &&
  	     (si->m_format == FORMAT_WIDGET_IFRAME ||
  	      si->m_format == FORMAT_WIDGET_APPEND ||
  	      si->m_format == FORMAT_WIDGET_AJAX ) ) {
  		int32_t sumLen = strLen;
  		if ( sumLen > 150 ) sumLen = 150;
  		if ( sumLen ) {
  			sb->safePrintf("<br>");
  			sb->safeTruncateEllipsis ( str , sumLen );
  		}
  	}
  
  	if ( si->m_format == FORMAT_HTML ) {
  		if ( printSummary ) {
  			sb->brify( str, strLen, cols );
  		}
  
  		// new line if not xml. even summary is empty we need it too like
  		// when showing xml docs - MDW 9/28/2014
  		sb->safePrintf( "<br>\n" );
  	}
  	else
  	if ( si->m_format == FORMAT_XML ) {
  		sb->safePrintf( "\t\t<sum><![CDATA[" );
  		cdataEncode(sb, str);
  		sb->safePrintf( "]]></sum>\n" );
  	}
  	else
  	if ( si->m_format == FORMAT_JSON ) {
  		sb->safePrintf( "\t\t\"sum\":\"" );
  		sb->jsonEncode( str );
  		sb->safePrintf( "\",\n" );
  	}
  
  	/////////
  	// 
  	// meta tag values for &dt=keywords ...
  	//
  	/////////
  	if ( mr->ptr_dbuf && mr->size_dbuf > 1 )
  		printMetaContent ( msg40 , ix,st,sb);
  
  	////////////
  	//
  	// print the URL
  	//
  	////////////
  
  	StackBuf<> tmpBuf;
  	char* displayUrl = Url::getDisplayUrl(url, &tmpBuf);
  	uint32_t displayUrlLen = tmpBuf.length();
  
  	// hack off the http:// if any for displaying it on screen
  	if ( displayUrlLen > 8 && strncmp ( displayUrl , "http://" , 7 )==0 ) {
  		displayUrl += 7; displayUrlLen -= 7; }
  
  	// . remove trailing /
  	// . only remove from root urls in case user cuts and 
  	//   pastes it for link: search
  	if ( displayUrl [ displayUrlLen - 1 ] == '/' ) {
  		// see if any other slash before us
  		int32_t j;
  		for ( j = displayUrlLen - 2 ; j >= 0 ; j-- )
  			if ( displayUrl[j] == '/' ) break;
  		// if there wasn't, we must have been a root url
  		// so hack off the last slash
  		if ( j < 0 ) displayUrlLen--;
  	}
  
  	if ( si->m_format == FORMAT_HTML ) {
  		sb->safePrintf ("<font color=gray>" );
  		//sb->htmlEncode ( url , strlen(url) , false );
  		// 20 for the date after it
  		sb->safeTruncateEllipsis ( displayUrl , 50 ); // cols - 30 );
  		// turn off the color
  		sb->safePrintf ( "</font>\n" );
  	}
  
  	// print url for widgets now
  	if ( (si->m_format == FORMAT_WIDGET_IFRAME ||
  	      si->m_format == FORMAT_WIDGET_APPEND ||
  	      si->m_format == FORMAT_WIDGET_AJAX ) ) {
  		//sb->safePrintf ("<br><font color=gray size=-1>" );
  		// print url for widgets in top left if we have a wide image
  		// otherwise it gets truncated below the title for some reason
  		if ( isWide )
  			sb->safePrintf ("<br><font color=white size=-1 "
  					"style=position:absolute;left:10px;"
  					"top:10px;background-color:black;>" );
  		else if ( mr->ptr_imgData )
  			sb->safePrintf ("<br><font color=gray size=-1 "
  					"style=position:absolute;left:%" PRId32"px;"
  					"top:10px;>"
  				       , (int32_t) PADDING + newdx + 10 );
  		else
  			sb->safePrintf ("<br><font color=gray size=-1>");
  		// print the url now, truncated to 50 chars
  		sb->safeTruncateEllipsis ( url , 50 ); // cols - 30 );
  		sb->safePrintf ( "</font>\n" );
  	}
  
  
  	if ( si->m_format == FORMAT_XML ) {
  		sb->safePrintf("\t\t<url><![CDATA[");
  		sb->safeMemcpy ( displayUrl , displayUrlLen );
  		sb->safePrintf("]]></url>\n");
  	}
  	if ( si->m_format == FORMAT_JSON ) {
  		sb->safePrintf("\t\t\"url\":\"");
  		sb->jsonEncode ( displayUrl , displayUrlLen );
  		sb->safePrintf("\",\n");
  	}
  
  	// now the last spidered date of the document
  	time_t ts = mr->m_lastSpidered;
  	if(si->m_format == FORMAT_HTML && ts>0) {
  		sb->safePrintf(" - indexed: ");
  		time_t now = getTime();
  		printTimeAgo(sb, now-ts, now, false);
  	}
  
  	// the date it was last modified
  	ts = mr->m_lastModified;
  	if(si->m_format == FORMAT_HTML && ts>0) {
  		sb->safePrintf(" - modified: ");
  		time_t now = getTime();
  		printTimeAgo(sb, now-ts, now, false);
  	}
  
  
  	//
  	// more xml stuff
  	//
  	if ( si->m_format == FORMAT_XML ) {
  		// doc size in Kilobytes
  		sb->safePrintf ("\t\t<size><![CDATA[%4.0fk]]></size>\n", (float)mr->m_contentLen/1024.0);
  		sb->safePrintf ( "\t\t<sizeInBytes>%" PRId32"</sizeInBytes>\n", mr->m_contentLen);
  		// . docId for possible cached link
  		// . might have merged a bunch together
  		sb->safePrintf("\t\t<docId>%" PRId64"</docId>\n",mr->m_docId );
  		sb->safePrintf("\t\t<docScore>%f</docScore>\n",docScore);
  
  		// . show the site root
  		// . for hompages.com/users/fred/mypage.html this will be
  		//   homepages.com/users/fred/
  		// . for www.xyz.edu/~foo/burp/ this will be
  		//   www.xyz.edu/~foo/ etc.
  		// seems like this isn't the way to do it, cuz Tagdb.cpp
  		// adds the "site" tag itself and we do not always have it
  		// in the XmlDoc::ptr_tagRec... so do it this way:
  		const char *site    = mr->ptr_site;
  		int32_t siteLen = mr->size_site-1;
  		//char *site=uu.getSite( &siteLen , si->m_coll, false, tagRec);
  		sb->safePrintf("\t\t<site><![CDATA[");
  		if ( site && siteLen > 0 ) sb->safeMemcpy ( site , siteLen );
  		sb->safePrintf("]]></site>\n");
  		//int32_t sh = hash32 ( site , siteLen );
  		//sb->safePrintf ("\t\t<siteHash32>%" PRIu32"</siteHash32>\n",sh);
  		//int32_t dh = uu.getDomainHash32 ();
  		//sb->safePrintf ("\t\t<domainHash32>%" PRIu32"</domainHash32>\n",dh);
  		// spider date
  		sb->safePrintf ( "\t\t<spidered>%" PRIu32"</spidered>\n", (uint32_t)mr->m_lastSpidered);
  		// backwards compatibility for buzz
  		sb->safePrintf ( "\t\t<firstIndexedDateUTC>%" PRIu32 "</firstIndexedDateUTC>\n", (uint32_t)mr->m_firstIndexedDate);
  		sb->safePrintf( "\t\t<contentHash32>%" PRIu32 "</contentHash32>\n", (uint32_t)mr->m_contentHash32);
  		// pub date
  		int32_t datedbDate = mr->m_datedbDate;
  		// show the datedb date as "<pubDate>" for now
  		if ( datedbDate != -1 )
  			sb->safePrintf ( "\t\t<pubdate>%" PRIu32"</pubdate>\n", (uint32_t)datedbDate);
  	}
  
  	if ( si->m_format == FORMAT_JSON ) {
  		// doc size in Kilobytes
  		sb->safePrintf ( "\t\t\"size\":\"%4.0fk\",\n", (float)mr->m_contentLen/1024.0);
  		sb->safePrintf ( "\t\t\"sizeInBytes\":%" PRId32 ",\n", mr->m_contentLen);
  		// . docId for possible cached link
  		// . might have merged a bunch together
  		sb->safePrintf("\t\t\"docId\":%" PRId64",\n", mr->m_docId );
  		sb->safePrintf("\t\t\"docScore\":%f,\n", docScore);
  		sb->safePrintf("\t\t\"flags\":%u,\n", docFlags);
  		sb->safePrintf("\t\t\"cacheAvailable\":%s,\n", printCached?"true":"false");
  		sb->safePrintf("\t\t\"isAdult\":%s,\n", mr->m_isAdult?"true":"false");
  
  		// . show the site root
  		// . for hompages.com/users/fred/mypage.html this will be
  		//   homepages.com/users/fred/
  		// . for www.xyz.edu/~foo/burp/ this will be
  		//   www.xyz.edu/~foo/ etc.
  		// seems like this isn't the way to do it, cuz Tagdb.cpp
  		// adds the "site" tag itself and we do not always have it
  		// in the XmlDoc::ptr_tagRec... so do it this way:
  		const char *site = mr->ptr_site;
  		int32_t siteLen = mr->size_site-1;
  		//char *site=uu.getSite( &siteLen , si->m_coll, false, tagRec);
  		sb->safePrintf("\t\t\"site\":\"");
  		if ( site && siteLen > 0 ) sb->safeMemcpy ( site , siteLen );
  		sb->safePrintf("\",\n");
  		// spider date
  		sb->safePrintf ( "\t\t\"spidered\":%" PRIu32 ",\n", (uint32_t)mr->m_lastSpidered);
  		// backwards compatibility for buzz
  		sb->safePrintf ( "\t\t\"firstIndexedDateUTC\":%" PRIu32 ",\n", (uint32_t) mr->m_firstIndexedDate);
  		sb->safePrintf( "\t\t\"contentHash32\":%" PRIu32 ",\n", (uint32_t)mr->m_contentHash32);
  		// pub date
  		int32_t datedbDate = mr->m_datedbDate;
  		// show the datedb date as "<pubDate>" for now
  		if ( datedbDate != -1 )
  			sb->safePrintf ( "\t\t\"pubdate\":%" PRIu32 ",\n", (uint32_t)datedbDate);
  	}
  
  
  	// . we also store the outlinks in a linkInfo structure
  	// . we can call LinkInfo::set ( Links *outlinks ) to set it
  	//   in the msg20
  	LinkInfo *outlinks = (LinkInfo *)mr->ptr_outlinks;
  	// NULLify if empty
  	if ( mr->size_outlinks <= 0 ) outlinks = NULL;
  	// only for xml for now
  	if ( si->m_format == FORMAT_HTML ) outlinks = NULL;
  	Inlink *k = NULL;
  	// do we need absScore2 for outlinks?
  	while ( outlinks &&
  		(k =outlinks->getNextInlink(k))) 
  		// print it out
  		sb->safePrintf("\t\t<outlink docId=\"%" PRId64"\" hostId=\"%" PRIu32"\" indexed=\"%" PRId32"\" pubdate=\"%" PRId32"\" ",
  			k->m_docId ,
  			(uint32_t)k->m_ip,//hostHash, but use ip for now
  			(int32_t)k->m_firstIndexedDate ,
  			(int32_t)k->m_datedbDate );
  
  	if ( si->m_format == FORMAT_XML ) {
  		// result
  		sb->safePrintf("\t\t<language><![CDATA[%s]]></language>\n", getLanguageString(mr->m_language));
  		sb->safePrintf("\t\t<langAbbr>%s</langAbbr>\n", getLanguageAbbr(mr->m_language));
  	}
  
  	if ( si->m_format == FORMAT_JSON ) {
  		// result
  		sb->safePrintf("\t\t\"language\":\"%s\",\n", getLanguageString(mr->m_language));
  		sb->safePrintf("\t\t\"langAbbr\":\"%s\",\n", getLanguageAbbr(mr->m_language));
  	}
  
  	//
  	// end more xml stuff
  	//
  
  	
  	if ( si->m_format == FORMAT_HTML ) {
  		int32_t lang = mr->m_language;
  		if ( lang ) sb->safePrintf(" - %s",getLanguageString(lang));
  
  		sb->safePrintf("<br>\n");
  	}
  
  	// if searching multiple collections  - federated search
  	CollectionRec *scr = g_collectiondb.getRec ( mr->m_collnum );
  	const char *coll = "UNKNOWN";
  	if ( scr ) coll = scr->m_coll;
  
  	if ( si->m_format == FORMAT_HTML && printCached ) {
  		sb->safePrintf ( "<a href=\"/get?q=%s&qlang=%s&c=%s&d=%" PRId64 "&cnsp=0\">cached</a>\n",
  			st->m_qesb.getBufStart() ,
  			si->m_defaultSortLang,		// "qlang" parm
  			coll ,
  			mr->m_docId );
  	}
  
  	// unhide the divs on click
  	int32_t placeHolder = -1;
  	int32_t placeHolderLen = 0;
  	if ( si->m_format == FORMAT_HTML && si->m_getDocIdScoringInfo ) {
  		// place holder for backlink table link
  		placeHolder = sb->length();
  		sb->safePrintf (" - <a onclick="
  			       "\""
  			       "var e = document.getElementById('bl%" PRId32"');"
  			       "if ( e.style.display == 'none' ){"
  			       "e.style.display = '';"
  			       "}"
  			       "else {"
  			       "e.style.display = 'none';"
  			       "}"
  			       "\""
  			       " "
  			       "style="
  			       "cursor:hand;"
  			       "cursor:pointer;"
  			       "color:blue;>"
  			       "<u>00000 backlinks</u>"
  			       "</a>\n"
  			       , ix 
  			       );
  		placeHolderLen = sb->length() - placeHolder;
  
  		// unhide the scoring table on click
  		sb->safePrintf (" - <a onclick="
  			       "\""
  			       "var e = document.getElementById('sc%" PRId32"');"
  			       "if ( e.style.display == 'none' ){"
  			       "e.style.display = '';"
  			       "}"
  			       "else {"
  			       "e.style.display = 'none';"
  			       "}"
  			       "\""
  			       " "
  			       "style="
  			       "cursor:hand;"
  			       "cursor:pointer;"
  			       "color:blue;>"
  			       "scoring"
  			       "</a>\n"
  			       ,ix
  			       );
  	}
  
  	if ( si->m_format == FORMAT_HTML ) {
  		// reindex
  		sb->safePrintf(" - <a style=color:blue; href=\"/addurl?urls=");
  		urlEncode(sb, url, strlen(url), false);
  		uint64_t rand64 = gettimeofdayInMilliseconds();
  		sb->safePrintf("&c=%s&rand64=%" PRIu64"\">respider</a>\n", coll,rand64);
  
  		sb->safePrintf (" - "
  				"<a style=color:blue; "
  				"href=\"/search?sb=1&c=%s&"
  				"q=gbfieldmatch%%3AgbssUrl%%3A"
  				, coll 
  				);
  		// do not include ending \0
  		urlEncode(sb, mr->ptr_ubuf, mr->size_ubuf-1, false);
  		sb->safePrintf("\">spider info</a>\n");
  
  
  		//
  		// show rainbow sections link
  		//
  
  		sb->safePrintf ( " - <a style=color:blue; href=\""
  				 "/get?"
  				 "page=4&"	// show rainbow sections
  				 "q=%s&"
  				 "qlang=%s&"
  				 "c=%s&"
  				 "d=%" PRId64"&"
  				 "cnsp=0\">"
  				 "sections</a>\n", 
  				 st->m_qesb.getBufStart() , 
  				 si->m_defaultSortLang,	// "qlang" parm
  				 coll , 
  				 mr->m_docId ); 
  
  		sb->safePrintf ( " - <a style=color:blue; href=\""
  				 "/get?"
  				 "page=1&"	// show rainbow sections
  				 //"q=%s&"
  				 //"qlang=%s&"
  				 "c=%s&"
  				 "d=%" PRId64"&"
  				 "cnsp=0\">"
  				 "page info</a>\n", 
  				 //st->m_qe , 
  				 // "qlang" parm
  				 //si->m_defaultSortLang,
  				 coll , 
  				 mr->m_docId ); 
  
  		sb->safePrintf ( " - <a style=color:blue; href=\""
  				 "/get?"
  				 "page=5&"	// show rainbow sections
  				 //"q=%s&"
  				 //"qlang=%s&"
  				 "c=%s&"
  				 "d=%" PRId64"&"
  				 "cnsp=0\">"
  				 "term info</a>\n", 
  				 //st->m_qe , 
  				 // "qlang" parm
  				 //si->m_defaultSortLang,
  				 coll , 
  				 mr->m_docId ); 
  
  		char ipbuf1[16];
  		char ipbuf2[16];
  		sb->safePrintf(//"<br>"
  			      " - <a style=color:blue; href=\"/search?"
  			      "c=%s&sc=1&dr=0&q=ip:%s&"
  			      "n=100&usecache=0\">%s</a>\n",
  			      coll,iptoa(mr->m_ip,ipbuf1), iptoa(mr->m_ip,ipbuf2) );
  		// ip domain link
  		unsigned char *us = (unsigned char *)&mr->m_ip;//urlip;
  		sb->safePrintf (" - <a style=color:blue; "
  				"href=\"/search?c=%s&sc=1&dr=0&n=100&"
  				"q=ip:%" PRId32".%" PRId32".%" PRId32"&"
  				"usecache=0\">%" PRId32".%" PRId32".%" PRId32"</a>\n",
  				coll,
  				(int32_t)us[0],(int32_t)us[1],(int32_t)us[2],
  				(int32_t)us[0],(int32_t)us[1],(int32_t)us[2]);
  	}
  
  	char dbuf [ MAX_URL_LEN ];
  	int32_t dlen = uu.getDomainLen();
  
  	if ( si->m_format == FORMAT_HTML ) {
  		memcpy ( dbuf , uu.getDomain() , dlen );
  		dbuf [ dlen ] = '\0';
  		// newspaperarchive urls have no domain
  		if ( dlen == 0 ) {
  			dlen = uu.getHostLen();
  			memcpy ( dbuf , uu.getHost() , dlen );
  			dbuf [ dlen ] = '\0';
  		}
  
  		// admin always gets the site: option so he can ban
  		sb->safePrintf (" - "
  			       " <a style=color:blue; href=\"/search?"
  			       "q=site%%3A%s&sc=0&c=%s\">"
  			       "domain</a>\n" ,
  				dbuf ,
  				coll );
  	}
  
  
  	if ( si->m_format == FORMAT_HTML && si->m_doSiteClustering ) {
  		char hbuf [ MAX_URL_LEN ];
  		int32_t hlen = uu.getHostLen();
  		memcpy ( hbuf , uu.getHost() , hlen );
  		hbuf [ hlen ] = '\0';
  
  		// make the cgi parm to add to the original url
  		StackBuf<512> qq;
  		qq.safePrintf("q=");
  		urlEncode(&qq, "site:");
  		urlEncode(&qq, hbuf);
  		urlEncode(&qq, " | ");
  		qq.safeStrcpy(st->m_qesb.getBufStart());
  		qq.nullTerm();
  		// get the original url and add/replace in query
  		StackBuf<512> newUrl;
  		replaceParm ( qq.getBufStart() , &newUrl , hr );
  		// put show more results from this site link
  		sb->safePrintf (" - <nobr><a href=\"%s\">"
  			       "more from this site</a></nobr>"
  				, newUrl.getBufStart()
  				);
  	}
  
  
  	if (si->m_format == FORMAT_HTML && isAdmin ){
  		const char *un = "";
  		int32_t  banVal = 1;
  		if ( mr->m_isBanned ) {
  			un = "UN";
  			banVal = 0;
  		}
  		// don't put on a separate line because then it is too
  		// easy to mis-click on it
  		sb->safePrintf(//"<br>"
  			       " - "
  			      " <a style=color:green; href=\"/admin/tagdb?"
  			      "user=admin&"
  			      "tagtype0=manualban&"
  			      "tagdata0=%" PRId32"&"
  			      "u=%s&c=%s\">"
  			      "<nobr>%sBAN %s"
  			      "</nobr></a>\n"
  			      , banVal
  			      , dbuf
  			      , coll
  			      , un
  			      , dbuf );
  		dlen = uu.getHostLen();
  		memcpy ( dbuf , uu.getHost() , dlen );
  		dbuf [ dlen ] = '\0';
  		sb->safePrintf(" - "
  			      " <a style=color:green; href=\"/admin/tagdb?"
  			      "user=admin&"
  			      "tagtype0=manualban&"
  			      "tagdata0=%" PRId32"&"
  			      "u=%s&c=%s\">"
  			      "<nobr>%sBAN %s</nobr></a>\n"
  			      , banVal
  			      , dbuf
  			      , coll
  			      , un
  			      , dbuf
  			      );
  	}
  
  	if ( mr->size_metadataBuf && si->m_format == FORMAT_JSON) {
  		sb->safePrintf("\t\t\"metadata\":[");
  		//sb->safeMemcpy(mr->ptr_metadataBuf, mr->size_metadataBuf);
  		sb->safeStrcpy(mr->ptr_metadataBuf);
  		// without this \n we seem to lose our ] i guess it gets
  		// backed up over
  		sb->safePrintf("],\n");
  	}
  
  
  	if ( mr->size_metadataBuf && si->m_format == FORMAT_HTML) {
  		sb->safePrintf("<br>");
  
  		Json md;
  		JsonItem *ji = md.parseJsonStringIntoJsonItems(mr->ptr_metadataBuf);
  		StackBuf<1024> nameBuf;
  		for ( ; ji ; ji = ji->m_next ) {
  			if(ji->isInArray()) continue;
  			if(ji->m_type == JT_ARRAY) continue;
  			ji->getCompoundName ( nameBuf ) ;
  			if(nameBuf.length() == 0) {
  				continue;
  			}
  			//nameBuf.replaceChar('-', '_');
  			nameBuf.nullTerm();
  
  			int32_t valLen;
  			const char* valBuf = ji->getValueAsString(&valLen);
  			StackBuf<1024> queryBuf;
  			// log("compound name is %s %d %d",nameBuf.getBufStart(),
  			// nameBuf.length(), valLen);
  
  			queryBuf.safePrintf("/search?q=%s:%%22",nameBuf.getBufStart());
  			urlEncode(&queryBuf, valBuf, valLen);
  			queryBuf.safePrintf("%%22&c=%s",coll);
  			queryBuf.nullTerm();
  			sb->safePrintf(" - <a href=\"%s\">%s:\"", queryBuf.getBufStart(),
  						   nameBuf.getBufStart());
  			sb->safeMemcpy(valBuf, valLen);
  			sb->safeStrcpy("\"</a>");
  		}
  	}
  
  
  	// end serp div
  	if ( si->m_format == FORMAT_WIDGET_IFRAME ||
  	     si->m_format == FORMAT_WIDGET_APPEND ||
  	     si->m_format == FORMAT_WIDGET_AJAX )
  		sb->safePrintf("</div><hr>");
  	
  
  	if ( si->m_format == FORMAT_HTML )
  		sb->safePrintf ( "<br><br>\n");
  
  	// search result spacer
  	if ( si->m_format == FORMAT_WIDGET_IFRAME ||
  	     si->m_format == FORMAT_WIDGET_APPEND ||
  	     si->m_format == FORMAT_WIDGET_AJAX   )
  		sb->safePrintf("<div style=line-height:%" PRId32"px;><br></div>",
  			       (int32_t)SERP_SPACER);
  
  
  	// inc it
  	*numPrintedSoFar = *numPrintedSoFar + 1;
  
  	// done?
  	DocIdScore *dp = msg40->getScoreInfo(ix);
  	if ( ! dp ) {
  		if ( si->m_format == FORMAT_XML ) 
  			sb->safePrintf ("\t</result>\n\n");
  		if ( si->m_format == FORMAT_JSON ) {
  			// remove last ,\n
  			sb->m_length -= 2;
  			sb->safePrintf ("\n\t}\n");
  		}
  		// wtf?
  		//g_process.shutdownAbort(true);
  		// at least close up the table
  		if ( si->m_format != FORMAT_HTML ) return true;
  
  		sb->safePrintf("</table>\n");
  		//blank out the "xxxxx backlinks" part
  		memset(sb->getBufStart() + placeHolder, ' ', placeHolderLen);
  
  		return true;
  	}
  
  
  	//
  	// scoring info tables
  	//
  
  	int32_t nr = dp->m_numRequiredTerms;
  	if ( nr == 1 ) nr = 0;
  	// print breakout tables here for distance matrix
  	// final score calc
  	StackBuf<1024> ft;
  
  	// put in a hidden div so you can unhide it
  	if ( si->m_format == FORMAT_HTML )
  		sb->safePrintf("<div id=bl%" PRId32" style=display:none;>\n", ix );
  
  	// print xml and html inlinks
  	int32_t numInlinks = 0;
  	printInlinkText ( sb , mr , si , &numInlinks );
  
  
  	if ( si->m_format == FORMAT_HTML ) {
  		sb->safePrintf("</div>");
  		sb->safePrintf("<div id=sc%" PRId32" style=display:none;>\n", ix );
  	}
  
  
  	// if pair changes then display the sum
  	int32_t lastTermNum1 = -1;
  	int32_t lastTermNum2 = -1;
  
  	float minScore = -1;
  
  	// display all the PairScores
  	for ( int32_t i = 0 ; i < dp->m_numPairs ; i++ ) {
  		float totalPairScore = 0.0;
  		// print all the top winners for this pair
  		PairScore *fps = &dp->m_pairScores[i];
  		// if same combo as last time skip
  		if ( fps->m_qtermNum1 == lastTermNum1 &&
  		     fps->m_qtermNum2 == lastTermNum2 )
  			continue;
  		lastTermNum1 = fps->m_qtermNum1;
  		lastTermNum2 = fps->m_qtermNum2;
  		bool firstTime = true;
  		// print all pairs for this combo
  		for ( int32_t j = i ; j < dp->m_numPairs ; j++ ) {
  			// get it
  			PairScore *ps = &dp->m_pairScores[j];
  			// stop if different pair now
  			if ( ps->m_qtermNum1 != fps->m_qtermNum1 ) break;
  			if ( ps->m_qtermNum2 != fps->m_qtermNum2 ) break;
  			// skip if 0. neighborhood terms have weight of 0 now
  			if ( almostEqualFloat(ps->m_finalScore, 0.0) ) continue;
  			// first time?
  			if ( firstTime && si->m_format == FORMAT_HTML ) {
  				const Query *q = &si->m_q;
  				printTermPairs ( sb , q , ps );
  				printScoresHeader ( sb );
  				firstTime = false;
  			}
  			// print it
  			printPairScore ( sb , si , ps , mr );
  
  			// add it up
  			totalPairScore += ps->m_finalScore;
  		}
  		if ( ft.length() ) ft.safePrintf(" , ");
  		ft.safePrintf("%f",totalPairScore);
  		// min?
  		if ( minScore < 0.0 || totalPairScore < minScore )
  			minScore = totalPairScore;
  		// we need to set "ft" for xml stuff below
  		if ( si->m_format != FORMAT_HTML ) continue;
  
  		sb->safePrintf("<tr><td><b>%.04f</b></td><td colspan=20>total of above scores</td></tr>", totalPairScore);
  		// close table from printScoresHeader
  		if ( ! firstTime ) sb->safePrintf("</table><br>");
  	}
  
  	int32_t lastTermNum = -1;
  
  	int32_t numSingles = dp->m_numSingles;
  	// do not print this if we got pairs
  	if ( dp->m_numPairs ) numSingles = 0;
  
  	for ( int32_t i = 0 ; i < numSingles ; i++ ) {
  		float totalSingleScore = 0.0;
  		// print all the top winners for this single
  		SingleScore *fss = &dp->m_singleScores[i];
  		// if same combo as last time skip
  		if ( fss->m_qtermNum == lastTermNum ) continue;
  		// do not reprint for this query term num
  		lastTermNum = fss->m_qtermNum;
  		bool firstTime = true;
  		// print all singles for this combo
  		for ( int32_t j = i ; j < dp->m_numSingles ; j++ ) {
  			// get it
  			SingleScore *ss = &dp->m_singleScores[j];
  			// stop if different single now
  			if ( ss->m_qtermNum != fss->m_qtermNum ) break;
  			// skip if 0. skip neighborhoods i guess
  			if ( almostEqualFloat(ss->m_finalScore, 0.0) ) continue;
  			// first time?
  			if ( firstTime && si->m_format == FORMAT_HTML ) {
  				const Query *q = &si->m_q;
  				printSingleTerm ( sb , q , ss );
  				printScoresHeader ( sb );
  				firstTime = false;
  			}
  
  			// print it
  			printSingleScore ( sb , si , ss , mr );
  			if(si->m_format == FORMAT_JSON && j+1 < dp->m_numSingles)
  				sb->safePrintf(",\n");
  			else
  				sb->safePrintf("\n");
  
  			// add up
  			totalSingleScore += ss->m_finalScore;
  		}
  		
  		if ( ft.length() ) ft.safePrintf(" , ");
  		ft.safePrintf("%f",totalSingleScore);
  
  		// min?
  		if ( minScore < 0.0 || totalSingleScore < minScore )
  			minScore = totalSingleScore;
  
  		// we need to set "ft" for xml stuff below
  		if ( si->m_format != FORMAT_HTML ) continue;
  
  		sb->safePrintf("<tr><td><b>%.04f</b></td><td colspan=20>total of above scores</td></tr>", totalSingleScore);
  		// close table from printScoresHeader
  		if ( ! firstTime ) sb->safePrintf("</table><br>");
  	}
  
  
  
  	if ( si->m_format == FORMAT_HTML ) {
  		sb->safePrintf("<table border=1 cellpadding=3>");
  		sb->safePrintf("<tr><td colspan=10><b><center>Ranking bits</center></b></td></tr>");
  		sb->safePrintf("<tr><td>Bit</td><td>Score multiplier</td><td>Rank adjustment</td></tr>");
  
  		if(docFlags) {
  			for(int bit=0; bit < 26; bit++) {
  				if( (1<<bit) & docFlags ) {
  					sb->safePrintf("<tr><td>%d</td><td>%f</td><td>%d</td></tr>", bit, si->m_baseScoringParameters.m_flagScoreMultiplier[bit], si->m_baseScoringParameters.m_flagRankAdjustment[bit]);
  				}
  			}
  		}
  		else {
  			sb->safePrintf("<tr><td colspan=10>None set for document</td></tr>");
  		}
  		sb->safePrintf("</table><br>");
  	}
  
  
  
  	const char *ff = "";
  	const char *ff2 = "sum";
  
  	// final score!!!
  	if ( si->m_format == FORMAT_XML ) {
  		sb->safePrintf ("\t\t<siteRank>%" PRId32"</siteRank>\n", (int32_t)dp->m_siteRank );
  
  		sb->safePrintf ("\t\t<numGoodSiteInlinks>%" PRId32 "</numGoodSiteInlinks>\n", (int32_t)mr->m_siteNumInlinks );
  
  		struct tm *timeStruct3;
  		time_t pageInlinksLastUpdated = mr->m_pageInlinksLastUpdated;
  		struct tm tm_buf;
  		timeStruct3 = gmtime_r(&pageInlinksLastUpdated, &tm_buf);
  		char tmp3[64];
  		strftime(tmp3, 64, "%b-%d-%Y(%H:%M:%S)", timeStruct3);
  
  		// -1 means unknown
  		if ( mr->m_pageNumInlinks >= 0 ) {
  			// how many inlinks, external and internal, we have
  			// to this page not filtered in any way!!!
  			sb->safePrintf("\t\t<numTotalPageInlinks>%" PRId32 "</numTotalPageInlinks>\n", mr->m_pageNumInlinks);
  		}
  		// how many inlinking ips we got, including our own if
  		// we link to ourself
  		sb->safePrintf("\t\t<numUniqueIpsLinkingToPage>%" PRId32 "</numUniqueIpsLinkingToPage>\n", mr->m_pageNumUniqueIps);
  		// how many inlinking cblocks we got, including our own if
  		// we link to ourself
  		sb->safePrintf("\t\t<numUniqueCBlocksLinkingToPage>%" PRId32 "</numUniqueCBlocksLinkingToPage>\n", mr->m_pageNumUniqueCBlocks);
  		
  		// how many "good" inlinks. i.e. inlinks whose linktext we
  		// count and index.
  		sb->safePrintf("\t\t<numGoodPageInlinks>%" PRId32 "</numGoodPageInlinks>\n", mr->m_pageNumGoodInlinks);
  		sb->safePrintf("\t\t<pageInlinksLastComputedUTC>%" PRIu32 "</pageInlinksLastComputedUTC>\n", (uint32_t)mr->m_pageInlinksLastUpdated);
  
  		float score = msg40->getScore(ix);
  		sb->safePrintf("\t\t<finalScore>%f</finalScore>\n", score);
  		sb->safePrintf ("\t\t<finalScoreEquationCanonical>"
  			       "<![CDATA["
  			       "Final Score = (siteRank*%.01f+1) * "
  			       "(%s of above matrix scores)"
  			       "]]>"
  			       "</finalScoreEquationCanonical>\n"
  			       , si->m_baseScoringParameters.m_siteRankMultiplier, ff2);
  
  		sb->safePrintf ("\t\t<finalScoreEquation>"
  			       "<![CDATA["
  			       "<b>%.03f</b> = (%" PRId32"*%.01f+1) "
  			       , dp->m_finalScore, (int32_t)dp->m_siteRank, si->m_baseScoringParameters.m_siteRankMultiplier);
  
  		
  		sb->safePrintf(" * %.01f", si->m_baseScoringParameters.m_languageWeights[mr->m_language]);
  
  		// the actual min then
  		sb->safePrintf(" * %.03f",minScore);
  
  		// no longer list all the scores
  		sb->safePrintf("]]></finalScoreEquation>\n");
  		sb->safePrintf ("\t</result>\n\n");
  		return true;
  	}
  
  	if ( si->m_format != FORMAT_HTML ) return true;
  
  	const char *cc = getCountryCode ( mr->m_country );
  	if ( mr->m_country == 0 ) cc = "Unknown";
  
  	float langBoost = si->m_baseScoringParameters.m_languageWeights[mr->m_language];
  
  
  
  	sb->safePrintf("<table border=1 cellpadding=3>");
  	sb->safePrintf("<tr><td colspan=10><b><center>Final score</center></b></td></tr>");
  	sb->safePrintf("<tr><td>DocId</td><td>%" PRId64"</td></tr>", dp->m_docId);
  	sb->safePrintf("<tr><td>Site</td><td>%s</td></tr>", mr->ptr_site);
  	sb->safePrintf("<tr><td>Language</td><td><font color=green><b>%s</b></font></td></tr>", getLanguageString(mr->m_language)); // use page language
  	sb->safePrintf("<tr><td>Language boost</td><td><font color=green><b>%.01f</b></font></td></tr>", langBoost);
  	sb->safePrintf("<tr><td>Country</td><td>%s</td></tr>", cc);
  	sb->safePrintf("<tr><td>Original SiteRank</td><td><font color=blue>%" PRId32 "</font></td></tr>", (int32_t)dp->m_siteRank);
  	sb->safePrintf("<tr><td>Adjusted SiteRank</td><td><font color=blue>%f</font></td></tr>", dp->m_adjustedSiteRank);
  	if( dp->m_usePageTemperature ) {
  		sb->safePrintf("<tr><td>Page temperature</td><td>%f</td></tr>", dp->m_pageTemperature);
  	}
  	else {
  		sb->safePrintf("<tr><td>Page temperature</td><td>Not used</td></tr>");
  	}
  
  
  	sb->safePrintf("<tr><td colspan=100>");
  
  	// list all final scores starting with pairs
  	sb->safePrintf("<b>%f</b> = (<font color=blue>%" PRId32"</font>*%.01f+1)", dp->m_finalScore, (int32_t)dp->m_siteRank, si->m_baseScoringParameters.m_siteRankMultiplier);
  
  	// Check if user specified a query language
  	if ( si->m_queryLangId != 0 ) {
  		// Query language same as document language?
  		if( si->m_queryLangId == mr->m_language ) {
  			sb->safePrintf(" * <font color=green><b>%.01f</b></font>", langBoost);
  		}
  		else
  		if( mr->m_language == 0 ) {
  			// Document language unknown, use the unknown language weight
  			sb->safePrintf(" * <font color=green><b>%.01f</b></font>", langBoost);
  		}
  	}
  
  	// list all final scores starting with pairs
  	sb->safePrintf(" * %s(", ff);
  	sb->safeMemcpy ( &ft );
  	sb->safePrintf(")</td></tr>");
  
  
  	sb->safePrintf("</table><br>");
  	// put in a hidden div so you can unhide it
  	sb->safePrintf("</div>\n");
  
  	// result is in a table so we can put the result # in its own column
  	sb->safePrintf("</td></tr></table>");
  
  
  
  	// space out 0000 backlinks
  	char *p = sb->getBufStart() + placeHolder;
  	if ( numInlinks == 0 ) 
  		memset(p, ' ', placeHolderLen);
  	else if ( numInlinks > 0 && numInlinks < 99999 ) {
  		char *ss = strstr ( p, "00000" );
  		if ( ss ) {
  			char c = ss[5];
  			sprintf(ss,"%5" PRId32,numInlinks);
  			ss[5] = c;
  		}
  		// print "1 backlink" not "1 backlinks"
  		if ( numInlinks == 1 ) {
  			char *xx = strstr(p,"backlinks");
  			if ( xx ) xx[8] = ' ';
  		}
  	}
  
  	return true;
  }
  
  
  
  
  static bool printPairScore(SafeBuf *sb, const SearchInput *si, const PairScore *ps, Msg20Reply *mr) {
  
  	// shortcut
  	const Query *q = &si->m_q;
  
  	int32_t qtn1 = ps->m_qtermNum1;
  	int32_t qtn2 = ps->m_qtermNum2;
  	
  	unsigned char de1 = ps->m_densityRank1;
  	unsigned char de2 = ps->m_densityRank2;
  	float dnw1 = getDensityWeight(de1);
  	float dnw2 = getDensityWeight(de2);
  	
  	int32_t hg1 = ps->m_hashGroup1;
  	int32_t hg2 = ps->m_hashGroup2;
  	
  	
  	float hgw1 = getHashGroupWeight(hg1);
  	float hgw2 = getHashGroupWeight(hg2);
  	
  	int32_t wp1 = ps->m_wordPos1;
  	int32_t wp2 = ps->m_wordPos2;
  	
  	unsigned char wr1 = ps->m_wordSpamRank1;
  	float wsw1 = getWordSpamWeight(wr1);
  	unsigned char wr2 = ps->m_wordSpamRank2;
  	float wsw2 = getWordSpamWeight(wr2);
  	
  	// HACK for inlink text!
  	if ( hg1 == HASHGROUP_INLINKTEXT )
  		wsw1 = getLinkerWeight(wr1);
  	if ( hg2 == HASHGROUP_INLINKTEXT )
  		wsw2 = getLinkerWeight(wr2);
  	
  	const char *syn1 = "no";
  	const char *syn2 = "no";
  	float sw1 = 1.0;
  	float sw2 = 1.0;
  	if ( ps->m_isSynonym1 ) {
  		syn1 = "yes";
  		sw1  = g_conf.m_baseScoringParameters.m_synonymWeight;
  	}
  	if ( ps->m_isSynonym2 ) {
  		syn2 = "yes";
  		sw2  = g_conf.m_baseScoringParameters.m_synonymWeight;
  	}
  
  	const char *bs1  = "no";
  	const char *bs2  = "no";
  	if ( ps->m_isHalfStopWikiBigram1 ) bs1 = "yes";
  	if ( ps->m_isHalfStopWikiBigram2 ) bs2 = "yes";
  	float wbw1 = 1.0;
  	float wbw2 = 1.0;
  	if ( ps->m_isHalfStopWikiBigram1 ) wbw1 = WIKI_BIGRAM_WEIGHT;
  	if ( ps->m_isHalfStopWikiBigram2 ) wbw2 = WIKI_BIGRAM_WEIGHT;
  
  	QueryTerm *qt1 = &q->m_qterms[qtn1];
  	QueryTerm *qt2 = &q->m_qterms[qtn2];
  
  	int64_t tf1 = qt1->m_termFreq;
  	int64_t tf2 = qt2->m_termFreq;
  	float tfw1 = ps->m_tfWeight1;
  	float tfw2 = ps->m_tfWeight2;
  	
  	const char *wp = "no";
  	float wiw = 1.0;
  	if ( ps->m_inSameWikiPhrase ) {
  		wp = "yes";
  		wiw = WIKI_WEIGHT; // 0.50;
  	}
  	int32_t a = ps->m_wordPos2;
  	int32_t b = ps->m_wordPos1;
  	const char *bes = "";
  	if ( a < b ) {
  		a = ps->m_wordPos1;
  		b = ps->m_wordPos2;
  		// out of query order penalty!
  		bes = "+ <b>1.0</b>";
  	}
  	
  	if ( si->m_format == FORMAT_XML ) {
  		sb->safePrintf("\t\t<pairInfo>\n");
  		
  		sb->safePrintf("\t\t\t<densityRank1>%" PRId32
  			      "</densityRank1>\n",
  			      (int32_t)de1);
  		sb->safePrintf("\t\t\t<densityRank2>%" PRId32
  			      "</densityRank2>\n",
  			      (int32_t)de2);
  		sb->safePrintf("\t\t\t<densityWeight1>%f"
  			      "</densityWeight1>\n",
  			      dnw1);
  		sb->safePrintf("\t\t\t<densityWeight2>%f"
  			      "</densityWeight2>\n",
  			      dnw2);
  		
  		sb->safePrintf("\t\t\t<term1><![CDATA[");
  		sb->safeMemcpy ( q->m_qterms[qtn1].m_term ,
  				q->m_qterms[qtn1].m_termLen );
  		sb->safePrintf("]]></term1>\n");
  		sb->safePrintf("\t\t\t<term2><![CDATA[");
  		sb->safeMemcpy ( q->m_qterms[qtn2].m_term ,
  				q->m_qterms[qtn2].m_termLen );
  		sb->safePrintf("]]></term2>\n");
  		
  		sb->safePrintf("\t\t\t<location1><![CDATA[%s]]>"
  			      "</location1>\n",
  			      getHashGroupString(hg1));
  		sb->safePrintf("\t\t\t<location2><![CDATA[%s]]>"
  			      "</location2>\n",
  			      getHashGroupString(hg2));
  		sb->safePrintf("\t\t\t<locationWeight1>%.01f"
  			      "</locationWeight1>\n",
  			      hgw1 );
  		sb->safePrintf("\t\t\t<locationWeight2>%.01f"
  			      "</locationWeight2>\n",
  			      hgw2 );
  		
  		sb->safePrintf("\t\t\t<wordPos1>%" PRId32
  			      "</wordPos1>\n", wp1 );
  		sb->safePrintf("\t\t\t<wordPos2>%" PRId32
  			      "</wordPos2>\n", wp2 );
  
  		sb->safePrintf("\t\t\t<isSynonym1>"
  			      "<![CDATA[%s]]>"
  			      "</isSynonym1>\n",
  			      syn1);
  		sb->safePrintf("\t\t\t<isSynonym2>"
  			      "<![CDATA[%s]]>"
  			      "</isSynonym2>\n",
  			      syn2);
  		sb->safePrintf("\t\t\t<synonymWeight1>%.01f"
  			      "</synonymWeight1>\n",
  			      sw1);
  		sb->safePrintf("\t\t\t<synonymWeight2>%.01f"
  			      "</synonymWeight2>\n",
  			      sw2);
  		
  		// word spam / link text weight
  		const char *r1 = "wordSpamRank1";
  		const char *r2 = "wordSpamRank2";
  		const char *t1 = "wordSpamWeight1";
  		const char *t2 = "wordSpamWeight2";
  		if ( hg1 == HASHGROUP_INLINKTEXT ) {
  			r1 = "inlinkSiteRank1";
  			t1 = "inlinkTextWeight1";
  		}
  		if ( hg2 == HASHGROUP_INLINKTEXT ) {
  			r2 = "inlinkSiteRank2";
  			t2 = "inlinkTextWeight2";
  		}
  		sb->safePrintf("\t\t\t<%s>%" PRId32"</%s>\n",
  			      r1,(int32_t)wr1,r1);
  		sb->safePrintf("\t\t\t<%s>%" PRId32"</%s>\n",
  			      r2,(int32_t)wr2,r2);
  		sb->safePrintf("\t\t\t<%s>%.02f</%s>\n",
  			      t1,wsw1,t1);
  		sb->safePrintf("\t\t\t<%s>%.02f</%s>\n",
  			      t2,wsw2,t2);
  		
  
  		// if offsite inlink text show the inlinkid for matching
  		// to an <inlink>
  		LinkInfo *info = (LinkInfo *)mr->ptr_linkInfo;//inlinks;
  		Inlink *k = info ? info->getNextInlink(NULL) : NULL;
  		for (;k&&hg1==HASHGROUP_INLINKTEXT ; k=info->getNextInlink(k)){
  			if ( ! k->getLinkText() ) continue;
  			if ( k->m_wordPosStart > wp1 ) continue;
  			if ( k->m_wordPosStart + 50 < wp1 ) continue;
  			// got it. we HACKED this to put the id
  			// in k->m_siteHash
  			sb->safePrintf("\t\t\t<inlinkId1>%" PRId32
  				      "</inlinkId1>\n",
  				      k->m_siteHash);
  		}
  
  		k = info ? info->getNextInlink(NULL) : NULL;
  		for (;k&&hg2==HASHGROUP_INLINKTEXT ; k=info->getNextInlink(k)){
  			if ( ! k->getLinkText() ) continue;
  			if ( k->m_wordPosStart > wp2 ) continue;
  			if ( k->m_wordPosStart + 50 < wp2 ) continue;
  			// got it. we HACKED this to put the id
  			// in k->m_siteHash
  			sb->safePrintf("\t\t\t<inlinkId2>%" PRId32
  				      "</inlinkId2>\n",
  				      k->m_siteHash);
  		}
  
  		// term freq
  		sb->safePrintf("\t\t\t<termFreq1>%" PRId64
  			      "</termFreq1>\n",tf1);
  		sb->safePrintf("\t\t\t<termFreq2>%" PRId64
  			      "</termFreq2>\n",tf2);
  		sb->safePrintf("\t\t\t<termFreqWeight1>%f"
  			      "</termFreqWeight1>\n",tfw1);
  		sb->safePrintf("\t\t\t<termFreqWeight2>%f"
  			      "</termFreqWeight2>\n",tfw2);
  		
  		sb->safePrintf("\t\t\t<isWikiBigram1>"
  			      "%" PRId32"</isWikiBigram1>\n",
  			      (int32_t)(ps->m_isHalfStopWikiBigram1));
  		sb->safePrintf("\t\t\t<isWikiBigram2>"
  			      "%" PRId32"</isWikiBigram2>\n",
  			      (int32_t)(ps->m_isHalfStopWikiBigram2));
  		
  		sb->safePrintf("\t\t\t<wikiBigramWeight1>%.01f"
  			      "</wikiBigramWeight1>\n",
  			      wbw1);
  		sb->safePrintf("\t\t\t<wikiBigramWeight2>%.01f"
  			      "</wikiBigramWeight2>\n",
  			      wbw2);
  		
  		sb->safePrintf("\t\t\t<inSameWikiPhrase>"
  			      "<![CDATA[%s]]>"
  			      "</inSameWikiPhrase>\n",
  			      wp);
  		
  		sb->safePrintf("\t\t\t<queryDist>"
  			      "%" PRId32
  			      "</queryDist>\n",
  			      ps->m_qdist );
  		
  		sb->safePrintf("\t\t\t<wikiWeight>"
  			      "%.01f"
  			      "</wikiWeight>\n",
  			      wiw );
  		
  		sb->safePrintf("\t\t\t<score>%f</score>\n",
  			      ps->m_finalScore);
  		
  		sb->safePrintf("\t\t\t<equationCanonical>"
  			      "<![CDATA["
  			      "score = "
  			      " 100 * "
  			      " locationWeight1" // hgw
  			      " * "
  			      " locationWeight2" // hgw
  			      " * "
  			      " synonymWeight1" // synweight
  			      " * "
  			      " synonymWeight2" // synweight
  			      " * "
  			      
  			      " wikiBigramWeight1"
  			      " * "
  			      " wikiBigramWeight2"
  			      " * "
  			      
  			      //"diversityWeight1"
  			      //" * "
  			      //"diversityWeight2"
  			      //" * "
  			      "densityWeight1" //density weight
  			      " * "
  			      "densityWeight2" //density weight
  			      " * "
  			      "%s" // wordspam weight
  			      " * "
  			      "%s" // wordspam weight
  			      " * "
  			      "termFreqWeight1" // tfw
  			      " * "
  			      "termFreqWeight2" // tfw
  			      " / ( ||wordPos1 - wordPos2| "
  			      " - queryDist| + 1.0 ) * "
  			      "wikiWeight"
  			      "]]>"
  			      "</equationCanonical>\n"
  			      , t1
  			      , t2
  			      );
  		
  		sb->safePrintf("\t\t\t<equation>"
  			      "<![CDATA["
  			      "%f="
  			      "100*"
  			      "<font color=orange>%.1f</font>"//hashgroupweight
  			      "*"
  			      "<font color=orange>%.1f</font>"//hashgroupweight
  			      "*"
  			      "<font color=blue>%.1f</font>" // syn weight
  			      "*"
  			      "<font color=blue>%.1f</font>" // syn weight
  			      "*"
  			      
  			      "<font color=green>%.1f</font>"//wikibigramweight
  			      "*"
  			      "<font color=green>%.1f</font>"//wikibigramweight
  			      "*"
  			      
  			      "<font color=purple>%.02f</font>"//density weight
  			      "*"
  			      "<font color=purple>%.02f</font>"//density weight
  			      "*"
  			      "<font color=red>%.02f</font>" // wordspam weight
  			      "*"
  			      "<font color=red>%.02f</font>" // wordspam weight
  			      "*"
  			      "<font color=magenta>%.02f</font>"//tf weight
  			      "*"
  			      "<font color=magenta>%.02f</font>"//tf weight
  			      , ps->m_finalScore
  			      , hgw1
  			      , hgw2
  			      , sw1
  			      , sw2
  			      , wbw1
  			      , wbw2
  			      , dnw1
  			      , dnw2
  			      , wsw1
  			      , wsw2
  			      , tfw1
  			      , tfw2
  			      );
  		
  		if ( ps->m_fixedDistance )
  			sb->safePrintf(
  				      "/<b>%" PRId32"</b> "
  				      , (int32_t)FIXED_DISTANCE );
  		else
  			sb->safePrintf(
  				      "/"
  				      "(((<font color=darkgreen>%" PRId32"</font>"
  				      "-<font color=darkgreen>%" PRId32"</font>"
  				      ")-<font color=lime>%" PRId32"</font>)+1.0%s)"
  				      ,
  				      a,b,ps->m_qdist,bes);
  		// wikipedia weight
  		if ( !almostEqualFloat(wiw, 1.0) )
  			sb->safePrintf("*%.01f", wiw );
  		sb->safePrintf("]]>"
  			      "</equation>\n" );
  		sb->safePrintf("\t\t</pairInfo>\n");
  		return true; // continue;
  	}
  
  	//
  	// print first term in first row
  	//
  	sb->safePrintf("<tr><td rowspan=3>");
  
  	sb->safePrintf("<a onclick=\""
  		      "var e = document.getElementById('poo');"
  		      "if ( e.style.display == 'none' ){"
  		      "e.style.display = '';"
  		      "}"
  		      "else {"
  		      "e.style.display = 'none';"
  		      "}"
  		      "\">"
  		      );
  	sb->safePrintf("%.04f</a></td>",ps->m_finalScore);
  
  	sb->safePrintf("<td>"
  		      "%s <font color=orange>"
  		      "%.01f</font></td>"
  		      , getHashGroupString(hg1)
  		      , hgw1 );
  
  	// the word position
  	sb->safePrintf("<td>");
  
  	sb->safePrintf("<a href=\"/get?d=");
  
  	sb->safePrintf("<a href=\"/get?d=");
  
  	sb->safePrintf("%" PRId64
  		       "&page=4"
  		       //"&page=sections&"
  		       "&hipos=%" PRId32
  		       "&c=%s#hipos\">"
  		       "%" PRId32"</a></td>"
  		       "</a></td>"
  		       ,mr->m_docId
  		       ,(int32_t)ps->m_wordPos1
  		       ,si->m_cr->m_coll
  		       ,(int32_t)ps->m_wordPos1);
  
  	sb->safePrintf("<td>%s <font color=blue>%.02f</font></td>",syn1,sw1);
  
  	sb->safePrintf("<td>%s <font color=green>%.02f</font></td>",bs1,wbw1);
  
  
  	// density
  	sb->safePrintf("<td>%" PRId32" <font color=purple>"
  		      "%.02f</font></td>",
  		      (int32_t)de1,dnw1);
  	// word spam
  	if ( hg1 == HASHGROUP_INLINKTEXT ) {
  		sb->safePrintf("<td>&nbsp;</td>");
  		sb->safePrintf("<td>%" PRId32" <font color=red>"
  			      "%.02f</font></td>",
  			      (int32_t)wr1,wsw1);
  	}
  	else {
  		sb->safePrintf("<td>%" PRId32, (int32_t)wr1);
  		//if ( wsw1 != 1.0 )
  			sb->safePrintf( " <font color=red>"
  				       "%.02f</font>",  wsw1);
  		sb->safePrintf("</td>");
  		sb->safePrintf("<td>&nbsp;</td>");
  	}
  	
  	// term freq
  	sb->safePrintf("<td id=tf>%" PRId64" <font color=magenta>"
  		      "%.02f</font></td>",
  		      tf1,tfw1);
  	// inSamePhraseId distInQuery phraseWeight
  	sb->safePrintf("<td>%s</td><td>%" PRId32"</td><td>%.01f</td>"
  		       ,wp,ps->m_qdist,wiw);
  	// end the row
  	sb->safePrintf("</tr>");
  	//
  	// print 2nd term in 2nd row
  	//
  	sb->safePrintf("<tr><td>");
  
  	sb->safePrintf(
  		      "%s <font color=orange>"
  		      "%.01f</font></td>"
  		      , getHashGroupString(hg2)
  		      , hgw2 );
  
  	// the word position
  	sb->safePrintf("<td>");
  
  	sb->safePrintf("<a href=\"/get?d=");
  
  	sb->safePrintf("%" PRId64
  		      "&page=4&"
  		      "hipos=%" PRId32"&c=%s#hipos\">"
  		      "%" PRId32"</a></td>"
  		      "</a></td>"
  		      ,mr->m_docId
  		      ,(int32_t)ps->m_wordPos2
  		      ,si->m_cr->m_coll
  		      ,(int32_t)ps->m_wordPos2);
  
  	sb->safePrintf("<td>%s <font color=blue>%.02f</font></td>",syn2,sw2);
  
  	sb->safePrintf("<td>%s <font color=green>%.02f</font></td>",bs2,wbw2);
  	
  	// density
  	sb->safePrintf("<td>%" PRId32" <font color=purple>"
  		      "%.02f</font></td>",
  		      (int32_t)de2,dnw2);
  	// word spam
  	if ( hg2 == HASHGROUP_INLINKTEXT ) {
  		sb->safePrintf("<td>&nbsp;</td>");
  		sb->safePrintf("<td>%" PRId32" <font color=red>"
  			      "%.02f</font></td>",
  			      (int32_t)wr2,wsw2);
  	}
  	else {
  		sb->safePrintf("<td>%" PRId32, (int32_t)wr2);
  		//if ( wsw2 != 1.0 )
  			sb->safePrintf( " <font color=red>"
  				       "%.02f</font>",  wsw2);
  		sb->safePrintf("</td>");
  		sb->safePrintf("<td>&nbsp;</td>");
  	}
  	// term freq
  	sb->safePrintf("<td id=tf>%" PRId64" <font color=magenta>"
  		      "%.02f</font></td>",
  		      tf2,tfw2);
  	// inSamePhraseId distInQuery phraseWeight
  	sb->safePrintf("<td>%s</td><td>%" PRId32"</td><td>%.01f</td>"
  		       ,wp,ps->m_qdist,wiw);
  	// end the row
  	sb->safePrintf("</tr>");
  	sb->safePrintf("<tr><td ");
  
  	sb->safePrintf("colspan=50>" //  style=\"display:none\">"
  		      "%.03f "
  		      "= "
  		      //" ( "
  		      "100*"
  		      "<font color=orange>%.1f"
  		      "</font>"
  		      "*"
  		      "<font color=orange>%.1f"
  		      "</font>"
  		      "*"
  		      //"(%" PRId32" - "
  		      , ps->m_finalScore
  		      //, idstr
  		      , hgw1
  		      , hgw2
  		      //, (int32_t)MAXWORDPOS+1
  		      );
  	sb->safePrintf("<font color=blue>%.1f</font>"
  		      "*"
  		      " <font color=blue>%.1f</font>"
  		      "*"
  		      
  		      // wiki bigram weight
  		      "<font color=green>%.02f</font>"
  		      "*"
  		      "<font color=green>%.02f</font>"
  		      "*"
  		      
  		      "<font color=purple>%.02f</font>"
  		      "*"
  		      "<font color=purple>%.02f</font>"
  		      "*"
  		      "<font color=red>%.02f</font>"
  		      "*"
  		      " <font color=red>%.02f</font>"
  		      "*"
  		      "<font color=magenta>%.02f</font>"
  		      "*"
  		      "<font color=magenta>%.02f</font>"
  		      , sw1
  		      , sw2
  		      , wbw1
  		      , wbw2
  		      , dnw1
  		      , dnw2
  		      , wsw1
  		      , wsw2
  		      , tfw1
  		      , tfw2
  		      );
  	if ( ps->m_fixedDistance )
  		sb->safePrintf(
  			      "/<b>%" PRId32"</b> "
  			      , (int32_t)FIXED_DISTANCE );
  	else
  		sb->safePrintf(
  			      "/"
  			      "(((<font color=darkgreen>%" PRId32"</font>"
  			      "-<font color=darkgreen>%" PRId32"</font>)-"
  			      "<font color=lime>%" PRId32"</font>) + 1.0%s)"
  			      ,
  			      a,b,ps->m_qdist,bes);
  	// wikipedia weight
  	if ( !almostEqualFloat(wiw, 1.0) )
  		sb->safePrintf("*%.01f", wiw );
  	sb->safePrintf( // end formula
  		      "</td></tr>"
  		      //"</table>"
  		      //"<br>");
  		      );
  	return true;
  }
  
  static bool printSingleTerm(SafeBuf *sb, const Query *q, const SingleScore *ss) {
  
  	int32_t qtn = ss->m_qtermNum;
  
  	sb->safePrintf("<table border=1 cellpadding=3>");
  	sb->safePrintf("<tr><td colspan=50><center><b>");
  	// link to rainbow page
  	//sb->safePrintf("<a href=\"/print?u=");
  	//sb->urlEncode( mr->ptr_ubuf );
  	//sb->safePrintf("&page=4&recycle=1&c=%s\">",coll);
  	if ( q->m_qterms[qtn].m_isPhrase )
  		sb->pushChar('\"');
  	sb->safeMemcpy ( q->m_qterms[qtn].m_term ,
  			q->m_qterms[qtn].m_termLen );
  	if ( q->m_qterms[qtn].m_isPhrase )
  		sb->pushChar('\"');
  	//sb->safePrintf("</a>");
  	sb->safePrintf("</b></center></td></tr>");
  	return true;
  }
  
  static bool printTermPairs(SafeBuf *sb, const Query *q, const PairScore *ps) {
  	// print pair text
  	int32_t qtn1 = ps->m_qtermNum1;
  	int32_t qtn2 = ps->m_qtermNum2;
  	sb->safePrintf("<table cellpadding=3 border=1>"
  		      "<tr><td colspan=20><center><b>");
  	if ( q->m_qterms[qtn1].m_isPhrase )
  		sb->pushChar('\"');
  	sb->safeMemcpy ( q->m_qterms[qtn1].m_term ,
  			q->m_qterms[qtn1].m_termLen );
  	if ( q->m_qterms[qtn1].m_isPhrase )
  		sb->pushChar('\"');
  	sb->safePrintf("</b> vs <b>");
  	if ( q->m_qterms[qtn2].m_isPhrase )
  		sb->pushChar('\"');
  	sb->safeMemcpy ( q->m_qterms[qtn2].m_term ,
  			q->m_qterms[qtn2].m_termLen );
  	if ( q->m_qterms[qtn2].m_isPhrase )
  		sb->pushChar('\"');
  	return true;
  }
  
  static bool printScoresHeader ( SafeBuf *sb ) {
  
  	sb->safePrintf("<tr>"
  		      "<td>score</td>"
  		      "<td>location</td>"
  		      "<td>wordPos</td>"
  		      "<td>synonym</td>"
  		      "<td>wikibigram</td>"
  		      //"<td>diversityRank</td>"
  		      "<td>density</td>"
  		      "<td>spam</td>"
  		      "<td>inlinkPR</td>" // nlinkSiteRank</td>"
  		      "<td>termFreq</td>"
  		       "<td>inSamePhrase</td>"
  		       "<td>distInQuery</td>"
  		       "<td>phraseWeight</td>"
  		      "</tr>\n" 
  		      );
  	return true;
  }
  
  static bool printSingleScore(SafeBuf *sb, const SearchInput *si, const SingleScore *ss, Msg20Reply *mr) {
  
  	// shortcut
  	const Query *q = &si->m_q;
  
  	//SafeBuf ft;
  	// store in final score calc
  	//if ( ft.length() ) ft.safePrintf(" + ");
  	//ft.safePrintf("%f",ss->m_finalScore);
  	const char *syn = "no";
  	float sw = 1.0;
  	if ( ss->m_isSynonym ) {
  		syn = "yes";
  		sw = g_conf.m_baseScoringParameters.m_synonymWeight;
  	}
  	//char bf = ss->m_bflags;
  	float wbw = 1.0;
  	const char *bs = "no";
  	if ( ss->m_isHalfStopWikiBigram ) {
  		bs = "yes";
  		wbw = WIKI_BIGRAM_WEIGHT;
  	}
  	float hgw = getHashGroupWeight(ss->m_hashGroup);
  	float dnw = getDensityWeight(ss->m_densityRank);
  	float wsw = getWordSpamWeight(ss->m_wordSpamRank);
  	// HACK for inlink text!
  	if ( ss->m_hashGroup == HASHGROUP_INLINKTEXT )
  		wsw = getLinkerWeight(ss->m_wordSpamRank);
  	
  	//int64_t tf = ss->m_termFreq;//ss->m_listSize;
  	int32_t qtn = ss->m_qtermNum;
  	//int64_t tf = msg40->m_msg3a.m_termFreqs[qtn];
  	QueryTerm *qt = &q->m_qterms[qtn];
  	int64_t tf = qt->m_termFreq;
  	float tfw = ss->m_tfWeight;
  	
  	if ( si->m_format == FORMAT_JSON ) {
  		sb->safePrintf("\t\t\"terminfo\": {\n");
  
  		sb->safePrintf("\t\t\t\"densityRank\": %" PRId32",\n", (int32_t)ss->m_densityRank);
  		sb->safePrintf("\t\t\t\"densityWeight\": %f,\n", dnw);
  		sb->safePrintf("\t\t\t\"term\": \"%*.*s\",\n", (int)q->m_qterms[qtn].m_termLen, (int)q->m_qterms[qtn].m_termLen,q->m_qterms[qtn].m_term);
  		
  		sb->safePrintf("\t\t\t\"location\": \"%s\",\n", getHashGroupString(ss->m_hashGroup));
  		sb->safePrintf("\t\t\t\"locationWeight\": %.01f,\n", hgw );
  		sb->safePrintf("\t\t\t\"wordPos\": %" PRId32",\n", (int32_t)ss->m_wordPos );
  		sb->safePrintf("\t\t\t\"isSynonym\": \"%s\",\n", syn);
  		sb->safePrintf("\t\t\t\"synonymWeight\": %.01f,\n", sw);
  		sb->safePrintf("\t\t\t\"isWikiBigram\": %" PRId32",\n", (int32_t)(ss->m_isHalfStopWikiBigram) );
  		sb->safePrintf("\t\t\t\"wikiBigramWeight\": %.01f,\n", (float)WIKI_BIGRAM_WEIGHT);
  		// word spam
  		if ( ss->m_hashGroup == HASHGROUP_INLINKTEXT ) {
  			sb->safePrintf("\t\t\t\"inlinkSiteRank>%" PRId32",\n", (int32_t)ss->m_wordSpamRank);
  			sb->safePrintf("\t\t\t\"inlinkTextWeight\": %.02f,\n", wsw);
  		}
  		else {
  			sb->safePrintf("\t\t\t\"wordSpamRank\": %" PRId32",\n", (int32_t)ss->m_wordSpamRank);
  			sb->safePrintf("\t\t\t\"wordSpamWeight\": %.02f,\n", wsw);
  		}
  
  
  		// if offsite inlink text show the inlinkid for matching
  		// to an <inlink>
  		LinkInfo *info = (LinkInfo *)mr->ptr_linkInfo;//inlinks;
  		Inlink *k = info ? info->getNextInlink(NULL) : NULL;
  		sb->safePrintf("\t\t\t\"inlinkIds\": [,\n");
  		for ( ; k && ss->m_hashGroup==HASHGROUP_INLINKTEXT ;
  		      k=info->getNextInlink(k)){
  			if ( ! k->getLinkText() ) continue;
  			if ( k->m_wordPosStart > ss->m_wordPos ) continue;
  			if ( k->m_wordPosStart + 50 < ss->m_wordPos ) continue;
  			// got it. we HACKED this to put the id
  			// in k->m_siteHash
  			sb->safePrintf("\t\t\t\t\"%" PRId32",\n", k->m_siteHash);
  		}
  		sb->safePrintf("\t\t\t],\n");
  
  		// term freq
  		sb->safePrintf("\t\t\t\"termFreq\": %" PRId64",\n", tf);
  		sb->safePrintf("\t\t\t\"termFreqWeight\": %f,\n", tfw);
  		
  		sb->safePrintf("\t\t\t\"score\": %f,\n", ss->m_finalScore);
  		
  		sb->safePrintf("\t\t\t\"equationCanonical\": \""
  			      "score = "
  			      " 100 * "
  			      " locationWeight" // hgw
  			      " * "
  			      " locationWeight" // hgw
  			      " * "
  			      " synonymWeight" // synweight
  			      " * "
  			      " synonymWeight" // synweight
  			      " * "
  			      " wikiBigramWeight"
  			      " * "
  			      " wikiBigramWeight"
  			      " * "
  			      //" diversityWeight" // divweight
  			      //" * "
  			      //" diversityWeight" // divweight
  			      //" * "
  			      "densityWeight" // density weight
  			      " * "
  			      "densityWeight" // density weight
  			      " * "
  			      "wordSpamWeight" // wordspam weight
  			      " * "
  			      "wordSpamWeight" // wordspam weight
  			      " * "
  			      "termFreqWeight" // tfw
  			      " * "
  			      "termFreqWeight" // tfw
  			      "\",\n"
  			      );
  		
  		sb->safePrintf("\t\t\t\"equation>\": \""
  			      "%f="
  			      "100*"
  			      "%.1f" // hgw
  			      "*"
  			      "%.1f" // hgw
  			      "*"
  			      "%.1f" // synweight
  			      "*"
  			      "%.1f" // synweight
  			      "*"
  			      "%.02f" // wikibigram weight
  			      "*"
  			      "%.02f" // wikibigram weight
  			      "*"
  			      "%.02f" // density weight
  			      "*"
  			      "%.02f" // density weight
  			      "*"
  			      "%.02f" // wordspam weight
  			      "*"
  			      "%.02f" // wordspam weight
  			      "*"
  			      "%.02f" // tfw
  			      "*"
  			      "%.02f" // tfw
  			      "\"\n"
  			      , ss->m_finalScore
  			      , hgw
  			      , hgw
  			      , sw
  			      , sw
  			      , wbw
  			      , wbw
  			      , dnw
  			      , dnw
  			      , wsw
  			      , wsw
  			      , tfw
  			      , tfw
  			      );
  		sb->safePrintf("\t\t}");
  		return true;
  	}
  
  	if ( si->m_format == FORMAT_XML ) {
  		sb->safePrintf("\t\t<termInfo>\n");
  
  		sb->safePrintf("\t\t\t<densityRank>%" PRId32
  			      "</densityRank>\n",
  			      (int32_t)ss->m_densityRank);
  		sb->safePrintf("\t\t\t<densityWeight>%f"
  			      "</densityWeight>\n",
  			      dnw);
  		sb->safePrintf("\t\t\t<term><![CDATA[");
  		sb->safeMemcpy ( q->m_qterms[qtn].m_term ,
  				q->m_qterms[qtn].m_termLen );
  		sb->safePrintf("]]></term>\n");
  		
  		sb->safePrintf("\t\t\t<location><![CDATA[%s]]>"
  			      "</location>\n",
  			      getHashGroupString(ss->m_hashGroup));
  		sb->safePrintf("\t\t\t<locationWeight>%.01f"
  			      "</locationWeight>\n",
  			      hgw );
  		sb->safePrintf("\t\t\t<wordPos>%" PRId32
  			      "</wordPos>\n", (int32_t)ss->m_wordPos );
  		sb->safePrintf("\t\t\t<isSynonym>"
  			      "<![CDATA[%s]]>"
  			      "</isSynonym>\n",
  			      syn);
  		sb->safePrintf("\t\t\t<synonymWeight>%.01f"
  			      "</synonymWeight>\n",
  			      sw);
  		sb->safePrintf("\t\t\t<isWikiBigram>%" PRId32
  			      "</isWikiBigram>\n",
  			      (int32_t)(ss->m_isHalfStopWikiBigram) );
  		sb->safePrintf("\t\t\t<wikiBigramWeight>%.01f"
  			      "</wikiBigramWeight>\n",
  			      (float)WIKI_BIGRAM_WEIGHT);
  		// word spam
  		if ( ss->m_hashGroup == HASHGROUP_INLINKTEXT ) {
  			sb->safePrintf("\t\t\t<inlinkSiteRank>%" PRId32
  				      "</inlinkSiteRank>\n",
  				      (int32_t)ss->m_wordSpamRank);
  			sb->safePrintf("\t\t\t<inlinkTextWeight>%.02f"
  				      "</inlinkTextWeight>\n",
  				      wsw);
  		}
  		else {
  			sb->safePrintf("\t\t\t<wordSpamRank>%" PRId32
  				      "</wordSpamRank>\n",
  				      (int32_t)ss->m_wordSpamRank);
  			sb->safePrintf("\t\t\t<wordSpamWeight>%.02f"
  				      "</wordSpamWeight>\n",
  				      wsw);
  		}
  
  
  		// if offsite inlink text show the inlinkid for matching
  		// to an <inlink>
  		LinkInfo *info = (LinkInfo *)mr->ptr_linkInfo;//inlinks;
  		Inlink *k = info ? info->getNextInlink(NULL) : NULL;
  		for ( ; k && ss->m_hashGroup==HASHGROUP_INLINKTEXT ; 
  		      k=info->getNextInlink(k)){
  			if ( ! k->getLinkText() ) continue;
  			if ( k->m_wordPosStart > ss->m_wordPos ) continue;
  			if ( k->m_wordPosStart + 50 < ss->m_wordPos ) continue;
  			// got it. we HACKED this to put the id
  			// in k->m_siteHash
  			sb->safePrintf("\t\t\t<inlinkId>%" PRId32
  				      "</inlinkId>\n",
  				      k->m_siteHash);
  		}
  
  		// term freq
  		sb->safePrintf("\t\t\t<termFreq>%" PRId64
  			      "</termFreq>\n",tf);
  		sb->safePrintf("\t\t\t<termFreqWeight>%f"
  			      "</termFreqWeight>\n",tfw);
  		
  		sb->safePrintf("\t\t\t<score>%f</score>\n",
  			      ss->m_finalScore);
  		
  		sb->safePrintf("\t\t\t<equationCanonical>"
  			      "<![CDATA["
  			      "score = "
  			      " 100 * "
  			      " locationWeight" // hgw
  			      " * "
  			      " locationWeight" // hgw
  			      " * "
  			      " synonymWeight" // synweight
  			      " * "
  			      " synonymWeight" // synweight
  			      " * "
  			      
  			      " wikiBigramWeight"
  			      " * "
  			      " wikiBigramWeight"
  			      " * "
  			      
  			      //" diversityWeight" // divweight
  			      //" * "
  			      //" diversityWeight" // divweight
  			      //" * "
  			      "densityWeight" // density weight
  			      " * "
  			      "densityWeight" // density weight
  			      " * "
  			      "wordSpamWeight" // wordspam weight
  			      " * "
  			      "wordSpamWeight" // wordspam weight
  			      " * "
  			      "termFreqWeight" // tfw
  			      " * "
  			      "termFreqWeight" // tfw
  			      //" / ( 3.0 )"
  			      "]]>"
  			      "</equationCanonical>\n"
  			      );
  		
  		sb->safePrintf("\t\t\t<equation>"
  			      "<![CDATA["
  			      "%f="
  			      "100*"
  			      "%.1f" // hgw
  			      "*"
  			      "%.1f" // hgw
  			      "*"
  			      
  			      "%.1f" // synweight
  			      "*"
  			      "%.1f" // synweight
  			      "*"
  			      
  			      
  			      "%.02f" // wikibigram weight
  			      "*"
  			      "%.02f" // wikibigram weight
  			      "*"
  			      
  			      "%.02f" // density weight
  			      "*"
  			      "%.02f" // density weight
  			      "*"
  			      "%.02f" // wordspam weight
  			      "*"
  			      "%.02f" // wordspam weight
  			      "*"
  			      "%.02f" // tfw
  			      "*"
  			      "%.02f" // tfw
  			      //" / ( 3.0 )"
  			      "]]>"
  			      "</equation>\n"
  			      , ss->m_finalScore
  			      , hgw
  			      , hgw
  			      , sw
  			      , sw
  			      , wbw
  			      , wbw
  			      , dnw
  			      , dnw
  			      , wsw
  			      , wsw
  			      , tfw
  			      , tfw
  			      );
  		sb->safePrintf("\t\t</termInfo>\n");
  		return true;
  	}
  
  
  
  	sb->safePrintf("<tr>"
  		      "<td rowspan=2>%.03f</td>\n"
  		      "<td>%s <font color=orange>%.1f"
  		      "</font></td\n>"
  		      // wordpos
  		      "<td>"
  		      "<a href=\"/get?d=" 
  		      , ss->m_finalScore
  		      , getHashGroupString(ss->m_hashGroup)
  		      , hgw
  		      );
  	//sb->urlEncode( mr->ptr_ubuf );
  	sb->safePrintf("%" PRId64,mr->m_docId );
  	sb->safePrintf("&page=4&"
  		      "hipos=%" PRId32"&c=%s#hipos\">"
  		      ,(int32_t)ss->m_wordPos
  		      ,si->m_cr->m_coll);
  	sb->safePrintf("%" PRId32"</a></td>\n"
  		      "<td>%s <font color=blue>%.1f"
  		      "</font></td>\n" // syn
  		      
  		      // wikibigram?/weight
  		      "<td>%s <font color=green>%.02f</font></td>\n"
  		      
  		      //"<td>%" PRId32"/<font color=green>%f"
  		      //"</font></td>" // diversity
  		      "<td>%" PRId32" <font color=purple>"
  		      "%.02f</font></td>\n" // density
  		      , (int32_t)ss->m_wordPos
  		      , syn
  		      , sw // synonym weight
  		      , bs
  		      , wbw
  		      //, (int32_t)ss->m_diversityRank
  		      //, dvw
  		      , (int32_t)ss->m_densityRank
  		      , dnw
  		      );
  	if ( ss->m_hashGroup == HASHGROUP_INLINKTEXT ) {
  		sb->safePrintf("<td>&nbsp;</td>"
  			      "<td>%" PRId32" <font color=red>%.02f"
  			      "</font></td>\n" // wordspam
  			      , (int32_t)ss->m_wordSpamRank
  			      , wsw
  			      );
  	}
  	else {
  		sb->safePrintf("<td>%" PRId32" <font color=red>%.02f"
  			      "</font></td>" // wordspam
  			      "<td>&nbsp;</td>\n"
  			      , (int32_t)ss->m_wordSpamRank
  			      , wsw
  			      );
  		
  	}
  	
  	sb->safePrintf("<td id=tf>%" PRId64" <font color=magenta>"
  		      "%.02f</font></td>\n" // termfreq
  		      "</tr>\n"
  		      , tf
  		      , tfw
  		      );
  	// last row is the computation of score
  	sb->safePrintf("<tr><td colspan=50>"
  		      "%.03f "
  		      " = "
  		      //" %" PRId32" * "
  		      "100 * "
  		      " <font color=orange>%.1f</font>"
  		      " * "
  		      " <font color=orange>%.1f</font>"
  		      " * "
  		      " <font color=blue>%.1f</font>"
  		      " * "
  		      " <font color=blue>%.1f</font>"
  		      " * "
  		      " <font color=green>%.02f</font>"//wikibigramwght
  		      " * "
  		      " <font color=green>%.02f</font>"
  		      " * "
  		      "<font color=purple>%.02f</font>"
  		      " * "
  		      "<font color=purple>%.02f</font>"
  		      " * "
  		      "<font color=red>%.02f</font>"
  		      " * "
  		      "<font color=red>%.02f</font>"
  		      " * "
  		      "<font color=magenta>%.02f</font>"
  		      " * "
  		      "<font color=magenta>%.02f</font>"
  		      //" / ( 3.0 )"
  		      // end formula
  		      "</td></tr>\n"
  		      , ss->m_finalScore
  		      //, (int32_t)MAXWORDPOS+1
  		      , hgw
  		      , hgw
  		      , sw
  		      , sw
  		      , wbw
  		      , wbw
  		      //, dvw
  		      //, dvw
  		      , dnw
  		      , dnw
  		      , wsw
  		      , wsw
  		      , tfw
  		      , tfw
  		      );
  	//sb->safePrintf("</table>"
  	//	      "<br>");
  	return true;
  }
  
  // if catId >= 1 then print the dmoz radio button
  static bool printLogoAndSearchBox(SafeBuf *sb, HttpRequest *hr, const SearchInput *si) {
  	const char *coll = hr->getString("c");
  	if ( ! coll ) coll = "";
  
  	// if there's a ton of sites use the post method otherwise
  	// they won't fit into the http request, the browser will reject
  	// sending such a large request with "GET"
  	const char *method = "GET";
  	if ( si && si->m_sites && strlen(si->m_sites)>800 ) {
  		method = "POST";
  	}
  
  	sb->safePrintf(
  		      //
  		      // search box
  		      //
  		      "<form name=f method=%s action=/search>\n\n" 
  
  		      // propagate the collection if they re-search
  		      "<input name=c type=hidden value=\"%s\">"
  		       , method
  		      , coll
  		      );
  
  	// propagate prepend
  	const char *prepend = hr->getString("prepend");
  	if ( prepend ) {
  		sb->safePrintf("<input name=prepend type=hidden value=\"");
  		sb->htmlEncode ( prepend, strlen(prepend), false);
  		sb->safePrintf("\">");
  	}
  	
  
  	// put search box in a box
  	sb->safePrintf(
  		       "<br>"
  		       "<br>"
  		       "<br>"
  		       "<div style="
  		       "background-color:#fcc714;"
  		       "border-style:solid;"
  		       "border-width:3px;"
  		       "border-color:blue;"
  		       //"background-color:blue;"
  		       "padding:20px;"
  		       "border-radius:20px;"
  		       ">");
  
  
  	sb->safePrintf (
  			//"<div style=margin-left:5px;margin-right:5px;>
  			"<input size=40 type=text name=q "
  
  			"style=\""
  			//"width:%" PRId32"px;"
  			"height:26px;"
  			"padding:0px;"
  			"font-weight:bold;"
  			"padding-left:5px;"
  			//"border-radius:10px;"
  			"margin:0px;"
  			"border:1px inset lightgray;"
  			"background-color:#ffffff;"
  			"font-size:18px;"
  			"\" "
  
  
  			"value=\""
  			);
  
  	// contents of search box
  	int32_t  qlen;
  	const char *qstr = hr->getString("q",&qlen,"",NULL);
  	sb->htmlEncode ( qstr , qlen , false );
  
  	// if it was an advanced search, this can be empty
  	if ( qlen == 0 && si && si->m_displayQuery )
  		sb->htmlEncode ( si->m_displayQuery );
  
  	sb->safePrintf ("\">"
  			"&nbsp; &nbsp;"
  
  			"<div onclick=document.f.submit(); "
  
  			" onmouseover=\""
  			"this.style.backgroundColor='lightgreen';"
  			"this.style.color='black';\""
  			" onmouseout=\""
  			"this.style.backgroundColor='green';"
  			"this.style.color='white';\" "
  
  			"style=border-radius:28px;"
  			"cursor:pointer;"
  			"cursor:hand;"
  			"border-color:white;"
  			"border-style:solid;"
  			"border-width:3px;"
  			"padding:12px;"
  			"width:20px;"
  			"height:20px;"
  			"display:inline-block;"
  			"background-color:green;color:white;>"
  			"<b style=margin-left:-5px;font-size:18px;"
  			">GO</b>"
  			"</div>"
  			);
  
  	sb->safePrintf(	"</div>"
  			"<br>"
  			"<br>"
  		       );
  
  
  	printSearchFiltersBar ( sb , hr );
  	
  
  	sb->safePrintf( "</form>\n" );
  	return true;
  }
  
  
  
  class MenuItem {
  public:
  	int32_t  m_menuNum;
  	const char *m_title;
  	// we append this to the url
  	const char *m_cgi;
  	char  m_tmp[25];
  };
  
  
  const static int MAX_MENU_ENTRIES = 200;
  
  static MenuItem s_mi[MAX_MENU_ENTRIES];
  static int32_t s_num = 0;
  
  static int getNextMenuEntryIndex(int curr_index)
  {
  	int i = curr_index+1;
  	if( i >= MAX_MENU_ENTRIES )
  	{
  		g_process.shutdownAbort(true);
  	}
  	return i;
  }
  
  static bool printSearchFiltersBar ( SafeBuf *sb , HttpRequest *hr ) {
  	SafeBuf cu;
  	hr->getCurrentUrl ( cu );
  
  
  	sb->safePrintf("<script>"
  		       "function show(id){"
  		       "var e = document.getElementById(id);"
  		       "if ( e.style.display == 'none' ){"
  		       "e.style.display = '';"
  		       "}"
  		       "else {"
  		       "e.style.display = 'none';"
  		       "}"
  		       "}"
  		       "</script>"
  		       );
  
  
  	static bool s_init = false;
  
  	if ( ! s_init ) {
  
  		int32_t menuNum = 0;
  		int32_t n = 0;
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Any time";
  		s_mi[n].m_cgi      = "secsback=0";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Past 24 hours";
  		s_mi[n].m_cgi      = "secsback=86400";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Past week";
  		s_mi[n].m_cgi      = "secsback=604800";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Past month";
  		s_mi[n].m_cgi      = "secsback=2592000";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Past year";
  		s_mi[n].m_cgi      = "secsback=31536000";
  		n = getNextMenuEntryIndex(n);
  
  		++menuNum;
  
  		// sort by
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Sorted by relevance";
  		s_mi[n].m_cgi      = "sortby=0";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Sorted by date";
  		s_mi[n].m_cgi      = "sortby=1";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Reverse sorted by date";
  		s_mi[n].m_cgi      = "sortby=2";
  		n = getNextMenuEntryIndex(n);
  
  		++menuNum;
  
  		// languages
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Any language";
  		s_mi[n].m_cgi      = "qlang=xx";
  		n = getNextMenuEntryIndex(n);
  
  		for ( int32_t i = 0 ; i < langLast ; i++ ) {
  			s_mi[n].m_menuNum  = menuNum;
  			s_mi[n].m_title    = getLanguageString(i);
  			const char *abbr = getLanguageAbbr(i);
  			snprintf(s_mi[n].m_tmp,10,"qlang=%s",abbr);
  			s_mi[n].m_cgi      = s_mi[n].m_tmp;
  			n = getNextMenuEntryIndex(n);
  		}
  
  		++menuNum;
  
  		// filetypes
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Any filetype";
  		s_mi[n].m_cgi      = "filetype=any";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "HTML";
  		s_mi[n].m_cgi      = "filetype=html";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "TEXT";
  		s_mi[n].m_cgi      = "filetype=txt";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "PDF";
  		s_mi[n].m_cgi      = "filetype=pdf";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Microsoft Word";
  		s_mi[n].m_cgi      = "filetype=doc";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "XML";
  		s_mi[n].m_cgi      = "filetype=xml";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "JSON";
  		s_mi[n].m_cgi      = "filetype=json";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Excel";
  		s_mi[n].m_cgi      = "filetype=xls";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "PostScript";
  		s_mi[n].m_cgi      = "filetype=ps";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Spider Status";
  		s_mi[n].m_cgi      = "filetype=status";
  		n = getNextMenuEntryIndex(n);
  
  		++menuNum;
  
  		// output
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Output HTML";
  		s_mi[n].m_cgi      = "format=html";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Output XML";
  		s_mi[n].m_cgi      = "format=xml";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Output JSON";
  		s_mi[n].m_cgi      = "format=json";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Output CSV";
  		s_mi[n].m_cgi      = "format=csv";
  		n = getNextMenuEntryIndex(n);
  
  		++menuNum;
  
  		// show/hide banned
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Hide banned results";
  		s_mi[n].m_cgi      = "sb=0";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Show banned results";
  		s_mi[n].m_cgi      = "sb=1";
  		n = getNextMenuEntryIndex(n);
  
  		++menuNum;
  
  		// spider status
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Hide Spider Log";
  		s_mi[n].m_cgi      = "splog=0";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Show Spider Log";
  		s_mi[n].m_cgi      = "q=type:status";
  		n = getNextMenuEntryIndex(n);
  
  		++menuNum;
  
  		// family filter
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Family Filter Off";
  		s_mi[n].m_cgi      = "ff=0";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Family Filter On";
  		s_mi[n].m_cgi      = "ff=1";
  		n = getNextMenuEntryIndex(n);
  
  		++menuNum;
  
  		// META TAGS
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "No Meta Tags";
  		s_mi[n].m_cgi      = "dt=";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Show Meta Tags";
  		s_mi[n].m_cgi      = "dt=keywords+description";
  		n = getNextMenuEntryIndex(n);
  
  		++menuNum;
  
  		// ADMIN
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Show Admin View";
  		s_mi[n].m_cgi      = "admin=1";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "Show User View";
  		s_mi[n].m_cgi      = "admin=0";
  		n = getNextMenuEntryIndex(n);
  
  		++menuNum;
  
  		// fx_country
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "fx_country (none)";
  		s_mi[n].m_cgi      = "fx_country=";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "de";
  		s_mi[n].m_cgi      = "fx_country=de";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "dk";
  		s_mi[n].m_cgi      = "fx_country=dk";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "fr";
  		s_mi[n].m_cgi      = "fx_country=fr";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "no";
  		s_mi[n].m_cgi      = "fx_country=no";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "se";
  		s_mi[n].m_cgi      = "fx_country=se";
  		n = getNextMenuEntryIndex(n);
  
  		++menuNum;
  
  		// fx_blang
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "fx_blang (none)";
  		s_mi[n].m_cgi      = "fx_blang=";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "da";
  		s_mi[n].m_cgi      = "fx_blang=da";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "de";
  		s_mi[n].m_cgi      = "fx_blang=de";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "en";
  		s_mi[n].m_cgi      = "fx_blang=en";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "en-US";
  		s_mi[n].m_cgi      = "fx_blang=en-US";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "no";
  		s_mi[n].m_cgi      = "fx_blang=no";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "se";
  		s_mi[n].m_cgi      = "fx_blang=se";
  		n = getNextMenuEntryIndex(n);
  
  		++menuNum;
  
  		// fx_fetld
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "fx_fetld (none)";
  		s_mi[n].m_cgi      = "fx_fetld=";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "com";
  		s_mi[n].m_cgi      = "fx_fetld=com";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "de";
  		s_mi[n].m_cgi      = "fx_fetld=de";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "dk";
  		s_mi[n].m_cgi      = "fx_fetld=dk";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "no";
  		s_mi[n].m_cgi      = "fx_fetld=no";
  		n = getNextMenuEntryIndex(n);
  
  		s_mi[n].m_menuNum  = menuNum;
  		s_mi[n].m_title    = "se";
  		s_mi[n].m_cgi      = "fx_fetld=se";
  		n = getNextMenuEntryIndex(n);
  
  		s_num = n;
  		s_init = true;
  	}
  
  
  	// we'll print the admin menu custom since it's mostly off-page links
  
  	// bar of drop down menus
  	sb->safePrintf("<div style=color:gray;>");
  
  	for ( int32_t i = 0 ; i <= s_mi[s_num-1].m_menuNum ; i++ ) {
  		// after 5 make a new line
  		if ( i % 5 == 0 ) sb->safePrintf("<br><br>");
  
  		printMenu ( sb , i , hr );
  	}
  
  	sb->safePrintf("</div>\n");
  	sb->safePrintf("<br>\n");
  
  	return true;
  }
  
  static bool printMenu ( SafeBuf *sb , int32_t menuNum , HttpRequest *hr ) {
  
  	bool firstOne = true;
  
  	MenuItem *first = NULL;
  
  	const char *src    = hr->getOrigUrlRequest();
  	int32_t  srcLen = hr->getOrigUrlRequestLen();
  
  	const char *frontTag = "";
  	const char *backTag = "";
  
  	bool isDefaultHeader = true;
  
  	// try to set first based on what's in the url
  	for ( int32_t i = 0 ; i < s_num ; i++ ) {
  		// shortcut
  		MenuItem *mi = &s_mi[i];
  		// skip if not our item
  		if ( mi->m_menuNum != menuNum ) continue;
  
  		// is it in the url
  		const char *match = strnstr ( src, mi->m_cgi, srcLen );
  
  		// or if empty quotes it is the true header like
  		// for 'hide spider log' option
  		if ( ! match ) {
  			isDefaultHeader = false;
  			continue;
  		}
  		// ensure ? or & preceeds
  		if ( match > src && match[-1] != '?' && match[-1] != '&' )
  			continue;
  		// and \0 or & follows
  		int32_t milen = strlen(mi->m_cgi);
  		if ( match+milen > src+srcLen ) continue;
  		if ( ! is_wspace_a(match[milen]) && match[milen] != '&' ) 
  			continue;
  		// got it
  		first = mi;
  		// do not highlight the orig header
  		if ( isDefaultHeader ) break;
  		frontTag = "<b style=color:maroon;>";
  		backTag = "</b>";
  		break;
  	}
  	
  
  	for ( int32_t i = 0 ; i < s_num ; i++ ) {
  
  		// shortcut
  		MenuItem *mi = &s_mi[i];
  
  		// skip if not our item
  		if ( mi->m_menuNum != menuNum ) continue;
  
  		if ( ! first ) first = mi;
  
  		if ( ! firstOne ) goto skip;
  
  		firstOne = false;
  
  		// for centering the dropdown
  		sb->safePrintf("<span style=position:relative;></span>");
  
  		// print hidden drop down menu
  		sb->safePrintf(
  			       "<span id=menu%" PRId32" style=\"display:none;"
  			       "position:absolute;"
  			       //"margin-left:-20px;"
  			       "margin-top:15px;"
  			       "width:150px;"
  			       "max-height:300px;"
  			       "overflow-y:auto;"
  			       "background-color:white;"
  			       "padding:10px;"
  			       "width=80px;border-width:1px;"
  			       "border-color:lightgray;"
  			       "box-shadow: -.5px 1px 1px gray;"
  			       "border-style:solid;color:gray;\" "
  
  			       //" onmouseout=\""
  			       //"this.style.display='none';\""
  
  			       // if clicking on scrollbar do not hide menu!
  			       " onmousedown=\"inmenuclick=1;\" "
  
  			       ">"
  			       , mi->m_menuNum
  			       );
  
  	skip:
  
  		
  		// . add our cgi to the original url
  		// . so if it has &qlang=de and they select &qlang=en
  		//   we have to replace it... etc.
  		StackBuf<512> newUrl;
  		replaceParm ( mi->m_cgi , &newUrl , hr );
  		newUrl += '\0';
  
  		// print each item in there
  		sb->safePrintf("<a href=%s>"
  			       "<div style=cursor:pointer;cursor:hand;"
  			       "padding-top:10px;"
  			       "padding-bottom:10px;"
  			       "color:gray;"
  
  			       " onmouseover=\""
  			       "this.style.backgroundColor='#e0e0e0';\" "
  			       " onmouseout=\""
  			       "this.style.backgroundColor='white';\" "
  
  			       // prevent the body onmousedown from 
  			       // hiding the menu
  			       " onmousedown=\"inmenuclick=1;\" "
  
  			       ">"
  			       "<nobr>"
  			       , newUrl.getBufStart()
  			       );
  
  		// print checkmark (check mark) next to selected one
  		// if not the default (trueHeader)
  		if ( mi == first ) // ! isDefaultHeader && mi == first )
  			sb->safePrintf("<b style=color:black;>&check;</b>");
  		else 
  			sb->safePrintf("&nbsp; &nbsp; ");
  
  		sb->safePrintf(" %s</nobr>"
  			       "</div>"
  			       "</a>"
  			       , mi->m_title );
  
  		//sb->safePrintf("<br><br>");
  	}
  
  	// wrap up the drop down
  	sb->safePrintf("</span>");
  	
  
  
  	// print heading or current selection i guess
  	sb->safePrintf(
  		       // separate menus with these two spaces
  		       " &nbsp; &nbsp; "
  		       // print the menu header that when clicked
  		       // will show the drop down
  		       "<span style=cursor:pointer;"
  		       "cursor:hand; "
  
  		       "onmousedown=\"this.style.color='red';"
  		       "inmenuclick=1;"
  		       "\" "
  
  		       "onmouseup=\"this.style.color='gray';"
  
  		       // close any other open menu
  		       "if ( openmenu !='') {"
  		       "document.getElementById(openmenu)."
  		       "style.display='none'; "
  		       "var saved=openmenu;"
  		       "openmenu='';"
  		       // don't reopen our same menu below!
  		       "if ( saved=='menu%" PRId32"') return;"
  		       "}"
  
  		       // show our menu
  		       "show('menu%" PRId32"'); "
  		       // we are now open
  		       "openmenu='menu%" PRId32"'; "
  
  		       "\""
  		       ">"
  
  		       "%s%s%s %c%c%c" 
  		       "</span>"
  		       , first->m_menuNum
  		       , first->m_menuNum
  		       , first->m_menuNum
  		       , frontTag
  		       , first->m_title
  		       , backTag
  		       // print triangle
  		       ,0xe2
  		       ,0x96
  		       ,0xbc
  		       );
  
  
  	return true;
  }
  
  static bool replaceParm ( const char *cgi , SafeBuf *newUrl , HttpRequest *hr ) { 
  	if ( ! cgi[0] ) return true;
  	// get original request url. this is not \0 terminated
  	const char *src    = hr->getOrigUrlRequest();
  	int32_t  srcLen = hr->getOrigUrlRequestLen();
  	return replaceParm2 ( cgi ,newUrl, src, srcLen );
  }
  
  static bool replaceParm2 ( const char *cgi , SafeBuf *newUrl , 
  			   const char *oldUrl , int32_t oldUrlLen ) {
  
  	const char *src    = oldUrl;
  	int32_t  srcLen = oldUrlLen;
  
  	const char *srcEnd = src + srcLen;
  
  	const char *equal = strstr(cgi,"=");
  	if ( ! equal ) {
  		log(LOG_WARN, "results: %s has no equal sign", cgi);
  		return false;
  	}
  	int32_t cgiLen = equal - cgi;
  
  	const char *p = src;
  
   tryagain:
  
  	const char *found = strncasestr ( p , cgi , srcEnd - p , cgiLen );
  
  	// if no ? or & before it it is bogus!
  	if ( found && found[-1] != '&' && found[-1] != '?' ) {
  		// try again
  		p = found + 1;
  		goto tryagain;
  	}
  		
  	// fix &s= replaceing &sb=
  	if ( found && found[cgiLen] != '=' ) {
  		// try again
  		p = found + 1;
  		goto tryagain;
  	}
  
  
  	// if no collision, just append it
  	if ( ! found ) {
  		if ( ! newUrl->safeMemcpy ( src , srcLen ) ) return false;
  		if ( ! newUrl->pushChar('&') ) return false;
  		if ( ! newUrl->safeStrcpy ( cgi ) ) return false;
  		if ( ! newUrl->nullTerm() ) return false;
  		return true;
  	}
  
  	// . otherwise we have to replace it
  	// . copy up to where it starts
  	if ( ! newUrl->safeMemcpy ( src , found-src ) ) return false;
  	// then insert our new cgi there
  	if ( ! newUrl->safeStrcpy ( cgi ) ) return false;
  	// then resume it
  	const char *foundEnd = strncasestr ( found , "&" , srcEnd - found );
  	// if nothing came after...
  	if ( ! foundEnd ) {
  		if ( ! newUrl->nullTerm() ) return false;
  		return true;
  	}
  	// copy over what came after
  	if ( ! newUrl->safeMemcpy ( foundEnd, srcEnd-foundEnd ) ) return false;
  	if ( ! newUrl->nullTerm() ) return false;
  	return true;
  }
  
  static bool printMetaContent ( Msg40 *msg40 , int32_t i , State0 *st, SafeBuf *sb ) {
  	// store the user-requested meta tags content
  	SearchInput *si = &st->m_si;
  	const char *pp      =      si->m_displayMetas;
  	const char *ppend   = pp + strlen(si->m_displayMetas);
  	Msg20 *m = msg40->m_msg20[i];//getMsg20(i);
  	Msg20Reply *mr = m->m_r;
  	char *dbuf    = mr->ptr_dbuf;//msg40->getDisplayBuf(i);
  	int32_t  dbufLen = mr->size_dbuf-1;//msg40->getDisplayBufLen(i);
  	char *dbufEnd = dbuf + (dbufLen-1);
  	char *dptr    = dbuf;
  	//bool  printedSomething = false;
  	// loop over the names of the requested meta tags
  	while ( pp < ppend && dptr < dbufEnd ) {
  		// . assure last byte of dbuf is \0
  		//   provided dbufLen > 0
  		// . this insures sprintf and strlen won't
  		//   crash on dbuf/dptr
  		if ( dbuf [ dbufLen ] != '\0' ) {
  			log(LOG_LOGIC,"query: Meta tag buffer has no \\0.");
  			break;
  		}
  		// skip initial spaces
  		while ( pp < ppend && is_wspace_a(*pp) ) pp++;
  		// break if done
  		if ( ! *pp ) break;
  		// that's the start of the meta tag name
  		const char *ss = pp;
  		// . find end of that meta tag name
  		// . can end in :<integer> -- specifies max len
  		while ( pp < ppend && ! is_wspace_a(*pp) && 
  			*pp != ':' ) pp++;
  		size_t sslen = (size_t)(pp-ss);
  		// if ':' was specified, skip the rest
  		if ( *pp == ':' ) while ( pp < ppend && ! is_wspace_a(*pp)) pp++;
  		// print the name
  		int32_t ddlen = dbufLen;
  		// newspaperarchive wants tags printed even if no value
  		// make sure the meta tag isn't fucked up
  		for ( int32_t ti = 0; ti < ddlen; ti++ ) {
  			if ( dptr[ti] == '"' ||
  			     dptr[ti] == '>' ||
  			     dptr[ti] == '<' ||
  			     dptr[ti] == '\r' ||
  			     dptr[ti] == '\n' ||
  			     dptr[ti] == '\0' ) {
  				ddlen = ti;
  				break;
  			}
  		}
  
  		if ( ddlen > 0 ) {
  			// ship it out
  			if ( si->m_format == FORMAT_XML ) {
  				sb->safePrintf ( "\t\t<display name=\"%.*s\">"
  					         "<![CDATA[", (int)sslen, ss);
  				cdataEncode(sb, dptr, ddlen);
  				sb->safePrintf ( "]]></display>\n" );
  			}
  			else if ( si->m_format == FORMAT_JSON ) {
  				sb->safePrintf ( "\t\t\"display.%.*s\":\"", (int)sslen, ss);
  				sb->jsonEncode ( dptr, ddlen );
  				sb->safePrintf ( "\",\n");
  			}
  			// otherwise, print in light gray
  			else {
  				sb->safePrintf("<font color=#c62939>"
  					      "<b>%.*s</b>: ", (int)sslen, ss);
  				sb->safeMemcpy ( dptr, ddlen );
  				sb->safePrintf ( "</font><br>" );
  			}
  		}
  		// point to next content of tag to display
  		dptr += ddlen + 1;
  	}
  	return true;
  }
  