text: // Matt Wells, copyright Jul 2001
  
  #include "gb-include.h"
  
  #include "StopWords.h"
  #include "HashTableX.h"
  #include "Threads.h"
  
  class Abbr {
  public:
  	char *m_str;
  	// MUST it have a word after it????
  	char  m_hasWordAfter;
  };
  
  // . i shrunk this list a lot
  // . see backups for the hold list
  static class Abbr s_abbrs99[] = {
  	{"hghway",0},//highway
  	{"hway",0},//highway
  	{"hwy",0},//highway
  	{"ln",0}, // lane
  	{"mil",0}, // military
  	{"pkway",0}, // parkway
  	{"pkwy",0},  // parkway
  	{"lp",0}, // Loop
  	{"phd",0}, // Loop
  	{"demon",0}, // demonstration
  	{"alz",0}, // alzheimer's
  
  	{"lang",0}, // language
  	{"gr",0}, // grade(s) "xmas concert gr. 1-5"
  	{"vars",0}, // varsity
  	{"avg",0}, // average
  	{"amer",0}, // america
  
  	{"bet",0}, // between 18th and 19th for piratecatradio.com
  	{"nr",0}, // near 6th street = nr. 6th street
  	{"appt",0},
  	{"tel",1},
  	{"intl",0},
  	{"div",1}, // div. II
  
  	{"int",1}, // Intermediate Dance
  	{"beg",1}, // Beginner Dance
  	{"adv",1}, // Advanced Dance
  
  	{"feat",1}, // featuring.
  	{"tdlr",0}, // toddler
  	{"schl",0}, // pre-schl
  
  	// times
  	{"am",0}, // unm.edu url puts {"7 am. - 9 am.{" time ranges!
  	{"pm",0},
  	{"mon",0},
  	{"tue",0},
  	{"tues",0},
  	{"wed",0},
  	{"wednes",0},
  	{"thu",0},
  	{"thur",0},
  	{"thurs",0},
  	{"fri",0},
  	{"sat",0},
  	{"sun",0},
  
  	{"Ala",0},
  	{"Ariz",0},
  	{"Assn",0},
  	{"Assoc",0},
  	{"asst",0}, // assistant
  	{"Atty",0},
  	{"Attn",1},
  	{"Aug",0},
  	{"Ave",0},
  	{"Bldg",0},
  	{"Bros",0}, // brothers
  	{"Blvd",0},
  	{"Calif",0},
  	{"Capt",1},
  	{"Cf",0},
  	{"Ch",0},
  	{"Co",0},
  	{"Col",0},
  	{"Colo",0},
  	{"Conn",0},
  	{"Mfg",0},
  	{"Corp",0},
  	{"DR",0},
  	{"Dec",0},
  	{"Dept",0},
  	{"Dist",0},
  	{"Dr",0},
  	{"Drs",0},
  	{"Ed",0},
  	{"Eq",0},
  	{"ext",0}, // extension
  	{"FEB",0},
  	{"Feb",0},
  	{"Fig",0},
  	{"Figs",0},
  	{"Fla",0},
  	{"Ft",1}, // ft. worth texas or feet
  	{"Ga",0},
  	{"Gen",0},
  	{"Gov",0},
  	{"HON",0},
  	{"Ill",0},
  	{"Inc",0},
  	{"JR",0},
  	{"Jan",0},
  	{"Jr",0},
  	{"Kan",0},
  	//{"Ky",0},
  	{"La",0},
  	{"Lt",0},
  	{"Ltd",0},
  	{"MR",1},
  	{"MRS",1},
  	{"Mar",0},
  	{"Mass",0},
  	{"Md",0},
  	{"Messrs",1},
  	{"Mich",0},
  	{"Minn",0},
  	{"Miss",0},
  	{"Mmes",0},
  	//{"Mo",0}, no more 2-letter state abbreviations
  	{"Mr",1},
  	{"Mrs",1},
  	{"Ms",1},
  	{"Msgr",1},
  	{"Mt",1},
  	{"NO",0},
  	{"No",0},
  	{"Nov",0},
  	{"Oct",0},
  	{"Okla",0},
  	{"Op",0},
  	{"Ore",0},
  	//{"Pa",0},
  	{"Pp",0},
  	{"Prof",1},
  	{"Prop",0},
  	{"Rd",0},
  	{"Ref",0},
  	{"Rep",0},
  	{"Reps",0},
  	{"Rev",0},
  	{"Rte",0},
  	{"Sen",0},
  	{"Sept",0},
  	{"Sr",0},
  	{"St",0},
  	{"ste",0},
  	{"Stat",0},
  	{"Supt",0},
  	{"Tech",0},
  	{"Tex",0},
  	{"Va",0},
  	{"Vol",0},
  	{"Wash",0},
  	//{"al",0},
  	{"av",0},
  	{"ave",0},
  	{"ca",0},
  	{"cc",0},
  	{"chap",0},
  	{"cm",0},
  	{"cu",0},
  	{"dia",0},
  	{"dr",0},
  	{"eqn",0},
  	{"etc",0},
  	{"fig",1},
  	{"figs",1},
  	{"ft",0}, // fort or feet or featuring
  	//{"gm",0},
  	{"hr",0},
  	//{"in",0},
  	//{"kc",0},
  	{"lb",0},
  	{"lbs",0},
  	{"mg",0},
  	{"ml",0},
  	{"mm",0},
  	{"mv",0},
  	//{"nw",0},
  	{"oz",0},
  	{"pl",0},
  	{"pp",0},
  	{"sec",0},
  	{"sq",0},
  	{"st",0},
  	{"vs",1},
  	{"yr",0},
  	{"yrs",0}, // 3 yrs old
  	// middle initials
  	{"a",0},
  	{"b",0},
  	{"c",0},
  	{"d",0},
  	{"e",0},
  	{"f",0},
  	{"g",0},
  	{"h",0},
  	{"i",0},
  	{"j",0},
  	{"k",0},
  	{"l",0},
  	{"m",0},
  	{"n",0},
  	{"o",0},
  	{"p",0},
  	{"q",0},
  	{"r",0},
  	{"s",0},
  	{"t",0},
  	{"u",0},
  	{"v",1}, // versus
  	{"w",0},
  	{"x",0},
  	{"y",0},
  	{"z",0}
  };
  
  static HashTableX s_abbrTable;
  static bool       s_abbrInitialized = false;
  
  /*
  static bool initTable ( HashTableX *table, char *words[], int32_t size ) {
  	// set up the hash table
  	if ( ! table->set ( 8 , 4 , size * 2,NULL,0,false,MAX_NICENESS,
  			    "abbrtbl") ) 
  		return log("build: Could not init abbreviation table.");
  	// now add in all the stop words
  	int32_t n = (int32_t)size/ sizeof(char *); 
  	for ( int32_t i = 0 ; i < n ; i++ ) {
  		char      *sw    = words[i];
  		//int32_t     swlen = gbstrlen ( sw );
  		int64_t  swh   = hash64Lower_utf8 ( sw );
  		if ( ! table->addTerm (&swh,i+1) ) return false;
  	}
  	return true;
  }
  */
  
  bool isAbbr ( int64_t h , bool *hasWordAfter ) {
  	if ( ! s_abbrInitialized ) {
  		// int16_tcut
  		HashTableX *t = &s_abbrTable;
  		// set up the hash table
  		int32_t n = ((int32_t)sizeof(s_abbrs99))/ ((int32_t)sizeof(Abbr));
  		if ( ! t->set ( 8,4,n*4, NULL,0,false,MAX_NICENESS,"abbrtbl")) 
  			return log("build: Could not init abbrev table.");
  		// now add in all the stop words
  		for ( int32_t i = 0 ; i < n ; i++ ) {
  			char      *sw    = s_abbrs99[i].m_str;
  			int64_t  swh   = hash64Lower_utf8 ( sw );
  			int32_t val = i + 1;
  			if ( ! t->addKey (&swh,&val) ) return false;
  		}
  		s_abbrInitialized = true;
  		// test it
  		int64_t h = hash64Lower_utf8("St");
  		if ( ! t->isInTable(&h) ) { char *xx=NULL;*xx=0; }
  		int32_t sc = s_abbrTable.getScore ( &h );
  		if ( sc >= n ) { char *xx=NULL;*xx=0; }
  	} 
  	// get from table
  	int32_t sc = s_abbrTable.getScore ( &h );
  	if ( sc <= 0 ) return false;
  	if ( hasWordAfter ) *hasWordAfter = s_abbrs99[sc-1].m_hasWordAfter;
  	return true;
  }		
  
  
  void resetAbbrTable ( ) {
  	s_abbrTable.reset();
  }
  
  