text: /*----------------------------------------------------------------------
  Compiler Generator Coco/R,
  Copyright (c) 1990, 2004 Hanspeter Moessenboeck, University of Linz
  extended by M. Loeberbauer & A. Woess, Univ. of Linz
  ported to C++ by Csaba Balazs, University of Szeged
  with improvements by Pat Terry, Rhodes University
  
  This program is free software; you can redistribute it and/or modify it
  under the terms of the GNU General Public License as published by the
  Free Software Foundation; either version 2, or (at your option) any
  later version.
  
  This program is distributed in the hope that it will be useful, but
  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  for more details.
  
  You should have received a copy of the GNU General Public License along
  with this program; if not, write to the Free Software Foundation, Inc.,
  59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  
  As an exception, it is allowed to write an extension of Coco/R that is
  used as a plugin in non-free software.
  
  If not otherwise stated, any source code generated by Coco/R (other than
  Coco/R itself) does not fall under the GNU General Public License.
  -----------------------------------------------------------------------*/
  
  #include "Scanner.h"
  
  namespace PascalLike {
  
  
  
  // string handling, wide character
  
  
  wchar_t* coco_string_create(const wchar_t* value) {
  	return coco_string_create(value, 0);
  }
  
  wchar_t* coco_string_create(const wchar_t *value, int startIndex) {
  	int valueLen = 0;
  	int len = 0;
  
  	if (value) {
  		valueLen = wcslen(value);
  		len = valueLen - startIndex;
  	}
  
  	return coco_string_create(value, startIndex, len);
  }
  
  wchar_t* coco_string_create(const wchar_t *value, int startIndex, int length) {
  	int len = 0;
  	wchar_t* data;
  
  	if (value) { len = length; }
  	data = new wchar_t[len + 1];
  	wcsncpy(data, &(value[startIndex]), len);
  	data[len] = 0;
  
  	return data;
  }
  
  wchar_t* coco_string_create_upper(const wchar_t* data) {
  	if (!data) { return NULL; }
  
  	int dataLen = 0;
  	if (data) { dataLen = wcslen(data); }
  
  	wchar_t *newData = new wchar_t[dataLen + 1];
  
  	for (int i = 0; i <= dataLen; i++) {
  		if ((L'a' <= data[i]) && (data[i] <= L'z')) {
  			newData[i] = data[i] + (L'A' - L'a');
  		}
  		else { newData[i] = data[i]; }
  	}
  
  	newData[dataLen] = L'\0';
  	return newData;
  }
  
  wchar_t* coco_string_create_lower(const wchar_t* data) {
  	if (!data) { return NULL; }
  	int dataLen = wcslen(data);
  	return coco_string_create_lower(data, 0, dataLen);
  }
  
  wchar_t* coco_string_create_lower(const wchar_t* data, int startIndex, int dataLen) {
  	if (!data) { return NULL; }
  
  	wchar_t* newData = new wchar_t[dataLen + 1];
  
  	for (int i = 0; i <= dataLen; i++) {
  		wchar_t ch = data[startIndex + i];
  		if ((L'A' <= ch) && (ch <= L'Z')) {
  			newData[i] = ch - (L'A' - L'a');
  		}
  		else { newData[i] = ch; }
  	}
  	newData[dataLen] = L'\0';
  	return newData;
  }
  
  wchar_t* coco_string_create_append(const wchar_t* data1, const wchar_t* data2) {
  	wchar_t* data;
  	int data1Len = 0;
  	int data2Len = 0;
  
  	if (data1) { data1Len = wcslen(data1); }
  	if (data2) {data2Len = wcslen(data2); }
  
  	data = new wchar_t[data1Len + data2Len + 1];
  
  	if (data1) { wcscpy(data, data1); }
  	if (data2) { wcscpy(data + data1Len, data2); }
  
  	data[data1Len + data2Len] = 0;
  
  	return data;
  }
  
  wchar_t* coco_string_create_append(const wchar_t *target, const wchar_t appendix) {
  	int targetLen = coco_string_length(target);
  	wchar_t* data = new wchar_t[targetLen + 2];
  	wcsncpy(data, target, targetLen);
  	data[targetLen] = appendix;
  	data[targetLen + 1] = 0;
  	return data;
  }
  
  void coco_string_delete(wchar_t* &data) {
  	delete [] data;
  	data = NULL;
  }
  
  int coco_string_length(const wchar_t* data) {
  	if (data) { return wcslen(data); }
  	return 0;
  }
  
  bool coco_string_endswith(const wchar_t* data, const wchar_t *end) {
  	int dataLen = wcslen(data);
  	int endLen = wcslen(end);
  	return (endLen <= dataLen) && (wcscmp(data + dataLen - endLen, end) == 0);
  }
  
  int coco_string_indexof(const wchar_t* data, const wchar_t value) {
  	const wchar_t* chr = wcschr(data, value);
  
  	if (chr) { return (chr-data); }
  	return -1;
  }
  
  int coco_string_lastindexof(const wchar_t* data, const wchar_t value) {
  	const wchar_t* chr = wcsrchr(data, value);
  
  	if (chr) { return (chr-data); }
  	return -1;
  }
  
  void coco_string_merge(wchar_t* &target, const wchar_t* appendix) {
  	if (!appendix) { return; }
  	wchar_t* data = coco_string_create_append(target, appendix);
  	delete [] target;
  	target = data;
  }
  
  bool coco_string_equal(const wchar_t* data1, const wchar_t* data2) {
  	return wcscmp( data1, data2 ) == 0;
  }
  
  int coco_string_compareto(const wchar_t* data1, const wchar_t* data2) {
  	return wcscmp(data1, data2);
  }
  
  int coco_string_hash(const wchar_t *data) {
  	int h = 0;
  	if (!data) { return 0; }
  	while (*data != 0) {
  		h = (h * 7) ^ *data;
  		++data;
  	}
  	if (h < 0) { h = -h; }
  	return h;
  }
  
  // string handling, ascii character
  
  wchar_t* coco_string_create(const char* value) {
  	int len = 0;
  	if (value) { len = strlen(value); }
  	wchar_t* data = new wchar_t[len + 1];
  	for (int i = 0; i < len; ++i) { data[i] = (wchar_t) value[i]; }
  	data[len] = 0;
  	return data;
  }
  
  char* coco_string_create_char(const wchar_t *value) {
  	int len = coco_string_length(value);
  	char *res = new char[len + 1];
  	for (int i = 0; i < len; ++i) { res[i] = (char) value[i]; }
  	res[len] = 0;
  	return res;
  }
  
  void coco_string_delete(char* &data) {
  	delete [] data;
  	data = NULL;
  }
  
  
  Token::Token() {
  	kind = 0;
  	pos  = 0;
  	col  = 0;
  	line = 0;
  	val  = NULL;
  	next = NULL;
  }
  
  Token::~Token() {
  	coco_string_delete(val);
  }
  
  
  int WSBuffer::Read()
  {
  	if (_pos >= _buf.size() ) return EoF;
  	return _buf[_pos++];
  }
  
  int WSBuffer::Peek()
  {
  	if (_pos >= _buf.size() ) return EoF;
  	return _buf[_pos];
  }
  
  int WSBuffer::GetPos()
  {
  	return _pos;
  }
  
  void WSBuffer::SetPos(int value)
  {
  	_pos = value;
  }
  
  Scanner::Scanner() {
  	buffer = new WSBuffer(_buf);
  	Init();
  }
  
  Scanner::~Scanner() {
  	char* cur = (char*) firstHeap;
  
  	while(cur != NULL) {
  		cur = *(char**) (cur + COCO_HEAP_BLOCK_SIZE);
  		free(firstHeap);
  		firstHeap = cur;
  	}
  	delete [] tval;
  	delete buffer;
  }
  
  void Scanner::Init() {
  	EOL    = '\n';
  	eofSym = 0;
  	maxT = 170;
  	noSym = 170;
  	int i;
  	for (i = 95; i <= 95; ++i) start.set(i, 1);
  	for (i = 97; i <= 122; ++i) start.set(i, 1);
  	for (i = 48; i <= 48; ++i) start.set(i, 44);
  	for (i = 51; i <= 55; ++i) start.set(i, 44);
  	for (i = 57; i <= 57; ++i) start.set(i, 44);
  	start.set(39, 2);
  	start.set(35, 3);
  	start.set(50, 45);
  	start.set(56, 46);
  	start.set(49, 47);
  	start.set(59, 20);
  	start.set(58, 48);
  	start.set(46, 49);
  	start.set(43, 50);
  	start.set(45, 51);
  	start.set(42, 52);
  	start.set(47, 53);
  	start.set(62, 54);
  	start.set(60, 55);
  	start.set(38, 56);
  	start.set(124, 57);
  	start.set(61, 58);
  	start.set(33, 59);
  	start.set(37, 60);
  	start.set(94, 38);
  	start.set(40, 39);
  	start.set(41, 40);
  	start.set(123, 61);
  	start.set(125, 43);
  	start.set(91, 68);
  	start.set(93, 69);
  	start.set(126, 70);
  	start.set(64, 71);
  	start.set(44, 72);
  		start.set(Buffer::EoF, -1);
  	keywords.set(L"abstract", 13);
  	keywords.set(L"and", 14);
  	keywords.set(L"array", 15);
  	keywords.set(L"begin", 16);
  	keywords.set(L"break", 17);
  	keywords.set(L"case", 18);
  	keywords.set(L"class", 19);
  	keywords.set(L"const", 20);
  	keywords.set(L"constant", 21);
  	keywords.set(L"constructor", 22);
  	keywords.set(L"continue", 23);
  	keywords.set(L"dec", 24);
  	keywords.set(L"destructor", 25);
  	keywords.set(L"div", 26);
  	keywords.set(L"do", 27);
  	keywords.set(L"downto", 28);
  	keywords.set(L"else", 29);
  	keywords.set(L"end", 30);
  	keywords.set(L"except", 31);
  	keywords.set(L"external", 32);
  	keywords.set(L"exit", 33);
  	keywords.set(L"false", 34);
  	keywords.set(L"far", 35);
  	keywords.set(L"file", 36);
  	keywords.set(L"finalization", 37);
  	keywords.set(L"finally", 38);
  	keywords.set(L"for", 39);
  	keywords.set(L"forward", 40);
  	keywords.set(L"function", 41);
  	keywords.set(L"goto", 42);
  	keywords.set(L"ident", 43);
  	keywords.set(L"if", 44);
  	keywords.set(L"inc", 45);
  	keywords.set(L"inherited", 46);
  	keywords.set(L"halt", 47);
  	keywords.set(L"high", 48);
  	keywords.set(L"implementation", 49);
  	keywords.set(L"interface", 50);
  	keywords.set(L"initialization", 51);
  	keywords.set(L"in", 52);
  	keywords.set(L"label", 53);
  	keywords.set(L"loophole", 54);
  	keywords.set(L"low", 55);
  	keywords.set(L"mod", 56);
  	keywords.set(L"not", 57);
  	keywords.set(L"object", 58);
  	keywords.set(L"of", 59);
  	keywords.set(L"on", 60);
  	keywords.set(L"or", 61);
  	keywords.set(L"origin", 62);
  	keywords.set(L"otherwise", 63);
  	keywords.set(L"overload", 64);
  	keywords.set(L"override", 65);
  	keywords.set(L"packed", 66);
  	keywords.set(L"private", 67);
  	keywords.set(L"procedure", 68);
  	keywords.set(L"program", 69);
  	keywords.set(L"protected", 70);
  	keywords.set(L"public", 71);
  	keywords.set(L"raise", 72);
  	keywords.set(L"record", 73);
  	keywords.set(L"repeat", 74);
  	keywords.set(L"return", 75);
  	keywords.set(L"set", 76);
  	keywords.set(L"shl", 77);
  	keywords.set(L"shr", 78);
  	keywords.set(L"static", 79);
  	keywords.set(L"str", 80);
  	keywords.set(L"then", 81);
  	keywords.set(L"to", 82);
  	keywords.set(L"true", 83);
  	keywords.set(L"try", 84);
  	keywords.set(L"type", 85);
  	keywords.set(L"until", 86);
  	keywords.set(L"unit", 87);
  	keywords.set(L"uses", 88);
  	keywords.set(L"var", 89);
  	keywords.set(L"virtual", 90);
  	keywords.set(L"while", 91);
  	keywords.set(L"with", 92);
  	keywords.set(L"xor", 93);
  	keywords.set(L"read", 94);
  	keywords.set(L"readln", 95);
  	keywords.set(L"write", 96);
  	keywords.set(L"writeln", 97);
  	keywords.set(L"function_block", 128);
  	keywords.set(L"end_function_block", 129);
  	keywords.set(L"var_input", 130);
  	keywords.set(L"var_output", 131);
  	keywords.set(L"var_in_out", 132);
  	keywords.set(L"var_external", 133);
  	keywords.set(L"var_const", 134);
  	keywords.set(L"end_var", 135);
  	keywords.set(L"end_function", 136);
  	keywords.set(L"end_if", 137);
  	keywords.set(L"end_for", 138);
  	keywords.set(L"end_while", 139);
  	keywords.set(L"end_repeat", 140);
  	keywords.set(L"end_case", 141);
  	keywords.set(L"end_program", 142);
  	keywords.set(L"end_type", 143);
  	keywords.set(L"end_record", 144);
  	keywords.set(L"end_class", 145);
  	keywords.set(L"retain", 146);
  	keywords.set(L"add", 147);
  	keywords.set(L"concat", 148);
  	keywords.set(L"eq", 149);
  	keywords.set(L"ne", 150);
  	keywords.set(L"ge", 151);
  	keywords.set(L"gt", 152);
  	keywords.set(L"le", 153);
  	keywords.set(L"lt", 154);
  	keywords.set(L"max", 155);
  	keywords.set(L"min", 156);
  	keywords.set(L"mul", 157);
  	keywords.set(L"mux", 158);
  
  
  	tvalLength = 128;
  	tval = new wchar_t[tvalLength]; // text of current token
  
  	// COCO_HEAP_BLOCK_SIZE byte heap + pointer to next heap block
  	heap = malloc(COCO_HEAP_BLOCK_SIZE + sizeof(void*));
  	firstHeap = heap;
  	heapEnd = (void**) (((char*) heap) + COCO_HEAP_BLOCK_SIZE);
  	*heapEnd = 0;
  	heapTop = heap;
  	if (sizeof(Token) > COCO_HEAP_BLOCK_SIZE) {
  		wprintf(L"--- Too small COCO_HEAP_BLOCK_SIZE\n");
  		exit(1);
  	}
  
  	pos = -1; line = 1; col = 0; charPos = -1;
  	oldEols = 0;
  	NextCh();
  
  
  	pt = tokens = CreateToken(); // first token is a dummy
  }
  void Scanner::ReInit() {
  	pos = -1; line = 1; col = 0; charPos = -1;
  	oldEols = 0;
  	buffer->SetPos(0);
  	NextCh();
  	pt = tokens = CreateToken();
  }
  
  
  void Scanner::NextCh() {
  	if (oldEols > 0) { ch = EOL; oldEols--; }
  	else {
  		pos = buffer->GetPos();
  		// buffer reads unicode chars, if UTF8 has been detected
  		ch = buffer->Read(); col++; charPos++;
  		// replace isolated '\r' by '\n' in order to make
  		// eol handling uniform across Windows, Unix and Mac
  		if (ch == L'\r' && buffer->Peek() != L'\n') ch = EOL;
  		if (ch == EOL) { line++; col = 0; }
  	}
  		valCh = ch;
  		if ('A' <= ch && ch <= 'Z') ch = ch - 'A' + 'a'; // ch.ToLower()
  }
  
  void Scanner::AddCh() {
  	if (tlen >= tvalLength) {
  		tvalLength *= 2;
  		wchar_t *newBuf = new wchar_t[tvalLength];
  		memcpy(newBuf, tval, tlen*sizeof(wchar_t));
  		delete [] tval;
  		tval = newBuf;
  	}
  	if (ch != Buffer::EoF) {
  		tval[tlen++] = valCh;
  		NextCh();
  	}
  }
  
  
  bool Scanner::Comment0() {
  	int level = 1, pos0 = pos, line0 = line, col0 = col, charPos0 = charPos;
  	NextCh();
  	if (ch == L'/') {
  		NextCh();
  		for(;;) {
  			if (ch == 10) {
  				level--;
  				if (level == 0) { oldEols = line - line0; NextCh(); return true; }
  				NextCh();
  			} else if (ch == buffer->EoF) return false;
  			else NextCh();
  		}
  	} else {
  		buffer->SetPos(pos0); NextCh(); line = line0; col = col0; charPos = charPos0;
  	}
  	return false;
  }
  
  bool Scanner::Comment1() {
  	int level = 1, pos0 = pos, line0 = line, col0 = col, charPos0 = charPos;
  	NextCh();
  	if (ch == L'*') {
  		NextCh();
  		for(;;) {
  			if (ch == L'*') {
  				NextCh();
  				if (ch == L')') {
  					level--;
  					if (level == 0) { oldEols = line - line0; NextCh(); return true; }
  					NextCh();
  				}
  			} else if (ch == buffer->EoF) return false;
  			else NextCh();
  		}
  	} else {
  		buffer->SetPos(pos0); NextCh(); line = line0; col = col0; charPos = charPos0;
  	}
  	return false;
  }
  
  bool Scanner::Comment2() {
  	int level = 1, pos0 = pos, line0 = line, col0 = col, charPos0 = charPos;
  	NextCh();
  		for(;;) {
  			if (ch == L'}') {
  				level--;
  				if (level == 0) { oldEols = line - line0; NextCh(); return true; }
  				NextCh();
  			} else if (ch == buffer->EoF) return false;
  			else NextCh();
  		}
  }
  
  
  void Scanner::CreateHeapBlock() {
  	void* newHeap;
  	char* cur = (char*) firstHeap;
  
  	while(((char*) tokens < cur) || ((char*) tokens > (cur + COCO_HEAP_BLOCK_SIZE))) {
  		cur = *((char**) (cur + COCO_HEAP_BLOCK_SIZE));
  		free(firstHeap);
  		firstHeap = cur;
  	}
  
  	// COCO_HEAP_BLOCK_SIZE byte heap + pointer to next heap block
  	newHeap = malloc(COCO_HEAP_BLOCK_SIZE + sizeof(void*));
  	*heapEnd = newHeap;
  	heapEnd = (void**) (((char*) newHeap) + COCO_HEAP_BLOCK_SIZE);
  	*heapEnd = 0;
  	heap = newHeap;
  	heapTop = heap;
  }
  
  Token* Scanner::CreateToken() {
  	Token *t;
  	if (((char*) heapTop + (int) sizeof(Token)) >= (char*) heapEnd) {
  		CreateHeapBlock();
  	}
  	t = (Token*) heapTop;
  	heapTop = (void*) ((char*) heapTop + sizeof(Token));
  	t->val = NULL;
  	t->next = NULL;
  	return t;
  }
  
  void Scanner::AppendVal(Token *t) {
  	int reqMem = (tlen + 1) * sizeof(wchar_t);
  	if (((char*) heapTop + reqMem) >= (char*) heapEnd) {
  		if (reqMem > COCO_HEAP_BLOCK_SIZE) {
  			wprintf(L"--- Too long token value\n");
  			exit(1);
  		}
  		CreateHeapBlock();
  	}
  	t->val = (wchar_t*) heapTop;
  	heapTop = (void*) ((char*) heapTop + reqMem);
  	t->valSize = tlen;
  	wcsncpy(t->val, tval, tlen);
  	t->val[tlen] = L'\0';
  }
  
  Token* Scanner::NextToken() {
  	while (ch == ' ' ||
  			(ch >= 9 && ch <= 10) || (ch >= 12 && ch <= 13)
  	) NextCh();
  	if ((ch == L'/' && Comment0()) || (ch == L'(' && Comment1()) || (ch == L'{' && Comment2())) return NextToken();
  	int apx = 0;
  	int recKind = noSym;
  	int recEnd = pos;
  	t = CreateToken();
  	t->pos = pos; t->col = col; t->line = line; t->charPos = charPos;
  	int state = start.state(ch);
  	tlen = 0; AddCh();
  
  	switch (state) {
  		case -1: { t->kind = eofSym; break; } // NextCh already done
  		case 0: {
  			case_0:
  			if (recKind != noSym) {
  				tlen = recEnd - t->pos;
  				SetScannerBehindT();
  			}
  			t->kind = recKind; break;
  		} // NextCh already done
  		case 1:
  			case_1:
  			recEnd = pos; recKind = 1;
  			if ((ch >= L'0' && ch <= L'9') || ch == L'_' || (ch >= L'a' && ch <= L'z')) {AddCh(); goto case_1;}
  			else {t->kind = 1; wchar_t *literal = coco_string_create_lower(tval, 0, tlen); t->kind = keywords.get(literal, t->kind); coco_string_delete(literal); break;}
  		case 2:
  			case_2:
  			if (ch <= L'&' || (ch >= L'(' && ch <= 65535)) {AddCh(); goto case_2;}
  			else if (ch == 39) {AddCh(); goto case_62;}
  			else {goto case_0;}
  		case 3:
  			if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_4;}
  			else {goto case_0;}
  		case 4:
  			case_4:
  			recEnd = pos; recKind = 3;
  			if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_4;}
  			else {t->kind = 3; break;}
  		case 5:
  			case_5:
  			if (ch == L'.') {apx++; AddCh(); goto case_6;}
  			else {goto case_0;}
  		case 6:
  			case_6:
  			{
  				tlen -= apx;
  				SetScannerBehindT();				buffer->SetPos(t->pos); NextCh(); line = t->line; col = t->col;
  				for (int i = 0; i < tlen; i++) NextCh();
  				t->kind = 4; break;}
  		case 7:
  			case_7:
  			recEnd = pos; recKind = 5;
  			if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_7;}
  			else if (ch == L'e') {AddCh(); goto case_8;}
  			else {t->kind = 5; break;}
  		case 8:
  			case_8:
  			if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_10;}
  			else if (ch == L'+' || ch == L'-') {AddCh(); goto case_9;}
  			else {goto case_0;}
  		case 9:
  			case_9:
  			if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_10;}
  			else {goto case_0;}
  		case 10:
  			case_10:
  			recEnd = pos; recKind = 5;
  			if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_10;}
  			else {t->kind = 5; break;}
  		case 11:
  			case_11:
  			if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_13;}
  			else if (ch == L'+' || ch == L'-') {AddCh(); goto case_12;}
  			else {goto case_0;}
  		case 12:
  			case_12:
  			if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_13;}
  			else {goto case_0;}
  		case 13:
  			case_13:
  			recEnd = pos; recKind = 5;
  			if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_13;}
  			else {t->kind = 5; break;}
  		case 14:
  			case_14:
  			if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_15;}
  			else {goto case_0;}
  		case 15:
  			case_15:
  			recEnd = pos; recKind = 6;
  			if ((ch >= L'0' && ch <= L'9') || ch == L'_') {AddCh(); goto case_15;}
  			else {t->kind = 6; break;}
  		case 16:
  			case_16:
  			if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_17;}
  			else {goto case_0;}
  		case 17:
  			case_17:
  			recEnd = pos; recKind = 7;
  			if ((ch >= L'0' && ch <= L'9') || ch == L'_') {AddCh(); goto case_17;}
  			else {t->kind = 7; break;}
  		case 18:
  			case_18:
  			if ((ch >= L'0' && ch <= L'9') || (ch >= L'a' && ch <= L'f')) {AddCh(); goto case_19;}
  			else {goto case_0;}
  		case 19:
  			case_19:
  			recEnd = pos; recKind = 8;
  			if ((ch >= L'0' && ch <= L'9') || ch == L'_' || (ch >= L'a' && ch <= L'f')) {AddCh(); goto case_19;}
  			else {t->kind = 8; break;}
  		case 20:
  			{t->kind = 9; break;}
  		case 21:
  			case_21:
  			{t->kind = 11; break;}
  		case 22:
  			case_22:
  			{t->kind = 102; break;}
  		case 23:
  			case_23:
  			{t->kind = 103; break;}
  		case 24:
  			case_24:
  			{t->kind = 104; break;}
  		case 25:
  			case_25:
  			{t->kind = 105; break;}
  		case 26:
  			case_26:
  			{t->kind = 106; break;}
  		case 27:
  			case_27:
  			{t->kind = 107; break;}
  		case 28:
  			case_28:
  			{t->kind = 108; break;}
  		case 29:
  			case_29:
  			{t->kind = 109; break;}
  		case 30:
  			case_30:
  			{t->kind = 110; break;}
  		case 31:
  			case_31:
  			{t->kind = 113; break;}
  		case 32:
  			case_32:
  			{t->kind = 114; break;}
  		case 33:
  			case_33:
  			{t->kind = 116; break;}
  		case 34:
  			case_34:
  			{t->kind = 118; break;}
  		case 35:
  			case_35:
  			{t->kind = 119; break;}
  		case 36:
  			case_36:
  			{t->kind = 125; break;}
  		case 37:
  			case_37:
  			{t->kind = 126; break;}
  		case 38:
  			{t->kind = 127; break;}
  		case 39:
  			{t->kind = 159; break;}
  		case 40:
  			{t->kind = 160; break;}
  		case 41:
  			case_41:
  			{t->kind = 161; break;}
  		case 42:
  			case_42:
  			{t->kind = 162; break;}
  		case 43:
  			{t->kind = 164; break;}
  		case 44:
  			case_44:
  			recEnd = pos; recKind = 4;
  			if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_44;}
  			else if (ch == L'_') {AddCh(); goto case_63;}
  			else if (ch == L'.') {apx++; AddCh(); goto case_64;}
  			else if (ch == L'e') {AddCh(); goto case_11;}
  			else {t->kind = 4; break;}
  		case 45:
  			recEnd = pos; recKind = 4;
  			if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_44;}
  			else if (ch == L'_') {AddCh(); goto case_63;}
  			else if (ch == L'.') {apx++; AddCh(); goto case_64;}
  			else if (ch == L'e') {AddCh(); goto case_11;}
  			else if (ch == L'#') {AddCh(); goto case_14;}
  			else {t->kind = 4; break;}
  		case 46:
  			recEnd = pos; recKind = 4;
  			if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_44;}
  			else if (ch == L'_') {AddCh(); goto case_63;}
  			else if (ch == L'.') {apx++; AddCh(); goto case_64;}
  			else if (ch == L'e') {AddCh(); goto case_11;}
  			else if (ch == L'#') {AddCh(); goto case_16;}
  			else {t->kind = 4; break;}
  		case 47:
  			recEnd = pos; recKind = 4;
  			if ((ch >= L'0' && ch <= L'5') || (ch >= L'7' && ch <= L'9')) {AddCh(); goto case_44;}
  			else if (ch == L'_') {AddCh(); goto case_63;}
  			else if (ch == L'.') {apx++; AddCh(); goto case_64;}
  			else if (ch == L'e') {AddCh(); goto case_11;}
  			else if (ch == L'6') {AddCh(); goto case_65;}
  			else {t->kind = 4; break;}
  		case 48:
  			recEnd = pos; recKind = 10;
  			if (ch == L'=') {AddCh(); goto case_22;}
  			else {t->kind = 10; break;}
  		case 49:
  			recEnd = pos; recKind = 12;
  			if (ch == L'.') {AddCh(); goto case_21;}
  			else {t->kind = 12; break;}
  		case 50:
  			recEnd = pos; recKind = 98;
  			if (ch == L'=') {AddCh(); goto case_23;}
  			else {t->kind = 98; break;}
  		case 51:
  			recEnd = pos; recKind = 99;
  			if (ch == L'=') {AddCh(); goto case_24;}
  			else {t->kind = 99; break;}
  		case 52:
  			recEnd = pos; recKind = 100;
  			if (ch == L'=') {AddCh(); goto case_26;}
  			else {t->kind = 100; break;}
  		case 53:
  			recEnd = pos; recKind = 101;
  			if (ch == L'=') {AddCh(); goto case_25;}
  			else {t->kind = 101; break;}
  		case 54:
  			recEnd = pos; recKind = 111;
  			if (ch == L'>') {AddCh(); goto case_66;}
  			else if (ch == L'=') {AddCh(); goto case_31;}
  			else {t->kind = 111; break;}
  		case 55:
  			recEnd = pos; recKind = 112;
  			if (ch == L'<') {AddCh(); goto case_67;}
  			else if (ch == L'=') {AddCh(); goto case_32;}
  			else if (ch == L'>') {AddCh(); goto case_33;}
  			else {t->kind = 112; break;}
  		case 56:
  			recEnd = pos; recKind = 121;
  			if (ch == L'=') {AddCh(); goto case_29;}
  			else if (ch == L'&') {AddCh(); goto case_35;}
  			else {t->kind = 121; break;}
  		case 57:
  			recEnd = pos; recKind = 120;
  			if (ch == L'=') {AddCh(); goto case_30;}
  			else if (ch == L'|') {AddCh(); goto case_34;}
  			else {t->kind = 120; break;}
  		case 58:
  			recEnd = pos; recKind = 115;
  			if (ch == L'=') {AddCh(); goto case_36;}
  			else {t->kind = 115; break;}
  		case 59:
  			recEnd = pos; recKind = 117;
  			if (ch == L'=') {AddCh(); goto case_37;}
  			else {t->kind = 117; break;}
  		case 60:
  			recEnd = pos; recKind = 124;
  			if (ch == L'}') {AddCh(); goto case_42;}
  			else {t->kind = 124; break;}
  		case 61:
  			recEnd = pos; recKind = 163;
  			if (ch == L'%') {AddCh(); goto case_41;}
  			else {t->kind = 163; break;}
  		case 62:
  			case_62:
  			recEnd = pos; recKind = 2;
  			if (ch == 39) {AddCh(); goto case_2;}
  			else {t->kind = 2; break;}
  		case 63:
  			case_63:
  			recEnd = pos; recKind = 4;
  			if ((ch >= L'0' && ch <= L'9') || ch == L'_') {AddCh(); goto case_63;}
  			else if (ch == L'.') {apx++; AddCh(); goto case_5;}
  			else {t->kind = 4; break;}
  		case 64:
  			case_64:
  			recEnd = pos; recKind = 5;
  			if ((ch >= L'0' && ch <= L'9')) {apx = 0; AddCh(); goto case_7;}
  			else if (ch == L'.') {apx++; AddCh(); goto case_6;}
  			else if (ch == L'e') {apx = 0; AddCh(); goto case_8;}
  			else {t->kind = 5; break;}
  		case 65:
  			case_65:
  			recEnd = pos; recKind = 4;
  			if ((ch >= L'0' && ch <= L'9')) {AddCh(); goto case_44;}
  			else if (ch == L'_') {AddCh(); goto case_63;}
  			else if (ch == L'.') {apx++; AddCh(); goto case_64;}
  			else if (ch == L'e') {AddCh(); goto case_11;}
  			else if (ch == L'#') {AddCh(); goto case_18;}
  			else {t->kind = 4; break;}
  		case 66:
  			case_66:
  			recEnd = pos; recKind = 123;
  			if (ch == L'=') {AddCh(); goto case_27;}
  			else {t->kind = 123; break;}
  		case 67:
  			case_67:
  			recEnd = pos; recKind = 122;
  			if (ch == L'=') {AddCh(); goto case_28;}
  			else {t->kind = 122; break;}
  		case 68:
  			{t->kind = 165; break;}
  		case 69:
  			{t->kind = 166; break;}
  		case 70:
  			{t->kind = 167; break;}
  		case 71:
  			{t->kind = 168; break;}
  		case 72:
  			{t->kind = 169; break;}
  
  	}
  	AppendVal(t);
  	return t;
  }
  
  void Scanner::SetScannerBehindT() {
  	buffer->SetPos(t->pos);
  	NextCh();
  	line = t->line; col = t->col; charPos = t->charPos;
  	for (int i = 0; i < tlen; i++) NextCh();
  }
  
  // get the next token (possibly a token already seen during peeking)
  Token* Scanner::Scan() {
  	if (tokens->next == NULL) {
  		return pt = tokens = NextToken();
  	} else {
  		pt = tokens = tokens->next;
  		return tokens;
  	}
  }
  
  // peek for the next token, ignore pragmas
  Token* Scanner::Peek() {
  	do {
  		if (pt->next == NULL) {
  			pt->next = NextToken();
  		}
  		pt = pt->next;
  	} while (pt->kind > maxT); // skip pragmas
  
  	return pt;
  }
  
  // make sure that peeking starts at the current scan position
  void Scanner::ResetPeek() {
  	pt = tokens;
  }
  
  } // namespace
  
  