text: //    abc_tokenizer.js: tokenizes an ABC Music Notation string to support abc_parse.
  //    Copyright (C) 2010-2018 Paul Rosen (paul at paulrosen dot net)
  //
  //    Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
  //    documentation files (the "Software"), to deal in the Software without restriction, including without limitation
  //    the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
  //    to permit persons to whom the Software is furnished to do so, subject to the following conditions:
  //
  //    The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
  //
  //    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
  //    BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  //    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
  //    DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  //    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  
  //var parseCommon = require('./abc_common'); 
  import { parseCommon } from './abc_common.js';
  
  // this is a series of functions that get a particular element out of the passed stream.
  // the return is the number of characters consumed, so 0 means that the element wasn't found.
  // also returned is the element found. This may be a different length because spaces may be consumed that aren't part of the string.
  // The return structure for most calls is { len: num_chars_consumed, token: str }
  var Tokenizer = function() {
  	this.skipWhiteSpace = function(str) {
  		for (var i = 0; i < str.length; i++) {
  		  if (!this.isWhiteSpace(str.charAt(i)))
  				return i;
  		}
  		return str.length;	// It must have been all white space
  	};
  	var finished = function(str, i) {
  		return i >= str.length;
  	};
  	this.eatWhiteSpace = function(line, index) {
  		for (var i = index; i < line.length; i++) {
  		  if (!this.isWhiteSpace(line.charAt(i)))
  				return i-index;
  		}
  		return i-index;
  	};
  
  	// This just gets the basic pitch letter, ignoring leading spaces, and normalizing it to a capital
  	this.getKeyPitch = function(str) {
  		var i = this.skipWhiteSpace(str);
  		if (finished(str, i))
  			return {len: 0};
  		switch (str.charAt(i)) {
  			case 'A':return {len: i+1, token: 'A'};
  			case 'B':return {len: i+1, token: 'B'};
  			case 'C':return {len: i+1, token: 'C'};
  			case 'D':return {len: i+1, token: 'D'};
  			case 'E':return {len: i+1, token: 'E'};
  			case 'F':return {len: i+1, token: 'F'};
  			case 'G':return {len: i+1, token: 'G'};
  //			case 'a':return {len: i+1, token: 'A'};
  //			case 'b':return {len: i+1, token: 'B'};
  //			case 'c':return {len: i+1, token: 'C'};
  //			case 'd':return {len: i+1, token: 'D'};
  //			case 'e':return {len: i+1, token: 'E'};
  //			case 'f':return {len: i+1, token: 'F'};
  //			case 'g':return {len: i+1, token: 'G'};
  		}
  		return {len: 0};
  	};
  
  	// This just gets the basic accidental, ignoring leading spaces, and only the ones that appear in a key
  	this.getSharpFlat = function(str) {
  		if (str === 'bass')
  			return {len: 0};
  		switch (str.charAt(0)) {
  			case '#':return {len: 1, token: '#'};
  			case 'b':return {len: 1, token: 'b'};
  		}
  		return {len: 0};
  	};
  
  	this.getMode = function(str) {
  		var skipAlpha = function(str, start) {
  			// This returns the index of the next non-alphabetic char, or the entire length of the string if not found.
  		  while (start < str.length && ((str.charAt(start) >= 'a' && str.charAt(start) <= 'z') || (str.charAt(start) >= 'A' && str.charAt(start) <= 'Z')))
  				start++;
  			return start;
  		};
  
  		var i = this.skipWhiteSpace(str);
  		if (finished(str, i))
  			return {len: 0};
  		var firstThree = str.substring(i,i+3).toLowerCase();
  		if (firstThree.length > 1 && firstThree.charAt(1) === ' ' || firstThree.charAt(1) === '^' || firstThree.charAt(1) === '_' || firstThree.charAt(1) === '=') firstThree = firstThree.charAt(0);	// This will handle the case of 'm'
  		switch (firstThree) {
  			case 'mix':return {len: skipAlpha(str, i), token: 'Mix'};
  			case 'dor':return {len: skipAlpha(str, i), token: 'Dor'};
  			case 'phr':return {len: skipAlpha(str, i), token: 'Phr'};
  			case 'lyd':return {len: skipAlpha(str, i), token: 'Lyd'};
  			case 'loc':return {len: skipAlpha(str, i), token: 'Loc'};
  			case 'aeo':return {len: skipAlpha(str, i), token: 'm'};
  			case 'maj':return {len: skipAlpha(str, i), token: ''};
  			case 'ion':return {len: skipAlpha(str, i), token: ''};
  			case 'min':return {len: skipAlpha(str, i), token: 'm'};
  			case 'm':return {len: skipAlpha(str, i), token: 'm'};
  		}
  		return {len: 0};
  	};
  
  	this.getClef = function(str, bExplicitOnly) {
  		var strOrig = str;
  		var i = this.skipWhiteSpace(str);
  		if (finished(str, i))
  			return {len: 0};
  		// The word 'clef' is optional, but if it appears, a clef MUST appear
  		var needsClef = false;
  		var strClef = str.substring(i);
  		if (parseCommon.startsWith(strClef, 'clef=')) {
  			needsClef = true;
  			strClef = strClef.substring(5);
  			i += 5;
  		}
  		if (strClef.length === 0 && needsClef)
  			return {len: i+5, warn: "No clef specified: " + strOrig};
  
  		var j = this.skipWhiteSpace(strClef);
  		if (finished(strClef, j))
  			return {len: 0};
  		if (j > 0) {
  			i += j;
  			strClef = strClef.substring(j);
  		}
  		var name = null;
  		if (parseCommon.startsWith(strClef, 'treble'))
  			name = 'treble';
  		else if (parseCommon.startsWith(strClef, 'bass3'))
  			name = 'bass3';
  		else if (parseCommon.startsWith(strClef, 'bass'))
  			name = 'bass';
  		else if (parseCommon.startsWith(strClef, 'tenor'))
  			name = 'tenor';
  		else if (parseCommon.startsWith(strClef, 'alto2'))
  			name = 'alto2';
  		else if (parseCommon.startsWith(strClef, 'alto1'))
  			name = 'alto1';
  		else if (parseCommon.startsWith(strClef, 'alto'))
  			name = 'alto';
  		else if (!bExplicitOnly && (needsClef && parseCommon.startsWith(strClef, 'none')))
  			name = 'none';
  		else if (parseCommon.startsWith(strClef, 'perc'))
  			name = 'perc';
  		else if (!bExplicitOnly && (needsClef && parseCommon.startsWith(strClef, 'C')))
  			name = 'tenor';
  		else if (!bExplicitOnly && (needsClef && parseCommon.startsWith(strClef, 'F')))
  			name = 'bass';
  		else if (!bExplicitOnly && (needsClef && parseCommon.startsWith(strClef, 'G')))
  			name = 'treble';
  		else
  			return {len: i+5, warn: "Unknown clef specified: " + strOrig};
  
  		strClef = strClef.substring(name.length);
  		j = this.isMatch(strClef, '+8');
  		if (j > 0)
  			name += "+8";
  		else {
  			j = this.isMatch(strClef, '-8');
  			if (j > 0)
  				name += "-8";
  		}
  		return {len: i+name.length, token: name, explicit: needsClef};
  	};
  
  	// This returns one of the legal bar lines
  	// This is called alot and there is no obvious tokenable items, so this is broken apart.
  	this.getBarLine = function(line, i) {
  		switch (line.charAt(i)) {
  			case ']':
  				++i;
  				switch (line.charAt(i)) {
  					case '|': return {len: 2, token: "bar_thick_thin"};
  					case '[':
  						++i;
  						if ((line.charAt(i) >= '1' && line.charAt(i) <= '9') || line.charAt(i) === '"')
  							return {len: 2, token: "bar_invisible"};
  						return {len: 1, warn: "Unknown bar symbol"};
  					default:
  						return {len: 1, token: "bar_invisible"};
  				}
  				break;
  			case ':':
  				++i;
  				switch (line.charAt(i)) {
  					case ':': return {len: 2, token: "bar_dbl_repeat"};
  					case '|':	// :|
  						++i;
  						switch (line.charAt(i)) {
  							case ']':	// :|]
  								++i;
  								switch (line.charAt(i)) {
  									case '|':	// :|]|
  										++i;
  										if (line.charAt(i) === ':')  return {len: 5, token: "bar_dbl_repeat"};
  										return {len: 3, token: "bar_right_repeat"};
  									default:
  										return {len: 3, token: "bar_right_repeat"};
  								}
  								break;
  							case '|':	// :||
  								++i;
  								if (line.charAt(i) === ':')  return {len: 4, token: "bar_dbl_repeat"};
  								return {len: 3, token: "bar_right_repeat"};
  							default:
  								return {len: 2, token: "bar_right_repeat"};
  						}
  						break;
  					default:
  						return {len: 1, warn: "Unknown bar symbol"};
  				}
  				break;
  			case '[':	// [
  				++i;
  				if (line.charAt(i) === '|') {	// [|
  					++i;
  					switch (line.charAt(i)) {
  						case ':': return {len: 3, token: "bar_left_repeat"};
  						case ']': return {len: 3, token: "bar_invisible"};
  						default: return {len: 2, token: "bar_thick_thin"};
  					}
  				} else {
  					if ((line.charAt(i) >= '1' && line.charAt(i) <= '9') || line.charAt(i) === '"')
  						return {len: 1, token: "bar_invisible"};
  					return {len: 0};
  				}
  				break;
  			case '|':	// |
  				++i;
  				switch (line.charAt(i)) {
  					case ']': return {len: 2, token: "bar_thin_thick"};
  					case '|': // ||
  						++i;
  						if (line.charAt(i) === ':') return {len: 3, token: "bar_left_repeat"};
  						return {len: 2, token: "bar_thin_thin"};
  					case ':':	// |:
  						var colons = 0;
  						while (line.charAt(i+colons) === ':') colons++;
  						return { len: 1+colons, token: "bar_left_repeat"};
  					default: return {len: 1, token: "bar_thin"};
  				}
  				break;
  		}
  		return {len: 0};
  	};
  
  	// this returns all the characters in the string that match one of the characters in the legalChars string
  	this.getTokenOf = function(str, legalChars) {
  		for (var i = 0; i < str.length; i++) {
  			if (legalChars.indexOf(str.charAt(i)) < 0)
  				return {len: i, token: str.substring(0, i)};
  		}
  		return {len: i, token: str};
  	};
  
  	this.getToken = function(str, start, end) {
  		// This returns the next set of chars that doesn't contain spaces
  		var i = start;
  		while (i < end && !this.isWhiteSpace(str.charAt(i)))
  			i++;
  		return str.substring(start, i);
  	};
  
  	// This just sees if the next token is the word passed in, with possible leading spaces
  	this.isMatch = function(str, match) {
  		var i = this.skipWhiteSpace(str);
  		if (finished(str, i))
  			return 0;
  		if (parseCommon.startsWith(str.substring(i), match))
  			return i+match.length;
  		return 0;
  	};
  
  	this.getPitchFromTokens = function(tokens) {
  		var ret = { };
  		var pitches = {A: 5, B: 6, C: 0, D: 1, E: 2, F: 3, G: 4, a: 12, b: 13, c: 7, d: 8, e: 9, f: 10, g: 11};
  		ret.position = pitches[tokens[0].token];
  		if (ret.position === undefined)
  			return { warn: "Pitch expected. Found: " + tokens[0].token };
  		tokens.shift();
  		while (tokens.length) {
  			switch (tokens[0].token) {
  				case ',': ret.position -= 7; tokens.shift(); break;
  				case '\'': ret.position += 7; tokens.shift(); break;
  				default: return ret;
  			}
  		}
  		return ret;
  	};
  
  	this.getKeyAccidentals2 = function(tokens) {
  		var accs;
  		// find and strip off all accidentals in the token list
  		while (tokens.length > 0) {
  			var acc;
  			if (tokens[0].token === '^') {
  				acc = 'sharp';
  				tokens.shift();
  				if (tokens.length === 0) return {accs: accs, warn: 'Expected note name after ' + acc};
  				switch (tokens[0].token) {
  					case '^': acc = 'dblsharp'; tokens.shift(); break;
  					case '/': acc = 'quartersharp'; tokens.shift(); break;
  				}
  			} else if (tokens[0].token === '=') {
  				acc = 'natural';
  				tokens.shift();
  			} else if (tokens[0].token === '_') {
  				acc = 'flat';
  				tokens.shift();
  				if (tokens.length === 0) return {accs: accs, warn: 'Expected note name after ' + acc};
  				switch (tokens[0].token) {
  					case '_': acc = 'dblflat'; tokens.shift(); break;
  					case '/': acc = 'quarterflat'; tokens.shift(); break;
  				}
  			} else {
  				// Not an accidental, we'll assume that a later parse will recognize it.
  				return { accs: accs };
  			}
  			if (tokens.length === 0) return {accs: accs, warn: 'Expected note name after ' + acc};
  			switch (tokens[0].token.charAt(0))
  			{
  				case 'a':
  				case 'b':
  				case 'c':
  				case 'd':
  				case 'e':
  				case 'f':
  				case 'g':
  				case 'A':
  				case 'B':
  				case 'C':
  				case 'D':
  				case 'E':
  				case 'F':
  				case 'G':
  					if (accs === undefined)
  						accs = [];
  					accs.push({ acc: acc, note: tokens[0].token.charAt(0) });
  					if (tokens[0].token.length === 1)
  						tokens.shift();
  					else
  						tokens[0].token = tokens[0].token.substring(1);
  					break;
  				default:
  					return {accs: accs, warn: 'Expected note name after ' + acc + ' Found: ' + tokens[0].token };
  			}
  		}
  		return { accs: accs };
  	};
  
  	// This gets an accidental marking for the key signature. It has the accidental then the pitch letter.
  	this.getKeyAccidental = function(str) {
  		var accTranslation = {
  			'^': 'sharp',
  			'^^': 'dblsharp',
  			'=': 'natural',
  			'_': 'flat',
  			'__': 'dblflat',
  			'_/': 'quarterflat',
  			'^/': 'quartersharp'
  		};
  		var i = this.skipWhiteSpace(str);
  		if (finished(str, i))
  			return {len: 0};
  		var acc = null;
  		switch (str.charAt(i))
  		{
  			case '^':
  			case '_':
  			case '=':
  				acc = str.charAt(i);
  				break;
  			default:return {len: 0};
  		}
  		i++;
  		if (finished(str, i))
  			return {len: 1, warn: 'Expected note name after accidental'};
  		switch (str.charAt(i))
  		{
  			case 'a':
  			case 'b':
  			case 'c':
  			case 'd':
  			case 'e':
  			case 'f':
  			case 'g':
  			case 'A':
  			case 'B':
  			case 'C':
  			case 'D':
  			case 'E':
  			case 'F':
  			case 'G':
  				return {len: i+1, token: {acc: accTranslation[acc], note: str.charAt(i)}};
  			case '^':
  			case '_':
  			case '/':
  				acc += str.charAt(i);
  				i++;
  				if (finished(str, i))
  					return {len: 2, warn: 'Expected note name after accidental'};
  				switch (str.charAt(i))
  				{
  					case 'a':
  					case 'b':
  					case 'c':
  					case 'd':
  					case 'e':
  					case 'f':
  					case 'g':
  					case 'A':
  					case 'B':
  					case 'C':
  					case 'D':
  					case 'E':
  					case 'F':
  					case 'G':
  						return {len: i+1, token: {acc: accTranslation[acc], note: str.charAt(i)}};
  					default:
  						return {len: 2, warn: 'Expected note name after accidental'};
  				}
  				break;
  			default:
  				return {len: 1, warn: 'Expected note name after accidental'};
  		}
  	};
  
  	this.isWhiteSpace = function(ch) {
  		return ch === ' ' || ch === '\t' || ch === '\x12';
  	};
  
  	this.getMeat = function(line, start, end) {
  		// This removes any comments starting with '%' and trims the ends of the string so that there are no leading or trailing spaces.
  		// it returns just the start and end characters that contain the meat.
  		var comment = line.indexOf('%', start);
  		if (comment >= 0 && comment < end)
  			end = comment;
  		while (start < end && (line.charAt(start) === ' ' || line.charAt(start) === '\t' || line.charAt(start) === '\x12'))
  			start++;
  		while (start < end && (line.charAt(end-1) === ' ' || line.charAt(end-1) === '\t' || line.charAt(end-1) === '\x12'))
  			end--;
  		return {start: start, end: end};
  	};
  
  	var isLetter = function(ch) {
  		return (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z');
  	};
  
  	var isNumber = function(ch) {
  		return (ch >= '0' && ch <= '9');
  	};
  
  	this.tokenize = function(line, start, end, alphaUntilWhiteSpace) {
  		// this returns all the tokens inside the passed string. A token is a punctuation mark, a string of digits, a string of letters.
  		//  Quoted strings are one token.
  		//  If there is a minus sign next to a number, then it is included in the number.
  		// If there is a period immediately after a number, with a number immediately following, then a float is returned.
  		// The type of token is returned: quote, alpha, number, punct
  		// If alphaUntilWhiteSpace is true, then the behavior of the alpha token changes.
  
  		var ret = this.getMeat(line, start, end);
  		start = ret.start;
  		end = ret.end;
  		var tokens = [];
  		var i;
  		while (start < end) {
  			if (line.charAt(start) === '"') {
  				i = start+1;
  				while (i < end && line.charAt(i) !== '"') i++;
  				tokens.push({ type: 'quote', token: line.substring(start+1, i), start: start+1, end: i});
  				i++;
  			} else if (isLetter(line.charAt(start))) {
  				i = start+1;
  				if (alphaUntilWhiteSpace)
  					while (i < end && !this.isWhiteSpace(line.charAt(i))) i++;
  				else
  					while (i < end && isLetter(line.charAt(i))) i++;
  				tokens.push({ type: 'alpha', token: line.substring(start, i), continueId: isNumber(line.charAt(i)), start: start, end: i});
  				start = i + 1;
  			} else if (line.charAt(start) === '.' && isNumber(line.charAt(i+1))) {
  				i = start+1;
  				var int2 = null;
  				var float2 = null;
  				while (i < end && isNumber(line.charAt(i))) i++;
  
  				float2 = parseFloat(line.substring(start, i));
  				tokens.push({ type: 'number', token: line.substring(start, i), intt: int2, floatt: float2, continueId: isLetter(line.charAt(i)), start: start, end: i});
  				start = i + 1;
  			} else if (isNumber(line.charAt(start)) || (line.charAt(start) === '-' && isNumber(line.charAt(i+1)))) {
  				i = start+1;
  				var intt = null;
  				var floatt = null;
  				while (i < end && isNumber(line.charAt(i))) i++;
  				if (line.charAt(i) === '.' && isNumber(line.charAt(i+1))) {
  					i++;
  					while (i < end && isNumber(line.charAt(i))) i++;
  				} else
  					intt = parseInt(line.substring(start, i));
  
  				floatt = parseFloat(line.substring(start, i));
  				tokens.push({ type: 'number', token: line.substring(start, i), intt: intt, floatt: floatt, continueId: isLetter(line.charAt(i)), start: start, end: i});
  				start = i + 1;
  			} else if (line.charAt(start) === ' ' || line.charAt(start) === '\t') {
  				i = start+1;
  			} else {
  				tokens.push({ type: 'punct', token: line.charAt(start), start: start, end: start+1});
  				i = start+1;
  			}
  			start = i;
  		}
  		return tokens;
  	};
  
  	this.getVoiceToken = function(line, start, end) {
  		// This finds the next token. A token is delimited by a space or an equal sign. If it starts with a quote, then the portion between the quotes is returned.
  		var i = start;
  		while (i < end && this.isWhiteSpace(line.charAt(i)) || line.charAt(i) === '=')
  			i++;
  
  		if (line.charAt(i) === '"') {
  			var close = line.indexOf('"', i+1);
  			if (close === -1 || close >= end)
  				return {len: 1, err: "Missing close quote"};
  			return {len: close-start+1, token: this.translateString(line.substring(i+1, close))};
  		} else {
  			var ii = i;
  			while (ii < end && !this.isWhiteSpace(line.charAt(ii)) && line.charAt(ii) !== '=')
  				ii++;
  			return {len: ii-start+1, token: line.substring(i, ii)};
  		}
  	};
  
  	var charMap = {
  		"`a": 'à', "'a": "á", "^a": "â", "~a": "ã", "\"a": "ä", "oa": "å", "aa": "å", "=a": "ā", "ua": "ă", ";a": "ą",
  		"`e": 'è', "'e": "é", "^e": "ê", "\"e": "ë", "=e": "ē", "ue": "ĕ", ";e": "ę", ".e": "ė",
  		"`i": 'ì', "'i": "í", "^i": "î", "\"i": "ï", "=i": "ī", "ui": "ĭ", ";i": "į",
  		"`o": 'ò', "'o": "ó", "^o": "ô", "~o": "õ", "\"o": "ö", "=o": "ō", "uo": "ŏ", "/o": "ø",
  		"`u": 'ù', "'u": "ú", "^u": "û", "~u": "ũ", "\"u": "ü", "ou": "ů", "=u": "ū", "uu": "ŭ", ";u": "ų",
  		"`A": 'À', "'A": "Á", "^A": "Â", "~A": "Ã", "\"A": "Ä", "oA": "Å", "AA": "Å", "=A": "Ā", "uA": "Ă", ";A": "Ą",
  		"`E": 'È', "'E": "É", "^E": "Ê", "\"E": "Ë", "=E": "Ē", "uE": "Ĕ", ";E": "Ę", ".E": "Ė",
  		"`I": 'Ì', "'I": "Í", "^I": "Î", "~I": "Ĩ", "\"I": "Ï", "=I": "Ī", "uI": "Ĭ", ";I": "Į", ".I": "İ",
  		"`O": 'Ò', "'O": "Ó", "^O": "Ô", "~O": "Õ", "\"O": "Ö", "=O": "Ō", "uO": "Ŏ", "/O": "Ø",
  		"`U": 'Ù', "'U": "Ú", "^U": "Û", "~U": "Ũ", "\"U": "Ü", "oU": "Ů", "=U": "Ū", "uU": "Ŭ", ";U": "Ų",
  		"ae": "æ", "AE": "Æ", "oe": "œ", "OE": "Œ", "ss": "ß",
  		"'c": "ć", "^c": "ĉ", "uc": "č", "cc": "ç", ".c": "ċ", "cC": "Ç", "'C": "Ć", "^C": "Ĉ", "uC": "Č", ".C": "Ċ",
  		"~N": "Ñ", "~n": "ñ",
  		"=s": "š", "vs": "š",
  		"DH": "Ð", "dh": "ð",
  		"HO": "Ő", "Ho": "ő", "HU": "Ű", "Hu": "ű",
  		"'Y": "Ý", "'y": "ý", "^Y": "Ŷ", "^y": "ŷ", "\"Y": "Ÿ", "\"y": "ÿ",
  		"vS": "Š", "vZ": "Ž", "vz": 'ž'
  
  // More chars: Ĳ ĳ Ď ď Đ đ Ĝ ĝ Ğ ğ Ġ ġ Ģ ģ Ĥ ĥ Ħ ħ Ĵ ĵ Ķ ķ ĸ Ĺ ĺ Ļ ļ Ľ ľ Ŀ ŀ Ł ł Ń ń Ņ ņ Ň ň ŉ Ŋ ŋ Ŕ ŕ Ŗ ŗ Ř ř Ś ś Ŝ ŝ Ş ş Š Ţ ţ Ť ť Ŧ ŧ Ŵ ŵ Ź ź Ż ż Ž
  	};
  	var charMap1 = {
  		"#": "♯",
  		"b": "♭",
  		"=": "♮"
  	};
  	var charMap2 = {
  		"201": "♯",
  		"202": "♭",
  		"203": "♮",
  		"241": "¡",
  		"242": "¢", "252": "a", "262": "2", "272": "o", "302": "Â", "312": "Ê", "322": "Ò", "332": "Ú", "342": "â", "352": "ê", "362": "ò", "372": "ú",
  		"243": "£", "253": "«", "263": "3", "273": "»", "303": "Ã", "313": "Ë", "323": "Ó", "333": "Û", "343": "ã", "353": "ë", "363": "ó", "373": "û",
  		"244": "¤", "254": "¬", "264": "  ́", "274": "1⁄4", "304": "Ä", "314": "Ì", "324": "Ô", "334": "Ü", "344": "ä", "354": "ì", "364": "ô", "374": "ü",
  		"245": "¥", "255": "-", "265": "μ", "275": "1⁄2", "305": "Å", "315": "Í", "325": "Õ", "335": "Ý",  "345": "å", "355": "í", "365": "õ", "375": "ý",
  		"246": "¦", "256": "®", "266": "¶", "276": "3⁄4", "306": "Æ", "316": "Î", "326": "Ö", "336": "Þ", "346": "æ", "356": "î", "366": "ö", "376": "þ",
  		"247": "§", "257": " ̄", "267": "·", "277": "¿", "307": "Ç", "317": "Ï", "327": "×", "337": "ß", "347": "ç", "357": "ï", "367": "÷", "377": "ÿ",
  		"250": " ̈", "260": "°", "270": " ̧", "300": "À", "310": "È", "320": "Ð", "330": "Ø", "340": "à", "350": "è", "360": "ð", "370": "ø",
  		"251": "©", "261": "±", "271": "1", "301": "Á", "311": "É", "321": "Ñ", "331": "Ù", "341": "á", "351": "é", "361": "ñ", "371": "ù" };
  	this.translateString = function(str) {
  		var arr = str.split('\\');
  		if (arr.length === 1) return str;
  		var out = null;
  		parseCommon.each(arr, function(s) {
  			if (out === null)
  				out = s;
  			else {
  				var c = charMap[s.substring(0, 2)];
  				if (c !== undefined)
  					out += c + s.substring(2);
  				else {
  					c = charMap2[s.substring(0, 3)];
  					if (c !== undefined)
  						out += c + s.substring(3);
  					else {
  						c = charMap1[s.substring(0, 1)];
  						if (c !== undefined)
  							out += c + s.substring(1);
  						else
  							out += "\\" + s;
  					}
  				}
  			}
  		});
  		return out;
  	};
  	this.getNumber = function(line, index) {
  		var num = 0;
  		while (index < line.length) {
  			switch (line.charAt(index)) {
  				case '0':num = num*10;index++;break;
  				case '1':num = num*10+1;index++;break;
  				case '2':num = num*10+2;index++;break;
  				case '3':num = num*10+3;index++;break;
  				case '4':num = num*10+4;index++;break;
  				case '5':num = num*10+5;index++;break;
  				case '6':num = num*10+6;index++;break;
  				case '7':num = num*10+7;index++;break;
  				case '8':num = num*10+8;index++;break;
  				case '9':num = num*10+9;index++;break;
  				default:
  					return {num: num, index: index};
  			}
  		}
  		return {num: num, index: index};
  	};
  
  	this.getFraction = function(line, index) {
  		var num = 1;
  		var den = 1;
  		if (line.charAt(index) !== '/') {
  			var ret = this.getNumber(line, index);
  			num = ret.num;
  			index = ret.index;
  		}
  		if (line.charAt(index) === '/') {
  			index++;
  			if (line.charAt(index) === '/') {
  				var div = 0.5;
  				while (line.charAt(index++) === '/')
  					div = div /2;
  				return {value: num * div, index: index-1};
  			} else {
  				var iSave = index;
  				var ret2 = this.getNumber(line, index);
  				if (ret2.num === 0 && iSave === index)	// If we didn't use any characters, it is an implied 2
  					ret2.num = 2;
  				if (ret2.num !== 0)
  					den = ret2.num;
  				index = ret2.index;
  			}
  		}
  
  		return {value: num/den, index: index};
  	};
  
  	this.theReverser = function(str) {
  		if (parseCommon.endsWith(str, ", The"))
  			return "The " + str.substring(0, str.length-5);
  		if (parseCommon.endsWith(str, ", A"))
  			return "A " + str.substring(0, str.length-3);
  		return str;
  	};
  
  	this.stripComment = function(str) {
  		var i = str.indexOf('%');
  		if (i >= 0)
  			return parseCommon.strip(str.substring(0, i));
  		return parseCommon.strip(str);
  	};
  
  	this.getInt = function(str) {
  		// This parses the beginning of the string for a number and returns { value: num, digits: num }
  		// If digits is 0, then the string didn't point to a number.
  		var x = parseInt(str);
  		if (isNaN(x))
  			return {digits: 0};
  		var s = "" + x;
  		var i = str.indexOf(s);	// This is to account for leading spaces
  		return {value: x, digits: i+s.length};
  	};
  
  	this.getFloat = function(str) {
  		// This parses the beginning of the string for a number and returns { value: num, digits: num }
  		// If digits is 0, then the string didn't point to a number.
  		var x = parseFloat(str);
  		if (isNaN(x))
  			return {digits: 0};
  		var s = "" + x;
  		var i = str.indexOf(s);	// This is to account for leading spaces
  		return {value: x, digits: i+s.length};
  	};
  
  	this.getMeasurement = function(tokens) {
  		if (tokens.length === 0) return { used: 0 };
  		var used = 1;
  		var num = '';
  		if (tokens[0].token === '-') {
  			tokens.shift();
  			num = '-';
  			used++;
  		}
  		else if (tokens[0].type !== 'number') return { used: 0 };
  		num += tokens.shift().token;
  		if (tokens.length === 0) return { used: 1, value: parseInt(num) };
  		var x = tokens.shift();
  		if (x.token === '.') {
  			used++;
  			if (tokens.length === 0) return { used: used, value: parseInt(num) };
  			if (tokens[0].type === 'number') {
  				x = tokens.shift();
  				num = num + '.' + x.token;
  				used++;
  				if (tokens.length === 0) return { used: used, value: parseFloat(num) };
  			}
  			x = tokens.shift();
  		}
  		switch (x.token) {
  			case 'pt': return { used: used+1, value: parseFloat(num) };
  			case 'cm': return { used: used+1, value: parseFloat(num)/2.54*72 };
  			case 'in': return { used: used+1, value: parseFloat(num)*72 };
  			default: tokens.unshift(x); return { used: used, value: parseFloat(num) };
  		}
  		return { used: 0 };
  	};
  	var substInChord = function(str)
  	{
  		while ( str.indexOf("\\n") !== -1)
  		{
  			str = str.replace("\\n", "\n");
  		}
  		return str;
  	};
  	this.getBrackettedSubstring = function(line, i, maxErrorChars, _matchChar)
  	{
  		// This extracts the sub string by looking at the first character and searching for that
  		// character later in the line (or search for the optional _matchChar).
  		// For instance, if the first character is a quote it will look for
  		// the end quote. If the end of the line is reached, then only up to the default number
  		// of characters are returned, so that a missing end quote won't eat up the entire line.
  		// It returns the substring and the number of characters consumed.
  		// The number of characters consumed is normally two more than the size of the substring,
  		// but in the error case it might not be.
  		var matchChar = _matchChar || line.charAt(i);
  		var pos = i+1;
  		while ((pos < line.length) && (line.charAt(pos) !== matchChar))
  			++pos;
  		if (line.charAt(pos) === matchChar)
  			return [pos-i+1,substInChord(line.substring(i+1, pos)), true];
  		else	// we hit the end of line, so we'll just pick an arbitrary num of chars so the line doesn't disappear.
  		{
  			pos = i+maxErrorChars;
  			if (pos > line.length-1)
  				pos = line.length-1;
  			return [pos-i+1, substInChord(line.substring(i+1, pos)), false];
  		}
  	};
  };
  
  //module.exports = Tokenizer;  
  export { Tokenizer }; 
  