text: # -*- coding: utf-8 -*-
  # Requires Python 2.7 or later
  
  import io, os, sys, unittest
  
  if sys.platform == "win32":
  	import XiteWin as Xite
  else:
  	import XiteQt as Xite
  
  keywordsHTML = [
  b"b body content head href html link meta "
  	b"name rel script strong title type xmlns",
  b"function",
  b"sub"
  ]
  
  keywordsPerl = [
  b"NULL __FILE__ __LINE__ __PACKAGE__ __DATA__ __END__ AUTOLOAD "
  b"BEGIN CORE DESTROY END EQ GE GT INIT LE LT NE CHECK abs accept "
  b"alarm and atan2 bind binmode bless caller chdir chmod chomp chop "
  b"chown chr chroot close closedir cmp connect continue cos crypt "
  b"dbmclose dbmopen defined delete die do dump each else elsif endgrent "
  b"endhostent endnetent endprotoent endpwent endservent eof eq eval "
  b"exec exists exit exp fcntl fileno flock for foreach fork format "
  b"formline ge getc getgrent getgrgid getgrnam gethostbyaddr gethostbyname "
  b"gethostent getlogin getnetbyaddr getnetbyname getnetent getpeername "
  b"getpgrp getppid getpriority getprotobyname getprotobynumber getprotoent "
  b"getpwent getpwnam getpwuid getservbyname getservbyport getservent "
  b"getsockname getsockopt glob gmtime goto grep gt hex if index "
  b"int ioctl join keys kill last lc lcfirst le length link listen "
  b"local localtime lock log lstat lt map mkdir msgctl msgget msgrcv "
  b"msgsnd my ne next no not oct open opendir or ord our pack package "
  b"pipe pop pos print printf prototype push quotemeta qu "
  b"rand read readdir readline readlink readpipe recv redo "
  b"ref rename require reset return reverse rewinddir rindex rmdir "
  b"scalar seek seekdir select semctl semget semop send setgrent "
  b"sethostent setnetent setpgrp setpriority setprotoent setpwent "
  b"setservent setsockopt shift shmctl shmget shmread shmwrite shutdown "
  b"sin sleep socket socketpair sort splice split sprintf sqrt srand "
  b"stat study sub substr symlink syscall sysopen sysread sysseek "
  b"system syswrite tell telldir tie tied time times truncate "
  b"uc ucfirst umask undef unless unlink unpack unshift untie until "
  b"use utime values vec wait waitpid wantarray warn while write "
  b"xor "
  b"given when default break say state UNITCHECK __SUB__ fc"
  ]
  
  class TestLexers(unittest.TestCase):
  
  	def setUp(self):
  		self.xite = Xite.xiteFrame
  		self.ed = self.xite.ed
  		self.ed.ClearAll()
  		self.ed.EmptyUndoBuffer()
  
  	def AsStyled(self, withWindowsLineEnds):
  		text = self.ed.Contents()
  		data = io.BytesIO()
  		prevStyle = -1
  		for o in range(self.ed.Length):
  			styleNow = self.ed.GetStyleAt(o)
  			if styleNow != prevStyle:
  				styleBuf = "{%0d}" % styleNow
  				data.write(styleBuf.encode('utf-8'))
  				prevStyle = styleNow
  			data.write(text[o:o+1])
  		if withWindowsLineEnds:
  			return data.getvalue().replace(b"\n", b"\r\n")
  		else:
  			return data.getvalue()
  
  	def LexExample(self, name, lexerName, keywords, fileMode="b"):
  		self.ed.ClearAll()
  		self.ed.EmptyUndoBuffer()
  		self.ed.SetCodePage(65001)
  		self.ed.LexerLanguage = lexerName
  		mask = 0xff
  		for i in range(len(keywords)):
  			self.ed.SetKeyWords(i, keywords[i])
  
  		nameExample = os.path.join("examples", name)
  		namePrevious = nameExample +".styled"
  		nameNew = nameExample +".new"
  		with open(nameExample, "rb") as f:
  			prog = f.read()
  		if fileMode == "t" and sys.platform == "win32":
  			prog = prog.replace(b"\r\n", b"\n")
  		BOM = b"\xEF\xBB\xBF"
  		if prog.startswith(BOM):
  			prog = prog[len(BOM):]
  		lenDocument = len(prog)
  		self.ed.AddText(lenDocument, prog)
  		self.ed.Colourise(0, lenDocument)
  		self.assertEquals(self.ed.EndStyled, lenDocument)
  		try:
  			with open(namePrevious, "rb") as f:
  				prevStyled = f.read()
  			if fileMode == "t" and sys.platform == "win32":
  				prog = prog.replace(b"\r\n", b"\n")
  		except EnvironmentError:
  			prevStyled = ""
  		progStyled = self.AsStyled(fileMode == "t" and sys.platform == "win32")
  		if progStyled != prevStyled:
  			with open(nameNew, "wb") as f:
  				f.write(progStyled)
  			print("Incorrect lex for " + name)
  			print(progStyled)
  			print(prevStyled)
  			self.assertEquals(progStyled, prevStyled)
  			# The whole file doesn't parse like it did before so don't try line by line
  			# as that is likely to fail many times.
  			return
  
  		if fileMode == "b":	# "t" files are large and this is a quadratic check
  			# Try partial lexes from the start of every line which should all be identical.
  			for line in range(self.ed.LineCount):
  				lineStart = self.ed.PositionFromLine(line)
  				self.ed.StartStyling(lineStart, mask)
  				self.assertEquals(self.ed.EndStyled, lineStart)
  				self.ed.Colourise(lineStart, lenDocument)
  				progStyled = self.AsStyled(fileMode == "t" and sys.platform == "win32")
  				if progStyled != prevStyled:
  					print("Incorrect partial lex for " + name + " at line " + line)
  					with open(nameNew, "wb") as f:
  						f.write(progStyled)
  					self.assertEquals(progStyled, prevStyled)
  					# Give up after one failure
  					return
  
  	# Test lexing just once from beginning to end in text form.
  	# This is used for test cases that are too long to be exhaustively tested by lines and
  	# may be sensitive to line ends so are tested as if using Unix LF line ends.
  	def LexLongCase(self, name, lexerName, keywords, fileMode="b"):
  		self.LexExample(name, lexerName, keywords, "t")
  
  	def testCXX(self):
  		self.LexExample("x.cxx", b"cpp", [b"int"])
  
  	def testPython(self):
  		self.LexExample("x.py", b"python",
  			[b"class def else for if import in print return while"])
  
  	def testHTML(self):
  		self.LexExample("x.html", b"hypertext", keywordsHTML)
  
  	def testASP(self):
  		self.LexExample("x.asp", b"hypertext", keywordsHTML)
  
  	def testPHP(self):
  		self.LexExample("x.php", b"hypertext", keywordsHTML)
  
  	def testVB(self):
  		self.LexExample("x.vb", b"vb", [b"as dim or string"])
  
  	def testLua(self):
  		self.LexExample("x.lua", b"lua", [b"function end"])
  
  	def testRuby(self):
  		self.LexExample("x.rb", b"ruby", [b"class def end"])
  
  	def testPerl(self):
  		self.LexExample("x.pl", b"perl", keywordsPerl)
  
  	def testPerl52(self):
  		self.LexLongCase("perl-test-5220delta.pl", b"perl", keywordsPerl)
  
  	def testPerlPrototypes(self):
  		self.LexLongCase("perl-test-sub-prototypes.pl", b"perl", keywordsPerl)
  
  	def testD(self):
  		self.LexExample("x.d", b"d",
  			[b"keyword1", b"keyword2", b"", b"keyword4", b"keyword5",
  			b"keyword6", b"keyword7"])
  
  if __name__ == '__main__':
  	Xite.main("lexTests")
  