# Copyright (c) 2013 Nicolas Dandrimont <nicolas.dandrimont@crans.org>
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

from rply import LexerGenerator


lg = LexerGenerator()


# A regexp for something that should end a quoting/unquoting operator
# i.e. a space or a closing brace/paren/curly
end_quote = r'(?![\s\)\]\}])'


lg.add('LPAREN', r'\(')
lg.add('RPAREN', r'\)')
lg.add('LBRACKET', r'\[')
lg.add('RBRACKET', r'\]')
lg.add('LCURLY', r'\{')
lg.add('RCURLY', r'\}')
lg.add('QUOTE', r'\'%s' % end_quote)
lg.add('QUASIQUOTE', r'`%s' % end_quote)
lg.add('UNQUOTESPLICE', r'~@%s' % end_quote)
lg.add('UNQUOTE', r'~%s' % end_quote)
lg.add('HASHBANG', r'#!.*[^\r\n]')
lg.add('HASHREADER', r'#.')


lg.add('STRING', r'''(?x)
    (?:u|r|ur|ru)? # prefix
    "  # start string
    (?:
       | [^"\\]             # non-quote or backslash
       | \\.                # or escaped single character
       | \\x[0-9a-fA-F]{2}  # or escaped raw character
       | \\u[0-9a-fA-F]{4}  # or unicode escape
       | \\U[0-9a-fA-F]{8}  # or long unicode escape
    )* # one or more times
    "  # end string
''')


lg.add('IDENTIFIER', r'[^()\[\]{}\'"\s;]+')


lg.ignore(r';.*[\r\n]+')
lg.ignore(r'\s+')


lexer = lg.build()

hy/lex/lexer.py
# Copyright (c) 2013 Nicolas Dandrimont <nicolas.dandrimont@crans.org>
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

from rply import LexerGenerator


lg = LexerGenerator()


# A regexp for something that should end a quoting/unquoting operator
# i.e. a space or a closing brace/paren/curly
end_quote = r'(?![\s\)\]\}])'


lg.add('LPAREN', r'\(')
lg.add('RPAREN', r'\)')
lg.add('LBRACKET', r'\[')
lg.add('RBRACKET', r'\]')
lg.add('LCURLY', r'\{')
lg.add('RCURLY', r'\}')
lg.add('QUOTE', r'\'%s' % end_quote)
lg.add('QUASIQUOTE', r'`%s' % end_quote)
lg.add('UNQUOTESPLICE', r'~@%s' % end_quote)
lg.add('UNQUOTE', r'~%s' % end_quote)
lg.add('HASHBANG', r'#!.*[^\r\n]')
lg.add('HASHREADER', r'#.')


lg.add('STRING', r'''(?x)
    (?:u|r|ur|ru)? # prefix
    "  # start string
    (?:
       | [^"\\]             # non-quote or backslash
       | \\.                # or escaped single character
       | \\x[0-9a-fA-F]{2}  # or escaped raw character
       | \\u[0-9a-fA-F]{4}  # or unicode escape
       | \\U[0-9a-fA-F]{8}  # or long unicode escape
    )* # one or more times
    "  # end string
''')


lg.add('IDENTIFIER', r'[^()\[\]{}\'"\s;]+')


lg.ignore(r';.*(?=\r|\n|$)')
lg.ignore(r'\s+')


lexer = lg.build()

hy/lex/lexer.py
diff --git a/hy/lex/lexer.py b/hy/lex/lexer.py
index d4a56383b..8032fc03c 100644
--- a/hy/lex/lexer.py
+++ b/hy/lex/lexer.py
@@ -60,7 +60,7 @@
 lg.add('IDENTIFIER', r'[^()\[\]{}\'"\s;]+')
 
 
-lg.ignore(r';.*[\r\n]+')
+lg.ignore(r';.*(?=\r|\n|$)')
 lg.ignore(r'\s+')
 
 
diff --git a/tests/lex/test_lex.py b/tests/lex/test_lex.py
index bda1e3f11..af9f28633 100644
--- a/tests/lex/test_lex.py
+++ b/tests/lex/test_lex.py
@@ -260,3 +260,9 @@ def test_reader_macro():
     assert entry[0][0] == HySymbol("dispatch_reader_macro")
     assert entry[0][1] == HyExpression([HySymbol("quote"), HyString("^")])
     assert len(entry[0]) == 3
+
+
+def test_lex_comment_382():
+    """Ensure that we can tokenize sources with a comment at the end"""
+    entry = tokenize("foo ;bar\n;baz")
+    assert entry == [HySymbol("foo")]

problem with comment parsing
I was translating some code to Hy from a textbook on Python programming (http://inventwithpython.com/pygame/index.html) and ran into a problem with this direct translation.

```
(import pygame sys)
(import [pygame.locals [*]])

(pygame.init)
(setv *displaysurf* (pygame.display.set_mode (, 400 300)))
(pygame.display.set_caption "Hello World!")
(while True ; main game loop
       (do (foreach [event (pygame.event.get)]
                    (if (= event.type QUIT)
                        (do (pygame.quit)
                            (sys.exit))))
           (pygame.display.update)))
```

I get a parse error if the end-of-line comment ("main game loop") appears where it does.  It works if I remove it.

The following interaction with the prompt also surprised me.

```
=> ; some comment
hy.lex.exceptions.LexException: Could not identify the next token at line -1, column -1
```

Fixing this isn't critical, but it should probably be fixed.  I do occasionally type something, realize I need to do something else first, comment it, press Enter, type whatever setup I needed, press Enter, then press Up twice, uncomment the line, and Enter to run it.

problem with comment parsing
I was translating some code to Hy from a textbook on Python programming (http://inventwithpython.com/pygame/index.html) and ran into a problem with this direct translation.

```
(import pygame sys)
(import [pygame.locals [*]])

(pygame.init)
(setv *displaysurf* (pygame.display.set_mode (, 400 300)))
(pygame.display.set_caption "Hello World!")
(while True ; main game loop
       (do (foreach [event (pygame.event.get)]
                    (if (= event.type QUIT)
                        (do (pygame.quit)
                            (sys.exit))))
           (pygame.display.update)))
```

I get a parse error if the end-of-line comment ("main game loop") appears where it does.  It works if I remove it.

The following interaction with the prompt also surprised me.

```
=> ; some comment
hy.lex.exceptions.LexException: Could not identify the next token at line -1, column -1
```

Fixing this isn't critical, but it should probably be fixed.  I do occasionally type something, realize I need to do something else first, comment it, press Enter, type whatever setup I needed, press Enter, then press Up twice, uncomment the line, and Enter to run it.

