/*
 * Copyright 2000-2016 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.jetbrains.python.parsing;

import java.util.EnumSet;
import java.util.Set;

import org.jetbrains.annotations.NonNls;
import javax.annotation.Nullable;
import com.intellij.lang.ITokenTypeRemapper;
import com.intellij.lang.PsiBuilder;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.TokenSet;
import com.intellij.util.text.CharArrayUtil;
import com.jetbrains.python.PyElementTypes;
import com.jetbrains.python.PyTokenTypes;
import com.jetbrains.python.psi.LanguageLevel;
import com.jetbrains.python.psi.PyElementType;


/**
 * @author yole
 */
public class StatementParsing extends Parsing implements ITokenTypeRemapper
{
	private static final Logger LOG = Logger.getInstance("#com.jetbrains.python.parsing.StatementParsing");
	@NonNls
	protected static final String TOK_FUTURE_IMPORT = "__future__";
	@NonNls
	protected static final String TOK_WITH_STATEMENT = "with_statement";
	@NonNls
	protected static final String TOK_NESTED_SCOPES = "nested_scopes";
	@NonNls
	protected static final String TOK_PRINT_FUNCTION = "print_function";
	@NonNls
	protected static final String TOK_WITH = "with";
	@NonNls
	protected static final String TOK_AS = "as";
	@NonNls
	protected static final String TOK_PRINT = "print";
	@NonNls
	protected static final String TOK_NONE = "None";
	@NonNls
	protected static final String TOK_TRUE = "True";
	@NonNls
	protected static final String TOK_DEBUG = "__debug__";
	@NonNls
	protected static final String TOK_FALSE = "False";
	@NonNls
	protected static final String TOK_NONLOCAL = "nonlocal";
	@NonNls
	protected static final String TOK_EXEC = "exec";
	@NonNls
	protected static final String TOK_ASYNC = "async";
	@NonNls
	protected static final String TOK_AWAIT = "await";

	private static final String EXPRESSION_EXPECTED = "Expression expected";
	public static final String IDENTIFIER_EXPECTED = "Identifier expected";

	protected enum Phase
	{
		NONE, FROM, FUTURE, IMPORT
	} // 'from __future__ import' phase

	private Phase myFutureImportPhase = Phase.NONE;
	private boolean myExpectAsKeyword = false;

	public enum FUTURE
	{
		ABSOLUTE_IMPORT, DIVISION, GENERATORS, NESTED_SCOPES, WITH_STATEMENT, PRINT_FUNCTION
	}

	protected Set<FUTURE> myFutureFlags = EnumSet.noneOf(FUTURE.class);

	public static class ImportTypes
	{
		public final IElementType statement;
		public final IElementType element;
		public IElementType starElement;

		public ImportTypes(IElementType statement, IElementType element, IElementType starElement)
		{
			this.statement = statement;
			this.element = element;
			this.starElement = starElement;
		}
	}

	public StatementParsing(ParsingContext context, @Nullable FUTURE futureFlag)
	{
		super(context);
		if(futureFlag != null)
		{
			myFutureFlags.add(futureFlag);
		}
	}

	private void setExpectAsKeyword(boolean expectAsKeyword)
	{
		myExpectAsKeyword = expectAsKeyword;
		myBuilder.setTokenTypeRemapper(this);  // clear cached token type
	}

	public void parseStatement()
	{

		while(myBuilder.getTokenType() == PyTokenTypes.STATEMENT_BREAK)
		{
			myBuilder.advanceLexer();
		}

		final IElementType firstToken;
		firstToken = myBuilder.getTokenType();

		if(firstToken == null)
		{
			return;
		}

		if(firstToken == PyTokenTypes.WHILE_KEYWORD)
		{
			parseWhileStatement();
			return;
		}
		if(firstToken == PyTokenTypes.IF_KEYWORD)
		{
			parseIfStatement(PyTokenTypes.IF_KEYWORD, PyTokenTypes.ELIF_KEYWORD, PyTokenTypes.ELSE_KEYWORD, PyElementTypes.IF_STATEMENT);
			return;
		}
		if(firstToken == PyTokenTypes.FOR_KEYWORD)
		{
			parseForStatement(myBuilder.mark());
			return;
		}
		if(firstToken == PyTokenTypes.TRY_KEYWORD)
		{
			parseTryStatement();
			return;
		}
		if(firstToken == PyTokenTypes.DEF_KEYWORD)
		{
			getFunctionParser().parseFunctionDeclaration(myBuilder.mark(), false);
			return;
		}
		if(firstToken == PyTokenTypes.AT)
		{
			getFunctionParser().parseDecoratedDeclaration();
			return;
		}
		if(firstToken == PyTokenTypes.CLASS_KEYWORD)
		{
			parseClassDeclaration();
			return;
		}
		if(firstToken == PyTokenTypes.WITH_KEYWORD)
		{
			parseWithStatement(myBuilder.mark());
			return;
		}
		if(firstToken == PyTokenTypes.ASYNC_KEYWORD)
		{
			parseAsyncStatement();
			return;
		}

		parseSimpleStatement();
	}

	protected void parseSimpleStatement()
	{
		PsiBuilder builder = myContext.getBuilder();
		final IElementType firstToken = builder.getTokenType();
		if(firstToken == null)
		{
			return;
		}
		if(firstToken == PyTokenTypes.PRINT_KEYWORD && hasPrintStatement())
		{
			parsePrintStatement(builder);
			return;
		}
		if(firstToken == PyTokenTypes.ASSERT_KEYWORD)
		{
			parseAssertStatement();
			return;
		}
		if(firstToken == PyTokenTypes.BREAK_KEYWORD)
		{
			parseKeywordStatement(builder, PyElementTypes.BREAK_STATEMENT);
			return;
		}
		if(firstToken == PyTokenTypes.CONTINUE_KEYWORD)
		{
			parseKeywordStatement(builder, PyElementTypes.CONTINUE_STATEMENT);
			return;
		}
		if(firstToken == PyTokenTypes.DEL_KEYWORD)
		{
			parseDelStatement();
			return;
		}
		if(firstToken == PyTokenTypes.EXEC_KEYWORD)
		{
			parseExecStatement();
			return;
		}
		if(firstToken == PyTokenTypes.GLOBAL_KEYWORD)
		{
			parseNameDefiningStatement(PyElementTypes.GLOBAL_STATEMENT);
			return;
		}
		if(firstToken == PyTokenTypes.NONLOCAL_KEYWORD)
		{
			parseNameDefiningStatement(PyElementTypes.NONLOCAL_STATEMENT);
			return;
		}
		if(firstToken == PyTokenTypes.IMPORT_KEYWORD)
		{
			parseImportStatement(PyElementTypes.IMPORT_STATEMENT, PyElementTypes.IMPORT_ELEMENT);
			return;
		}
		if(firstToken == PyTokenTypes.FROM_KEYWORD)
		{
			parseFromImportStatement();
			return;
		}
		if(firstToken == PyTokenTypes.PASS_KEYWORD)
		{
			parseKeywordStatement(builder, PyElementTypes.PASS_STATEMENT);
			return;
		}
		if(firstToken == PyTokenTypes.RETURN_KEYWORD)
		{
			parseReturnStatement(builder);
			return;
		}
		if(firstToken == PyTokenTypes.RAISE_KEYWORD)
		{
			parseRaiseStatement();
			return;
		}
		PsiBuilder.Marker exprStatement = builder.mark();
		if(builder.getTokenType() == PyTokenTypes.YIELD_KEYWORD)
		{
			getExpressionParser().parseYieldOrTupleExpression(false);
			checkEndOfStatement();
			exprStatement.done(PyElementTypes.EXPRESSION_STATEMENT);
			return;
		}
		else if(getExpressionParser().parseExpressionOptional())
		{
			IElementType statementType = PyElementTypes.EXPRESSION_STATEMENT;
			if(PyTokenTypes.AUG_ASSIGN_OPERATIONS.contains(builder.getTokenType()))
			{
				statementType = PyElementTypes.AUG_ASSIGNMENT_STATEMENT;
				builder.advanceLexer();
				if(!getExpressionParser().parseYieldOrTupleExpression(false))
				{
					builder.error(EXPRESSION_EXPECTED);
				}
			}
			else if(atToken(PyTokenTypes.EQ) || (atToken(PyTokenTypes.COLON) && myContext.getLanguageLevel().isPy3K()))
			{
				exprStatement.rollbackTo();
				exprStatement = builder.mark();
				getExpressionParser().parseExpression(false, true);
				LOG.assertTrue(builder.getTokenType() == PyTokenTypes.EQ || builder.getTokenType() == PyTokenTypes.COLON, builder.getTokenType());

				if(builder.getTokenType() == PyTokenTypes.COLON)
				{
					statementType = PyElementTypes.TYPE_DECLARATION_STATEMENT;
					getFunctionParser().parseParameterAnnotation();
				}

				if(builder.getTokenType() == PyTokenTypes.EQ)
				{
					statementType = PyElementTypes.ASSIGNMENT_STATEMENT;
					builder.advanceLexer();

					while(true)
					{
						PsiBuilder.Marker maybeExprMarker = builder.mark();
						final boolean isYieldExpr = builder.getTokenType() == PyTokenTypes.YIELD_KEYWORD;
						if(!getExpressionParser().parseYieldOrTupleExpression(false))
						{
							maybeExprMarker.drop();
							builder.error(EXPRESSION_EXPECTED);
							break;
						}
						if(builder.getTokenType() == PyTokenTypes.EQ)
						{
							if(isYieldExpr)
							{
								maybeExprMarker.drop();
								builder.error("Cannot assign to 'yield' expression");
							}
							else
							{
								maybeExprMarker.rollbackTo();
								getExpressionParser().parseExpression(false, true);
								LOG.assertTrue(builder.getTokenType() == PyTokenTypes.EQ, builder.getTokenType());
							}
							builder.advanceLexer();
						}
						else
						{
							maybeExprMarker.drop();
							break;
						}
					}
				}
			}

			checkEndOfStatement();
			exprStatement.done(statementType);
			return;
		}
		else
		{
			exprStatement.drop();
		}

		builder.advanceLexer();
		reportParseStatementError(builder, firstToken);
	}

	protected void reportParseStatementError(PsiBuilder builder, IElementType firstToken)
	{
		if(firstToken == PyTokenTypes.INCONSISTENT_DEDENT)
		{
			builder.error("Unindent does not match any outer indentation level");
		}
		else if(firstToken == PyTokenTypes.INDENT)
		{
			builder.error("Unexpected indent");
		}
		else
		{
			builder.error("Statement expected, found " + firstToken.toString());
		}
	}

	private boolean hasPrintStatement()
	{
		return myContext.getLanguageLevel().hasPrintStatement() && !myFutureFlags.contains(FUTURE.PRINT_FUNCTION);
	}

	protected void checkEndOfStatement()
	{
		PsiBuilder builder = myContext.getBuilder();
		final ParsingScope scope = getParsingContext().getScope();
		if(builder.getTokenType() == PyTokenTypes.STATEMENT_BREAK)
		{
			builder.advanceLexer();
			scope.setAfterSemicolon(false);
		}
		else if(builder.getTokenType() == PyTokenTypes.SEMICOLON)
		{
			if(!scope.isSuite())
			{
				builder.advanceLexer();
				scope.setAfterSemicolon(true);
				if(builder.getTokenType() == PyTokenTypes.STATEMENT_BREAK)
				{
					builder.advanceLexer();
					scope.setAfterSemicolon(false);
				}
			}
		}
		else if(!builder.eof())
		{
			builder.error("End of statement expected");
		}
	}

	private void parsePrintStatement(final PsiBuilder builder)
	{
		LOG.assertTrue(builder.getTokenType() == PyTokenTypes.PRINT_KEYWORD);
		final PsiBuilder.Marker statement = builder.mark();
		builder.advanceLexer();
		if(builder.getTokenType() == PyTokenTypes.GTGT)
		{
			final PsiBuilder.Marker target = builder.mark();
			builder.advanceLexer();
			getExpressionParser().parseSingleExpression(false);
			target.done(PyElementTypes.PRINT_TARGET);
		}
		else
		{
			getExpressionParser().parseSingleExpression(false);
		}
		while(builder.getTokenType() == PyTokenTypes.COMMA)
		{
			builder.advanceLexer();
			if(getEndOfStatementsTokens().contains(builder.getTokenType()))
			{
				break;
			}
			getExpressionParser().parseSingleExpression(false);
		}
		checkEndOfStatement();
		statement.done(PyElementTypes.PRINT_STATEMENT);
	}

	protected void parseKeywordStatement(PsiBuilder builder, IElementType statementType)
	{
		final PsiBuilder.Marker statement = builder.mark();
		builder.advanceLexer();
		checkEndOfStatement();
		statement.done(statementType);
	}

	private void parseReturnStatement(PsiBuilder builder)
	{
		LOG.assertTrue(builder.getTokenType() == PyTokenTypes.RETURN_KEYWORD);
		final PsiBuilder.Marker returnStatement = builder.mark();
		builder.advanceLexer();
		if(builder.getTokenType() != null && !getEndOfStatementsTokens().contains(builder.getTokenType()))
		{
			getExpressionParser().parseExpression();
		}
		checkEndOfStatement();
		returnStatement.done(PyElementTypes.RETURN_STATEMENT);
	}

	private void parseDelStatement()
	{
		assertCurrentToken(PyTokenTypes.DEL_KEYWORD);
		final PsiBuilder.Marker delStatement = myBuilder.mark();
		myBuilder.advanceLexer();
		if(!getExpressionParser().parseSingleExpression(false))
		{
			myBuilder.error("Expression expected");
		}
		while(myBuilder.getTokenType() == PyTokenTypes.COMMA)
		{
			myBuilder.advanceLexer();
			if(!getEndOfStatementsTokens().contains(myBuilder.getTokenType()))
			{
				if(!getExpressionParser().parseSingleExpression(false))
				{
					myBuilder.error("Expression expected");
				}
			}
		}

		checkEndOfStatement();
		delStatement.done(PyElementTypes.DEL_STATEMENT);
	}

	private void parseRaiseStatement()
	{
		assertCurrentToken(PyTokenTypes.RAISE_KEYWORD);
		final PsiBuilder.Marker raiseStatement = myBuilder.mark();
		myBuilder.advanceLexer();
		if(!getEndOfStatementsTokens().contains(myBuilder.getTokenType()))
		{
			getExpressionParser().parseSingleExpression(false);
			if(myBuilder.getTokenType() == PyTokenTypes.COMMA)
			{
				myBuilder.advanceLexer();
				getExpressionParser().parseSingleExpression(false);
				if(myBuilder.getTokenType() == PyTokenTypes.COMMA)
				{
					myBuilder.advanceLexer();
					getExpressionParser().parseSingleExpression(false);
				}
			}
			else if(myBuilder.getTokenType() == PyTokenTypes.FROM_KEYWORD)
			{
				myBuilder.advanceLexer();
				if(!getExpressionParser().parseSingleExpression(false))
				{
					myBuilder.error("Expression expected");
				}
			}
		}
		checkEndOfStatement();
		raiseStatement.done(PyElementTypes.RAISE_STATEMENT);
	}

	private void parseAssertStatement()
	{
		assertCurrentToken(PyTokenTypes.ASSERT_KEYWORD);
		final PsiBuilder.Marker assertStatement = myBuilder.mark();
		myBuilder.advanceLexer();
		if(getExpressionParser().parseSingleExpression(false))
		{
			if(myBuilder.getTokenType() == PyTokenTypes.COMMA)
			{
				myBuilder.advanceLexer();
				if(!getExpressionParser().parseSingleExpression(false))
				{
					myContext.getBuilder().error(EXPRESSION_EXPECTED);
				}
			}
			checkEndOfStatement();
		}
		else
		{
			myContext.getBuilder().error(EXPRESSION_EXPECTED);
		}
		assertStatement.done(PyElementTypes.ASSERT_STATEMENT);
	}

	protected void parseImportStatement(IElementType statementType, IElementType elementType)
	{
		final PsiBuilder builder = myContext.getBuilder();
		final PsiBuilder.Marker importStatement = builder.mark();
		builder.advanceLexer();
		parseImportElements(elementType, true, false, false);
		checkEndOfStatement();
		importStatement.done(statementType);
	}

	/*
	Really parses two forms:
	from identifier import id, id... -- may be either relative or absolute
	from . import identifier -- only relative
	 */
	private void parseFromImportStatement()
	{
		PsiBuilder builder = myContext.getBuilder();
		assertCurrentToken(PyTokenTypes.FROM_KEYWORD);
		myFutureImportPhase = Phase.FROM;
		final PsiBuilder.Marker fromImportStatement = builder.mark();
		builder.advanceLexer();
		boolean from_future = false;
		boolean had_dots = parseRelativeImportDots();
		IElementType statementType = PyElementTypes.FROM_IMPORT_STATEMENT;
		if(had_dots && parseOptionalDottedName() || parseDottedName())
		{
			final ImportTypes types = checkFromImportKeyword();
			statementType = types.statement;
			final IElementType elementType = types.element;
			if(myFutureImportPhase == Phase.FUTURE)
			{
				myFutureImportPhase = Phase.IMPORT;
				from_future = true;
			}
			if(builder.getTokenType() == PyTokenTypes.MULT)
			{
				final PsiBuilder.Marker star_import_mark = builder.mark();
				builder.advanceLexer();
				star_import_mark.done(types.starElement);
			}
			else if(builder.getTokenType() == PyTokenTypes.LPAR)
			{
				builder.advanceLexer();
				parseImportElements(elementType, false, true, from_future);
				checkMatches(PyTokenTypes.RPAR, ") expected");
			}
			else
			{
				parseImportElements(elementType, false, false, from_future);
			}
		}
		else if(had_dots)
		{ // from . import ...
			final ImportTypes types = checkFromImportKeyword();
			statementType = types.statement;
			parseImportElements(types.element, false, false, from_future);
		}
		checkEndOfStatement();
		fromImportStatement.done(statementType);
		myFutureImportPhase = Phase.NONE;
	}

	protected ImportTypes checkFromImportKeyword()
	{
		checkMatches(PyTokenTypes.IMPORT_KEYWORD, "'import' expected");
		return new ImportTypes(PyElementTypes.FROM_IMPORT_STATEMENT, PyElementTypes.IMPORT_ELEMENT, PyElementTypes.STAR_IMPORT_ELEMENT);
	}

	/**
	 * Parses option dots before imported name.
	 *
	 * @return true iff there were dots.
	 */
	private boolean parseRelativeImportDots()
	{
		PsiBuilder builder = myContext.getBuilder();
		boolean had_dots = false;
		while(builder.getTokenType() == PyTokenTypes.DOT)
		{
			had_dots = true;
			builder.advanceLexer();
		}
		return had_dots;
	}

	private void parseImportElements(IElementType elementType, boolean is_module_import, boolean in_parens, final boolean from_future)
	{
		PsiBuilder builder = myContext.getBuilder();
		while(true)
		{
			final PsiBuilder.Marker asMarker = builder.mark();
			if(is_module_import)
			{ // import _
				if(!parseDottedNameAsAware(true, false))
				{
					asMarker.drop();
					break;
				}
			}
			else
			{ // from X import _
				String token_text = parseIdentifier(getReferenceType());
				if(from_future)
				{
					// TODO: mark all known future feature names
					if(TOK_WITH_STATEMENT.equals(token_text))
					{
						myFutureFlags.add(FUTURE.WITH_STATEMENT);
					}
					else if(TOK_NESTED_SCOPES.equals(token_text))
					{
						myFutureFlags.add(FUTURE.NESTED_SCOPES);
					}
					else if(TOK_PRINT_FUNCTION.equals(token_text))
					{
						myFutureFlags.add(FUTURE.PRINT_FUNCTION);
					}
				}
			}
			setExpectAsKeyword(true); // possible 'as' comes as an ident; reparse it as keyword if found
			if(builder.getTokenType() == PyTokenTypes.AS_KEYWORD)
			{
				builder.advanceLexer();
				setExpectAsKeyword(false);
				parseIdentifier(PyElementTypes.TARGET_EXPRESSION);
			}
			asMarker.done(elementType);
			setExpectAsKeyword(false);
			if(builder.getTokenType() == PyTokenTypes.COMMA)
			{
				builder.advanceLexer();
				if(in_parens && builder.getTokenType() == PyTokenTypes.RPAR)
				{
					break;
				}
			}
			else
			{
				break;
			}
		}
	}

	@Nullable
	private String parseIdentifier(final IElementType elementType)
	{
		final PsiBuilder.Marker idMarker = myBuilder.mark();
		if(myBuilder.getTokenType() == PyTokenTypes.IDENTIFIER)
		{
			String id_text = myBuilder.getTokenText();
			myBuilder.advanceLexer();
			idMarker.done(elementType);
			return id_text;
		}
		else
		{
			myBuilder.error(IDENTIFIER_EXPECTED);
			idMarker.drop();
		}
		return null;
	}

	public boolean parseOptionalDottedName()
	{
		return parseDottedNameAsAware(false, true);
	}

	public boolean parseDottedName()
	{
		return parseDottedNameAsAware(false, false);
	}

	// true if identifier was parsed or skipped as optional, false on error
	protected boolean parseDottedNameAsAware(boolean expect_as, boolean optional)
	{
		if(myBuilder.getTokenType() != PyTokenTypes.IDENTIFIER)
		{
			if(optional)
			{
				return true;
			}
			myBuilder.error(IDENTIFIER_EXPECTED);
			return false;
		}
		PsiBuilder.Marker marker = myBuilder.mark();
		myBuilder.advanceLexer();
		marker.done(getReferenceType());
		boolean old_expect_AS_kwd = myExpectAsKeyword;
		setExpectAsKeyword(expect_as);
		while(myBuilder.getTokenType() == PyTokenTypes.DOT)
		{
			marker = marker.precede();
			myBuilder.advanceLexer();
			checkMatches(PyTokenTypes.IDENTIFIER, IDENTIFIER_EXPECTED);
			marker.done(getReferenceType());
		}
		setExpectAsKeyword(old_expect_AS_kwd);
		return true;
	}

	private void parseNameDefiningStatement(final PyElementType elementType)
	{
		final PsiBuilder.Marker globalStatement = myBuilder.mark();
		myBuilder.advanceLexer();
		parseIdentifier(PyElementTypes.TARGET_EXPRESSION);
		while(myBuilder.getTokenType() == PyTokenTypes.COMMA)
		{
			myBuilder.advanceLexer();
			parseIdentifier(PyElementTypes.TARGET_EXPRESSION);
		}
		checkEndOfStatement();
		globalStatement.done(elementType);
	}

	private void parseExecStatement()
	{
		assertCurrentToken(PyTokenTypes.EXEC_KEYWORD);
		final PsiBuilder.Marker execStatement = myBuilder.mark();
		myBuilder.advanceLexer();
		getExpressionParser().parseExpression(true, false);
		if(myBuilder.getTokenType() == PyTokenTypes.IN_KEYWORD)
		{
			myBuilder.advanceLexer();
			getExpressionParser().parseSingleExpression(false);
			if(myBuilder.getTokenType() == PyTokenTypes.COMMA)
			{
				myBuilder.advanceLexer();
				getExpressionParser().parseSingleExpression(false);
			}
		}
		checkEndOfStatement();
		execStatement.done(PyElementTypes.EXEC_STATEMENT);
	}

	protected void parseIfStatement(PyElementType ifKeyword, PyElementType elifKeyword, PyElementType elseKeyword, PyElementType elementType)
	{
		assertCurrentToken(ifKeyword);
		final PsiBuilder.Marker ifStatement = myBuilder.mark();
		final PsiBuilder.Marker ifPart = myBuilder.mark();
		myBuilder.advanceLexer();
		if(!getExpressionParser().parseSingleExpression(false))
		{
			myBuilder.error("expression expected");
		}
		parseColonAndSuite();
		ifPart.done(PyElementTypes.IF_PART_IF);
		PsiBuilder.Marker elifPart = myBuilder.mark();
		while(myBuilder.getTokenType() == elifKeyword)
		{
			myBuilder.advanceLexer();
			if(!getExpressionParser().parseSingleExpression(false))
			{
				myBuilder.error("expression expected");
			}
			parseColonAndSuite();
			elifPart.done(PyElementTypes.IF_PART_ELIF);
			elifPart = myBuilder.mark();
		}
		elifPart.drop(); // we always kept an open extra elif
		final PsiBuilder.Marker elsePart = myBuilder.mark();
		if(myBuilder.getTokenType() == elseKeyword)
		{
			myBuilder.advanceLexer();
			parseColonAndSuite();
			elsePart.done(PyElementTypes.ELSE_PART);
		}
		else
		{
			elsePart.drop();
		}
		ifStatement.done(elementType);
	}

	private boolean expectColon()
	{
		if(myBuilder.getTokenType() == PyTokenTypes.COLON)
		{
			myBuilder.advanceLexer();
			return true;
		}
		else if(myBuilder.getTokenType() == PyTokenTypes.STATEMENT_BREAK)
		{
			myBuilder.error("Colon expected");
			return true;
		}
		final PsiBuilder.Marker marker = myBuilder.mark();
		while(!atAnyOfTokens(null, PyTokenTypes.DEDENT, PyTokenTypes.STATEMENT_BREAK, PyTokenTypes.COLON))
		{
			myBuilder.advanceLexer();
		}
		boolean result = matchToken(PyTokenTypes.COLON);
		if(!result && atToken(PyTokenTypes.STATEMENT_BREAK))
		{
			myBuilder.advanceLexer();
		}
		marker.error("Colon expected");
		return result;
	}

	private void parseForStatement(PsiBuilder.Marker endMarker)
	{
		assertCurrentToken(PyTokenTypes.FOR_KEYWORD);
		parseForPart();
		final PsiBuilder.Marker elsePart = myBuilder.mark();
		if(myBuilder.getTokenType() == PyTokenTypes.ELSE_KEYWORD)
		{
			myBuilder.advanceLexer();
			parseColonAndSuite();
			elsePart.done(PyElementTypes.ELSE_PART);
		}
		else
		{
			elsePart.drop();
		}
		endMarker.done(PyElementTypes.FOR_STATEMENT);
	}

	protected void parseForPart()
	{
		final PsiBuilder.Marker forPart = myBuilder.mark();
		myBuilder.advanceLexer();
		getExpressionParser().parseExpression(true, true);
		checkMatches(PyTokenTypes.IN_KEYWORD, "'in' expected");
		getExpressionParser().parseExpression();
		parseColonAndSuite();
		forPart.done(PyElementTypes.FOR_PART);
	}

	private void parseWhileStatement()
	{
		assertCurrentToken(PyTokenTypes.WHILE_KEYWORD);
		final PsiBuilder.Marker statement = myBuilder.mark();
		final PsiBuilder.Marker whilePart = myBuilder.mark();
		myBuilder.advanceLexer();
		if(!getExpressionParser().parseSingleExpression(false))
		{
			myBuilder.error(EXPRESSION_EXPECTED);
		}
		parseColonAndSuite();
		whilePart.done(PyElementTypes.WHILE_PART);
		final PsiBuilder.Marker elsePart = myBuilder.mark();
		if(myBuilder.getTokenType() == PyTokenTypes.ELSE_KEYWORD)
		{
			myBuilder.advanceLexer();
			parseColonAndSuite();
			elsePart.done(PyElementTypes.ELSE_PART);
		}
		else
		{
			elsePart.drop();
		}
		statement.done(PyElementTypes.WHILE_STATEMENT);
	}

	private void parseTryStatement()
	{
		assertCurrentToken(PyTokenTypes.TRY_KEYWORD);
		final PsiBuilder.Marker statement = myBuilder.mark();
		final PsiBuilder.Marker tryPart = myBuilder.mark();
		myBuilder.advanceLexer();
		parseColonAndSuite();
		tryPart.done(PyElementTypes.TRY_PART);
		boolean haveExceptClause = false;
		if(myBuilder.getTokenType() == PyTokenTypes.EXCEPT_KEYWORD)
		{
			haveExceptClause = true;
			while(myBuilder.getTokenType() == PyTokenTypes.EXCEPT_KEYWORD)
			{
				final PsiBuilder.Marker exceptBlock = myBuilder.mark();
				myBuilder.advanceLexer();
				if(myBuilder.getTokenType() != PyTokenTypes.COLON)
				{
					if(!getExpressionParser().parseSingleExpression(false))
					{
						myBuilder.error(EXPRESSION_EXPECTED);
					}
					setExpectAsKeyword(true);
					if(myBuilder.getTokenType() == PyTokenTypes.COMMA || myBuilder.getTokenType() == PyTokenTypes.AS_KEYWORD)
					{
						myBuilder.advanceLexer();
						if(!getExpressionParser().parseSingleExpression(true))
						{
							myBuilder.error(EXPRESSION_EXPECTED);
						}
					}
				}
				parseColonAndSuite();
				exceptBlock.done(PyElementTypes.EXCEPT_PART);
			}
			final PsiBuilder.Marker elsePart = myBuilder.mark();
			if(myBuilder.getTokenType() == PyTokenTypes.ELSE_KEYWORD)
			{
				myBuilder.advanceLexer();
				parseColonAndSuite();
				elsePart.done(PyElementTypes.ELSE_PART);
			}
			else
			{
				elsePart.drop();
			}
		}
		final PsiBuilder.Marker finallyPart = myBuilder.mark();
		if(myBuilder.getTokenType() == PyTokenTypes.FINALLY_KEYWORD)
		{
			myBuilder.advanceLexer();
			parseColonAndSuite();
			finallyPart.done(PyElementTypes.FINALLY_PART);
		}
		else
		{
			finallyPart.drop();
			if(!haveExceptClause)
			{
				myBuilder.error("'except' or 'finally' expected");
				// much better to have a statement of incorrectly determined type
				// than "TRY" and "COLON" tokens attached to nothing
			}
		}
		statement.done(PyElementTypes.TRY_EXCEPT_STATEMENT);
	}

	private void parseColonAndSuite()
	{
		if(expectColon())
		{
			parseSuite();
		}
		else
		{
			final PsiBuilder.Marker mark = myBuilder.mark();
			mark.done(PyElementTypes.STATEMENT_LIST);
		}
	}

	private void parseWithStatement(PsiBuilder.Marker endMarker)
	{
		assertCurrentToken(PyTokenTypes.WITH_KEYWORD);
		myBuilder.advanceLexer();
		while(true)
		{
			PsiBuilder.Marker withItem = myBuilder.mark();
			getExpressionParser().parseExpression();
			setExpectAsKeyword(true);
			if(myBuilder.getTokenType() == PyTokenTypes.AS_KEYWORD)
			{
				myBuilder.advanceLexer();
				if(!getExpressionParser().parseSingleExpression(true))
				{
					myBuilder.error("Identifier expected");
					// 'as' is followed by a target
				}
			}
			withItem.done(PyElementTypes.WITH_ITEM);
			if(!matchToken(PyTokenTypes.COMMA))
			{
				break;
			}
		}
		parseColonAndSuite();
		endMarker.done(PyElementTypes.WITH_STATEMENT);
	}

	private void parseClassDeclaration()
	{
		final PsiBuilder.Marker classMarker = myBuilder.mark();
		parseClassDeclaration(classMarker);
	}

	public void parseClassDeclaration(PsiBuilder.Marker classMarker)
	{
		assertCurrentToken(PyTokenTypes.CLASS_KEYWORD);
		myBuilder.advanceLexer();
		parseIdentifierOrSkip(PyTokenTypes.LPAR, PyTokenTypes.COLON);
		if(myBuilder.getTokenType() == PyTokenTypes.LPAR)
		{
			getExpressionParser().parseArgumentList();
		}
		else
		{
			final PsiBuilder.Marker inheritMarker = myBuilder.mark();
			inheritMarker.done(PyElementTypes.ARGUMENT_LIST);
		}
		final ParsingContext context = getParsingContext();
		context.pushScope(context.getScope().withClass());
		parseColonAndSuite();
		context.popScope();
		classMarker.done(PyElementTypes.CLASS_DECLARATION);
	}

	private void parseAsyncStatement()
	{
		assertCurrentToken(PyTokenTypes.ASYNC_KEYWORD);
		final PsiBuilder.Marker marker = myBuilder.mark();
		myBuilder.advanceLexer();
		final IElementType token = myBuilder.getTokenType();
		if(token == PyTokenTypes.DEF_KEYWORD)
		{
			getFunctionParser().parseFunctionDeclaration(marker, true);
		}
		else if(token == PyTokenTypes.WITH_KEYWORD)
		{
			parseWithStatement(marker);
		}
		else if(token == PyTokenTypes.FOR_KEYWORD)
		{
			parseForStatement(marker);
		}
		else
		{
			marker.drop();
			myBuilder.error("'def' or 'with' or 'for' expected");
		}
	}

	public void parseSuite()
	{
		parseSuite(null, null);
	}

	public void parseSuite(@Nullable PsiBuilder.Marker endMarker, @Nullable IElementType elType)
	{
		if(myBuilder.getTokenType() == PyTokenTypes.STATEMENT_BREAK)
		{
			myBuilder.advanceLexer();

			final PsiBuilder.Marker marker = myBuilder.mark();
			final boolean indentFound = myBuilder.getTokenType() == PyTokenTypes.INDENT;
			if(indentFound)
			{
				myBuilder.advanceLexer();
				if(myBuilder.eof())
				{
					myBuilder.error("Indented block expected");
				}
				else
				{
					while(!myBuilder.eof() && myBuilder.getTokenType() != PyTokenTypes.DEDENT)
					{
						parseStatement();
					}
				}
			}
			else
			{
				myBuilder.error("Indent expected");
			}

			marker.done(PyElementTypes.STATEMENT_LIST);
			marker.setCustomEdgeTokenBinders(LeadingCommentsBinder.INSTANCE, FollowingCommentBinder.INSTANCE);
			if(endMarker != null)
			{
				endMarker.done(elType);
			}
			if(indentFound && !myBuilder.eof())
			{
				checkMatches(PyTokenTypes.DEDENT, "Dedent expected");
			}
		}
		else
		{
			final PsiBuilder.Marker marker = myBuilder.mark();
			if(myBuilder.eof())
			{
				myBuilder.error("Statement expected");
			}
			else
			{
				final ParsingContext context = getParsingContext();
				context.pushScope(context.getScope().withSuite());
				parseSimpleStatement();
				context.popScope();
				while(matchToken(PyTokenTypes.SEMICOLON))
				{
					if(matchToken(PyTokenTypes.STATEMENT_BREAK))
					{
						break;
					}
					context.pushScope(context.getScope().withSuite());
					parseSimpleStatement();
					context.popScope();
				}
			}
			marker.done(PyElementTypes.STATEMENT_LIST);
			if(endMarker != null)
			{
				endMarker.done(elType);
			}
		}
	}

	public IElementType filter(final IElementType source, final int start, final int end, final CharSequence text)
	{
		if((myExpectAsKeyword || myContext.getLanguageLevel().hasWithStatement()) &&
				source == PyTokenTypes.IDENTIFIER && isWordAtPosition(text, start, end, TOK_AS))
		{
			return PyTokenTypes.AS_KEYWORD;
		}
		else if( // filter
				(myFutureImportPhase == Phase.FROM) &&
						source == PyTokenTypes.IDENTIFIER &&
						isWordAtPosition(text, start, end, TOK_FUTURE_IMPORT))
		{
			myFutureImportPhase = Phase.FUTURE;
			return source;
		}
		else if(hasWithStatement() &&
				source == PyTokenTypes.IDENTIFIER &&
				isWordAtPosition(text, start, end, TOK_WITH))
		{
			return PyTokenTypes.WITH_KEYWORD;
		}
		else if(hasPrintStatement() && source == PyTokenTypes.IDENTIFIER &&
				isWordAtPosition(text, start, end, TOK_PRINT))
		{
			return PyTokenTypes.PRINT_KEYWORD;
		}
		else if(myContext.getLanguageLevel().isPy3K() && source == PyTokenTypes.IDENTIFIER)
		{
			if(isWordAtPosition(text, start, end, TOK_NONE))
			{
				return PyTokenTypes.NONE_KEYWORD;
			}
			if(isWordAtPosition(text, start, end, TOK_TRUE))
			{
				return PyTokenTypes.TRUE_KEYWORD;
			}
			if(isWordAtPosition(text, start, end, TOK_FALSE))
			{
				return PyTokenTypes.FALSE_KEYWORD;
			}
			if(isWordAtPosition(text, start, end, TOK_DEBUG))
			{
				return PyTokenTypes.DEBUG_KEYWORD;
			}
			if(isWordAtPosition(text, start, end, TOK_NONLOCAL))
			{
				return PyTokenTypes.NONLOCAL_KEYWORD;
			}
			if(myContext.getLanguageLevel().isAtLeast(LanguageLevel.PYTHON35))
			{
				if(isWordAtPosition(text, start, end, TOK_ASYNC))
				{
					if(myContext.getScope().isAsync() || myBuilder.lookAhead(1) == PyTokenTypes.DEF_KEYWORD)
					{
						return PyTokenTypes.ASYNC_KEYWORD;
					}
				}
				if(isWordAtPosition(text, start, end, TOK_AWAIT))
				{
					if(myContext.getScope().isAsync())
					{
						return PyTokenTypes.AWAIT_KEYWORD;
					}
				}
			}
		}
		else if(!myContext.getLanguageLevel().isPy3K() && source == PyTokenTypes.IDENTIFIER)
		{
			if(isWordAtPosition(text, start, end, TOK_EXEC))
			{
				return PyTokenTypes.EXEC_KEYWORD;
			}
		}
		return source;
	}

	protected TokenSet getEndOfStatementsTokens()
	{
		return PyTokenTypes.END_OF_STATEMENT;
	}

	private static boolean isWordAtPosition(CharSequence text, int start, int end, final String tokenText)
	{
		return CharArrayUtil.regionMatches(text, start, end, tokenText) && end - start == tokenText.length();
	}

	private boolean hasWithStatement()
	{
		return myContext.getLanguageLevel().hasWithStatement() || myFutureFlags.contains(FUTURE.WITH_STATEMENT);
	}
}
