   end
   
   def tokenize(lines) do
-    split_lines(lines)
-    |> remove_comments()
-    |> split_all_words(@jack_symbols)
-    |> concat_str_constants(nil)
-    |> Enum.map(&to_token_tup/1)
+    Enum.join(lines)
+    |> bite(%Env{})
   end
 
   def xml_word({:symbol, "<"}),        do: "<symbol> &lt; </symbol>"

defmodule Tokenizer do

  @jack_keywords ["class", "constructor", "function", "method",
                  "field", "static", "var", "int", "char", "boolean",

                 "+", "-", "*", "/", "&", "|", "<", ">", "=",
                 "~"]

  @doc """
  Chomps off the front character and tokenizes based on what it finds
  """
  # Starts chomping off a multi-line comment
  def bite("/*" <> rest, %Env{comments: level}=env) do
    bite(rest, %Env{env | comments: level + 1})
  end

  # Finishes chomping off a multi-line comment
  def bite("*/" <> rest, %Env{comments: level}=env) do
    bite(rest, %{env | comments: level - 1})
  end

  # Continues chomping off a multi-line comment
  def bite(<<_>> <> rest, %Env{comments: level}=env) when level > 0 do
    bite(rest, env)
  end

  # Starts storing a string constant
  def bite("\"" <> rest, %Env{str_const: nil}) do
    bite(rest, %Env{str_const: "\""})
  end

  # Finishes storing a string constant
  def bite("\"" <> rest, %Env{str_const: str_const}=env) do
    [to_token_tup(str_const <> "\"")]
    ++ bite(rest, %Env{env | str_const: nil})
  end

  # Continues storing a string constant
  def bite(<<f>> <> rest, %Env{str_const: str_const}=env) when not is_nil(str_const) do
    bite(rest, %Env{env | str_const: str_const <> <<f>>})
  end

  # Hasn't saved any characters, so just saves the Jack symbol
  def bite(<<f>> <> rest, %Env{acc: ""}=env) when <<f>> in @jack_symbols do
    [to_token_tup(<<f>>)]
    ++ bite(rest, %Env{env | acc: ""})
  end

  # Emits saved characters and Jack symbol
  def bite(<<f>> <> rest, %Env{acc: acc}=env) when <<f>> in @jack_symbols do
    [to_token_tup(acc), to_token_tup(<<f>>)]
    ++ bite(rest, %Env{env | acc: ""})
  end

  # Handles multiple spaces
  def bite(<<f>> <> rest, %Env{acc: ""}=env) when f == ?\s do
    bite(rest, %Env{env | acc: ""})
  end

  # Emits saved characters lying before a space
  def bite(<<f>> <> rest, %Env{acc: acc}=env) when f == ?\s do
    [to_token_tup(acc)]
    ++ bite(rest, %Env{env | acc: ""})
  end

  # Save characters until we encounter a delimiter
  def bite(<<f>> <> rest, %Env{acc: acc}=env) when not <<f>> in @jack_symbols and f != ?\s do
    bite(rest, %Env{env | acc: acc <> <<f>>})
  end

  # Nothing left to chomp off
  def bite("", _), do: []
  
  def to_token_tup(t) when t in @jack_symbols,  do: {:symbol,  t}
  def to_token_tup(t) when t in @jack_keywords, do: {:keyword, t}
  def to_token_tup(t) do
