-- | Given a starting token, and an effectful getNext function, returns the list
-- | of tokens resulting from calling getNext until it reaches Stop.
tokensFrom :: Token -> GetNextToken -> IO [Token]
tokensFrom startToken getNext = do
  nextToken <- getNext startToken   -- nextToken :: Token
  case nextToken of
    Stop  -> return []
    token -> liftA2 (:) (pure token) (tokensFrom token getNext)

-- | Converts a list of tokens into a string, putting spaces between words
-- | but not before punctuation.
tokensFrom :: Token -> GetNextToken -> IO [Token]
tokensFrom startToken getNext = do
  nextToken <- getNext startToken   -- nextToken :: Token
  case nextToken of
    Stop  -> return []
    token -> liftA2 (:) (pure token) (tokensFrom token getNext)

-- | Converts a list of tokens into a string, putting spaces between words
-- | but not before punctuation.
tokensFrom startToken getNext = do
  nextToken <- getNext startToken   -- nextToken :: Token
  case nextToken of
    Stop  -> return []
    token -> liftA2 (:) (pure token) (tokensFrom token getNext)

-- | Converts a list of tokens into a string, putting spaces between words
-- | but not before punctuation.