<reponame>chedbrandh/glabra<gh_stars>0
#!/usr/bin/env python
#
# The MIT License (MIT)
#
# Copyright (c) 2015 <NAME>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import argparse
import itertools

from glabra import text
from glabra import analyze

WORD_DELIM_PATTERN = r"\s|\,|:|;|\""
FREQUENCY_DELIM_PATTERN = r"\r|\n"
FREQUENCY_GROUPING_PATTERN = r"^\s*(.+?)\s*(\d*.?\d+)\s*$"

BOUNDS_DEFAULT = ["4:0,100"]
NUM_RANDOM_DEFAULT = 10

DESCRIPTION = \
r"""Generate new words from one or more files of training data.

Two different types of files can be parsed. Either files are read only as a
sequence of words, or they can be read as data sets of word-frequency pairs.

A delimiter pattern determines where a word starts and where it ends. For
frequency files, a frequency grouping pattern determines what to interpret as
a word and what to interpret as the frequency associated with that word.

For example, this list of U.S. cities may be parsed with the word delimiter
matching a space, a comma, and a new line character.

--word-delim "\s|\,|\n"

"Abilene, Akron, Albuquerque, Alexandria, Allentown, Amarillo, Anaheim,
Anchorage, Ann Arbor, Antioch, Arlington, Arvada, Athens, Atlanta, Augusta,
Aurora, Austin, Bakersfield, Baltimore, Baton Rouge..."

And this frequency file listing the most common U.S. family names in 2000, can
be parsed with the following.

--frequency-delim "\n" --frequency-grouping "(.+):(\d+)"

"Smith:2376207
Johnson:1857160
Williams:1534042
Brown:1380145
Jones:1362755
..."

Bounds define which n-grams in the training data that will be used to generate
new words. The n-grams are specified by length and frequency percentile.

For example all 3-grams in all words generated with --bound 3:50,100 will
belong to the 50% most frequent 3-grams in the training data.

Words can be generated by either requesting some number of random words or by
requesting all words with some limit. The lengths of the words generated are
based on the lengths of words in the training data.

Occasionally words that already exist in the training data may be generated. If
"unique" words are requested, the returned words are guaranteed to not appear
in the training data, and they are guaranteed to not be returned twice. This
means that fewer than the number of requested random words may be returned."""

def main():

  parser = get_arg_parser()
  args = parser.parse_args()

  bounds = text.parse_bounds(args.bounds or BOUNDS_DEFAULT)

  # print help and exit if no files were specified
  if len(args.filenames or []) + len(args.freq_filenames or []) == 0:
    print "Must specify at least one filename."
    parser.print_help()
    exit(1)

  # decode delim strings passed in from command line
  args.word_delim = args.word_delim.decode("string-escape")
  args.freq_delim = args.freq_delim.decode("string-escape")

  # parse files and analyze contents
  sequence_analyzer = get_sequence_analyzer(
    args.filenames, args.freq_filenames, args.word_delim, args.freq_delim, args.freq_grouping)

  # create the generator
  text_generator = text.TextGenerator(bounds, sequence_analyzer)

  # exit if no words can be generated
  if text_generator.is_empty():
    print "Could not generate any words. Try larger files or expanding bounds."
    exit(1)

  # print random words if all_limit has not been set, else print all
  if args.all_limit == None:
    for word in text_generator.get_random_texts(args.num_random, args.unique):
      print word
  else:
    for word in itertools.islice(text_generator.get_all_texts(args.unique), args.all_limit):
      print word

def get_sequence_analyzer(filenames, freq_filenames, word_delim, freq_delim, freq_grouping):
  """Parse and analyzer word files.

  Files are parsed using the provided patterns. One SequenceAnalyzer is
  created for each file, they are then weighted equally, meaning that, for
  example, one file containg 1000x more words than another will NOT have 1000x
  more influence on the generated words.

  Args:
    filenames: Files with words separated by word_delim.
    freq_filesnames: Files with words and frequencies.
    word_delim: Pattern for word separation.
    freq_delim: Pattern for word-frequencey separation.
    freq_grouping: Reqex for matching word and frequency.

  Returns:
    A SequenceAnalyzer with all files weighted equally.
  """
  # analyze data files, both frequency files and others
  sequence_analyzers = []
  for filename in filenames or []:
    sequence_analyzers.append(text.get_sequence_analyzer(filename, word_delim))
  for filename in freq_filenames or []:
    sequence_analyzers.append(text.get_sequence_analyzer(
      filename,
      freq_delim,
      element_delimiter_pattern=None,
      frequency_grouping_pattern=freq_grouping))

  # merge all analyzers into one
  return analyze.merge_normalized_sequence_analyzers(sequence_analyzers)

def get_arg_parser():
  parser = argparse.ArgumentParser(
    formatter_class=argparse.RawDescriptionHelpFormatter,
    description=DESCRIPTION)
  parser.add_argument("--filename",
    dest="filenames",
    metavar="FILENAME",
    action="append",
    help="File with words to parse.")
  parser.add_argument("--frequency-filename",
    dest="freq_filenames",
    metavar="FILENAME",
    action="append",
    help="File with word and frequencies to parse.")
  parser.add_argument("--word-delim",
    dest="word_delim",
    metavar="WORD_DELIMITER_PATTERN",
    default=WORD_DELIM_PATTERN,
    help="Delimiter pattern for words. (default: %(default)s)")
  parser.add_argument("--frequency-delim",
    dest="freq_delim",
    metavar="FREQUENCY_DELIMITER_PATTERN",
    default=FREQUENCY_DELIM_PATTERN,
    help="Delimiter pattern for word-frequency files. (default: %(default)s)")
  parser.add_argument("--frequency-grouping",
    dest="freq_grouping",
    metavar="FREQUENCY_GROUPING_PATTERN",
    default=FREQUENCY_GROUPING_PATTERN,
    help="Pattern for grouping word-frequency files. (default: %(default)s)")
  parser.add_argument("--bound",
    dest="bounds",
    metavar="BOUND",
    action="append",
    help="One or more bounds for inclusion of n-grams. (default: %s)" % BOUNDS_DEFAULT)
  parser.add_argument("--get-random-words",
    dest="num_random",
    metavar="NUMBER_OF_RANDOM_WORDS",
    default=NUM_RANDOM_DEFAULT,
    type=int,
    help="Number of random words to generate. (default: %(default)s)")
  parser.add_argument("--get-all-words",
    dest="all_limit",
    metavar="ALL_WORDS_LIMIT",
    default=None,
    type=int,
    help="Generate all possible words with some limit. (default: %(default)s)")
  parser.add_argument("--unique",
    dest="unique",
    action="store_true",
    help="Don't return words in training data.")
  return parser

if __name__ == "__main__":
  main()
