lib/megahal/megahal.rb
def _segment(line)
    # split the sentence into an array of alternating words and word-separators
    sequence =
      if _character_segmentation(line)
        line.split(/([[:word:]])/)
      else
        line.split(/([[:word:]]+)/)
      end
    # ensure the array starts with and ends with a word-separator, even if it's the blank one
    sequence << "" if sequence.last =~ /[[:word:]]+/
    sequence.unshift("") if sequence.first =~ /[[:word:]]+/
    # join trigrams of word-separator-word if the separator is a single ' or -
    # this means "don't" and "hob-goblin" become single words
    while index = sequence[1..-2].index { |item| item =~ /^['-]$/ } do
      sequence[index+1] = sequence[index, 3].join
      sequence[index] = nil
      sequence[index+2] = nil
      sequence.compact!
    end
    # split the alternating sequence into two arrays of word-separators and words
    sequence.partition.with_index { |symbol, index| index.even? }
  end
# This segments a sentence into two arrays representing word-separators and
# the original words themselves/
This segments a sentence into two arrays representing word-separators and
the original words themselves
This segments a sentence into two arrays representing word - separators and the original words themselves
def _segment(line)
    sequence =
      if _character_segmentation(line)
        line.split(/([[:word:]])/)
      else
        line.split(/([[:word:]]+)/)
      end
    sequence << "" if sequence.last =~ /[[:word:]]+/
    sequence.unshift("") if sequence.first =~ /[[:word:]]+/
    while index = sequence[1..-2].index { |item| item =~ /^['-]$/ } do
      sequence[index+1] = sequence[index, 3].join
      sequence[index] = nil
      sequence[index+2] = nil
      sequence.compact!
    end
    sequence.partition.with_index { |symbol, index| index.even? }
  end
def _segment ( line ) sequence = if _character_segmentation ( line ) line . split ( / ([[:word:]]) / ) else line . split ( / ([[:word:]]+) / ) end sequence << "" if sequence . last =~ / [[:word:]]+ / sequence . unshift ( "" ) if sequence . first =~ / [[:word:]]+ / while index = sequence [ 1 .. - 2 ] . index { | item | item =~ / ^['-]$ / } do sequence [ index + 1 ] = sequence [ index , 3 ] . join sequence [ index ] = nil sequence [ index + 2 ] = nil sequence . compact! end sequence . partition . with_index { | symbol , index | index . even? } end
This segments a sentence into two arrays representing word-separators and
the original words themselves
This segments a sentence into two arrays representing word - separators and the original words themselves
# split the sentence into an array of alternating words and word-separators # ensure the array starts with and ends with a word-separator, even if it's the blank one # join trigrams of word-separator-word if the separator is a single ' or - # this means "don't" and "hob-goblin" become single words # split the alternating sequence into two arrays of word-separators and words
line