diff --git "a/uroman/lib/NLP/English.pm" "b/uroman/lib/NLP/English.pm"
new file mode 100644--- /dev/null
+++ "b/uroman/lib/NLP/English.pm"
@@ -0,0 +1,3112 @@
+################################################################
+# #
+# English #
+# #
+################################################################
+
+package NLP::English;
+
+use File::Basename;
+use File::Spec;
+
+# tok v1.3.7 (May 16, 2019)
+
+$chinesePM = NLP::Chinese;
+$ParseEntry = NLP::ParseEntry;
+$util = NLP::utilities;
+$utf8 = NLP::UTF8;
+$logfile = "";
+# $logfile2 = (-d "/nfs/isd/ulf/smt/agile") ? "/nfs/isd/ulf/smt/agile/minilog" : "";
+# $util->init_log($logfile2);
+
+$currency_symbol_list = "\$|\xC2\xA5|\xE2\x82\xAC|\xE2\x82\xA4";
+$english_resources_skeleton_dir = "";
+%dummy_ht = ();
+
+sub build_language_hashtables {
+ local($caller, $primary_entity_style_filename, $data_dir) = @_;
+
+ unless ($data_dir) {
+ $default_data_dir = "/nfs/nlg/users/textmap/brahms-ml/arabic/bin/modules/NLP";
+ $data_dir = $default_data_dir if -d $default_data_dir;
+ }
+ my $english_word_filename = "$data_dir/EnglishWordlist.txt";
+ my $default_entity_style_MT_filename = "$data_dir/EntityStyleMT-zh.txt";
+ my $entity_style_all_filename = "$data_dir/EntityStyleAll.txt";
+ my $EnglishNonNameCapWords_filename = "$data_dir/EnglishNonNameCapWords.txt";
+ $english_resources_skeleton_dir = "$data_dir/EnglishResources/skeleton";
+ %english_annotation_ht = ();
+ %annotation_english_ht = ();
+ %english_ht = ();
+ $CardinalMaxWithoutComma = 99999;
+ $CardinalMaxNonLex = 9999000;
+
+ $primary_entity_style_filename = $default_entity_style_MT_filename unless defined($primary_entity_style_filename);
+ if ($primary_entity_style_filename =~ /^(ar|zh)$/) {
+ $languageCode = $primary_entity_style_filename;
+ $primary_entity_style_filename
+ = File::Spec->catfile($data_dir, "EntityStyleMT-$languageCode.txt");
+ }
+
+ open(IN,$english_word_filename) || die "Can't open $english_word_filename";
+ while (<IN>) {
+ next unless $_ =~ /^s*[^#\s]/; # unless blank/comment line
+ $_ =~ s/\s+$//;
+ $line = $_;
+ @lines = ($line);
+ if (($line =~ /::gpe:/)
+ && (($annotation) = ($line =~ /^.*?::(.*)$/))
+ && (($pre_annotation, $singular_english, $post_annotation) = ($annotation =~ /^(.*)::plural-of:([^:]+)(|::.*)\s*$/))) {
+ $derived_annotation = $singular_english . "::$pre_annotation$post_annotation";
+ # print STDERR "derived_annotation: $derived_annotation\n";
+ push(@lines, $derived_annotation);
+ }
+ foreach $line (@lines) {
+ ($english,@slots) = split("::",$line);
+ next unless defined($english);
+ $english =~ s/\s+$//;
+ $lc_english = $english;
+ $lc_english =~ tr/[A-Z]/[a-z]/;
+ $annotation = "::" . join("::",@slots) . "::";
+ $english_annotation_ht{$english} = $annotation;
+ $english_annotation_ht{$lc_english} = $annotation;
+ $english_annotation_ht{"_ALT_"}->{$english}->{$annotation} = 1;
+ $english_annotation_ht{"_ALT_"}->{$lc_english}->{$annotation} = 1;
+ $synt = "";
+ foreach $slot_value (@slots) {
+ ($slot,$value) = ($slot_value =~ /\s*(\w[^:]+):\s*(\S.*)$/);
+ next unless defined($value);
+ $slot =~ s/\s+$//;
+ $value =~ s/\s+$//;
+ $synt = $value if $slot eq "synt";
+ if (defined($annotation_english_ht{$slot_value})) {
+ push(@{$annotation_english_ht{$slot_value}},$english);
+ } else {
+ my @elist = ($english);
+ $annotation_english_ht{$slot_value} = \@elist;
+ }
+ if ($synt && defined($slot_value) && ($slot ne "synt")) {
+ $annot = "synt:$synt" . "::$slot_value";
+ if (defined($annotation_english_ht{$annot})) {
+ push(@{$annotation_english_ht{$annot}},$english);
+ } else {
+ my @elist = ($english);
+ $annotation_english_ht{$annot} = \@elist;
+ }
+ $english_annotation_ht{"_EN_SYNT_"}->{$english}->{$synt}->{$slot} = $value;
+ }
+ }
+ }
+ }
+ close(IN);
+
+ if (open(IN,$EnglishNonNameCapWords_filename)) {
+ while (<IN>) {
+ next unless $_ =~ /^s*[^#\s]/; # unless blank/comment line
+ $_ =~ s/\s+$//;
+ $english_ht{(lc $_)}->{COMMON_NON_NAME_CAP} = 1;
+ }
+ close(IN);
+ } else {
+ print STDERR "Can't open $EnglishNonNameCapWords_filename\n";
+ }
+
+ foreach $style ("primary", "all") {
+ if ($style eq "primary") {
+ $entity_style_filename = $primary_entity_style_filename || $default_entity_style_MT_filename;
+ } elsif ($style eq "all") {
+ $entity_style_filename = $entity_style_all_filename;
+ } else {
+ next;
+ }
+ %ht = ();
+ open(IN,$entity_style_filename) || die("Can't open $entity_style_filename (stylefile)");
+ my $n_entries = 0;
+ while (<IN>) {
+ next unless $_ =~ /^s*[^#\s]/; # unless blank/comment line
+ $_ =~ s/\s+$//;
+ ($slot,$value_string) = ($_ =~ /^([^:]+):\s*(\S.*)$/);
+ next unless defined($value_string);
+ if (defined($ht{$slot})) {
+ print STDERR "Warning: ignoring duplicate entry for $slot in $entity_style_filename\n";
+ next;
+ }
+ @values = split("::", $value_string);
+ foreach $value (@values) {
+ $value =~ s/^\s+//g;
+ $value =~ s/\s+$//g;
+ }
+ my @values_copy = @values;
+ $ht{$slot} = \@values_copy;
+ $n_entries++;
+ }
+ # print STDERR "Processed $n_entries entries in $entity_style_filename\n";
+ close(IN);
+ if ($style eq "primary") {
+ %english_entity_style_ht = %ht;
+ } elsif ($style eq "all") {
+ %english_entity_style_all_ht = %ht;
+ }
+ }
+
+ if (defined($raw = $english_entity_style_ht{CardinalMaxWithoutComma})
+ && (@styles = @{$raw}) && ($n = $styles[0]) && ($n =~ /^\d+$/) && ($n >= 999)) {
+ $CardinalMaxWithoutComma = $n;
+ }
+ if (defined($raw = $english_entity_style_ht{CardinalMaxNonLex})
+ && (@styles = @{$raw}) && ($n = $styles[0]) && ($n =~ /^\d+$/) && ($n >= 999999)) {
+ $CardinalMaxNonLex = $n;
+ }
+
+ return (*english_annotation_ht,*annotation_english_ht,*english_entity_style_ht);
+}
+
+sub read_language_variations {
+ local($this, $filename, *ht) = @_;
+
+ my $n = 0;
+ my $line_number = 0;
+ if (open(IN, $filename)) {
+ while (<IN>) {
+ $line_number++;
+ $us = $util->slot_value_in_double_colon_del_list($_, "us");
+ $uk = $util->slot_value_in_double_colon_del_list($_, "uk");
+ $formal = $util->slot_value_in_double_colon_del_list($_, "formal");
+ $informal = $util->slot_value_in_double_colon_del_list($_, "informal");
+ if ($us && $uk) {
+ $ht{VARIATION_UK_US}->{$uk}->{$us} = 1;
+ $n++;
+ }
+ if ($informal && $formal) {
+ $ht{VARIATION_INFORMAL_FORMAL}->{$informal}->{$formal} = 1;
+ $n++;
+ }
+ }
+ close(IN);
+ # print STDERR "Read $n spelling variation entries from $filename\n";
+ }
+}
+
+sub entity_style_listing {
+ local($caller,$attr) = @_;
+
+ if (defined($l = $english_entity_style_ht{$attr})) {
+ @sl = @{$l};
+ if (($#sl == 0) && ($sl[0] eq "all")) {
+ if (defined($al = $english_entity_style_all_ht{$attr})) {
+ return @{$al};
+ } else {
+ return ();
+ }
+ } else {
+ return @sl;
+ }
+ } else {
+ return ();
+ }
+}
+
+sub is_abbreviation {
+ local($caller,$noun) = @_;
+
+ $result = defined($annotation_s = $english_annotation_ht{$noun})
+ && ($annotation_s =~ /::abbreviation:true::/);
+# print "is_abbreviation($noun): $result\n";
+ return $result;
+}
+
+sub noun_adv_sem {
+ local($caller,$noun) = @_;
+
+ return "" unless defined($annotation_s = $english_annotation_ht{$noun});
+ ($adv_sem) = ($annotation_s =~ /::adv_sem:([-_a-z]+)::/);
+ return "" unless defined($adv_sem);
+ return $adv_sem;
+}
+
+sub numeral_value {
+ local($caller,$numeral) = @_;
+
+ return "" unless defined($annotation_s = $english_annotation_ht{$numeral});
+ ($value) = ($annotation_s =~ /::value:(\d+)::/);
+ return "" unless defined($value);
+ return $value;
+}
+
+sub annot_slot_value {
+ local($caller,$lex, $slot) = @_;
+
+ return "" unless defined($annotation_s = $english_annotation_ht{$lex});
+ ($value) = ($annotation_s =~ /::$slot:([-_a-z]+)(?:::.*|)\s*$/i);
+ return "" unless defined($value);
+ return $value;
+}
+
+sub annot_slot_values {
+ local($caller,$lex, $slot) = @_;
+
+ return () unless @annotations = keys %{$english_annotation_ht{"_ALT_"}->{$lex}};
+ @annot_slot_values = ();
+ foreach $annotation_s (@annotations) {
+ ($value) = ($annotation_s =~ /::$slot:([^:]+)(?:::.*|)\s*$/i);
+ if (defined($value)) {
+ $value =~ s/\s*$//;
+ push(@annot_slot_values, $value);
+ }
+ }
+ return @annot_slot_values;
+}
+
+# quick and dirty
+sub noun_number_form {
+ local($caller,$noun,$number) = @_;
+
+ $noun = "rupee" if $noun =~ /^Rs\.?$/;
+ $noun = "kilometer" if $noun =~ /^km$/;
+ $noun = "kilogram" if $noun =~ /^kg$/;
+ $noun = "meter" if $noun =~ /^m$/;
+ $noun = "second" if $noun =~ /^(s|secs?\.?)$/;
+ $noun = "minute" if $noun =~ /^(mins?\.?)$/;
+ $noun = "hour" if $noun =~ /^(h|hrs?\.?)$/;
+ $noun = "year" if $noun =~ /^(yrs?\.?)$/;
+ $noun = "degree" if $noun =~ /^(deg\.?)$/;
+ $noun = "foot" if $noun =~ /^(feet|ft\.?)$/;
+ $noun = "square kilometer" if $noun =~ /^sq\.? km/;
+ $noun =~ s/metre$/meter/;
+ $noun =~ s/litre$/liter/;
+ $noun =~ s/gramme$/gram/;
+ $noun =~ s/tonne$/ton/;
+ return $noun if $noun =~ /\$$/;
+ return $noun unless $number =~ /^[0-9.]+$/;
+ return $noun if $util->member($noun,"percent"); # no change in plural
+ return $noun if $noun =~ /\b(yuan|renminbi|RMB|rand|won|yen|ringgit|birr)$/; # no change in plural
+ return $noun if $number <= 1;
+
+ return $noun if $caller->is_abbreviation($noun);
+
+ $noun =~ s/^(hundred|thousand|million|billion|trillion)\s+//;
+ return $noun if $noun =~ /^(dollar|kilometer|pound|ton|year)s$/i;
+
+ $original_noun = $noun;
+ #check for irregular plural
+ $annot = "synt:noun::plural-of:$noun";
+ if (defined($annotation_english_ht{$annot})) {
+ @elist = @{$annotation_english_ht{$annot}};
+ return $elist[0] if @elist;
+ }
+
+ $noun = $noun . "s";
+ return $noun if $noun =~ /(a|e|o|u)ys$/; # days, keys, toys, guys
+ $noun =~ s/ys$/ies/; # babies
+ $noun =~ s/ss$/ses/; # buses
+ $noun =~ s/xs$/xes/; # taxes
+ $noun =~ s/shs$/shes/; # dishes
+ $noun =~ s/chs$/ches/; # churches
+ $noun =~ s/mans$/men/; # women
+ # print STDERR "NNF: $original_noun($number): $noun\n";
+ return $noun;
+}
+
+# quick and dirty
+sub lex_candidates {
+ local($caller,$surf) = @_;
+
+ @lex_cands = ($surf);
+ $lex_cand = $surf;
+ $lex_cand =~ s/ies$/y/;
+ push(@lex_cands,$lex_cand) unless $util->member($lex_cand, @lex_cands);
+ $lex_cand = $surf;
+ $lex_cand =~ s/s$//;
+ push(@lex_cands,$lex_cand) unless $util->member($lex_cand, @lex_cands);
+ $lex_cand = $surf;
+ $lex_cand =~ s/es$//;
+ push(@lex_cands,$lex_cand) unless $util->member($lex_cand, @lex_cands);
+ $lex_cand = $surf;
+ $lex_cand =~ s/\.$//;
+ push(@lex_cands,$lex_cand) unless $util->member($lex_cand, @lex_cands);
+ $lex_cand = $surf;
+ $lex_cand =~ s/men$/man/;
+ push(@lex_cands,$lex_cand) unless $util->member($lex_cand, @lex_cands);
+
+ return @lex_cands;
+}
+
+# quick and dirty
+sub pos_tag {
+ local($caller,$surf) = @_;
+
+ return CD if ($surf =~ /^-?[0-9,\.]+$/);
+ return NN if ($surf =~ /^($currency_symbol_list\d)/);
+ @lex_candidates = $caller->lex_candidates($surf);
+# print " lex_candidates: @lex_candidates\n";
+ foreach $lex_cand (@lex_candidates) {
+ if (defined($annotation_s = $english_annotation_ht{$lex_cand})) {
+# print " annotation: $annotation_s\n";
+ ($synt) = ($annotation_s =~ /::synt:([^:]+)::/);
+ if (defined($synt)) {
+ if ($synt eq "art") {
+ return "DT";
+ } elsif ($synt eq "adj") {
+ ($grade) = ($annotation_s =~ /::grade:([^:]+)::/);
+ if (defined($grade) && ($grade eq "superlative")) {
+ return "JJS";
+ } elsif (defined($grade) && ($grade eq "comparative")) {
+ return "JJR";
+ } else {
+ return "JJ";
+ }
+ } elsif ($synt eq "noun") {
+ if ($lex_cand eq $surf) {
+ return "NN";
+ } else {
+ return "NNS";
+ }
+ } elsif ($synt eq "name") {
+ return "NNP";
+ } elsif ($synt eq "cardinal") {
+ return "CD";
+ } elsif ($synt eq "ordinal") {
+ return "JJ";
+ } elsif ($synt eq "prep") {
+ return "IN";
+ } elsif ($synt eq "conj") {
+ return "CC";
+ } elsif ($synt eq "wh_pron") {
+ return "WP";
+ } elsif ($synt eq "adv") {
+ return "RB";
+ } elsif ($synt eq "genetive_particle") {
+ return "POS";
+ } elsif ($synt eq "ordinal_particle") {
+ return "NN";
+ } elsif ($synt eq "suffix_particle") {
+ return "NN";
+ } elsif ($synt =~ /^int(erjection)?$/) {
+ return "UH";
+ } elsif (($synt =~ /^punctuation$/)
+ && $util->is_rare_punctuation_string_p($surf)) {
+ return "SYM";
+ } elsif ($synt =~ /\bverb$/) {
+ if ($surf =~ /^(is)$/) {
+ return "VBZ";
+ } else {
+ return "VB";
+ }
+ }
+ }
+ }
+ }
+ return "";
+}
+
+sub indef_art_filter {
+ local($caller,$surf) = @_;
+
+ # check article in lexical annotation
+ # e.g. hour::synt:noun::unit:temporal::indef-article:an
+ # uniform::synt:noun::indef-article:a
+ ($surf_article,$word) = ($surf =~ /^(an?) (\S+)\s*/);
+ if (defined($surf_article)
+ && defined($word)
+ && defined($annotation = $english_annotation_ht{$word})) {
+ ($ann_article) = ($annotation =~ /::indef-article:([^:]+)::/);
+ if (defined($ann_article)) {
+ return ($surf_article eq $ann_article) ? $surf : "";
+ }
+ }
+ return "" if $surf =~ /\ban [bcdfghjklmnpqrstvwxyz]/;
+ return "" if $surf =~ /\ban (US)\b/;
+ return "" if $surf =~ /\ba [aeio]/;
+ return "" if $surf =~ /\ba (under)/;
+ return $surf;
+}
+
+sub wordlist_synt {
+ local($caller,$word) = @_;
+
+ return "" unless defined($annotation = $english_annotation_ht{$word});
+ ($synt) = ($annotation =~ /::synt:([^:]+)::/);
+ return $synt || "";
+}
+
+sub qualifier_filter {
+ local($caller,$surf) = @_;
+
+ return "" if $surf =~ /\b(over|more than|approximately) (million|billion|trillion)/;
+ return "" if $surf =~ /\b(over) (once|twice)/;
+ return $surf;
+}
+
+sub quantity_filter {
+ local($caller,$surf) = @_;
+
+ return "" if $surf =~ /^(a|an)-/; # avoid "the a-week meeting"
+ return $surf;
+}
+
+sub value_to_english {
+ local($caller,$number) = @_;
+
+ $result = "";
+
+ $annot = "value:$number";
+ if (defined($annotation_english_ht{$annot})) {
+ @elist = @{$annotation_english_ht{$annot}};
+ $result = $elist[0] if @elist;
+ }
+# print "value_to_english($number)=$result\n";
+ return $result;
+}
+
+sub value_to_english_ordinal {
+ local($caller,$number) = @_;
+
+ $result = "";
+
+ $annot = "synt:ordinal::value:$number";
+ if (defined($annotation_english_ht{$annot})) {
+ @elist = @{$annotation_english_ht{$annot}};
+ $result = $elist[0] if @elist;
+ } else {
+ $annot = "value:$number";
+ if (defined($annotation_english_ht{$annot})) {
+ @elist = @{$annotation_english_ht{$annot}};
+ $cardinal = $elist[0] if @elist;
+ $result = $cardinal . "th";
+ $result =~ s/yth$/ieth/;
+ }
+ }
+# print "value_to_english($number)=$result\n";
+ return $result;
+}
+
+sub english_with_synt_slot_value {
+ local($caller, $english, $synt, $slot) = @_;
+
+ return $english_annotation_ht{"_EN_SYNT_"}->{$english}->{$synt}->{$slot};
+}
+
+sub english_with_synt_slot_value_defined {
+ local($caller, $synt, $slot) = @_;
+
+ @englishes_with_synt_slot_value_defined = ();
+ foreach $english (keys %{$english_annotation_ht{"_EN_SYNT_"}}) {
+ push(@englishes_with_synt_slot_value_defined, $english)
+ if defined($english_annotation_ht{"_EN_SYNT_"}->{$english}->{$synt}->{$slot})
+ && ! $util->member($english, @englishes_with_synt_slot_value_defined)
+ }
+ return @englishes_with_synt_slot_value_defined;
+}
+
+sub number_composed_surface_form {
+ local($caller,$number,$leave_num_section_p) = @_;
+
+ return "" unless $number =~ /^\d+$/;
+ $leave_num_section_p = 0 unless defined($leave_num_section_p);
+ $anchor = "1000000000000000000000000";
+ while (($number < $anchor) && ($anchor >= 1000000)) {
+ $anchor =~ s/000//;
+ }
+# print "number_composed_surface_form number: $number anchor:$anchor\n";
+ return "" unless $anchor >= 1000000;
+ return "" unless $english = $caller->value_to_english($anchor);
+ $ending = $anchor;
+ $ending =~ s/^1000//;
+ return "" unless ($number =~ /$ending$/) || (($number * 1000) % $anchor) == 0;
+ $num_section = $number / $anchor;
+ if (($num_section =~ /^[1-9]0?$/) && ! $leave_num_section_p) {
+ $num_section_english = $caller->value_to_english($num_section);
+ $num_section = $num_section_english if $num_section_english;
+ }
+ $num_section = $caller->commify($num_section); # only for extremely large numbers
+ return "$num_section $english";
+}
+
+sub de_scientify {
+ local($caller,$number) = @_;
+
+# print "de_scientify: $number\n";
+ if ($number =~ /[eE][-+]/) {
+ ($n,$exp) = ($number =~ /^(\d+)[eE]\+(\d+)$/);
+ if (defined($exp)) {
+ $result = $n;
+ foreach $i (0 .. $exp-1) {
+ $result .= "0"
+ }
+ return $result;
+ } else {
+ ($n,$f,$exp) = ($number =~ /^(\d+)\.(\d+)[eE]\+(\d+)$/);
+ if (defined($exp) && ($exp >= length($f))) {
+ $result = "$n$f";
+ foreach $i (0 .. $exp-1-length($f)) {
+ $result .= "0";
+ }
+ return $result;
+ }
+ }
+ }
+ return $number;
+}
+
+sub commify {
+ local($caller,$number) = @_;
+
+ my $text = reverse $number;
+ $text =~ s/(\d\d\d)(?=\d)(?!\d*\.)/$1,/g;
+ return scalar reverse $text;
+}
+
+my %plural_rough_number_ht = (
+ 10 => "tens",
+ 12 => "dozens",
+ 20 => "scores",
+ 100 => "hundreds",
+ 1000 => "thousands",
+ 10000 => "tens of thousands",
+ 100000 => "hundreds of thousands",
+ 1000000 => "millions",
+ 10000000 => "tens of millions",
+ 100000000 => "hundreds of millions",
+ 1000000000 => "billions",
+ 10000000000 => "tens of billions",
+ 100000000000 => "hundreds of billions",
+ 1000000000000 => "trillions",
+ 10000000000000 => "tens of trillions",
+ 100000000000000 => "hundreds of trillions",
+);
+
+sub plural_rough_plural_number {
+ local($caller,$number) = @_;
+
+ return $plural_rough_number_ht{$number} || "";
+}
+
+my %roman_numeral_ht = (
+ "I" => 1,
+ "II" => 2,
+ "III" => 3,
+ "IIII" => 4,
+ "IV" => 4,
+ "V" => 5,
+ "VI" => 6,
+ "VII" => 7,
+ "VIII" => 8,
+ "VIIII" => 9,
+ "IX" => 9,
+ "X" => 10,
+ "XX" => 20,
+ "XXX" => 30,
+ "XXXX" => 40,
+ "XL" => 40,
+ "L" => 50,
+ "LX" => 60,
+ "LXX" => 70,
+ "LXXX" => 80,
+ "LXXXX" => 90,
+ "XC" => 90,
+ "C" => 100,
+ "CC" => 200,
+ "CCC" => 300,
+ "CCCC" => 400,
+ "CD" => 400,
+ "D" => 500,
+ "DC" => 600,
+ "DCC" => 700,
+ "DCCC" => 800,
+ "DCCCC" => 900,
+ "CM" => 900,
+ "M" => 1000,
+ "MM" => 2000,
+ "MMM" => 3000,
+ "MMM" => 3000,
+);
+
+sub roman_numeral_value {
+ local($caller,$s) = @_;
+
+ if (($m, $c, $x, $i) = ((uc $s) =~ /^(M{0,3})(C{1,4}|CD|DC{0,4}|CM|)(X{1,4}|XL|LX{0,4}|XC|)(I{1,4}|IV|VI{0,4}|IX|)$/)) {
+ $sum = ($roman_numeral_ht{$m} || 0)
+ + ($roman_numeral_ht{$c} || 0)
+ + ($roman_numeral_ht{$x} || 0)
+ + ($roman_numeral_ht{$i} || 0);
+ return $sum;
+ } else {
+ return 0;
+ }
+}
+
+sub number_surface_forms {
+ local($caller,$number,$pe) = @_;
+
+ print STDERR "Warning from number_surface_forms: $number not a number\n"
+ if $logfile && !($number =~ /^(\d+(\.\d+)?|\.\d+)$/);
+ # $util->log("number_surface_forms number:$number", $logfile);
+ # $util->log(" surf:$surf", $logfile) if $surf = ($pe && $pe->surf);
+
+ $pe = "" unless defined($pe);
+
+ @num_style_list = @{$english_entity_style_ht{"FollowSourceLanguageNumberStyle"}};
+ $follow_num_style = $util->member("yes", @num_style_list)
+ && (! (($number =~ /^([1-9]|10)$/) &&
+ $util->member("except-small-numbers", @num_style_list)));
+ $num_style = ($pe) ? $pe->get("num_style") : "";
+ if ($follow_num_style) {
+ if ($num_style =~ /digits_plus_alpha/) {
+ if ($number =~ /^[1-9]\d?\d?000$/) {
+ $digital_portion = $number;
+ $digital_portion =~ s/000$//;
+ return ("$digital_portion thousand");
+ } elsif ($number =~ /^[1-9]\d?\d?000000$/) {
+ $digital_portion = $number;
+ $digital_portion =~ s/000000$//;
+ return ("$digital_portion million");
+ } elsif ($number =~ /^[1-9]\d?\d?000000000$/) {
+ $digital_portion = $number;
+ $digital_portion =~ s/000000000$//;
+ return ("$digital_portion billion");
+ }
+ } elsif ($num_style eq "digits") {
+ if ($number =~ /^\d{1,4}$/) {
+ return ($number);
+ }
+ }
+ }
+
+ $number = $caller->de_scientify($number);
+
+ $composed_form = $caller->number_composed_surface_form($number);
+ $composed_form2 = $caller->number_composed_surface_form($number,1);
+ $lex_form = $caller->value_to_english($number);
+ $commified_form = $caller->commify($number);
+
+ if ($lex_form) {
+ if ($number >= 1000000) {
+ @result = ("one $lex_form", "1 $lex_form", "a $lex_form", $lex_form, $commified_form);
+ push(@result, $commified_form) if ($number <= $CardinalMaxNonLex);
+ } elsif ($number >= 100) {
+ @result = ($commified_form, "one $lex_form", "a $lex_form", $lex_form);
+ } elsif ($number >= 10) {
+ @result = ($number, $lex_form);
+ } elsif ($number == 1) {
+ @result = ("a", "an", $lex_form);
+ } elsif ($number == 0) {
+ @result = ($number, $lex_form);
+ } else {
+ @result = ($lex_form);
+ }
+ } elsif ($composed_form) {
+ if ($composed_form eq $composed_form2) {
+ @result = ($composed_form);
+ } elsif (($number >= 10000000) && ($composed_form2 =~ /^[1-9]0/)) {
+ @result = ($composed_form2, $composed_form);
+ } else {
+ @result = ($composed_form, $composed_form2);
+ }
+ push(@result, $commified_form) if $number <= $CardinalMaxNonLex;
+ } else {
+ ($ten,$one) = ($number =~ /^([2-9])([1-9])$/);
+ ($hundred) = ($number =~ /^([1-9])00$/) unless defined($one);
+ ($thousand) = ($number =~ /^([1-9]\d?)000$/) unless defined($one) || defined($hundred);
+ if (defined($one) && defined($ten)
+ && ($part1 = $caller->value_to_english($ten * 10))
+ && ($part2 = $caller->value_to_english($one))) {
+ $wordy_form = "$part1-$part2";
+ @result = ($commified_form, $wordy_form);
+ } elsif (defined($hundred)
+ && ($part1 = $caller->value_to_english($hundred))) {
+ $wordy_form = "$part1 hundred";
+ @result = ($commified_form, $wordy_form);
+ } elsif (defined($thousand)
+ && ($part1 = $caller->value_to_english($thousand))) {
+ $wordy_form = "$part1 thousand";
+ @result = ($commified_form, $wordy_form);
+ } elsif ($number =~ /^100000$/) {
+ @result = ($commified_form, "one hundred thousand", "a hundred thousand", "hundred thousand");
+ } elsif ($pe && ($pe->surf eq $number) && ($number =~ /^\d\d\d\d(\.\d+)?$/)) {
+ @result = ($number);
+ push(@result, $commified_form) unless $commified_form eq $number;
+ } elsif ($number =~ /^\d{4,5}$/) {
+ if ($commified_form eq $number) {
+ @result = ($number);
+ } else {
+ @result = ($commified_form, $number);
+ }
+ } else {
+ @result = ($commified_form);
+ }
+ }
+ push (@result, $number)
+ unless $util->member($number, @result) || ($number > $CardinalMaxWithoutComma);
+# $util->log("number_surface_forms result:@result", $logfile);
+
+ # filter according to num_style
+ if ($follow_num_style) {
+ my @filtered_result = ();
+ foreach $r (@result) {
+ push(@filtered_result, $r)
+ if (($num_style eq "digits") && ($r =~ /^\d+$/))
+ || (($num_style eq "alpha") && ($r =~ /^[-\@ a-z]*$/i))
+ || (($num_style eq "digits_plus_alpha") && ($r =~ /\d.*[a-z]/i));
+ }
+ @result = @filtered_result if @filtered_result;
+ }
+
+ if ($pe && $pe->childGloss("and")) {
+ @new_result = ();
+ foreach $r (@result) {
+ if ($r =~ /^and /) {
+ push(@new_result, $r);
+ } else {
+ push(@new_result, "and $r");
+ }
+ }
+ @result = @new_result;
+ }
+ return @result;
+}
+
+sub number_range_surface_forms {
+ local($caller,$pe) = @_;
+
+ $value = $pe->value;
+ $value_coord = $pe->get("value-coord");
+ unless ($value_coord) {
+ return $caller->number_surface_forms($value);
+ }
+ $prefix = "";
+ if ($conj = $pe->get("conj")) {
+ $connector = $conj;
+ } else {
+ $connector = ($value_coord == $value + 1) ? "or" : "to";
+ }
+ if ($pe->get("between")) {
+ $prefix = "between ";
+ $connector = "and";
+ }
+
+ $pe1 = $pe->child("head");
+ $pe2 = $pe->child("coord");
+ @result1 = $caller->number_surface_forms($value, $pe1);
+ @result2 = $caller->number_surface_forms($value_coord, $pe2);
+ @num_style_list = @{$english_entity_style_ht{"FollowSourceLanguageNumberStyle"}};
+ $follow_num_style = 1 if $util->member("yes", @num_style_list);
+
+ # between two thousand and three thousand => between two and three thousand
+ # 3 million to 5 million => 3 to 5 million
+ if ($follow_num_style && ($#result1 == 0) && ($#result2 == 0)) {
+ $range = $prefix . $result1[0] . " $connector " . $result2[0];
+ $util->log(" range1: $range", $logfile);
+ $gazillion = "thousand|million|billion|trillion";
+ ($a,$gaz1,$b,$gaz2) = ($range =~ /^(.+) ($gazillion) ($connector .+) ($gazillion)$/);
+ if (defined($a) && defined($gaz1) && defined($b) && defined($gaz2) && ($gaz1 eq $gaz2)) {
+ $range = "$a $b $gaz1";
+ $util->log(" range2: $range", $logfile);
+ return ($range);
+ }
+ }
+
+ @result = ();
+ foreach $result1 (@result1) {
+ next if ($value >= 1000) && ($result1 =~ /^\d+$/);
+ foreach $result2 (@result2) {
+ next if $result1 =~ /^an?\b/;
+ push(@result, "$prefix$result1 $connector $result2")
+ if ($result1 =~ /^[a-z]+$/) && ($result2 =~ /^[a-z]+$/);
+ next if ($result1 =~ /^[a-z]/) || ($result2 =~ /^[a-z]/);
+ next if ($value_coord >= 1000) && ($result2 =~ /^\d+$/);
+ ($digits1,$letters1) = ($result1 =~ /^(\d+(?:.\d+)?) ([a-z].*)$/);
+ ($digits2,$letters2) = ($result2 =~ /^(\d+(?:.\d+)?) ([a-z].*)$/);
+ if (defined($digits1) && defined($letters1)
+ && defined($digits2) && defined($letters2)
+ && ($letters1 eq $letters2)) {
+ push(@result, "$prefix$digits1 $connector $digits2 $letters1");
+ } elsif (($result1 =~ /^\d{1,3}$/) && ($result2 =~ /^\d{1,3}$/) && !$prefix) {
+ push(@result, "$result1-$result2");
+ if ($connector eq "to") {
+ my $span = "$result1 to $result2";
+ push(@result, $span) unless $util->member($span, @result);
+ }
+ } else {
+ push(@result, "$prefix$result1 $connector $result2");
+ }
+ }
+ }
+ unless (@result) {
+ $result1 = (@result1) ? $result1[0] : $value;
+ $result2 = (@result2) ? $result2[0] : $value_coord;
+ @result = "$prefix$result1 $connector $result2";
+ }
+ return @result;
+}
+
+sub q_number_surface_forms {
+ local($caller,$pe) = @_;
+
+ $surf = $pe->surf;
+ return ($pe->gloss) unless $value = $pe->value;
+ if (($value >= 1961) && ($value <= 2030)
+ &&
+ (($pe->get("struct") eq "sequence of digits")
+ ||
+ ($surf =~ /^\d+$/))) {
+ $value = "$prefix $value" if $prefix = $pe->get("prefix");
+ @result = ("$value");
+ } else {
+ @result = $caller->number_surface_forms($value,$pe);
+ @result = $caller->qualify_entities($pe,@result);
+ }
+ return @result;
+}
+
+sub ordinal_surface_forms {
+ local($caller,$number,$exclude_cardinals_p,$exclude_adverbials_p, $pe) = @_;
+
+ if (defined($os = $english_entity_style_ht{"Ordinal"})) {
+ @ordinal_styles = @{$os};
+ } else {
+ return ();
+ }
+ $exclude_cardinals_p = 0 unless defined($exclude_cardinals_p);
+ @num_style_list = @{$english_entity_style_ht{"FollowSourceLanguageNumberStyle"}};
+ $follow_num_style = 1 if $util->member("yes", @num_style_list);
+ $num_style = ($pe) ? $pe->get("num_style") : "";
+ $alpha_ok = ! ($follow_num_style && ($num_style =~ /^digits$/));
+ my $c_number = $caller->commify($number);
+ my $lex_form = "";
+ $lex_form = $caller->value_to_english_ordinal($number) if $alpha_ok;
+ my $adverbial_form
+ = (($number =~ /^\d+$/) && ($number >= 1) && ($number <= 10)
+ && $lex_form && $util->member("secondly", @ordinal_styles))
+ ? $lex_form . "ly" : "";
+ my $num_form = $caller->numeric_ordinal_form($number);
+ my $c_num_form = $caller->numeric_ordinal_form($c_number);
+ my @result = ();
+
+# print "lex_form: $lex_form num_form:$num_form c_num_form:$c_num_form\n";
+ if ($lex_form && $util->member("second", @ordinal_styles)) {
+ if (! $util->member("2nd", @ordinal_styles)) {
+ @result = ($lex_form);
+ } elsif ($c_num_form ne $num_form) {
+ @result = ($c_num_form, $lex_form, $num_form);
+ } elsif ($number >= 10) {
+ @result = ($num_form, $lex_form);
+ } else {
+ @result = ($lex_form, $num_form);
+ }
+ } elsif ($util->member("2nd", @ordinal_styles)) {
+ if ($c_num_form ne $num_form) {
+ @result = ($c_num_form, $num_form);
+ } else {
+ @result = ($num_form);
+ }
+ }
+ unless ($number =~ /^\d+$/) {
+ print STDERR "Warning: $number not an integer (for ordinal)\n";
+ }
+ unless ($exclude_cardinals_p) {
+ $incl_num_card = $util->member("2", @ordinal_styles);
+ $incl_lex_card = $util->member("two", @ordinal_styles);
+ foreach $card ($caller->number_surface_forms($number)) {
+ if ($card =~ /^an?$/) {
+ # don't include
+ } elsif ($card =~ /^[0-9,]+$/) {
+ push(@result, $card) if $incl_num_card;
+ } else {
+ push(@result, $card) if $incl_lex_card && $alpha_ok;
+ }
+ }
+ }
+ push(@result,$adverbial_form) if $adverbial_form && ! $exclude_adverbials_p;
+ push(@result, $num_form) unless @result;
+ return @result;
+}
+
+sub ordinal_surface_form {
+ local($caller,$number,$exclude_cardinals_p,$exclude_adverbials_p, $pe) = @_;
+
+ my @surf_forms = $caller->ordinal_surface_forms($number,$exclude_cardinals_p,$exclude_adverbials_p, $pe);
+ return (@surf_forms) ? $surf_forms[0] : $caller->numeric_ordinal_form($number);
+}
+
+sub fraction_surface_forms {
+ local($caller,$pe,$modp) = @_;
+
+ my @result = ();
+ $numerator = $pe->get("numerator");
+ $denominator = $pe->get("denominator");
+# print "numerator: $numerator denominator:$denominator\n";
+ @surf_nums = $caller->number_surface_forms($numerator,$pe);
+ @surf_nums = ("one") if $numerator == 1;
+ @surf_dens = $caller->ordinal_surface_forms($denominator,1,1);
+ @surf_dens = ("half") if $denominator == 2;
+ @surf_dens = ("quarter") if $denominator == 4;
+ @surf_dens = ("tenth") if $denominator == 10;
+# print "surf_nums: @surf_nums surf_dens: @surf_dens\n";
+ @fraction_patterns = @{$english_entity_style_ht{"Fraction"}};
+ if (@surf_nums && @surf_dens) {
+ $surf_num = $surf_nums[0];
+ $surf_den = $surf_dens[0];
+ $surf_num_den = "";
+ foreach $sd (@surf_dens) {
+ $surf_num_den = $sd if $sd =~ /^\d/;
+ }
+ $surf_den_w_proper_number = $caller->noun_number_form($surf_den, $numerator);
+ foreach $fp (@fraction_patterns) {
+ if ($fp eq "one tenth") {
+ push(@result, $surf_num . " " . $surf_den_w_proper_number) unless $modp;
+ } elsif ($fp eq "one-tenth") {
+ if ($modp) {
+ push(@result, $surf_num . "-" . $surf_den);
+ } else {
+ push(@result, $surf_num . "-" . $surf_den_w_proper_number);
+ }
+ } elsif ($fp eq "1/10") {
+ push(@result, $numerator . "/" . $denominator);
+ } elsif ($fp eq "1/10th") {
+ push(@result, $numerator . "/" . $surf_num_den) if $surf_num_den;
+ }
+ }
+ return @result;
+ } else {
+ return ($pe->gloss);
+ }
+}
+
+sub currency_surface_forms {
+ local($caller,$pe) = @_;
+
+ @currency_surf_forms = ();
+ return @currency_surf_forms unless $pe->sem =~ /monetary quantity/;
+ $unit = $pe->get("unit");
+ return ($pe->gloss) unless $quant = $pe->get("quant");
+ return ($pe->gloss) if $pe->childSem("head") eq "currency symbol";
+ $quant_pe = $pe->child("quant");
+ if ($unit =~ /^(US|Hongkong) dollar$/) {
+ @units = $caller->entity_style_listing($unit);
+ } elsif ($unit eq "yuan") {
+ @units = $caller->entity_style_listing("Chinese yuan");
+ @rmb_pos = @{$english_entity_style_ht{"Chinese RMB position"}};
+ @rmb_pos = ("before-number", "after-number") if $util->member("all",@units);
+ } else {
+ @units = ($unit);
+ }
+ if (($pe->sem =~ /range$/) && $quant_pe) {
+ @quants = $caller->number_range_surface_forms($quant_pe);
+ } else {
+ @quants = $caller->number_surface_forms($quant, $quant_pe);
+ }
+ @quants = ($quant) unless @quants;
+ # print STDERR "units: @units \n";
+ foreach $q (@quants) {
+ foreach $u_sing (@units) {
+ $u = ($modp) ? $u_sing : $caller->noun_number_form($u_sing, $quant);
+# print " q: $q unit: $u value: $quant\n";
+ if ($u eq "RMB") {
+ if ($util->member("before-number", @rmb_pos)) {
+ if ($q =~ /^\d/) {
+ push(@currency_surf_forms, "RMB" . $q);
+ }
+ }
+ if ($util->member("after-number", @rmb_pos)) {
+ push(@currency_surf_forms, $q . " RMB");
+ }
+ } elsif ($u =~ /\$$/) {
+ if ($q =~ /^\d/) {
+ $currency_surf_form = $u . $q;
+ push(@currency_surf_forms, $currency_surf_form);
+ }
+ } else {
+ $new_form = "$q $u";
+ push(@currency_surf_forms, $new_form) if $caller->indef_art_filter($new_form);
+ }
+ }
+ }
+ @currency_surf_forms = $caller->qualify_entities($pe,@currency_surf_forms);
+
+ # print STDERR "currency_surface_forms: @currency_surf_forms \n";
+ return @currency_surf_forms;
+}
+
+sub age_surface_forms {
+ local($caller,$pe, $modp) = @_;
+
+ $gloss = $pe->gloss;
+ @age_surf_forms = ();
+ return @age_surf_forms unless $pe->sem =~ /age quantity/;
+ $unit = $pe->get("unit");
+ return ($gloss) unless $quant = $pe->get("quant");
+ $temporal_quant_pe = $pe->child("head");
+ $synt = $pe->synt;
+ if ($synt =~ /parenthetical/) {
+ if ($pe->get("slashed")) {
+ @age_markers = $caller->entity_style_listing("ParentheticalAgeFormatSlashed");
+ @age_markers = $caller->entity_style_listing("ParentheticalAgeFormat") unless @age_markers;
+ } else {
+ @age_markers = $caller->entity_style_listing("ParentheticalAgeFormat");
+ }
+ return ($gloss) unless @age_markers;
+ foreach $a (@age_markers) {
+ $age_surf_form = $a;
+ $age_surf_form =~ s/8/$quant/;
+ push(@age_surf_forms, $age_surf_form);
+ }
+ } elsif (($quant =~ /^\d+$/) && ($temporal_quant_pe->sem eq "age unit")) {
+ @quants = $caller->number_surface_forms($quant);
+ @quants = ($quant) if $pe->childSurf("quant") =~ /^\d+$/;
+ foreach $quant2 (@quants) {
+ if ($modp) {
+ push(@age_surf_forms, "$quant2-year-old");
+ } else {
+ $plural_marker = ($quant >= 2) ? "s" : "";
+ push(@age_surf_forms, "$quant2 year$plural_marker old");
+ }
+ }
+ } elsif ($temporal_quant_pe && ($temporal_quant_pe->sem eq "temporal quantity")) {
+ @temporal_quants = $caller->quantity_surface_forms($temporal_quant_pe, $modp);
+ foreach $temporal_quant (@temporal_quants) {
+ push(@age_surf_forms, $temporal_quant . (($modp) ? "-" : " ") . "old");
+ }
+ } else {
+ return ($gloss);
+ }
+
+ @age_surf_forms = ($gloss) unless @age_surf_forms;
+ return @age_surf_forms;
+}
+
+sub occurrence_surface_forms {
+ local($caller,$pe,$modp) = @_;
+
+ @quantity_surf_forms = ();
+ return ($pe->gloss) unless $quant = $pe->get("quant");
+ $quant_coord = $pe->get("quant-coord");
+ $quant_pe = $pe->child("quant");
+ $unit = "time";
+ if (($pe->sem =~ /range$/) && $quant_pe) {
+ @quants = $caller->number_range_surface_forms($quant_pe);
+ } else {
+ @quants = $caller->number_surface_forms($quant, $quant_pe);
+ }
+ @quants = ($quant) unless @quants;
+ if ($modp) {
+ return () if $pe->get("qualifier") || $quant_coord;
+ return ("one-time") if $quant eq "1";
+ return ("two-time", "two-fold", "2-fold") if $quant eq "2";
+ } else {
+ if ($quant_coord) {
+ return $caller->qualify_entities($pe, ("once or twice"))
+ if $quant eq "1" and $quant_coord eq "2";
+ } else {
+ return $caller->qualify_entities($pe, ("once")) if $quant eq "1";
+ return $caller->qualify_entities($pe, ("twice", "two times", "2 times",
+ "2-fold", "two fold")) if $quant eq "2";
+ }
+ }
+ foreach $q (@quants) {
+ $u = ($modp) ? $unit : $caller->noun_number_form($unit, $quant);
+ $new_form = "$q $u";
+ if ($modp) {
+ # for the time being, no "more than/over/..." in modifiers: more than 20-ton
+ if ($pe->get("qualifier")) {
+ $new_form = "";
+ } else {
+ $new_form =~ s/-/-to-/;
+ $new_form =~ s/ /-/g;
+ }
+ }
+ push(@quantity_surf_forms, $new_form) if $new_form;
+ push(@quantity_surf_forms, "$q-fold") if $q =~ /\d/ || ($quant <= 9);
+ }
+ @quantity_surf_forms = $caller->qualify_entities($pe,@quantity_surf_forms);
+
+ return @quantity_surf_forms;
+}
+
+sub quantity_surface_forms {
+ local($caller,$pe,$modp) = @_;
+
+ if ($pe->get("complex") eq "true") {
+ return () if $modp;
+ $quantity_surf_form = $pe->gloss;
+ return ($quantity_surf_form);
+ }
+
+ @quantity_surf_forms = ();
+ $sem = $pe->get("sem");
+ $scale = $pe->get("scale");
+ $scale_mod = $pe->get("scale_mod");
+ $unit = $pe->get("unit") || $scale;
+ $mod_gloss = $pe->get("mod");
+ return ($pe->gloss) unless $quant = $pe->get("quant");
+ $quant_coord = $pe->get("quant-coord");
+ $quant_comb = $quant_coord || $quant;
+ $quant_pe = $pe->child("quant");
+ if (defined($u_style = $english_entity_style_ht{"\u$unit"})) {
+ @units = @{$u_style};
+ } else {
+ @units = ($unit);
+ }
+ if (($pe->sem =~ /range$/) && $quant_pe) {
+ @quants = $caller->number_range_surface_forms($quant_pe);
+ } else {
+ @quants = $caller->number_surface_forms($quant, $quant_pe);
+ }
+ @quants = ($quant) unless @quants;
+ foreach $q (@quants) {
+ foreach $u_sing (@units) {
+ my $u = $u_sing;
+ if (($sem =~ /seismic quantity/) && $scale) {
+ $scale =~ s/(\w+)\s*/\u\L$1/g if $scale =~ /^(Richter|Mercalli)/i;
+ $u = "on the $scale_mod $scale scale";
+ $u =~ s/\s+/ /g;
+ } elsif (($u_sing =~ /\S/) && ! $modp) {
+ $u = $caller->noun_number_form($u_sing, $quant_comb);
+ }
+# print " q: $q unit: $u value: $quant modp: $modp\n";
+ @mods = ("");
+ @mods = ("consecutive", "in a row") if $mod_gloss eq "continuous";
+ foreach $mod (@mods) {
+ $pre_quant_mod = "";
+ $in_quant_mod = ($mod =~ /(consecutive)/) ? "$mod " : "";
+ $post_quant_mod = ($mod =~ /(in a row)/) ? " $mod" : "";
+ $new_form = "$pre_quant_mod$q $in_quant_mod$u$post_quant_mod";
+ if ($caller->is_abbreviation($u)) {
+ if (($pe->sem =~ /range/) && ($q =~ /^[-0-9,\. to]+$/)
+ && $modp && !($new_form =~ / (to|or) /)) {
+ $new_form =~ s/-/-to-/;
+ $new_form =~ s/ /-/g;
+ } elsif ($q =~ /^[-0-9,\.]+$/) {
+# $new_form =~ s/ //g;
+ } else {
+ $new_form = "";
+ }
+ } elsif ($modp) {
+ # for the time being, no "more than/over/..." in modifiers: more than 20-ton
+ if (($pe->get("qualifier")) || $mod) {
+ $new_form = "";
+ } elsif ($u =~ /(square|cubic|metric|short)/) {
+ # no hyphenation for the time being (based on CTE style)
+ } elsif (($pe->sem =~ /range/) && !($new_form =~ / (to|or) /)) {
+ $new_form =~ s/-/-to-/;
+ $new_form =~ s/ /-/g;
+ } else {
+ $new_form =~ s/ /-/g;
+ }
+ }
+ push(@quantity_surf_forms, $new_form)
+ if $new_form && $caller->quantity_filter($new_form) && $caller->indef_art_filter($new_form);
+ }
+ }
+ }
+ @quantity_surf_forms = $caller->qualify_entities($pe,@quantity_surf_forms);
+
+ # print STDERR "QSF unit:$unit sem:$sem Result(s): " . join("; ", @quantity_surf_forms) . "\n";
+ return @quantity_surf_forms;
+}
+
+sub qualify_entities {
+ local($caller,$pe,@surf_forms) = @_;
+
+ $prefix = $pe->get("prefix");
+ $prefix_clause = ($prefix) ? "$prefix " : "";
+ if ($qualifier = $pe->get("qualifier")) {
+ $qualifier =~ s/-/ /g;
+ $qualifier_key = $qualifier;
+ $qualifier_key =~ s/(\w+)\s*/\u\L$1/g;
+ # print "qualifier_key: $qualifier_key\n";
+ @new_list = ();
+ if (defined($value = $english_entity_style_ht{$qualifier_key})) {
+ @quals = @{$value};
+ # print STDERR " qk $qualifier_key in ht: @quals :: @surf_forms\n";
+ foreach $q (@quals) {
+ foreach $surf_form (@surf_forms) {
+ $new_form = "$prefix_clause$q $surf_form";
+ push(@new_list, $new_form) if $caller->qualifier_filter($new_form);
+ }
+ }
+ return @new_list if @new_list;
+ } else {
+ @keys = sort keys %english_entity_style_ht;
+ # print STDERR " did not find qk $qualifier_key in ht: @keys\n";
+ foreach $surf_form (@surf_forms) {
+ if (($qualifier =~ /^(couple|few|lot|many|number|several|some)$/i)
+ && (($art, $lex) = ($surf_form =~ /^(an?)\s+(\S|\S.*\S)\s*$/i))) {
+ $plural_form = $caller->noun_number_form($lex,2);
+ $new_form = "$prefix_clause$qualifier $plural_form";
+ } else {
+ $new_form = "$prefix_clause$qualifier $surf_form";
+ }
+ push(@new_list, $new_form) if $caller->qualifier_filter($new_form);
+ }
+ return @new_list if @new_list;
+ }
+ }
+ if ($prefix) {
+ @prefixed_surf_forms = ();
+ foreach $surf_form (@surf_forms) {
+ if ($surf_form =~ /^$prefix /) { # already prefixed
+ push(@prefixed_surf_forms, $surf_form);
+ } else {
+ push(@prefixed_surf_forms, "$prefix $surf_form");
+ }
+ }
+ return @prefixed_surf_forms;
+ } else {
+ return @surf_forms;
+ }
+}
+
+sub percent_surface_forms {
+ local($caller,$pe,$modp) = @_;
+
+ @percent_surf_forms = ();
+ return @percent_surf_forms unless $pe->sem eq "percentage";
+ $prefix = "";
+ $quant = $pe->gloss;
+ $quant =~ s/%$//;
+ $quant =~ s/^and //;
+ if ($pe->gloss =~ /^and /) {
+ $prefix = "and";
+ }
+ @percent_markers = $caller->entity_style_listing("Percentage");
+ @quants = $caller->number_surface_forms($quant);
+ @quants = ($quant) unless @quants;
+ foreach $p (@percent_markers) {
+ foreach $q (@quants) {
+ if ($p =~ /%$/) {
+ if ($q =~ /\d$/) {
+ $percent_surf_form = $q . "%";
+ $percent_surf_form = "$prefix $percent_surf_form" if $prefix;
+ push(@percent_surf_forms, $percent_surf_form);
+ push(@percent_surf_forms, "by $percent_surf_form") unless $modp || $percent_surf_form =~ /^and /;
+ }
+ } else {
+ if ((($p =~ /^\d/) && ($q =~ /^\d/))
+ ||
+ (($p =~ /^[a-z]/) && ($q =~ /^[a-z]/))) {
+ if ($p =~ /percentage point/) {
+ if ($quant == 1) {
+ $percent_surf_form = $q . " percentage point";
+ } else {
+ $percent_surf_form = $q . " percentage points";
+ }
+ } else {
+ $percent_surf_form = $q . " percent";
+ }
+ $percent_surf_form = "$prefix $percent_surf_form" if $prefix;
+ $percent_surf_form =~ s/ /-/g if $modp;
+ push(@percent_surf_forms, $percent_surf_form);
+ push(@percent_surf_forms, "by $percent_surf_form") unless $modp || $percent_surf_form =~ /^and /;
+ }
+ }
+ }
+ }
+ return @percent_surf_forms;
+}
+
+sub decade_century_surface_forms {
+ local($caller,$pe) = @_;
+
+ if ($pe->sem =~ /century/) {
+ $gloss = $pe->gloss;
+ return ("the $gloss", "in the $gloss", $gloss);
+ }
+ @decade_surf_forms = ();
+ return @decade_surf_forms unless $pe->sem =~ /year range\b.*\bdecade/;
+ @decade_markers = @{$english_entity_style_ht{"Decade"}};
+ @extend_decades = @{$english_entity_style_ht{"ExtendDecades"}};
+ @extended_decades = @{$english_entity_style_ht{"ExtendedDecade"}};
+ $extended_decade = (@extended_decades) ? $extended_decades[0] : "none";
+
+ $value = $pe->value;
+ $extended_value = "";
+ foreach $extend_decade (@extend_decades) {
+ if ($extend_decade =~ /$value$/) {
+ $extended_value = $extend_decade unless $extended_value eq $extend_decade;
+ last;
+ }
+ }
+ if ($sub = $pe->get("sub")) {
+ $sub_clause = "$sub ";
+ $sub_clause =~ s/(mid) /$1-/;
+ } else {
+ $sub_clause = "";
+ }
+
+ if (! $extended_value) {
+ @values = ($value);
+ } elsif ($extended_decade eq "ignore") {
+ @values = ($value);
+ } elsif ($extended_decade eq "only") {
+ @values = ($extended_value);
+ } elsif ($extended_decade eq "primary") {
+ @values = ($extended_value, $value);
+ } elsif ($extended_decade eq "secondary") {
+ @values = ($value, $extended_value);
+ } else {
+ @values = ($value);
+ }
+ foreach $v (@values) {
+ foreach $dm (@decade_markers) {
+ $dm_ending = $dm;
+ $dm_ending =~ s/^\d+//;
+ push (@decade_surf_forms, "the $sub_clause$v$dm_ending");
+ push (@decade_surf_forms, "in the $sub_clause$v$dm_ending");
+ push (@decade_surf_forms, "$sub_clause$v$dm_ending");
+ }
+ }
+ return @decade_surf_forms;
+}
+
+sub day_of_the_month_surface_forms {
+ local($caller,$pe) = @_;
+
+ @dom_surf_forms = ();
+ return @dom_surf_forms
+ unless ($pe->sem eq "day of the month")
+ && ($day_number = $pe->get("day-number"));
+ @dom_markers = @{$english_entity_style_ht{"DayOfTheMonth"}};
+ foreach $dm (@dom_markers) {
+ $ord = $caller->numeric_ordinal_form($day_number);
+ if ($dm eq "on the 5th") {
+ push (@dom_surf_forms, "on the $ord");
+ } elsif ($dm eq "the 5th") {
+ push (@dom_surf_forms, "the $ord");
+ } elsif ($dm eq "5th") {
+ push (@dom_surf_forms, $ord);
+ }
+ }
+ return @dom_surf_forms;
+}
+
+sub score_surface_forms {
+ local($caller,$pe) = @_;
+
+ @score_surf_forms = ();
+ if (($score1 = $pe->get("score1"))
+ && ($score2 = $pe->get("score2"))) {
+ @score_markers = @{$english_entity_style_ht{"ScoreMarker"}};
+ @score_markers = (":") unless @score_markers;
+ foreach $sm (@score_markers) {
+ push (@score_surf_forms, "$score1$sm$score2");
+ }
+ }
+ push(@score_surf_forms, $pe->gloss) unless @score_surf_forms;
+ return @score_surf_forms;
+}
+
+sub day_of_the_week_surface_forms {
+ local($caller,$pe) = @_;
+
+ @dom_surf_forms = ();
+ @dom_markers = @{$english_entity_style_ht{"DayOfTheWeek"}};
+ $gloss = $pe->get("gloss");
+ $weekday = $pe->get("weekday");
+ $weekday = $gloss if ($weekday eq "") && ($gloss =~ /^\S+$/);
+ $relday = $pe->get("relday");
+ $period = $pe->get("period");
+ foreach $dm (@dom_markers) {
+ if (($dm =~ /NOPERIOD/) && $period) {
+ $surf = ""; # bad combination
+ } elsif (($dm eq "Sunday") || ! $relday) {
+ $surf = $weekday;
+ $surf .= " $period" if $period;
+ } elsif ($dm =~ /morning/) {
+ if ($period) {
+ $surf = $dm;
+ $surf =~ s/tomorrow/$relday/;
+ $surf =~ s/morning/$period/;
+ $surf =~ s/Sunday/$weekday/;
+ } else {
+ $surf = ""; # bad combination
+ }
+ } else {
+ $surf = $dm;
+ if ($period) {
+ if ($relday eq "today") {
+ $core_surf = "this $period";
+ } else {
+ $core_surf = "$relday $period";
+ }
+ } else {
+ $core_surf = $relday;
+ }
+ $surf =~ s/tomorrow/$core_surf/;
+ $surf =~ s/Sunday/$weekday/;
+ }
+ $surf =~ s/yesterday night/last night/;
+ $surf =~ s/this noon, ($weekday)(,\s*)?/today, $1, at noon/;
+ $surf =~ s/this noon/today at noon/;
+ $surf =~ s/this night/tonight/;
+ $surf =~ s/\s*NOPERIOD\s*$//;
+ push (@dom_surf_forms, $surf) unless $util->member($surf, @dom_surf_forms) || ! $surf;
+ $on_weekday = "on $surf";
+ push (@dom_surf_forms, $on_weekday)
+ if ($surf eq $weekday) && ! $util->member($on_weekday, @dom_surf_forms);
+ }
+ return @dom_surf_forms;
+}
+
+sub date_surface_forms {
+ local($caller,$pe,$modp) = @_;
+
+ @date_surf_forms = ();
+ $sem = $pe->sem;
+ $synt = $pe->synt;
+ return @date_surf_forms unless $sem =~ /date(\+year)?/;
+ $day = $pe->get("day");
+ $weekday = $pe->get("weekday");
+ $month_name = $pe->get("month-name");
+ $month_number = $pe->get("month-number");
+ $year = $pe->get("year");
+ $era = $pe->get("era");
+ $era_clause = "";
+ $calendar_type = $pe->get("calendar");
+ $calendar_type_clause = "";
+ $calendar_type_clause = " AH" if $calendar_type eq "Islamic";
+ $ad_year = $year;
+ if ($era eq "Republic era") {
+ $ad_year = $year + 1911;
+ $era_clause = " (year $year of the $era)";
+ }
+ $rel = $pe->get("rel");
+ if ($sep = $pe->get("sep")) {
+ $date_surf_form = "$month_number$sep$day";
+ $date_surf_form .= "$sep$year" if $year;
+ $date_surf_form = "$weekday, $date_surf_form" if $weekday;
+ $date_surf_form = "on $date_surf_form" if $synt eq "pp";
+ return ($date_surf_form);
+ }
+ @date_months = @{$english_entity_style_ht{"DateMonth"}};
+ @date_days = @{$english_entity_style_ht{"DateDay"}};
+ @date_order = @{$english_entity_style_ht{"DateOrder"}};
+ foreach $m (@date_months) {
+ if ($m eq "September") {
+ $surf_month = $month_name;
+ } elsif ($m =~ /^Sep(\.)?$/) {
+ if ($month_name eq "May") {
+ $surf_month = $month_name;
+ } else {
+ $period_clause = ($m =~ /\.$/) ? "." : "";
+ $surf_month = substr($month_name, 0, 3) . $period_clause;
+ }
+ } elsif ($m =~ /^Sept(\.)?$/) {
+ if ($util->member($month_name, "February", "September")) {
+ $period_clause = ($m =~ /\.$/) ? "." : "";
+ $surf_month = substr($month_name, 0, 4) . $period_clause;
+ } else {
+ $surf_month = "";
+ }
+ } else {
+ $surf_month = "";
+ }
+ foreach $d (@date_days) {
+ if ($d =~ /^\d+$/) {
+ $surf_day = $day;
+ } elsif ($d =~ /^\d+[sthrd]+$/) {
+ $surf_day = $caller->numeric_ordinal_form($day);
+ } else {
+ $surf_day = "";
+ }
+ if ($surf_month && $surf_day) {
+ foreach $o (@date_order) {
+ if ($calendar_type eq "Islamic") {
+ $date_surf_form = "$surf_day $surf_month";
+ } elsif ($o eq "September 6, 1998") {
+ $date_surf_form = "$surf_month $surf_day";
+ } elsif ($o eq "6 September, 1998") {
+ $date_surf_form = "$surf_day $surf_month";
+ }
+ $date_surf_form = "$weekday, $date_surf_form" if $weekday;
+ $consider_on_p = 1;
+ if ($year) {
+ $date_surf_form .= "," unless $calendar_type eq "Islamic";
+ $date_surf_form .= " $ad_year$calendar_type_clause$era_clause";
+ } elsif ($rel) {
+ if ($rel eq "current") {
+ $date_surf_form = "this $date_surf_form";
+ } else {
+ $date_surf_form = "$rel $date_surf_form";
+ }
+ $consider_on_p = 0;
+ }
+ push(@date_surf_forms, $date_surf_form)
+ unless $util->member($date_surf_form, @date_surf_forms) || ($synt eq "pp");
+ if ($consider_on_p) {
+ $on_date_surf_form = "on $date_surf_form";
+ push(@date_surf_forms, $on_date_surf_form)
+ unless $modp || $util->member($on_date_surf_form, @date_surf_forms);
+ }
+
+ if (($synt eq "pp") && ($sem eq "date")) {
+ push(@date_surf_forms, $date_surf_form)
+ unless $util->member($date_surf_form, @date_surf_forms);
+ }
+ }
+ }
+ }
+ }
+ return @date_surf_forms;
+ # rel, last, next, this
+}
+
+sub numeric_ordinal_form {
+ local($caller,$cardinal) = @_;
+
+ return $cardinal . "th" if $cardinal =~ /1\d$/;
+ return $cardinal . "st" if $cardinal =~ /1$/;
+ return $cardinal . "nd" if $cardinal =~ /2$/;
+ return $cardinal . "rd" if $cardinal =~ /3$/;
+ return $cardinal . "h" if $cardinal =~ /t$/;
+ $cardinal =~ s/y$/ie/;
+ return $cardinal . "th";
+}
+
+sub guard_urls_x045 {
+ local($caller, $s) = @_;
+
+ # URLs (http/https/ftp/mailto)
+ my $result = "";
+ while (($pre, $url, $post) = ($s =~ /^(.*?)((?:(?:https?|ftp):\/\/|mailto:)[#%-;=?-Z_-z~]*[-a-zA-Z0-9\/#])(.*)$/)) {
+ $result .= "$pre\x04$url\x05";
+ $s = $post;
+ }
+ $result .= $s;
+
+ # emails
+ $s = $result;
+ $result = "";
+ while (($pre, $email, $post) = ($s =~ /^(.*?[ ,;:()\/\[\]{}<>|"'])([a-z][-_.a-z0-9]*[a-z0-9]\@[a-z][-_.a-z0-9]*[a-z0-9]\.(?:[a-z]{2,}))([ .,;:?!()\/\[\]{}<>|"'].*)$/i)) {
+ $result .= "$pre\x04$email\x05";
+ $s = $post;
+ }
+ $result .= $s;
+
+ # (Twitter style) #hashtag or @handle
+ $s = $result;
+ $result = "";
+ while (($pre, $hashtag, $post) = ($s =~ /^(.*?[ .,;()\[\]{}'])([#@](?:[a-z]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|HHERE)(?:[_a-z0-9]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])*(?:[a-z0-9]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF]))(.*)$/i)) {
+ $result .= "$pre\x04$hashtag\x05";
+ $s = $post;
+ }
+ $result .= $s;
+
+ # Keep together number+letter in: Fig. 4g; Chromosome 12p
+ $result =~ s/((?:\b(?:fig))(?:_DONTBREAK_)?\.?|\b(?:figures?|tables?|chromosomes?)|<xref\b[^<>]*\b(?:fig)\b[^<>]*>)\s*(\d+[a-z])\b/$1 \x04$2\x05/gi;
+
+ # special combinations, e.g. =/= emoticons such as :)
+ $s = $result;
+ $result = "";
+ while (($pre, $special, $post) = ($s =~ /^(.*?)(:-?\)|:-?\(|=\/=?|\?+\/\?+|=\[)(.*)$/)) {
+ $result .= "$pre\x04$special\x05";
+ $s = $post;
+ }
+ $result .= $s;
+
+ return $result;
+}
+
+sub guard_xml_tags_x0123 {
+ local($caller, $s) = @_;
+
+ my $result = "";
+ # xml tag might or might not already have "@" on left and/or right end: @<br>@
+ while (($pre, $tag, $post) = ($s =~ /^(.*?)(\@?<\/?(?:[a-z][-_:a-z0-9]*)(?:\s+[a-z][-_:a-z0-9]*="[^"]*")*\s*\/?>\@?|&(?:amp|gt|lt|quot);|\[(?:QUOTE|URL)=[^ \t\n\[\]]+\]|\[\/?(?:QUOTE|IMG|INDENT|URL)\]|<\$[-_a-z0-9]+\$>|<\!--.*?-->)(.*)$/si)) {
+ $result .= $pre;
+ if (($pre =~ /\S$/) && ($tag =~ /^\S/)) {
+ $result .= " \x01";
+ $result .= "\@" if ($tag =~ /^<[a-z]/i) && (! ($pre =~ /[,;(>]$/)); #)
+ } else {
+ $result .= "\x01";
+ }
+ $guarded_tag = $tag;
+ $guarded_tag =~ s/ /\x02/g;
+ # print STDERR "tag: $tag\nguarded_tag: $guarded_tag\n" if ($result =~ /Harvey/) || ($s =~ /Harvey/);
+ $result .= $guarded_tag;
+ if (($tag =~ /\S$/) && ($post =~ /^\S/)) { # (
+ $result .= "\@" if (($tag =~ /^<\//) || ($tag =~ /\/>$/)) && (! ($result =~ /\@$/)) && (! ($post =~ /^[,;)<]/));
+ $result .= "\x03 ";
+ } else {
+ $result .= "\x03";
+ }
+ $s = $post;
+ }
+ $result .= $s;
+ return $result;
+}
+
+sub restore_urls_x045_guarded_string {
+ local($caller, $s) = @_;
+
+ my $orig = $s;
+ while (($pre, $url, $post) = ($s =~ /^(.*?)\x04([^\x04\x05]*?)\x05(.*)$/)) {
+ $url =~ s/ \@([-:\/])/$1/g;
+ $url =~ s/([-:\/])\@ /$1/g;
+ $url =~ s/ //g;
+ $url =~ s/\x02/ /g;
+ $s = "$pre$url$post";
+ }
+ if ($s =~ /[\x04\x05]/) {
+ print STDERR "Removing unexpectedly unremoved x04/x05 marks from $s\n";
+ $s =~ s/[\x04\x05]//g;
+ }
+ return $s;
+}
+
+sub restore_xml_tags_x0123_guarded_string {
+ local($caller, $s) = @_;
+
+ my $result = "";
+ while (($pre, $tag, $post) = ($s =~ /^(.*?)\x01(.*?)\x03(.*)$/)) {
+ $result .= $pre;
+ $tag =~ s/ \@([-:\/])/$1/g;
+ $tag =~ s/([-:\/])\@ /$1/g;
+ $tag =~ s/ //g;
+ $tag =~ s/\x02/ /g;
+ $result .= $tag;
+ $s = $post;
+ }
+ $result .= $s;
+ return $result;
+}
+
+sub load_english_abbreviations {
+ local($caller, $filename, *ht, $verbose) = @_;
+ # e.g. /nfs/nlg/users/textmap/brahms-ml/arabic/data/EnglishAbbreviations.txt
+
+ $verbose = 1 unless defined($verbose);
+ my $n = 0;
+ if (open(IN, $filename)) {
+ while (<IN>) {
+ next if /^\# /;
+ s/\s*$//;
+ my @expansions;
+ if (@expansions = split(/\s*::\s*/, $_)) {
+ my $abbrev = shift @expansions;
+ $ht{IS_ABBREVIATION}->{$abbrev} = 1;
+ $ht{IS_LC_ABBREVIATION}->{(lc $abbrev)} = 1;
+ foreach $expansion (@expansions) {
+ $ht{ABBREV_EXPANSION}->{$abbrev}->{$expansion} = 1;
+ $ht{ABBREV_EXPANSION_OF}->{$expansion}->{$abbrev} = 1;
+ }
+ $n++;
+ }
+ }
+ close(IN);
+ print STDERR "Loaded $n entries from $filename\n" if $verbose;
+ } else {
+ print STDERR "Can't open $filename\n";
+ }
+}
+
+sub load_split_patterns {
+ local($caller, $filename, *ht) = @_;
+ # e.g. /nfs/nlg/users/textmap/brahms-ml/arabic/data/BioSplitPatterns.txt
+
+ my $n = 0;
+ if (open(IN, $filename)) {
+ while (<IN>) {
+ next if /^\# /;
+ s/\s*$//;
+ if (($s) = ($_ =~ /^SPLIT-DASH-X\s+(\S.*\S|\S)\s*$/)) {
+ $ht{SPLIT_DASH_X}->{$s} = 1;
+ $ht{LC_SPLIT_DASH_X}->{(lc $s)} = 1;
+ $n++;
+ } elsif (($s) = ($_ =~ /^SPLIT-X-DASH\s+(\S.*\S|\S)\s*$/)) {
+ $ht{SPLIT_X_DASH}->{$s} = 1;
+ $ht{LC_SPLIT_X_DASH}->{(lc $s)} = 1;
+ $n++;
+ } elsif (($s) = ($_ =~ /^DO-NOT-SPLIT-DASH-X\s+(\S.*\S|\S)\s*$/)) {
+ $ht{DO_NOT_SPLIT_DASH_X}->{$s} = 1;
+ $ht{LC_DO_NOT_SPLIT_DASH_X}->{(lc $s)} = 1;
+ $n++;
+ } elsif (($s) = ($_ =~ /^DO-NOT-SPLIT-X-DASH\s+(\S.*\S|\S)\s*$/)) {
+ $ht{DO_NOT_SPLIT_X_DASH}->{$s} = 1;
+ $ht{LC_DO_NOT_SPLIT_X_DASH}->{(lc $s)} = 1;
+ $n++;
+ } elsif (($s) = ($_ =~ /^DO-NOT-SPLIT\s+(\S.*\S|\S)\s*$/)) {
+ $ht{DO_NOT_SPLIT}->{$s} = 1;
+ $ht{LC_DO_NOT_SPLIT}->{(lc $s)} = 1;
+ $n++;
+ } elsif (($s) = ($_ =~ /^SPLIT\s+(\S.*\S|\S)\s*$/)) {
+ $ht{SPLIT}->{$s} = 1;
+ $ht{LC_SPLIT}->{(lc $s)} = 1;
+ $n++;
+ }
+ }
+ close(IN);
+ print STDERR "Loaded $n entries from $filename\n";
+ } else {
+ print STDERR "Can't open $filename\n";
+ }
+}
+
+sub guard_abbreviations_with_dontbreak {
+ local($caller, $s, *ht) = @_;
+
+ my $orig = $s;
+ my $result = "";
+ while (($pre,$potential_abbrev,$period,$post) = ($s =~ /^(.*?)((?:[a-z]+\.-?)*(?:[a-z]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])+)(\.)(.*)$/i)) {
+ if (($pre =~ /([-&\/0-9]|[-\/]\@ )$/)
+ && (! ($pre =~ /\b[DR](?: \@)?-(?:\@ )?$/))) { # D-Ariz.
+ $result .= "$pre$potential_abbrev$period";
+ } else {
+ $result .= $pre . $potential_abbrev;
+ $potential_abbrev_with_period = $potential_abbrev . $period;
+ if ($ht{IS_ABBREVIATION}->{$potential_abbrev_with_period}) {
+ $result .= "_DONTBREAK_";
+ } elsif ($ht{IS_LC_ABBREVIATION}->{(lc $potential_abbrev_with_period)}) {
+ $result .= "_DONTBREAK_";
+ }
+ $result .= $period;
+ }
+ $s = $post;
+ }
+ $result .= $s;
+ $result =~ s/\b([Nn])o\.(\s*\d)/$1o_DONTBREAK_.$2/g;
+ return $result;
+}
+
+$alpha = "(?:[a-z]|\xCE[\xB1-\xBF]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])";
+$alphanum = "(?:[a-z0-9]|\xCE[\xB1-\xBF]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])(?:[-_a-z0-9]|\xCE[\xB1-\xBF]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])*(?:[a-z0-9]|\xCE[\xB1-\xBF]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])|(?:[a-z0-9]|\xCE[\xB1-\xBF]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])";
+
+sub normalize_punctuation {
+ local($caller, $s) = @_;
+
+ $s =~ s/\xE2\x80[\x93\x94]/-/g; # ndash, mdash to hyphen
+ $s =~ s/ \@([-\/])/$1/g;
+ $s =~ s/([-\/])\@ /$1/g;
+ return $s;
+}
+
+sub update_replace_characters_based_on_context {
+ local($caller, $s) = @_;
+
+ # This is just a start. Collect stats over text with non-ASCII, e.g. K?ln.
+ # HHERE
+ my $rest = $s;
+ $s = "";
+ while (($pre, $left, $repl_char, $right, $post) = ($rest =~ /^(.*?\s+)(\S*)(\xEF\xBF\xBD)(\S*)(\s.*)$/)) {
+ $s .= "$pre$left";
+ if (($left =~ /[a-z]$/i) && ($right =~ /^s(?:[-.,:;?!].*)?$/i)) { # China's etc.
+ $repl_char = "\xE2\x80\x99"; # right single quotation mark
+ } elsif (($left =~ /n$/i) && ($right =~ /^t$/i)) { # don't etc.
+ $repl_char = "\xE2\x80\x99"; # right single quotation mark
+ } elsif (($left =~ /[a-z]\s*[.]$/i) && ($right eq "")) { # end of sentence
+ $repl_char = "\xE2\x80\x9D"; # right double quotation mark
+ } elsif (($left eq "") && ($right =~ /^[A-Z]/i)) { # start of word
+ $repl_char = "\xE2\x80\x9C"; # left double quotation mark
+ }
+ $s .= "$repl_char$right";
+ $rest = $post;
+ }
+ $s .= $rest;
+
+ return $s;
+}
+
+sub tokenize {
+ local($caller, $s, *ht, $control) = @_;
+
+ my $local_verbose = 0;
+ print "Point A: $s\n" if $local_verbose;
+ $control = "" unless defined($control);
+ my $bio_p = ($control =~ /\bbio\b/);
+
+ $s = $utf8->repair_misconverted_windows_to_utf8_strings($s);
+ print "Point A2: $s\n" if $local_verbose;
+ $s = $utf8->delete_weird_stuff($s);
+ print "Point B: $s\n" if $local_verbose;
+
+ # reposition xml-tag with odd space
+ $s =~ s/( +)((?:<\/[a-z][-_a-z0-9]*>)+)(\S)/$2$1$3/ig;
+ $s =~ s/(\S)((?:<[a-z][^<>]*>)+)( +)/$1$3$2/ig;
+ print "Point C: $s\n" if $local_verbose;
+
+ $a_value = $ht{IS_ABBREVIATION}->{"Fig."} || "n/a";
+ $s = $caller->guard_abbreviations_with_dontbreak($s, *ht);
+ my $standard_abbrev_s = "Adm|al|Apr|Aug|Calif|Co|Dec|Dr|etc|e.g|Feb|Febr|Gen|Gov|i.e|Jan|Ltd|Lt|Mr|Mrs|Nov|Oct|Pfc|Pres|Prof|Sen|Sept|U.S.A|U.S|vs";
+ my $pre;
+ my $core;
+ my $post;
+ $s = " $core " if ($pre,$core,$post) = ($s =~ /^(\s*)(.*?)(\s*)$/i);
+ $s =~ s/\xE2\x80\x89/ /g; # thin space
+ $standard_abbrev_s =~ s/\./\\\./g;
+ $s =~ s/[\x01-\x05]//g;
+ $s = $caller->guard_urls_x045($s);
+ $s = $caller->guard_xml_tags_x0123($s);
+ $s = $caller->update_replace_characters_based_on_context($s);
+ $s =~ s/((?:[a-zA-Z_]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])\.)([,;]) /$1 $2 /g;
+ $s =~ s/((?:[a-zA-Z_]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])\.)(\x04)/$1 $2/g;
+ if ($bio_p) {
+ $s =~ s/(\S)((?:wt\/|onc\/)?(?:[-+]|\?+|\xE2\x80[\x93\x94])\/(?:[-+]|\?+|\xE2\x80[\x93\x94]))/$1 $2/g;
+ $s =~ s/((?:[-+]|\xE2\x80[\x93\x94])\/(?:[-+]|\xE2\x80[\x93\x94]))(\S)/$1 $2/g;
+ }
+ print "Point D: $s\n" if $local_verbose;
+ $s =~ s/(~+)/ $1 /g;
+ $s =~ s/((?:\xE2\x80\xB9|\xE2\x80\xBA|\xC2\xAB|\xC2\xBB|\xE2\x80\x9E)+)/ $1 /g; # triangular bracket(s) "<" or ">" etc.
+ $s =~ s/(``)([A-Za-z])/$1 $2/g; # added Nov. 30, 2017
+ $s =~ s/((?:<|<)?=+(?:>|>)?)/ $1 /g; # include arrows
+ $s =~ s/(\\")/ $1 /g;
+ $s =~ s/([^\\])("+)/$1 $2 /g;
+ $s =~ s/([^\\])((?:\xE2\x80\x9C)+)/$1 $2 /g; # open "
+ $s =~ s/([^\\])((?:\xE2\x80\x9D)+)/$1 $2 /g; # close "
+ $s =~ s/((?:<|<)?-{2,}(?:>|>)?)/ $1 /g; # include arrows
+ $s =~ s/((?:\xE2\x80\xA6)+)/ $1 /g; # ellipsis
+ print "Point E: $s\n" if $local_verbose;
+ foreach $_ ((1..2)) {
+ # colon
+ $s =~ s/([.,;])(:+)/$1 \@$2/g;
+ $s =~ s/(:+)([.,;])/$1 \@\@ $2/g;
+ # # question mark/exclamation mark blocks
+ # $s =~ s/([^!?])([!?]+)([^!?])/$1 $2 $3/g;
+ }
+ print "Point F: $s\n" if $local_verbose;
+ $s =~ s/(\?)/ $1 /g;
+ $s =~ s/(\!)/ $1 /g;
+ $s =~ s/ +/ /g;
+ $s =~ s/(\$+|\xC2\xA3|\xE2\x82[\xA0-\xBE])/ $1 /g; # currency signs (Euro sign; British pound sign; Yen sign etc.)
+ $s =~ s/(\xC2\xA9|\xE2\x84\xA2)/ $1 /g; # copyright/trademark signs
+ $s =~ s/(\xC2\xB2)([-.,;:!?()])/$1 $2/g; # superscript 2
+ $s =~ s/([^ ])( )/$1 $2/g;
+ $s =~ s/( )([^ ])/$1 $2/g;
+ $s =~ s/(&#\d+|&#x[0-9A-F]+);/$1_DONTBREAK_;/gi;
+ $s =~ s/([\@\.]\S*\d)([a-z][A-z])/$1_DONTBREAK_$2/g; # email address, URL
+ $s =~ s/ ($standard_abbrev_s)\./ $1_DONTBREAK_\./gi;
+ $s =~ s/ ($standard_abbrev_s) \. (\S)/ $1_DONTBREAK_\. $2/gi;
+ $s =~ s/\b((?:[A-Za-z]\.){1,3}[A-Za-z])\.\s+/$1_DONTBREAK_\. /g; # e.g. a.m. O.B.E.
+ $s =~ s/([ ])([A-Z])\. ([A-Z])/$1$2_DONTBREAK_\. $3/; # e.g. George W. Bush
+ $s =~ s/(\S\.*?[ ])([A-Z])_DONTBREAK_\. (After|All|And|But|Each|Every|He|How|In|It|My|She|So|That|The|Then|There|These|They|This|Those|We|What|When|Which|Who|Why|You)([', ])/$1$2\. $3$4/; # Exceptions to previous line, e.g. "plan B. This"
+ $s =~ s/\b(degrees C|[Ff]ig\.? \d+ ?[A-Z]|(?:plan|Scud) [A-Z])_DONTBREAK_\./$1\./g; # Exception, e.g. "plan B";
+ $s =~ s/([^-_a-z0-9])(art|fig|no|p)((?:_DONTBREAK_)?\.)(\d)/$1$2$3 $4/gi; # Fig.2 No.14
+ $s =~ s/([^-_A-Za-z0-9])(\d+(?:\.\d+)?)(?:_DONTBREAK_)?(thousand|million|billion|trillion|min|mol|sec|kg|km|g|m|p)\b/$1$2 $3/g; # 3.4kg 1.7million 49.9p
+ $s =~ s/([^-_a-z0-9])((?:[1-9]|1[0-2])(?:[.:][0-5]\d)?)(?:_DONTBREAK_)?([ap]m\b|[ap]\.m(?:_DONTBREAK_)?\.)/$1$2 $3/gi; # 3.15pm 12:00p.m. 8am
+ print "Point H: $s\n" if $local_verbose;
+
+ $s =~ s/(\d)([a-z][A-z])/$1 $2/g;
+ $s =~ s/(\w|`|'|%|[a-zA-Z]\.|[a-zA-Z]_DONTBREAK_\.)(-|\xE2\x80\x93)(\w|`|')/$1 \@$2\@ $3/g;
+ $s =~ s/(\w|`|'|%|[a-zA-Z]\.|[a-zA-Z]_DONTBREAK_\.)(-|\xE2\x80\x93)(\w|`|')/$1 \@$2\@ $3/g;
+ $s =~ s/(\w)- /$1 \@- /g;
+ $s =~ s/ -(\w)/ -\@ $1/g;
+ $s =~ s/(\d):(\d)/$1 \@:\@ $2/g;
+ $s =~ s/(\d)\/(\d)/$1 \@\/\@ $2/g;
+ $s =~ s/($alphanum)\/([,;:!?])/$1 \@\/\@ $2/g;
+ $s =~ s/($alphanum)([-+]+)\/($alphanum)/$1$2 \@\/\@ $3/gi;
+ print "Point I: $s\n" if $local_verbose;
+ foreach $_ ((1..5)) {
+ $s =~ s/([ \/()])($alphanum) ?\/ ?($alphanum)([-+ \/().,;])/$1$2 \@\/\@ $3$4/gi;
+ }
+ $s =~ s/([a-zA-Z%\/\[\]]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF]|\x05|[a-zA-Z]_DONTBREAK_\.)([,;:!?])\s*(\S)/$1 $2 $3/g;
+ # asterisk
+ $s =~ s/( [(\[]?)(\*)([a-z0-9])/$1$2\@ $3/gi;
+ $s =~ s/([a-z0-9])(\*)([.,;:)\]]* )/$1 \@$2$3/gi;
+ print "Point J: $s\n" if $local_verbose;
+
+ # Arabic script
+ if ($s =~ /[\xD8-\xDB]/) {
+ for (my $i=0; $i <= 1; $i++) {
+ $s =~ s/([\xD8-\xDB][\x80-\xBF])([,;:!?.\(\)\[\]\/]|\xD8\x8C|\xD8\x9B|\xD8\x9F|\xD9\xAA|\xC2\xAB|\xC2\xBB|\xE2[\x80-\x9F][\x80-\xBF])/$1 $2/gi; # punctuation includes Arabic ,;?%
+ $s =~ s/([,;:!?.\(\)\[\]\/]|\xD8\x8C|\xD8\x9B|\xD8\x9F|\xD9\xAA|\xC2\xAB|\xC2\xBB|\xE2[\x80-\x9F][\x80-\xBF])([\xD8-\xDB][\x80-\xBF])/$1 $2/gi;
+ }
+ }
+ $s =~ s/(\d|[a-zA-Z]|[\xD8-\xDB][\x80-\xBF])([-])([\xD8-\xDB][\x80-\xBF])/$1 \@$2\@ $3/g;
+ $s =~ s/(\d|[a-zA-Z])([\xD8-\xDB][\x80-\xBF])/$1 \@\@ $2/g;
+ print "Point K: $s\n" if $local_verbose;
+
+ # misc. non-ASCII punctuation
+ $s =~ s/(\xC2[\xA1\xBF]|\xD5\x9D|\xD6\x89|\xD8[\x8C\x9B]|\xD8\x9F|\xD9[\xAA\xAC]|\xDB\x94|\xDC[\x80\x82])/ $1 /g;
+ $s =~ s/(\xE0\xA5[\xA4\xA5]|\xE0\xBC[\x84-\x86\x8D-\x8F\x91\xBC\xBD])/ $1 /g;
+ $s =~ s/(\xE1\x81[\x8A\x8B]|\xE1\x8D[\xA2-\xA6])/ $1 /g;
+ $s =~ s/(\xE1\x81[\x8A\x8B]|\xE1\x8D[\xA2-\xA6]|\xE1\x9F[\x94\x96])/ $1 /g;
+ $s =~ s/([^0-9])(5\xE2\x80\xB2)(-)([ACGTU])/$1 $2 \@$3\@ $4/g; # 5-prime-DNA-seq.
+ $s =~ s/([^0-9])([35]\xE2\x80\xB2)/$1 $2 /g; # prime (keep 3-prime/5-prime together for bio domain)
+ $s =~ s/([^0-9])(\xE2\x80\xB2)/$1 $2 /g; # prime
+ $s =~ s/(\xE2\x81\x99)/ $1 /g; # five dot punctuation
+ $s =~ s/(\xE3\x80[\x81\x82\x8A-\x91]|\xE3\x83\xBB|xEF\xB8\xB0|\xEF\xBC\x8C)/ $1 /g;
+ $s =~ s/(\xEF\xBC[\x81-\x8F\x9A\x9F])/ $1 /g; # CJK fullwidth punctuation (e.g. fullwidth exclamation mark)
+ print "Point L: $s\n" if $local_verbose;
+ # spaces
+ $s =~ s/((?:\xE3\x80\x80)+)/ $1 /g; # idiographic space
+ $s =~ s/((?:\xE1\x8D\xA1)+)/ $1 /g; # Ethiopic space
+
+ # isolate \xF0 and up from much more normal characters
+ $s =~ s/([\xF0-\xFE][\x80-\xBF]*)([\x00-\x7F\xC0-\xDF][\x80-\xBF]*)/$1 $2/g;
+ $s =~ s/([\x00-\x7F\xC0-\xDF][\x80-\xBF]*)([\xF0-\xFE][\x80-\xBF]*)/$1 $2/g;
+ print "Point M: $s\n" if $local_verbose;
+
+ $s =~ s/( \d+)([,;:!?] )/$1 $2/g;
+ $s =~ s/ ([,;()\[\]])([a-zA-Z0-9.,;])/ $1 $2/g;
+ $s =~ s/(\)+)([-\/])([a-zA-Z0-9])/$1 $2 $3/g;
+ $s =~ s/([0-9\*\[\]()]|\xE2\x80\xB2)([.,;:] )/$1 $2/g;
+ $s =~ s/([a-zA-Z%]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF]|\x05)([,;:.!?])([")]|''|\xE2\x80[\x99\x9D]|)(\s)/$1 $2 $3$4/g;
+ $s =~ s/([a-zA-Z%]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF]|\x05)([,;:.!?])([")]|''|\xE2\x80[\x99\x9D]|)\s*$/$1 $2 $3/g;
+ $s =~ s/([.,;:]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF]|\x05)('|\xE2\x80[\x99\x9D])/$1 $2/g;
+ $s =~ s/('|\xE2\x80[\x99\x9D])([.,;:]|\x04)/$1 $2/g;
+ $s =~ s/([(){}\[\]]|\xC2\xB1)/ $1 /g;
+ $s =~ s/([a-zA-Z0-9]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF]|\x05)\.\s*$/$1 ./g;
+ $s =~ s/([a-zA-Z]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF]|\x05)\.\s+/$1 . /g;
+ $s =~ s/([a-zA-Z]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF]|\x05)\.(\x04)/$1 . $2/g;
+ $s =~ s/([0-9]),\s+(\S)/$1 , $2/g;
+ $s =~ s/([a-zA-Z])(\$)/$1 $2/g;
+ $s =~ s/(\$|[~<=>]|\xC2\xB1|\xE2\x89[\xA4\xA5]|\xE2\xA9[\xBD\xBE])(\d)/$1 $2/g;
+ $s =~ s/(RMB)(\d)/$1 $2/g;
+ print "Point N: $s\n" if $local_verbose;
+ foreach $_ ((1..2)) {
+ $s =~ s/([ '"]|\xE2\x80\x9C)(are|could|did|do|does|had|has|have|is|should|was|were|would)(n't|n\xE2\x80\x99t)([ '"]|\xE2\x80\x9D)/$1 $2 $3 $4/gi;
+ $s =~ s/ (can)(not) / $1 $2 /gi;
+ $s =~ s/ (ca)\s*(n)('t|\xE2\x80\x99t) / $1$2 $2$3 /gi;
+ $s =~ s/ ([Ww])o\s*n('|\xE2\x80\x99)t / $1ill n$2t /g;
+ $s =~ s/ WO\s*N('|\xE2\x80\x99)T / WILL N$1T /g;
+ $s =~ s/ ([Ss])ha\s*n('|\xE2\x80\x99)t / $1hall n$2t /g;
+ $s =~ s/ SHAN('|\xE2\x80\x99)T / SHALL N$1T /g;
+ # $s =~ s/ ain('|\xE2\x80\x99)t / is n$1t /g;
+ # $s =~ s/ Ain('|\xE2\x80\x99)t / Is n$1t /g;
+ # $s =~ s/ AIN('|\xE2\x80\x99)T / IS N$1T /g;
+ }
+ print "Point O: $s\n" if $local_verbose;
+ $s =~ s/(\d)%/$1 %/g;
+ $s =~ s/ '(d|ll|m|re|s|ve|em) / '_DONTBREAK_$1 /g; # 'd = would; 'll = will; 'em = them
+ $s =~ s/ \xE2\x80\x99t(d|ll|m|re|s|ve) / \xE2\x80\x99t_DONTBREAK_$1 /g;
+ $s =~ s/([^0-9a-z'.])('|\xE2\x80\x98)([0-9a-z])/$1$2 $3/gi;
+ $s =~ s/([0-9a-z])(\.(?:'|\xE2\x80\x99))([^0-9a-z']|\xE2\x80\x99)/$1 $2$3/gi;
+ $s =~ s/([0-9a-z]_?\.?)((?:'|\xE2\x80\x99)(?:d|ll|m|re|s|ve|))([^0-9a-z'])/$1 $2$3/gi;
+ $s =~ s/([("]|\xE2\x80\x9C|'')(\w)/$1 $2/g;
+ print "Point P: $s\n" if $local_verbose;
+ $s =~ s/(\w|[.,;:?!])([")]|''|\xE2\x80\x9D)/$1 $2/g;
+ $s =~ s/ ([,;()\[\]])([a-zA-Z0-9.,;])/ $1 $2/g;
+ $s =~ s/([a-z0-9]) ?(\()([-+_ a-z0-9\/]+)(\))/$1 $2 $3 $4 /ig;
+ $s =~ s/([a-z0-9]) ?(\[)([-+_ a-z0-9\/]+)(\])/$1 $2 $3 $4 /ig;
+ $s =~ s/([a-z0-9]) ?(\{)([-+_ a-z0-9\/]+)(\})/$1 $2 $3 $4 /ig;
+ $s =~ s/([%])-(\d+(?:\.\d+)? ?%)/$1 \@-\@ $2/g;
+ $s =~ s/( )(art|No)_DONTBREAK_(\.{2,})/$1 $2$3/gi;
+ $s =~ s/(_DONTBREAK_\.)(\.{1,})/$1 $2/g;
+ print "Point Q: $s\n" if $local_verbose;
+ foreach $_ ((1 .. 2)) {
+ $s =~ s/(\s(?:[-a-z0-9()']|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])*)(\.{2,})((?:[-a-z0-9()?!:\/']|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])*\s|(?:[-a-z0-9()'\/]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])+\.\s)/$1 $2 $3/gi;
+ }
+ $s =~ s/0s\b/0 s/g;
+ $s =~ s/([0-9])(\x04)/$1 $2/g;
+ $s =~ s/ +/ /g;
+ print "Point R: $s\n" if $local_verbose;
+
+ if ($bio_p) {
+ foreach $_ ((1 .. 2)) {
+ $s =~ s/([a-z]) \@(-|\xE2\x80[\x93\x94])\@ (\d+(?:$alpha)?\d*\+?)([- \/])/$1$2$3$4/ig;
+ $s =~ s/([a-z]) \@(-|\xE2\x80[\x93\x94])\@ ((?:alpha|beta|kappa)\d+)([- \/])/$1$2$3$4/ig;
+ $s =~ s/([a-z]) \@(-|\xE2\x80[\x93\x94])\@ ((?:a|b|h|k)\d)([- \/])/$1$2$3$4/ig;
+ $s =~ s/([a-z0-9]) \@(-|\xE2\x80[\x93\x94])\@ ([a-z])([- \/])/$1$2$3$4/ig;
+ $s =~ s/([- \/])(\d*[a-z]) \@(-|\xE2\x80[\x93\x94])\@ ([a-z0-9])/$1$2$3$4/ig;
+ }
+ # mutation indicators such -/- etc.
+ $s =~ s/(\?\/) +(\?)/$1$2/g;
+ $s =~ s/([^ ?])((?:wt\/|onc\/)?(?:[-+]|\?+|\xE2\x80[\x93\x94])\/(?:[-+]|\?+|\xE2\x80[\x93\x94]))/$1 $2/g;
+ $s =~ s/((?:[-+]|\xE2\x80[\x93\x94])\/(?:[-+]|\xE2\x80[\x93\x94]))(\S)/$1 $2/g;
+
+ # Erk1/2
+ $rest = $s;
+ $s = "";
+ while (($pre, $stem, $slashed_number_s, $post) = ($rest =~ /^(.*?[^-_a-z0-9])([a-z][-_a-z]*)(\d+(?:(?: \@)?\/(?:\@ )?(?:\d+))+)([^-+a-z0-9].*|)$/i)) {
+ if ((($pre =~ /\x04[^\x05]*$/) && ($post =~ /^[^\x04]*\x05/))
+ || ($stem =~ /^(mid|pre|post|sub|to)$/i)) {
+ $s .= "$pre$stem$slashed_number_s";
+ } else {
+ $s .= $pre;
+ my @slashed_numbers = split(/(?: \@)?\/(?:\@ )?/, $slashed_number_s);
+ foreach $i ((0 .. $#slashed_numbers)) {
+ my $number = $slashed_numbers[$i];
+ $s .= "$stem$number";
+ $s .= " @\/@ " unless $i == $#slashed_numbers;
+ }
+ }
+ $rest = $post;
+ }
+ $s .= $rest;
+
+ # Erk-1/-2
+ while (($pre, $stem, $dash1, $number1, $dash2, $number2, $post) = ($s =~ /^(.*[^-_a-z0-9])([a-z][-_a-z]*)(?: \@)?(-|\xE2\x80[\x93\x94])(?:\@ )?(\d+)(?: \@)?\/(?:\@ )?(?:\@ )?(-|\xE2\x80[\x93\x94])(?:\@ )?(\d+)([^-+a-z0-9].*|)$/i)) {
+ $s = "$pre$stem$dash1$number1 \@\/\@ $stem$dash2$number2$post";
+ }
+ $rest = $s;
+ $s = "";
+ # IFN-a/b (Slac2-a/b/c)
+ while (($pre, $stem, $dash, $slashed_letter_s, $post) = ($rest =~ /^(.*[^-_a-z0-9])([a-z][-_a-z0-9]*)(-|\xE2\x80[\x93\x94])([a-z](?:(?: \@)?\/(?:\@ )?(?:[a-z]))+)([^-+a-z0-9].*|)$/i)) {
+ if (($pre =~ /\x04[^\x05]*$/) && ($post =~ /^[^\x04]*\x05/)) {
+ $s .= "$pre$stem$dash1$number1$dash2$number2";
+ } else {
+ $s .= $pre;
+ my @slashed_letters = split(/(?: \@)?\/(?:\@ )?/, $slashed_letter_s);
+ foreach $i ((0 .. $#slashed_letters)) {
+ my $letter = $slashed_letters[$i];
+ $s .= "$stem$dash$letter";
+ $s .= " @\/@ " unless $i == $#slashed_letters;
+ }
+ }
+ $rest = $post;
+ }
+ $s .= $rest;
+
+ # SPLIT X-induced
+ my $rest = $s;
+ my $new_s = "";
+ while (($pre, $dash, $right, $post) = ($rest =~ /^(.*?)(-|\xE2\x80[\x93\x94])([a-z]+)( .*|)$/i)) {
+ $new_s .= $pre;
+ if (($right eq "I") && ($pre =~ / [a-zA-Z][a-z]*$/)) {
+ # compatriots-I have a dream
+ $new_s .= " \@" . $dash . "\@ ";
+ } elsif ($ht{LC_SPLIT_DASH_X}->{($caller->normalize_punctuation(lc $right))}) {
+ $new_s .= " \@" . $dash . "\@ ";
+ } else {
+ $new_s .= $dash;
+ }
+ $new_s .= $right;
+ $rest = $post;
+ }
+ $new_s .= $rest;
+ $s = $new_s;
+
+ # SPLIT ubiquinated-X
+ $rest = $s;
+ $new_s = "";
+ while (($pre, $left, $dash, $post) = ($rest =~ /^(.*? |)([a-z0-9]+|'s)(-|\xE2\x80[\x93\x94])([a-z0-9].*)$/i)) {
+ $new_s .= "$pre$left";
+ if ($ht{LC_SPLIT_X_DASH}->{($caller->normalize_punctuation(lc $left))}) {
+ $new_s .= " \@" . $dash . "\@ ";
+ } else {
+ $new_s .= $dash;
+ }
+ $rest = $post;
+ }
+ $new_s .= $rest;
+ $s = $new_s;
+
+ # SPLIT low-frequency
+ $rest = $s;
+ $new_s = "";
+ if (($pre, $left, $dash, $right, $post) = ($rest =~ /^(.*?[- ]|)([a-z]+)([-\/]|\xE2\x80[\x93\x94])([a-z]+)([- ].*|)$/i)) {
+ }
+ while (($pre, $left, $dash, $right, $post) = ($rest =~ /^(.*?[-\/ ]|)([a-z]+)((?: \@)?(?:[-\/]|\xE2\x80[\x93\x94])(?:\@ )?)([a-z]+)([-\/ ].*|)$/i)) {
+ $x = $caller->normalize_punctuation(lc ($left . $dash . $right));
+ if ($ht{LC_SPLIT}->{($caller->normalize_punctuation(lc ($left . $dash . $right)))}) {
+ $pre =~ s/([-\/])$/ \@$1\@ /;
+ $post =~ s/^([-\/])/ \@$1\@ /;
+ $dash = $caller->normalize_punctuation($dash);
+ $new_s .= "$pre$left";
+ $new_s .= " \@" . $dash . "\@ ";
+ $new_s .= $right;
+ $rest = $post;
+ } elsif ($pre =~ /[-\/]$/) {
+ $new_s .= $pre;
+ $rest = "$left$dash$right$post";
+ } else {
+ $new_s .= "$pre$left";
+ $rest = "$dash$right$post";
+ }
+ }
+ $new_s .= $rest;
+ $s = $new_s;
+
+ # DO-NOT-SPLIT X-ras
+ $rest = $s;
+ $new_s = "";
+ while (($pre, $dash, $right, $post) = ($rest =~ /^(.*?) \@(-|\xE2\x80[\x93\x94])\@ ([a-z0-9]+)( .*|)$/i)) {
+ $new_s .= $pre;
+ if ($ht{LC_DO_NOT_SPLIT_DASH_X}->{($caller->normalize_punctuation(lc $right))}) {
+ $new_s .= $dash;
+ } else {
+ $new_s .= " \@" . $dash . "\@ ";
+ }
+ $new_s .= $right;
+ $rest = $post;
+ }
+ $new_s .= $rest;
+ $s = $new_s;
+
+ # DO-NOT-SPLIT Caco-X
+ $rest = $s;
+ $new_s = "";
+ while (($pre, $left, $dash, $post) = ($rest =~ /^(.*? |)([a-z0-9]+) \@([-\/]|\xE2\x80[\x93\x94]])\@ ([a-z0-9].*)$/i)) {
+ $new_s .= "$pre$left";
+ if ($ht{LC_DO_NOT_SPLIT_X_DASH}->{($caller->normalize_punctuation(lc $left))}) {
+ $new_s .= $dash;
+ } else {
+ $new_s .= " \@" . $dash . "\@ ";
+ }
+ $rest = $post;
+ }
+ $new_s .= $rest;
+ $s = $new_s;
+
+ # DO-NOT-SPLIT down-modulate (2 elements)
+ $rest = $s;
+ $new_s = "";
+ while (($pre, $left, $dash, $right, $post) = ($rest =~ /^(.*? |)([a-z0-9]+) \@([-\/]|\xE2\x80[\x93\x94]])\@ ([a-z0-9]+)( .*|)$/i)) {
+ $new_s .= "$pre$left";
+ if ($ht{LC_DO_NOT_SPLIT}->{($caller->normalize_punctuation(lc ($left . $dash . $right)))}) {
+ $new_s .= $dash;
+ } else {
+ $new_s .= " \@" . $dash . "\@ ";
+ }
+ $new_s .= $right;
+ $rest = $post;
+ }
+ $new_s .= $rest;
+ $s = $new_s;
+
+ # DO-NOT-SPLIT 14-3-3 (3 elements)
+ $rest = $s;
+ $new_s = "";
+ while (($pre, $left, $dash_group1, $dash1, $middle, $dash_group2, $dash2, $right, $post) = ($rest =~ /^(.*? |)([a-z0-9]+)((?: \@)?([-\/]|\xE2\x80[\x93\x94]])(?:\@ )?)([a-z0-9]+)((?: \@)?([-\/]|\xE2\x80[\x93\x94]])(?:\@ )?)([a-z0-9]+)( .*|)$/i)) {
+ $new_s .= "$pre$left";
+ if ($ht{LC_DO_NOT_SPLIT}->{($caller->normalize_punctuation(lc ($left . $dash1 . $middle . $dash2 . $right)))}) {
+ $new_s .= $dash1;
+ } else {
+ $new_s .= $dash_group1;
+ }
+ $new_s .= $middle;
+ if ($ht{LC_DO_NOT_SPLIT}->{($caller->normalize_punctuation(lc ($left . $dash1 . $middle . $dash2 . $right)))}) {
+ $new_s .= $dash2;
+ } else {
+ $new_s .= $dash_group2;
+ }
+ $new_s .= $right;
+ $rest = $post;
+ }
+ $new_s .= $rest;
+ $s = $new_s;
+
+ $s =~ s/ +/ /g;
+ }
+ print "Point S: $s\n" if $local_verbose;
+
+ $s =~ s/_DONTBREAK_//g;
+ $s =~ s/( )(ark|ill|mass|miss|wash|GA|LA|MO|OP|PA|VA|VT)(\.)( )/$1$2 $3$4/g;
+ print "Point T: $s\n" if $local_verbose;
+ $s = $caller->restore_urls_x045_guarded_string($s);
+ $s = $caller->restore_xml_tags_x0123_guarded_string($s);
+ print "Point U: $s\n" if $local_verbose;
+ $s =~ s/(https?|ftp)\s*(:)\s*(\/\/)/$1$2$3/gi;
+ $s =~ s/\b(mailto)\s*(:)\s*([a-z])/$1$2$3/gi;
+ $s =~ s/(\d)\s*(:)\s*([0-5]\d[^0-9])/$1$2$3/gi;
+ print "Point V: $s\n" if $local_verbose;
+ $s =~ s/(5\xE2\x80\xB2-[ACGT]+)\s*(-|\xE2\x80[\x93\x94])\s*(3\xE2\x80\xB2)/$1$2$3/g; # repair broken DNA sequence
+ $s =~ s/ (etc) \. / $1. /g; # repair most egrareous separations
+ print "Point W: $s\n" if $local_verbose;
+ $s = $caller->repair_separated_periods($s);
+ print "Point X: $s\n" if $local_verbose;
+ $s =~ s/^\s+//;
+ $s =~ s/\s+$//;
+ $s = "$pre$s$post" if defined($pre) && defined($post);
+ $s =~ s/ +/ /g;
+ print "Point Y: $s\n" if $local_verbose;
+
+ return $s;
+}
+
+sub tokenize_plus_for_noisy_text {
+ local($caller, $s, *ht, $control) = @_;
+
+ $control = "" unless defined($control);
+ my $pre;
+ my $code;
+ my $post;
+ $s = " $core " if ($pre,$core,$post) = ($s =~ /^(\s*)(.*?)(\s*)$/i);
+ foreach $i ((1 .. 2)) {
+ $s =~ s/ ([A-Z][a-z]+'?[a-z]+)(-) / $1 $2 /gi; # Example: Beijing-
+ $s =~ s/ (\d+(?:\.\d+)?)(-|:-|:|_|\.|'|;)([A-Z][a-z]+'?[a-z]+|[A-Z]{3,}) / $1 $2 $3 /gi; # Example: 3:-Maxkamado
+ $s =~ s/ (\d+(?:\.\d+)?)(')([A-Za-z]{3,}) / $1 $2 $3 /gi; # Example: 42'daqiiqo
+ $s =~ s/ (-|:-|:|_|\.)([A-Z][a-z]+'?[a-z]+|[A-Z]{3,}) / $1 $2 /gi; # Example: -Xassan
+ $s =~ s/ ((?:[A-Z]\.[A-Z]|[A-Z]|Amb|Col|Dr|Eng|Gen|Inj|Lt|Maj|Md|Miss|Mr|Mrs|Ms|Pres|Prof|Sen)\.)([A-Z][a-z]+|[A-Z]{2,}) / $1 $2 /gi; # Example: Dr.Smith
+ $s =~ s/ (\d+)(,)([a-z]{3,}) / $1 $2 $3 /gi; # Example: 24,October
+ $s =~ s/ (%)(\d+(?:\.\d+)?) / $1 $2 /gi; # Example: %0.6
+ $s =~ s/ ([A-Za-z][a-z]{3,}\d*)([.,\/]|:\()([A-Za-z][a-z]{3,}|[A-Z]{3,}) / $1 $2 $3 /gi; # Example: Windows8,falanqeeyaal
+ $s =~ s/ ([A-Za-z]{3,}\d*?|[A-Za-z]+'[A-Za-z]+)([,\/]|:\()([A-Za-z]{3,}|[A-Za-z]+'[A-Za-z]+) / $1 $2 $3 /gi; # Example: GAROOWE:(SHL
+ $s =~ s/ (\d[0-9.,]*\d)(;)([a-z]+) / $1 $2 $3 /gi; # Example: 2.1.2014;Waraka
+ }
+ $s =~ s/^\s+//;
+ $s =~ s/\s+$//;
+ $s = "$pre$s$post" if defined($pre) && defined($post);
+ return $s;
+}
+
+# preparation for sub repair_separated_periods:
+
+my $abbrev_s = "etc.|e.g.|i.e.|U.K.|S.p.A.|A.F.P.";
+my @abbrevs = split(/\|/, $abbrev_s);
+my @exp_abbrevs = ();
+foreach $abbrev (@abbrevs) {
+ if (($core,$period) = ($abbrev =~ /^(.*?)(\.|)$/)) {
+ $core =~ s/\./\\s*\\.\\s*/g;
+ $abbrev = $core;
+ $abbrev .= "\\b" if $abbrev =~ /[a-z]$/i; # don't split etcetera -> etc. etera
+ $abbrev .= "(?:\\s*\\.|)" if $period;
+ push(@exp_abbrevs, $abbrev);
+ }
+}
+my $exp_abbrev_s = join("|", @exp_abbrevs);
+
+sub repair_separated_periods {
+ local($caller,$s) = @_;
+
+ # separated or missing period
+ my $result = "";
+ while (($pre, $abbrev, $post) = ($s =~ /^(.*? |)($exp_abbrev_s)(.*)$/)) {
+ $abbrev =~ s/ //g;
+ $abbrev .= "." unless $abbrev =~ /\.$/;
+ $result .= "$pre$abbrev ";
+ $s = $post;
+ }
+ $result .= $s;
+ $result =~ s/ +/ /g;
+ return $result;
+}
+
+# provided by Alex Fraser
+sub fix_tokenize {
+ local($caller,$s) = @_;
+
+ ## change "2:15" to "2 @:@ 15"
+ $s =~ s/(\d)\:(\d)/$1 \@:\@ $2/g;
+
+ ## strip leading zeros from numbers
+ $s =~ s/(^|\s)0+(\d)/$1$2/g;
+
+ ## fix rule typo
+ $s =~ s/associatedpress/associated press/g;
+
+ ## fix _ entities
+ $s =~ s/hong_kong/hong kong/g;
+ $s =~ s/united_states/united states/g;
+
+ return $s;
+}
+
+sub de_mt_tokenize {
+ local($caller,$s) = @_;
+
+ $s =~ s/\s+\@([-:\/])/$1/g;
+ $s =~ s/([-:\/])\@\s+/$1/g;
+ $s =~ s/\s+\/\s+/\//g;
+ return $s;
+}
+
+sub surface_forms {
+ local($caller,$pe,$modp) = @_;
+
+ $sem = $pe->sem;
+ $surf = $pe->surf;
+ $synt = $pe->synt;
+ $value = $pe->value;
+ $gloss = $pe->gloss;
+# $util->log("surface_forms surf:$surf sem:$sem gloss:$gloss value:$value", $logfile);
+ if ($sem eq "integer") {
+ return ($gloss) if ($gloss =~ /several/) && !($value =~ /\S/);
+ print STDERR "Warning: $value not an integer\n" unless $value =~ /^\d+(e\+\d+)?$/;
+ if ($pe->get("reliable") =~ /sequence of digits/) {
+ $english = $value;
+ $english = "$prefix $english" if $prefix = $pe->get("prefix");
+ @result = ($english);
+ } else {
+ @result = $caller->q_number_surface_forms($pe);
+ }
+ } elsif ($sem eq "decimal number") {
+ @result = $caller->q_number_surface_forms($pe);
+ } elsif ($sem =~ /(integer|decimal number) range/) {
+ @result = $caller->number_range_surface_forms($pe);
+ } elsif ($sem eq "ordinal") {
+ if ($pe->get("definite")) {
+ $exclude_adverbials_p = 1;
+ } elsif (defined($chinesePM) && ($hao = $chinesePM->e2c("hao-day"))
+ && ($gc = $chinesePM->e2c("generic counter"))) {
+ $exclude_adverbials_p = ($surf =~ /($hao|$gc)$/);
+ } else {
+ $exclude_adverbials_p = 1;
+ }
+ @result = $caller->ordinal_surface_forms($pe->get("ordvalue") || $pe->value,0,$exclude_adverbials_p, $pe);
+ } elsif ($sem eq "fraction") {
+ @result = $caller->fraction_surface_forms($pe,$modp);
+ } elsif ($sem =~ /monetary quantity/) {
+ @result = $caller->currency_surface_forms($pe);
+ } elsif ($sem =~ /occurrence quantity/) {
+ @result = $caller->occurrence_surface_forms($pe,$modp);
+ } elsif ($sem =~ /score quantity/) {
+ @result = $caller->score_surface_forms($pe);
+ } elsif ($sem =~ /age quantity/) {
+ @result = $caller->age_surface_forms($pe, $modp);
+ } elsif ($sem =~ /quantity/) {
+ @result = $caller->quantity_surface_forms($pe,$modp);
+ } elsif ($sem eq "percentage") {
+ @result = $caller->percent_surface_forms($pe,$modp);
+ } elsif ($sem eq "percentage range") {
+ if ($gloss =~ /^and /) {
+ @result = ($gloss);
+ } else {
+ @result = ($gloss, "by $gloss", "of $gloss");
+ }
+ } elsif ($sem =~ /^(month of the year|month\+year|year)$/) {
+ if ($synt eq "pp") {
+ @result = ($gloss);
+ } elsif ($gloss =~ /^the (beginning|end) of/) {
+ @result = ($gloss, "at $gloss");
+ } elsif ($gloss =~ /^(last|this|current|next)/) {
+ @result = ($gloss);
+ } else {
+ # in November; in mid-November
+ @result = ($gloss, "in $gloss");
+ }
+ } elsif ($sem =~ /date(\+year)?$/) {
+ @result = $caller->date_surface_forms($pe,$modp);
+ } elsif ($sem =~ /year range\b.*\b(decade|century)$/) {
+ @result = $caller->decade_century_surface_forms($pe);
+ } elsif ($sem eq "day of the month") {
+ @result = $caller->day_of_the_month_surface_forms($pe);
+ } elsif ($sem =~ /period of the day\+day of the week/) {
+ @result = ($gloss);
+ push(@result, "on $gloss") if $gloss =~ /^the night/;
+ } elsif ($sem =~ /day of the week/) {
+ @result = $caller->day_of_the_week_surface_forms($pe);
+ } elsif ($sem =~ /^(time)$/) {
+ if ($gloss =~ /^at /) {
+ @result = ($gloss);
+ } else {
+ @result = ($gloss, "at $gloss");
+ }
+ } elsif ($sem =~ /^date range$/) {
+ if ($synt eq "pp") {
+ @result = ($gloss);
+ } elsif ($pe->get("between")) {
+ $b_gloss = "between $gloss";
+ $b_gloss =~ s/-/ and /;
+ @result = ($b_gloss, $gloss, "from $gloss");
+ } else {
+ @result = ($gloss, "from $gloss");
+ }
+ } elsif ($sem =~ /^date enumeration$/) {
+ if ($synt eq "pp") {
+ @result = ($gloss);
+ } else {
+ @result = ($gloss, "on $gloss");
+ }
+ } elsif ($pe->get("unknown-in-pc")) {
+ @result = ();
+ foreach $unknown_pos_en (split(/;;/, $pe->get("unknown-pos-en-list"))) {
+ ($engl) = ($unknown_pos_en =~ /^[^:]+:[^:]+:(.*)$/);
+ push(@result, $engl) if defined($engl) && ! $util->member($engl, @result);
+ }
+ @result = ($gloss) unless @result;
+ } elsif (($sem =~ /\b(name|unknown)\b/) && (($en_s = $pe->get("english")) =~ /[a-z]/i)) {
+ @result = split(/\s*\|\s*/, $en_s);
+ } elsif (($sem =~ /^proper\b/) && (($en_s = $pe->get("english")) =~ /[a-z]/i)) {
+ @result = split(/\s*\|\s*/, $en_s);
+ } else {
+ @result = ($gloss);
+ }
+
+ if (($sem =~ /^(date\+year|month\+year|year)$/)
+ && ($year = $pe->get("year"))
+ && ($year =~ /^\d\d$/)
+ && (@extend_years = @{$english_entity_style_ht{"ExtendYears"}})
+ && ($#extend_years == 1)
+ && ($extended_year_start = $extend_years[0])
+ && ($extended_year_end = $extend_years[1])
+ && ($extended_year_start <= $extended_year_end)
+ && ($extended_year_start + 99 >= $extended_year_end)
+ && ($extended_year_start =~ /^\d\d\d\d$/)
+ && ($extended_year_end =~ /^\d\d\d\d$/)) {
+ $century1 = substr($extended_year_start, 0, 2);
+ $century2 = substr($extended_year_end, 0, 2);
+ $exp_year1 = "$century1$year";
+ $exp_year2 = "$century2$year";
+ if (($extended_year_start <= $exp_year1) && ($exp_year1 <= $extended_year_end)) {
+ $exp_year = $exp_year1;
+ } elsif (($extended_year_start <= $exp_year2) && ($exp_year2 <= $extended_year_end)) {
+ $exp_year = $exp_year2;
+ } else {
+ $exp_year = "";
+ }
+ if ($exp_year) {
+ @new_glosses = ();
+ foreach $old_gloss (@result) {
+ $new_gloss = $old_gloss;
+ $new_gloss =~ s/\b$year$/$exp_year/;
+ push (@new_glosses, $new_gloss) unless $new_gloss eq $old_gloss;
+ }
+ push (@result, @new_glosses);
+ }
+ }
+
+ # tokenize as requested
+ @tokenize_list = @{$english_entity_style_ht{"Tokenize"}};
+ $tokenize_p = 1 if $util->member("yes", @tokenize_list)
+ || $util->member("all", @tokenize_list);
+ $dont_tokenize_p = 1 if $util->member("no", @tokenize_list)
+ || $util->member("all", @tokenize_list);
+ if ($tokenize_p) {
+ @new_result = ();
+ foreach $item (@result) {
+ $t_item = $caller->tokenize($item, *dummy_ht);
+ push(@new_result, $item) if $dont_tokenize_p && ($item ne $t_item);
+ push(@new_result, $t_item);
+ }
+ @result = @new_result;
+ }
+
+ # case as requested
+ @case_list = @{$english_entity_style_ht{"Case"}};
+ $lower_case_p = $util->member("lower", @case_list)
+ || $util->member("all", @case_list);
+ $reg_case_p = $util->member("regular", @case_list)
+ || $util->member("all", @case_list);
+ if ($lower_case_p) {
+ @new_result = ();
+ foreach $item (@result) {
+ $l_item = "\L$item";
+ push(@new_result, $item) if $reg_case_p && ($item ne $l_item);
+ push(@new_result, $l_item) unless $util->member($l_item, @new_result);
+ }
+ @result = @new_result;
+ }
+ # $value = "n/a" unless $value;
+ # print STDERR "SF surf:$surf sem:$sem gloss:$gloss value:$value Result(s): " . join("; ", @result) . "\n";
+ return @result;
+}
+
+sub case_list {
+ return @{$english_entity_style_ht{"Case"}};
+}
+
+sub right_cased_list {
+ local($caller, $word) = @_;
+
+ @case_list = @{$english_entity_style_ht{"Case"}};
+
+ @right_cased_core_list = ();
+ push(@right_cased_core_list, $word)
+ if ($util->member("regular", @case_list) || $util->member("all", @case_list))
+ && ! $util->member($word, @right_cased_core_list);
+ push(@right_cased_core_list, lc $word)
+ if ($util->member("lower", @case_list) || $util->member("all", @case_list))
+ && ! $util->member(lc $word, @right_cased_core_list);
+
+ return @right_cased_core_list;
+}
+
+sub string2surf_forms {
+ local($caller, $text, $lang, $alt_sep) = @_;
+
+ $alt_sep = " | " unless defined($alt_sep);
+ $lang = "zh" unless defined($lang);
+
+ if ($lang eq "zh") {
+ @pes = $chinesePM->parse_entities_in_string($text);
+ $n = $#pes + 1;
+# print " $n pes\n";
+ @pes = $chinesePM->select_reliable_entities(@pes);
+ my @res_surf_forms_copy = $caller->reliable_pes2surf_forms($alt_sep, @pes);
+ return @res_surf_forms_copy;
+ } else {
+ return ();
+ }
+}
+
+sub reliable_pe2surf_forms {
+ local($caller, $pe, $parent_reliant_suffices_p) = @_;
+
+ $parent_reliant_suffices_p = 0 unless defined($parent_reliant_suffices_p);
+ if ((defined($r = $pe->get("reliable")) && $r)
+ || ($parent_reliant_suffices_p && ($parent_pe = $pe->get("parent")) &&
+ $parent_pe->get("reliable"))) {
+ @surf_forms = $caller->surface_forms($pe);
+ if ((($pe->sem =~ /quantity( range)?$/) && !($pe->sem =~ /monetary quantity/))
+ || ($util->member($pe->sem, "percentage","fraction"))) {
+ foreach $mod_form ($caller->surface_forms($pe, 1)) {
+ push(@surf_forms, $mod_form) unless $util->member($mod_form, @surf_forms);
+ }
+ }
+ return @surf_forms;
+ }
+ return ();
+}
+
+sub reliable_pe2surf_form {
+ local($caller, $alt_sep, $pe) = @_;
+
+ if (@surf_forms = $caller->reliable_pe2surf_forms($pe)) {
+ return $pe->surf . " == " . join($alt_sep, @surf_forms);
+ } else {
+ return "";
+ }
+}
+
+sub reliable_pes2surf_forms {
+ local($caller, $alt_sep, @pes) = @_;
+
+ my @res_surf_forms = ();
+ foreach $pe (@pes) {
+ if ($new_surf_form = $caller->reliable_pe2surf_form($alt_sep, $pe)) {
+ push(@res_surf_forms, $new_surf_form);
+ }
+ }
+ return @res_surf_forms;
+}
+
+sub string_contains_ascii_letter {
+ local($caller,$string) = @_;
+ return $string =~ /[a-zA-Z]/;
+}
+
+sub string_starts_w_ascii_letter {
+ local($caller,$string) = @_;
+ return $string =~ /^[a-zA-Z]/;
+}
+
+sub en_lex_bin {
+ local($caller, $word) = @_;
+
+ $word =~ s/\s+//g;
+ $word =~ s/[-_'\/]//g;
+ $word =~ tr/A-Z/a-z/;
+ return "digit" if $word =~ /^\d/;
+ return "special" unless $word =~ /^[a-z]/;
+ return substr($word, 0, 1);
+}
+
+sub skeleton_bin {
+ local($caller, $sk_bin_control, $word) = @_;
+
+ $word =~ s/\s+//g;
+ $word =~ s/[-_'\/]//g;
+ $word =~ tr/A-Z/a-z/;
+ return "E" unless $word;
+ if ($sk_bin_control =~ /^v1/i) {
+ return $word if length($word) <= 2;
+ return substr($word, 0, 3) if $word =~ /^(b|f[lnrt]|gr|j[nr]|k|l[nt]|m|n[kmst]|r[knst]|s|t)/;
+ return substr($word, 0, 2);
+ } elsif ($sk_bin_control =~ /d6f$/) {
+ return $word if length($word) <= 6;
+ return substr($word, 0, 6);
+ } elsif ($sk_bin_control =~ /d5f$/) {
+ return $word if length($word) <= 5;
+ return substr($word, 0, 5);
+ } elsif ($sk_bin_control =~ /d4f$/) {
+ return $word if length($word) <= 4;
+ return substr($word, 0, 4);
+ } else {
+ return $word if length($word) <= 4;
+ return substr($word, 0, 5) if $word =~ /^(bnts|brnt|brst|brtk|brtn|brts|frst|frts|klts|kntr|knts|krst|krtn|krts|ksks|kstr|lktr|ntrs|sbrt|skrt|sntr|strn|strt|trns|trts|ts)/;
+ return substr($word, 0, 4);
+ }
+}
+
+sub skeleton_bin_sub_dir {
+ local($caller, $sk_bin_control, $skeleton_bin) = @_;
+
+ $sk_bin_control = "v1" unless defined($sk_bin_control);
+ return "" if $sk_bin_control =~ /^v1/i;
+ if ($sk_bin_control =~ /^2d4d\df$/) {
+ return "SH/SHOR" if (length($skeleton_bin) < 2);
+ return substr($skeleton_bin, 0, 2) . "/" . substr($skeleton_bin, 0, 2) . "SH" if (length($skeleton_bin) < 4);
+ return substr($skeleton_bin, 0, 2) . "/" . substr($skeleton_bin, 0, 4);
+ } elsif ($sk_bin_control =~ /^2d3d\df$/) {
+ return "SH/SHO" if (length($skeleton_bin) < 2);
+ return substr($skeleton_bin, 0, 2) . "/" . substr($skeleton_bin, 0, 2) . "S" if (length($skeleton_bin) < 3);
+ return substr($skeleton_bin, 0, 2) . "/" . substr($skeleton_bin, 0, 3);
+ }
+ $bin3 = "ts";
+ return "SH" if (length($skeleton_bin) < 2) || ($skeleton_bin =~ /^($bin3)$/);
+ return substr($skeleton_bin, 0, 3) if $skeleton_bin =~ /^($bin3)/;
+ return substr($skeleton_bin, 0, 2);
+}
+
+sub en_words_and_counts_matching_skeletons {
+ local($caller, $sk_bin_version, @skeletons) = @_;
+
+ return () unless @skeletons;
+
+ @rem_skeletons = sort @skeletons;
+ $previous_skeleton = "";
+ $current_skeleton = shift @rem_skeletons;
+ @list = ($current_skeleton);
+ @lists = ();
+
+ $current_bin = "";
+ while ($current_skeleton) {
+ unless ($current_skeleton eq $previous_skeleton) {
+ $current_skeleton_bin = $caller->skeleton_bin($sk_bin_version, $current_skeleton);
+ unless ($current_skeleton_bin eq $current_bin) {
+ # need to read from new file
+ close(IN) if $current_bin;
+ $current_bin = $current_skeleton_bin;
+ $current_bin_subdir
+ = $caller->skeleton_bin_sub_dir($sk_bin_version, $current_bin);
+ if ($current_bin_subdir) {
+ $en_skeleton_file = File::Spec->catfile($english_resources_skeleton_dir,
+ $current_bin_subdir,
+ "$current_bin.txt");
+ } else {
+ $en_skeleton_file = File::Spec->catfile($english_resources_skeleton_dir,
+ "$current_bin.txt");
+ }
+ # print STDERR " Perusing $en_skeleton_file ...\n";
+ if (open(IN, $en_skeleton_file)) {
+ $en_skeleton_file_exists = 1;
+ } else {
+ $en_skeleton_file_exists = 0;
+ print STDERR "Can't open $en_skeleton_file (Point A)\n";
+ }
+ }
+ $previous_skeleton = $current_skeleton;
+ }
+ $_ = <IN> if $en_skeleton_file_exists;
+ unless ($en_skeleton_file_exists && defined($_)) {
+ push(@lists, join(' ; ', @list));
+ if (@rem_skeletons) {
+ $current_skeleton = shift @rem_skeletons;
+ @list = ($current_skeleton);
+ } else {
+ $current_skeleton = "";
+ }
+ next;
+ }
+ ($skeleton) = ($_ =~ /^(\S+)\t/);
+ next unless defined($skeleton);
+ $skeletons_match_p = $caller->skeletons_match_p($skeleton, $current_skeleton);
+ next if ($skeleton lt $current_skeleton) && ! $skeletons_match_p;
+ if ($skeletons_match_p) {
+ ($token, $count) = ($_ =~ /^\S+\t(\S|\S[-' a-zA-Z]*\S)\t(\d+)\s*$/);
+ push(@list, "$token : $count") if defined($token) && defined($count);
+ } else {
+ while ($current_skeleton lt $skeleton) {
+ push(@lists, join(' ; ', @list));
+ unless (@rem_skeletons) {
+ close(IN) if $current_bin;
+ $current_skeleton = "";
+ last;
+ }
+ $current_skeleton = shift @rem_skeletons;
+ @list = ($current_skeleton);
+ }
+ if ($caller->skeletons_match_p($skeleton, $current_skeleton)) {
+ ($token, $count) = ($_ =~ /^\S+\t(\S|\S[-' a-zA-Z]*\S)\t(\d+)\s*$/);
+ push(@list, "$token : $count") if defined($token) && defined($count);
+ }
+ }
+ }
+ close(IN) if $current_bin;
+ return @lists;
+}
+
+sub skeletons_match_p {
+# one of the skeletons might have been cut off at max
+ local($caller, $skeleton1, $skeleton2, $max) = @_;
+
+ return 1 if $skeleton1 eq $skeleton2;
+
+ $max = 5 unless defined($max);
+ if ((length($skeleton1) > length($skeleton2)) && (length($skeleton2) == $max)) {
+ return ($skeleton1 =~ /^$skeleton2/) ? 1 : 0;
+ } elsif ((length($skeleton2) > length($skeleton1)) && (length($skeleton1) == $max)) {
+ return ($skeleton2 =~ /^$skeleton1/) ? 1 : 0;
+ } else {
+ return 0;
+ }
+}
+
+sub token_weird_or_too_long {
+ local($caller, *WARNING_FH, $token) = @_;
+
+ $lc_token = lc $token;
+ $norm_token = $lc_token;
+ $norm_token =~ s/[-' ,]//g;
+ $snippet4_5 = "";
+ $snippet4_5 = substr($norm_token, 4, 2) if length($norm_token) >= 10;
+ $snippet4_6 = "";
+ $snippet4_6 = substr($norm_token, 4, 3) if length($norm_token) >= 10;
+ if (($norm_token =~ /(kkk|vvv|www|xxx|yyy|zzz)/) ||
+ ($norm_token =~ /[acgt]{15,}/) || # DNA sequence
+ ($snippet4_5 && ($norm_token =~ /($snippet4_5){5,}/)) || # 2-letter repetition
+ ($snippet4_6 && ($norm_token =~ /($snippet4_6){4,}/)) || # 3-letter repetition
+ ($norm_token =~ /[bcdfghjklmnpqrstvwxz]{8,}/) || # too many consonants
+ ($token =~ /(DDD)/) ||
+ (($lc_token =~ /fff/) && ! ($lc_token =~ /schifff/))) {
+ print WARNING_FH "skipping (WEIRD): $_";
+ return 1;
+ }
+ if ((length($norm_token) >= 50) ||
+ ((length($norm_token) >= 28)
+
+ # typical German compound noun components
+ && ! ($norm_token =~ /entwicklung/)
+ && ! ($norm_token =~ /fabrik/)
+ && ! ($norm_token =~ /finanz/)
+ && ! ($norm_token =~ /forschung/)
+ && ! ($norm_token =~ /geschwindigkeit/)
+ && ! ($norm_token =~ /gesundheit/)
+ && ! ($norm_token =~ /gewohnheit/)
+ && ! ($norm_token =~ /schaft/)
+ && ! ($norm_token =~ /schifffahrt/)
+ && ! ($norm_token =~ /sicherheit/)
+ && ! ($norm_token =~ /vergangen/)
+ && ! ($norm_token =~ /versicherung/)
+ && ! ($norm_token =~ /unternehmen/)
+ && ! ($norm_token =~ /verwaltung/)
+
+ # Other Germanic languages
+ && ! ($norm_token =~ /aktiebolag/)
+ && ! ($norm_token =~ /aktieselskab/)
+ && ! ($norm_token =~ /ontwikkeling/)
+
+ # chemical
+ && ! ($norm_token =~ /phetamine/)
+ && ! ($norm_token =~ /ethyl/)
+
+ # medical
+ && ! ($norm_token =~ /^pneumonaultramicroscopicsilicovolcanoconios[ei]s$/)
+
+ # business
+ && ! ($norm_token =~ /PriceWaterhouse/)
+ )) {
+ print WARNING_FH "skipping (TOO LONG): $_";
+ return 1;
+ }
+ return 0;
+}
+
+sub xml_de_accent {
+ local($caller, $string) = @_;
+
+ # for the time being, unlauts are mapped to main vowel (without "e")
+
+ $string =~ s/\[2-7];/A/g;
+ $string =~ s/\Æ/Ae/g;
+ $string =~ s/\Ç/C/g;
+ $string =~ s/\[0-3];/E/g;
+ $string =~ s/\[4-7];/I/g;
+ $string =~ s/\Ð/Dh/g;
+ $string =~ s/\Ñ/N/g;
+ $string =~ s/\[0-4];/O/g;
+ $string =~ s/\Ø/O/g;
+ $string =~ s/\[7-9];/U/g;
+ $string =~ s/\Ü/U/g;
+ $string =~ s/\Ý/Y/g;
+ $string =~ s/\Þ/Th/g;
+
+ $string =~ s/\ß/ss/g;
+ $string =~ s/\[4-9];/a/g;
+ $string =~ s/\æ/ae/g;
+ $string =~ s/\ç/c/g;
+ $string =~ s/\[2-5];/e/g;
+ $string =~ s/\[6-9];/i/g;
+ $string =~ s/\ð/dh/g;
+ $string =~ s/\ñ/n/g;
+ $string =~ s/\[2-6];/o/g;
+ $string =~ s/\ø/o/g;
+ $string =~ s/\ù/u/g;
+ $string =~ s/\[0-2];/u/g;
+ $string =~ s/\ý/y/g;
+ $string =~ s/\þ/th/g;
+ $string =~ s/\ÿ/y/g;
+ $string =~ s/\xE2\x80\x99/'/g;
+
+ return $string;
+}
+
+sub de_accent {
+ local($caller, $string) = @_;
+
+ # for the time being, unlauts are mapped to main vowel (without "e")
+
+ $string =~ s/\xC3[\x80-\x85]/A/g;
+ $string =~ s/\xC3\x86/Ae/g;
+ $string =~ s/\xC3\x87/C/g;
+ $string =~ s/\xC3[\x88-\x8B]/E/g;
+ $string =~ s/\xC3[\x8C-\x8F]/I/g;
+ $string =~ s/\xC3\x90/Dh/g;
+ $string =~ s/\xC3\x91/N/g;
+ $string =~ s/\xC3[\x92-\x96]/O/g;
+ $string =~ s/\xC3\x98/O/g;
+ $string =~ s/\xC3[\x99-\x9C]/U/g;
+ $string =~ s/\xC3\x9D/Y/g;
+ $string =~ s/\xC3\x9E/Th/g;
+
+ $string =~ s/\xC3\x9F/ss/g;
+ $string =~ s/\xC3[\xA0-\xA5]/a/g;
+ $string =~ s/\xC3\xA6/ae/g;
+ $string =~ s/\xC3\xA7/c/g;
+ $string =~ s/\xC3[\xA8-\xAB]/e/g;
+ $string =~ s/\xC3[\xAC-\xAF]/i/g;
+ $string =~ s/\xC3\xB0/dh/g;
+ $string =~ s/\xC3\xB1/n/g;
+ $string =~ s/\xC3[\xB2-\xB6]/o/g;
+ $string =~ s/\xC3\xB8/o/g;
+ $string =~ s/\xC3[\xB9-\xBC]/u/g;
+ $string =~ s/\xC3\xBD/y/g;
+ $string =~ s/\xC3\xBE/th/g;
+ $string =~ s/\xC3\xBF/y/g;
+ $string =~ s/\xE2\x80\x99/'/g;
+
+ return $string;
+}
+
+sub common_non_name_cap_p {
+ local($caller, $word) = @_;
+ return defined($english_ht{(lc $word)}->{COMMON_NON_NAME_CAP});
+}
+
+sub language {
+ return "English";
+}
+
+sub language_id {
+ return "en";
+}
+
+sub parse_entities_in_string {
+ local($caller, $string) = @_;
+
+ $ParseEntry->set_current_lang("en");
+ @pes = $ParseEntry->init_ParseEntry_list($string);
+ @pes = $caller->lexical_heuristic(@pes);
+ @pes = $caller->base_number_heuristic(@pes);
+
+ return @pes;
+}
+
+sub lexical_heuristic {
+ local($caller, @pes) = @_;
+
+ $i = 0;
+ while ($i <= $#pes) {
+ $pe = $pes[$i];
+ if ($pe->undefined("synt")) {
+ if ($pe->surf =~ /^\d+(,\d\d\d)*\.\d+/) {
+ $pe->set("synt", "cardinal");
+ $pe->set("sem", "decimal number");
+ $value = $pe->surf;
+ $value =~ s/,//g;
+ $pe->set("value", $value);
+ } elsif ($pe->surf =~ /^\d+(,\d\d\d)*$/) {
+ $pe->set("synt", "cardinal");
+ $pe->set("sem", "integer");
+ $value = $pe->surf;
+ $value =~ s/,//g;
+ $pe->set("value", $value);
+ } elsif ($pe->surf =~ /^([-",\.;\s:()\/%]|\@[-:\/]\@|[-:\/]\@|\@[-:\/])$/) {
+ $pe->set("gloss", $pe->surf);
+ $pe->set("synt", "punctuation");
+ } else {
+ ($length,$english) = $caller->find_max_lex_match($i,3,@pes);
+ if ($length) {
+ if ($length > 1) {
+ @slot_value_list = ();
+ @children = splice(@pes,$i,$length);
+ @roles = $util->list_with_same_elem($length,"lex");
+ $pe = $ParseEntry->newParent(*slot_value_list,*children,*roles);
+ $pe->set("surf",$english);
+ $pe->set("eot",1) if $pe->eot_p;
+ splice(@pes,$i,0,$pe);
+ } else {
+ $pe = $pes[$i];
+ }
+ $annot_s = $english_annotation_ht{$english};
+ $annot_s =~ s/^\s*:+//;
+ $annot_s =~ s/^\s+//;
+ $annot_s =~ s/\s+$//;
+ $annot_s =~ s/#.*$//;
+ foreach $annot (split('::', $annot_s)) {
+ ($slot, $value) = ($annot =~ /^([^:]+):(.*)$/);
+ if (defined($slot) && defined($value)) {
+ $pe->set($slot, $value);
+ }
+ $pe->set("sem", "integer") if ($slot eq "synt") && ($value eq "cardinal");
+ }
+ $pe->set("ord-value", $ord_value)
+ if $ord_value = $english_annotation_ht{"_EN_SYNT_"}->{(lc $english)}->{"ordinal"}->{"value"};
+ $pe->set("card-value", $card_value)
+ if $card_value = $english_annotation_ht{"_EN_SYNT_"}->{(lc $english)}->{"cardinal"}->{"value"};
+ }
+ }
+ }
+ $i++;
+ }
+ return @pes;
+}
+
+# builds numbers, incl. integers, decimal numbers, fractions, percentages, ordinals
+sub base_number_heuristic {
+ local($caller, @pes) = @_;
+
+ $i = 0;
+ # $ParseEntry->print_pes("start base_number_heuristic",$i,@pes);
+ while ($i <= $#pes) {
+ # forty-five
+ ($head_pe, @pes) =
+ $ParseEntry->build_parse_entry("composite number plus","",$i,*pes,
+ ' :head :($pe->sem eq "integer") && ($pe->value =~ /^[1-9]0$/)',
+ 'optional:dummy:$pe->surf eq "\@-\@"',
+ ' :mod :($pe->sem eq "integer") && ($pe->value =~ /^[1-9]$/)');
+ if ($head_pe) { # match succeeded
+ $value1 = $head_pe->childValue("head");
+ $value2 = $head_pe->childValue("mod");
+ $head_pe->set("value", $value1 + $value2);
+ }
+ # six billion
+ ($head_pe, @pes) =
+ $ParseEntry->build_parse_entry("composite number 1000","",$i,*pes,
+ ' :mod :(($value1 = $pe->value) =~ /^\d+(.\d+)?$/) && ($value1 < 1000)',
+ ' :head:($value2 = $pe->value) =~ /^1(000)+$/');
+ if ($head_pe) { # match succeeded
+ $value1 = $head_pe->childValue("mod");
+ $value2 = $head_pe->childValue("head");
+ $head_pe->set("value", $value1 * $value2);
+ }
+ # twenty-second
+ ($head_pe, @pes) =
+ $ParseEntry->build_parse_entry("composite ordinal","",$i,*pes,
+ ' :mod :($pe->sem eq "integer") && ($pe->value =~ /^[1-9]0$/)',
+ 'optional:dummy:$pe->surf eq "\@-\@"',
+ ' :head :$pe->get("ord-value") =~ /^[1-9]$/');
+ if ($head_pe) { # match succeeded
+ $value1 = $head_pe->childSlot("head", "ord-value");
+ $value2 = $head_pe->childValue("mod");
+ $head_pe->set("value", $value1 + $value2);
+ }
+ $i++;
+ }
+
+ return @pes;
+}
+
+sub find_max_lex_match {
+ local($caller,$start,$maxlength,@pes) = @_;
+
+ while ($maxlength > 0) {
+ if (($english = $util->pes_subseq_surf($start,$maxlength,"en",@pes))
+ && defined($english_annotation_ht{$english})
+ && ($english =~ /\S/)) {
+ return ($maxlength, $english);
+ } else {
+ $maxlength--;
+ }
+ }
+ return (0,"");
+}
+
+sub select_reliable_entities {
+ local($caller, @pes) = @_;
+
+ foreach $i (0 .. $#pes) {
+ $pe = $pes[$i];
+ $surf = $pe->surf;
+
+ $pe->set("reliable",1);
+ }
+ return @pes;
+}
+
+sub negatives_p {
+ # (cool <-> uncool), (improper <-> proper), ...
+ local($caller, $s1, $s2) = @_;
+
+ my $g_s1 = $util->regex_guard($s1);
+ my $g_s2 = $util->regex_guard($s2);
+ return 1 if $s1 =~ /^[iu]n$g_s2$/;
+ return 1 if $s1 =~ /^il$g_s2$/ && ($s2 =~ /^l/);
+ return 1 if $s1 =~ /^im$g_s2$/ && ($s2 =~ /^[mp]/);
+
+ return 1 if $s2 =~ /^[iu]n$g_s1$/;
+ return 1 if $s2 =~ /^il$g_s1$/ && ($s1 =~ /^l/);
+ return 1 if $s2 =~ /^im$g_s1$/ && ($s1 =~ /^[mp]/);
+
+ return 0;
+}
+
+sub present_participle_p {
+ local($caller, $pe) = @_;
+
+ my $aux_pe = $pe->child("aux");
+ return $caller->present_participle_p($aux_pe) if $aux_pe;
+ my $head_pe = $pe->child("head");
+ return $caller->present_participle_p($head_pe) if $head_pe;
+ return ($pe->synt =~ /^VBG/);
+}
+
+
+%engl_value_ht = (
+ "monday" => 1,
+ "tuesday" => 2,
+ "wednesday" => 3,
+ "thursday" => 4,
+ "friday" => 5,
+ "saturday" => 6,
+ "sunday" => 7,
+
+ "january" => 1,
+ "february" => 2,
+ "march" => 3,
+ "april" => 4,
+ "may" => 5,
+ "june" => 6,
+ "july" => 7,
+ "august" => 8,
+ "september" => 9,
+ "october" => 10,
+ "november" => 11,
+ "december" => 12,
+
+ "spring" => 1,
+ "summer" => 2,
+ "fall" => 3,
+ "autumn" => 3,
+ "winter" => 4,
+
+ "morning" => 1,
+ "noon" => 2,
+ "afternoon" => 3,
+ "evening" => 4,
+ "night" => 5,
+
+ "picosecond" => 1,
+ "nanosecond" => 2,
+ "microsecond" => 3,
+ "millisecond" => 4,
+ "second" => 5,
+ "minute" => 6,
+ "hour" => 7,
+ "day" => 8,
+ "week" => 9,
+ "fortnight" => 10,
+ "month" => 11,
+ "year" => 12,
+ "decade" => 13,
+ "century" => 14,
+ "millennium" => 15,
+
+ "nanometer" => 2,
+ "micrometer" => 3,
+ "millimeter" => 4,
+ "centimeter" => 5,
+ "decimeter" => 6,
+ "meter" => 7,
+ "kilometer" => 8,
+ "inch" => 11,
+ "foot" => 12,
+ "yard" => 13,
+ "mile" => 14,
+ "lightyear" => 20,
+
+ "microgram" => 2,
+ "milligram" => 3,
+ "gram" => 4,
+ "kilogram" => 5,
+ "ton" => 6,
+ "ounce" => 14,
+);
+
+sub engl_order_value {
+ local($this, $s) = @_;
+
+ return $value = $engl_value_ht{(lc $s)} || 0;
+}
+
+1;
+