var result = {};

load('./lib/bopomofo_encoder.js');

if (!stringsAreUTF8()) {
  throw 'You need UTF-8 enabled SpiderMonkey to do cook the data.';
  quit();
}

var line;
while (line = readline()) {

  line = line.split(' ');

  if (line[1].indexOf('_punctuation_') !== -1) continue;

  var str = line[1]
    .replace(/([^\u02ca\u02c7\u02cb\u02d9])(\-|$)/g, '$1 $2')
    .replace(/\-/g, '');
  str = BopomofoEncoder.encode(str);

  switch (arguments[0]) {
    case 'words':
    default:
      if (str.length > 1) continue;
    break;
    case 'phrases':
      if (str.length === 1) continue;
    break;
  }

  if (!result[str]) {
    result[str] = [];
  }

  result[str].push([line[0], parseFloat(line[2])]);
}

for (syllables in result) {
  result[syllables] = result[syllables].sort(
    function(a, b) {
      return (b[1] - a[1]);
    }
  );
}

var jsonStr = JSON.stringify(result).replace(/\],/g, '],\n');

print(jsonStr);

quit(0);

var result = {};

load('./lib/bopomofo_encoder.js');

if (!stringsAreUTF8()) {
  throw 'You need UTF-8 enabled SpiderMonkey to do cook the data.';
  quit();
}

var line;
while (line = readline()) {

  line = line.split(' ');

  if (line[1].indexOf('_punctuation_') !== -1) continue;

  var str = BopomofoEncoder.encode(line[1].replace(/\-/g, ''));

  switch (arguments[0]) {
    case 'words':
    default:
      if (str.length > 1) continue;
    break;
    case 'phrases':
      if (str.length === 1) continue;
    break;
  }

  if (!result[str]) {
    result[str] = [];
  }

  result[str].push([line[0], parseFloat(line[2])]);
}

for (syllables in result) {
  result[syllables] = result[syllables].sort(
    function(a, b) {
      return (b[1] - a[1]);
    }
  );
}

var jsonStr = JSON.stringify(result).replace(/\],/g, '],\n');

print(jsonStr);

quit(0);

--- 
+++ 
@@ -14,7 +14,10 @@
 
   if (line[1].indexOf('_punctuation_') !== -1) continue;
 
-  var str = BopomofoEncoder.encode(line[1].replace(/\-/g, ''));
+  var str = line[1]
+    .replace(/([^\u02ca\u02c7\u02cb\u02d9])(\-|$)/g, '$1 $2')
+    .replace(/\-/g, '');
+  str = BopomofoEncoder.encode(str);
 
   switch (arguments[0]) {
     case 'words':
Make sure tone 1 is on record of the database
