message: Experimental prototype of SentencePieceTokenizer for TFLite support library
diff: @@ -22,6 +22,7 @@ import java.io.FileInputStream;
  import java.io.IOException;
  import java.io.InputStream;
  import java.io.InputStreamReader;
  +import java.nio.ByteBuffer;
  import java.nio.MappedByteBuffer;
  import java.nio.channels.FileChannel;
  import java.nio.charset.Charset;
  @@ -67,9 +68,10 @@ public class FileUtil {
  @NonNull Context context, @NonNull String filePath, Charset cs) throws IOException {
  SupportPreconditions.checkNotNull(context, "Context cannot be null.");
  SupportPreconditions.checkNotNull(filePath, "File path cannot be null.");
  - InputStream inputStream = context.getAssets().open(filePath);
  + try (InputStream inputStream = context.getAssets().open(filePath)) {
  return loadLabels(inputStream, cs);
  }
  + }
  /**
  * Loads labels from an input stream of an opened label file. See details for label files in
  @@ -97,14 +99,14 @@ public class FileUtil {
  public static List<String> loadLabels(@NonNull InputStream inputStream, Charset cs)
  throws IOException {
  List<String> labels = new ArrayList<>();
  - BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, cs));
  + try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, cs))) {
  String line;
  while ((line = reader.readLine()) != null) {
  labels.add(line);
  }
  - reader.close();
  return labels;
  }
  + }
  /**
  * Loads a vocabulary file (a single-column text file) into a list of strings.
  @@ -151,11 +153,29 @@ public class FileUtil {
  throws IOException {
  SupportPreconditions.checkNotNull(context, "Context should not be null.");
  SupportPreconditions.checkNotNull(filePath, "File path cannot be null.");
  - AssetFileDescriptor fileDescriptor = context.getAssets().openFd(filePath);
  - FileInputStream inputStream = new FileInputStream(fileDescriptor.getFileDescriptor());
  + try (AssetFileDescriptor fileDescriptor = context.getAssets().openFd(filePath);
  + FileInputStream inputStream = new FileInputStream(fileDescriptor.getFileDescriptor())) {
  FileChannel fileChannel = inputStream.getChannel();
  long startOffset = fileDescriptor.getStartOffset();
  long declaredLength = fileDescriptor.getDeclaredLength();
  return fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength);
  }
  }
  +
  + /**
  + * Loads a binary file from the asset folder.
  + *
  + * @param context Application context to access assets.
  + * @param filePath Asset path of the file.
  + * @return the byte array for the binary file.
  + * @throws IOException if an I/O error occurs when loading file.
  + */
  + @NonNull
  + public static byte[] loadByteFromFile(@NonNull Context context, @NonNull String filePath)
  + throws IOException {
  + ByteBuffer buffer = loadMappedFile(context, filePath);
  + byte[] byteArray = new byte[buffer.remaining()];
  + buffer.get(byteArray);
  + return byteArray;
  + }
  +}
  