Download data files for tests automatically.
@@ -6,7 +6,7 @@ dir_guard=@mkdir -p $(@D)
SHARED_RESOURCES = $(DATA_DIR)/gpt2-vocab.json $(DATA_DIR)/gpt2-merges.txt $(DATA_DIR)/bert-base-uncased-vocab.txt
BENCHMARK_RESOURCES = $(SHARED_RESOURCES) $(DATA_DIR)/big.txt $(DATA_DIR)/small.txt
-TESTS_RESOURCES = $(SHARED_RESOURCES)
+TESTS_RESOURCES = $(SHARED_RESOURCES) $(DATA_DIR)/unigram.json $(DATA_DIR)/unigram.model
.PHONY : build
build :
@@ -53,6 +53,10 @@ $(DATA_DIR)/bert-% :
$(dir_guard)
wget https://s3.amazonaws.com/models.huggingface.co/bert/bert-$* -O $@
+$(DATA_DIR)/unigram% :
+ $(dir_guard)
+ wget https://storage.googleapis.com/tokenizers/unigram$* -O $@
+
$(DATA_DIR)/big.txt :
$(dir_guard)
wget https://norvig.com/big.txt -O $@
