BPE

Author: springkim

.gitignore

@@ -49,4 +49,8 @@ perftest/other/
 perftest/test.py
 perftest/.*/
 docs/blackpaper/
-extract.py
+extract.py
+dataset/BPE
+dataset/txt
+perftest/performance_test_splade.py
+*.h5

CMakeLists.txt

@@ -33,7 +33,7 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
             -fstrict-aliasing -finline-functions
             -march=native -mtune=native)
 elseif (MSVC)
-    set(COMPILE_OPTIONS /O2 /W1 /GL /Ot /Ob3 /fp:fast /arch:AVX2 /Zc:__cplusplus /EHsc- /GR-)
+    set(COMPILE_OPTIONS /O2 /W1 /GL /Ot /Ob3 /fp:fast /arch:AVX2 /Zc:__cplusplus /EHsc- /GR- /link /STACK:4194304)
     set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
 endif ()
 

dataset/README.md

@@ -1,3 +1,8 @@
 # flash-tokenizer/dataset
 
-[View documentation for the dataset directory](./DATA.md)
+[View documentation for the dataset directory](./DATA.md)
+
+
+dataset/data/texts_ko/texts_ko.txt
+
+dataset/data/texts_ko/texts_ko.bert-base-cased.txt

dataset/config/bert-base-chinese/tokenizer_config.json

@@ -1 +1,4 @@
-{"do_lower_case": false, "model_max_length": 512}
+{
+  "do_lower_case": false,
+  "model_max_length": 512
+}

dataset/config/bert-base-japanese.bin

@@ -0,0 +1,20 @@
+{
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "tokenizer_class": "BertJapaneseTokenizer",
+  "type_vocab_size": 2,
+  "vocab_size": 32000
+}

dataset/config/bert-base-japanese/config.json

@@ -0,0 +1,6 @@
+{
+  "do_lower_case": false,
+  "subword_tokenizer_type": "wordpiece",
+  "word_tokenizer_type": "mecab",
+  "model_max_length": 512
+}