Fix for Possible bug with BPE RegEx #5

latitudegames · Apr 21, 2021 · 80a424b · 80a424b
1 parent 79387f4
commit 80a424b
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 4 deletions.
diff --git a/Encoder.js b/Encoder.js
@@ -65,7 +65,7 @@ function get_pairs(word) {
   return pairs
 }
 
-const pat = /'s|'t|'re|'ve|'m|'l l|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu
+const pat = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu
 
 const decoder = {}
 Object.keys(encoder).map(x => { decoder[encoder[x]] = x })
@@ -87,7 +87,7 @@ const cache = {}
 function bpe(token) {
   if (token in cache) {
     return cache[token]
-  }
+  }``
 
   let word = token.split('')
 

diff --git a/encoder.py b/encoder.py
@@ -48,7 +48,7 @@ def __init__(self, encoder, bpe_merges, errors="replace"):
         self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
         self.cache = {}
         self.pat = re.compile(
-            r"""'s|'t|'re|'ve|'m|'l l|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
+            r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
         )
 
     def bpe(self, token):

diff --git a/package-lock.json b/package-lock.json