Merge branch 'main' into snnn-patch-1

microsoft · Mar 1, 2025 · e9104c3 · e9104c3
2 parents defd6e4 + ee945bb
commit e9104c3
Showing 1 changed file with 15 additions and 0 deletions.
diff --git a/operators/tokenizer/bpe_utils.hpp b/operators/tokenizer/bpe_utils.hpp
@@ -397,6 +397,11 @@ class PreTokenizerWithRegEx {
       return {};
     }
 
+    if (j >= m_text.size())  {
+      auto res = m_text.substr(0, j);
+      m_text = m_text.substr(j);
+      return res;
+    }
     i = j;
     // (?i:'s|'t|'re|'ve|'m|'ll|'d)?
     if ((m_text[i] == U'\'') && ((i + 1) < m_text.size())) {
@@ -442,6 +447,11 @@ class PreTokenizerWithRegEx {
     } else {
       return {};
     }
+    if (i >= m_text.size()) {
+      auto res = m_text.substr(0, i);
+      m_text = m_text.substr(i);
+      return res;
+    }
 
     // [\p{Ll}\p{Lm}\p{Lo}\p{M}]*
     const ufal::unilib::unicode::category_t categories2 =
@@ -451,6 +461,11 @@ class PreTokenizerWithRegEx {
         if (!IsCategory(m_text[i], categories2)) break;
       }
     }
+    if (i >= m_text.size()) {
+      auto res = m_text.substr(0, i);
+      m_text = m_text.substr(i);
+      return res;
+    }
 
     // (?i:'s|'t|'re|'ve|'m|'ll|'d)?
     if ((m_text[i] == U'\'') && ((i + 1) < m_text.size())) {