Tidying up the patch.

microsoft · Jan 25, 2024 · 5c29422 · 5c29422
1 parent 73b867b
commit 5c29422
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/operators/tokenizer/sentencepiece_tokenizer.cc b/operators/tokenizer/sentencepiece_tokenizer.cc
@@ -85,6 +85,9 @@ OrtStatusPtr KernelSentencepieceTokenizer::Compute(const ortc::Tensor<std::strin
       }
     }
   }
+  instance_indices.push_back(content.size());
+
+  // Patch fairseq indices
   if (fairseq.has_value() && (*fairseq) && !add_rev) {
     // HF Fairseq Example (XLMRobertaTokenizer) : https://huggingface.co/transformers/v4.6.0/_modules/transformers/models/xlm_roberta/tokenization_xlm_roberta.html#XLMRobertaTokenizer
     //
@@ -106,7 +109,6 @@ OrtStatusPtr KernelSentencepieceTokenizer::Compute(const ortc::Tensor<std::strin
       }
     });
   }
-  instance_indices.push_back(content.size());
 
   // Setup output
   std::vector<int64_t> size_content(1);