fix the json_pointer warnings within its latest release

microsoft · Mar 4, 2025 · eeb8093 · eeb8093
1 parent 2cba60c
commit eeb8093
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 3 deletions.
diff --git a/operators/tokenizer/bpe_kernels.cc b/operators/tokenizer/bpe_kernels.cc
@@ -656,13 +656,16 @@ void JsonFastTokenizer::LoadSpmModelParams(const json& tok_json) {
       for (const auto& step : *decoders_node) {
         std::string type = step.value("type", "");
         if (type == "Replace") {
-          std::string target = step.value("/pattern/String"_json_pointer, "");
+            std::string target = "";
+            if (step.contains("pattern")) {
+              target = step["pattern"].value("String", "");
+            }
           if (target == spm_escaped_space) {
             json_conf_.spm_model_ = true;
           }
         }
         else if (type == "Strip") {
-          std::string content = step.value("/content"_json_pointer, "");
+          std::string content = step.value("content", "");
           if (content == " ") {
             json_conf_.add_dummy_prefix_ = true;
           }

diff --git a/operators/tokenizer/tokenizer_jsconfig.hpp b/operators/tokenizer/tokenizer_jsconfig.hpp
@@ -45,7 +45,6 @@ class TokenJsonConfig final {
   TokenJsonConfig() {}
   ~TokenJsonConfig() {}
   using json = nlohmann::json;
-  using json_pointer = nlohmann::json_pointer<std::string>;
   std::shared_ptr<json> added_tokens_decoder;
 
  public: