fix the json_pointer compiler warnings within its latest release (#904)

* fix the json_pointer warnings within its latest release * fix version info in cgmanifest as well
microsoft · Mar 5, 2025 · 4c3ae1b · 4c3ae1b
1 parent 2cba60c
commit 4c3ae1b
Show file tree

Hide file tree

Showing 5 changed files with 10 additions and 8 deletions.
diff --git a/ThirdPartyNotices.txt b/ThirdPartyNotices.txt
@@ -45,7 +45,7 @@ blingfire 0831265c1aca95ca02eca5bf1155e4251e545328
 
 _____
 
-dlib v19.24.6
+dlib v19.24.7
 
 Boost Software License - Version 1.0 - August 17th, 2003
 
@@ -122,7 +122,7 @@ Viatcheslav Ostapenko <[email protected]>
 
 _____
 
-nlohmann/json v3.10.5
+nlohmann/json v3.11.3
 
 MIT License
 

diff --git a/cgmanifest.json b/cgmanifest.json
@@ -34,7 +34,7 @@
       "component": {
         "type": "git",
         "git": {
-          "commitHash": "v19.24.6",
+          "commitHash": "v19.24.7",
           "repositoryUrl": "https://github.com/davisking/dlib.git"
         }
       }

diff --git a/cmake/externals/dlib.cmake b/cmake/externals/dlib.cmake
@@ -1,7 +1,7 @@
 FetchContent_Declare(
     dlib
-    URL https://github.com/davisking/dlib/archive/d3520131a6e0e0fb62bc556b0222b45d99caf905.zip
-    URL_HASH SHA1=26b0eb3063da744a11144ae620b61fd1fb90fb39
+    URL https://github.com/davisking/dlib/archive/refs/tags/v19.24.7.zip
+    URL_HASH SHA1=6c63ea576e2b525751b0dead27c6c1139c5100ae
     DOWNLOAD_EXTRACT_TIMESTAMP TRUE
     SOURCE_SUBDIR  not_set
 )

diff --git a/operators/tokenizer/bpe_kernels.cc b/operators/tokenizer/bpe_kernels.cc
@@ -656,13 +656,16 @@ void JsonFastTokenizer::LoadSpmModelParams(const json& tok_json) {
       for (const auto& step : *decoders_node) {
         std::string type = step.value("type", "");
         if (type == "Replace") {
-          std::string target = step.value("/pattern/String"_json_pointer, "");
+            std::string target = "";
+            if (step.contains("pattern")) {
+              target = step["pattern"].value("String", "");
+            }
           if (target == spm_escaped_space) {
             json_conf_.spm_model_ = true;
           }
         }
         else if (type == "Strip") {
-          std::string content = step.value("/content"_json_pointer, "");
+          std::string content = step.value("content", "");
           if (content == " ") {
             json_conf_.add_dummy_prefix_ = true;
           }

diff --git a/operators/tokenizer/tokenizer_jsconfig.hpp b/operators/tokenizer/tokenizer_jsconfig.hpp
@@ -45,7 +45,6 @@ class TokenJsonConfig final {
   TokenJsonConfig() {}
   ~TokenJsonConfig() {}
   using json = nlohmann::json;
-  using json_pointer = nlohmann::json_pointer<std::string>;
   std::shared_ptr<json> added_tokens_decoder;
 
  public: