Enable unit test build flag by default (#648)

* the new tokenizer API

* set the api cmake option off

* fix the pipeline

* more pipelines fixing

* fix the ci pipeline of macOS Python

* added token supporting

* fix the simdjson issue on gcc

* undo tfmtok

* remove the simdjson

.pipelines/ci.yml

@@ -180,7 +180,7 @@ stages:
     - bash: |
         set -e -x -u
         echo 'set (OCOS_ENABLE_BERT_TOKENIZER ON CACHE BOOL "" FORCE)' > cmake/_selectedoplist.cmake
-        ./build.sh -DOCOS_ENABLE_CPP_EXCEPTIONS=OFF -DOCOS_ENABLE_SELECTED_OPLIST=ON
+        ./build.sh -DOCOS_ENABLE_CPP_EXCEPTIONS=OFF -DOCOS_ENABLE_SELECTED_OPLIST=ON -DOCOS_ENABLE_CTEST=OFF
       displayName: Build ort-extensions with only one operator was selected
 
 - stage: MacOSBuilds
@@ -689,7 +689,8 @@ stages:
         -DOCOS_ENABLE_SPM_TOKENIZER=ON \
         -DOCOS_BUILD_PYTHON=OFF \
         -DOCOS_ENABLE_CV2=OFF \
-        -DOCOS_ENABLE_VISION=OFF
+        -DOCOS_ENABLE_VISION=OFF \
+        -DOCOS_ENABLE_CTEST=OFF
       displayName: build the customop library with onnxruntime
     # TODO add unittest for webassembly
 

.pyproject/cmdclass.py

@@ -239,12 +239,14 @@ def build_cmake(self, extension):
                     pass
 
         if sys.platform.startswith("darwin"):
+            cmake_args += ["-DCMAKE_OSX_DEPLOYMENT_TARGET=10.15"]
             # Cross-compile support for macOS - respect ARCHFLAGS if set
             archs = re.findall(r"-arch (\S+)", os.environ.get("ARCHFLAGS", ""))
             if archs:
                 cmake_args += [
                     "-DCMAKE_OSX_ARCHITECTURES={}".format(";".join(archs))]
 
+
         # overwrite the Python module info if the auto-detection doesn't work.
         # export Python3_INCLUDE_DIRS=/opt/python/cp38-cp38
         # export Python3_LIBRARIES=/opt/python/cp38-cp38

CMakeLists.txt

@@ -47,7 +47,7 @@ include(CheckLanguage)
 
 option(CC_OPTIMIZE "Allow compiler optimizations, Set to OFF to disable" ON)
 option(OCOS_ENABLE_PYTHON "Enable Python component building, (deprecated)" OFF)
-option(OCOS_ENABLE_CTEST "Enable C++ test" OFF)
+option(OCOS_ENABLE_CTEST "Enable C++ test" ON)
 option(OCOS_ENABLE_CPP_EXCEPTIONS "Enable C++ Exception" ON)
 option(OCOS_ENABLE_TF_STRING "Enable String Operator Set" ON)
 option(OCOS_ENABLE_RE2_REGEX "Enable StringRegexReplace and StringRegexSplit" ON)
@@ -67,6 +67,7 @@ option(OCOS_ENABLE_AZURE "Enable the operators for azure execution provider" OFF
 
 option(OCOS_ENABLE_STATIC_LIB "Enable generating static library" OFF)
 option(OCOS_ENABLE_SELECTED_OPLIST "Enable including the selected_ops tool file" OFF)
+
 option(OCOS_BUILD_PYTHON "Enable building the Python package" OFF)
 option(OCOS_BUILD_JAVA "Enable building the Java package" OFF)
 option(OCOS_BUILD_ANDROID "Enable building the Android package" OFF)