Skip to content

Commit

Permalink
onnxruntime: simplify patching
Browse files Browse the repository at this point in the history
Replace the onnxruntime_external_deps.cmake and its FetchContent calls entirely with find_package() calls against Conan targets.
  • Loading branch information
valgur committed Oct 24, 2023
1 parent 71bd29a commit ee980e5
Show file tree
Hide file tree
Showing 11 changed files with 138 additions and 769 deletions.
115 changes: 115 additions & 0 deletions recipes/onnxruntime/all/cmake/onnxruntime_external_deps.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
# Replacement for /~https://github.com/microsoft/onnxruntime/blob/v1.16.1/cmake/external/onnxruntime_external_deps.cmake

if(NOT onnxruntime_DISABLE_ABSEIL)
find_package(absl REQUIRED CONFIG)
list(APPEND onnxruntime_EXTERNAL_LIBRARIES absl::absl)
include_directories(${absl_INCLUDE_DIRS})
endif()

find_package(re2 REQUIRED CONFIG)
list(APPEND onnxruntime_EXTERNAL_LIBRARIES re2::re2)

#flatbuffers 1.11.0 does not have flatbuffers::IsOutRange, therefore we require 1.12.0+
find_package(Flatbuffers REQUIRED CONFIG)
list(APPEND onnxruntime_EXTERNAL_LIBRARIES flatbuffers::flatbuffers)

find_package(Protobuf 3.21.12 REQUIRED CONFIG)
list(APPEND onnxruntime_EXTERNAL_LIBRARIES protobuf::libprotobuf)
set(ONNX_CUSTOM_PROTOC_EXECUTABLE protoc)
set(PROTOC_EXECUTABLE protoc)

find_package(date REQUIRED CONFIG)
list(APPEND onnxruntime_EXTERNAL_LIBRARIES date_interface)

find_package(Boost REQUIRED CONFIG)
list(APPEND onnxruntime_EXTERNAL_LIBRARIES Boost::mp11)

find_package(nlohmann_json REQUIRED CONFIG)
list(APPEND onnxruntime_EXTERNAL_LIBRARIES nlohmann_json::nlohmann_json)

find_package(cpuinfo CONFIG)
set(CPUINFO_SUPPORTED ${cpuinfo_FOUND})

if (NOT WIN32)
find_package(nsync REQUIRED CONFIG)
list(APPEND onnxruntime_EXTERNAL_LIBRARIES nsync::nsync_cpp)
include_directories(${nsync_INCLUDE_DIRS})
endif()

find_package(Microsoft.GSL 4.0 REQUIRED CONFIG)
list(APPEND onnxruntime_EXTERNAL_LIBRARIES Microsoft.GSL::GSL)
include_directories(${Microsoft.GSL_INCLUDE_DIRS})

find_package(safeint REQUIRED CONFIG)
list(APPEND onnxruntime_EXTERNAL_LIBRARIES safeint::safeint)

find_package(ONNX REQUIRED CONFIG)
list(APPEND onnxruntime_EXTERNAL_LIBRARIES onnx onnx_proto)

find_package(Eigen3 REQUIRED CONFIG)
list(APPEND onnxruntime_EXTERNAL_LIBRARIES Eigen3::Eigen)
set(eigen_INCLUDE_DIRS ${Eigen3_INCLUDE_DIRS})

if(WIN32)
find_package(wil REQUIRED CONFIG)
list(APPEND onnxruntime_EXTERNAL_LIBRARIES WIL::WIL)
endif()

# XNNPACK EP
if (onnxruntime_USE_XNNPACK)
if (onnxruntime_DISABLE_CONTRIB_OPS)
message(FATAL_ERROR "XNNPACK EP requires the internal NHWC contrib ops to be available "
"but onnxruntime_DISABLE_CONTRIB_OPS is ON")
endif()
find_package(xnnpack REQUIRED CONFIG)
list(APPEND onnxruntime_EXTERNAL_LIBRARIES xnnpack::xnnpack)
endif()

if (onnxruntime_USE_MIMALLOC)
find_package(mimalloc REQUIRED CONFIG)
add_definitions(-DUSE_MIMALLOC)
endif()

# The source code of onnx_proto is generated, we must build this lib first before starting to compile the other source code that uses ONNX protobuf types.
# The other libs do not have the problem. All the sources are already there. We can compile them in any order.
set(onnxruntime_EXTERNAL_DEPENDENCIES
onnx_proto
flatbuffers::flatbuffers
)

if (onnxruntime_RUN_ONNX_TESTS)
add_definitions(-DORT_RUN_EXTERNAL_ONNX_TESTS)
endif()

if(onnxruntime_ENABLE_ATEN)
message("Aten fallback is enabled.")
find_package(dlpack REQUIRED CONFIG)
endif()

if(onnxruntime_ENABLE_TRAINING OR (onnxruntime_ENABLE_TRAINING_APIS AND onnxruntime_BUILD_UNIT_TESTS))
find_package(cxxopts REQUIRED CONFIG)
endif()

set(onnxruntime_LINK_DIRS )
if (onnxruntime_USE_CUDA)
#TODO: combine onnxruntime_CUDNN_HOME and onnxruntime_CUDA_HOME, assume they are the same
if (WIN32)
if(onnxruntime_CUDNN_HOME)
list(APPEND onnxruntime_LINK_DIRS ${onnxruntime_CUDNN_HOME}/lib ${onnxruntime_CUDNN_HOME}/lib/x64)
endif()
list(APPEND onnxruntime_LINK_DIRS ${onnxruntime_CUDA_HOME}/x64/lib64)
else()
if(onnxruntime_CUDNN_HOME)
list(APPEND onnxruntime_LINK_DIRS ${onnxruntime_CUDNN_HOME}/lib ${onnxruntime_CUDNN_HOME}/lib64)
endif()
list(APPEND onnxruntime_LINK_DIRS ${onnxruntime_CUDA_HOME}/lib64)
endif()
endif()

if(onnxruntime_USE_SNPE)
include(external/find_snpe.cmake)
list(APPEND onnxruntime_EXTERNAL_LIBRARIES ${SNPE_NN_LIBS})
endif()

file(TO_NATIVE_PATH ${CMAKE_BINARY_DIR} ORT_BINARY_DIR)
file(TO_NATIVE_PATH ${PROJECT_SOURCE_DIR} ORT_SOURCE_DIR)
25 changes: 0 additions & 25 deletions recipes/onnxruntime/all/conandata.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,43 +13,21 @@ sources:
sha256: "f998352b131bb89fa7dd1f1d87ddbafe647dfaddd11929b6b5168b3f4ef857de"
patches:
"1.16.1":
- patch_file: "patches/1.16.0-0001-cmake-dependencies.patch"
patch_description: "CMake: ensure conan dependencies are used"
patch_type: "conan"
- patch_file: "patches/1.14.1-0004-abseil-no-string-view.patch"
patch_description: "allow to build with abseil built without c++17 support"
patch_type: "portability"
- patch_file: "patches/1.16.1-0002-cuda-gsl.patch"
patch_description: "CMake: fix GSL and cuda"
patch_type: "conan"
"1.16.0":
- patch_file: "patches/1.16.0-0001-cmake-dependencies.patch"
patch_description: "CMake: ensure conan dependencies are used"
patch_type: "conan"
- patch_file: "patches/1.14.1-0004-abseil-no-string-view.patch"
patch_description: "allow to build with abseil built without c++17 support"
patch_type: "portability"
- patch_file: "patches/1.16.0-0002-cuda-gsl.patch"
patch_description: "CMake: fix GSL and cuda"
patch_type: "conan"
"1.15.1":
- patch_file: "patches/1.15.1-0001-cmake-dependencies.patch"
patch_description: "CMake: ensure conan dependencies are used"
patch_type: "conan"
- patch_file: "patches/1.14.1-0004-abseil-no-string-view.patch"
patch_description: "allow to build with abseil built without c++17 support"
patch_type: "portability"
- patch_file: "patches/1.15.1-0002-cuda-gsl.patch"
patch_description: "CMake: fix GSL and cuda"
patch_type: "conan"
"1.14.1":
- patch_file: "patches/1.14.1-0001-cmake-dependencies.patch"
patch_description: "CMake: ensure conan dependencies are used (upstreamed future versions)"
patch_type: "conan"
patch_source: "/~https://github.com/microsoft/onnxruntime/pull/15323"
- patch_file: "patches/1.14.1-0002-cmake-dependencies.patch"
patch_description: "CMake: ensure conan dependencies are used"
patch_type: "conan"
- patch_file: "patches/1.14.1-0003-amx-gas-version.patch"
patch_description: "Check GNU AS supports AMX before enabling it"
patch_type: "portability"
Expand All @@ -61,6 +39,3 @@ patches:
patch_description: "Ensures the forward compatibility with the newest versions of re2 library."
patch_type: "portability"
patch_source: "/~https://github.com/microsoft/onnxruntime/commit/126e7bf15fa4af8621814b82a3f7bd0d786f0239.patch"
- patch_file: "patches/1.14.1-0006-cuda-gsl.patch"
patch_description: "CMake: fix GSL and cuda"
patch_type: "conan"
34 changes: 23 additions & 11 deletions recipes/onnxruntime/all/conanfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
from conan.errors import ConanInvalidConfiguration
from conan.tools.apple import is_apple_os
from conan.tools.cmake import CMake, CMakeDeps, CMakeToolchain, cmake_layout
from conan.tools.files import apply_conandata_patches, copy, export_conandata_patches, get, rmdir
from conan.tools.files import apply_conandata_patches, copy, export_conandata_patches, get, rmdir, replace_in_file
from conan.tools.build import check_min_cppstd
from conan.tools.scm import Version
from conan.tools.env import VirtualBuildEnv
Expand Down Expand Up @@ -61,6 +61,7 @@ def _compilers_minimum_version(self):

def export_sources(self):
export_conandata_patches(self)
copy(self, "cmake/*", src=self.recipe_folder, dst=self.export_sources_folder)

def config_options(self):
if self.settings.os == "Windows":
Expand All @@ -86,21 +87,23 @@ def requirements(self):
self.requires("abseil/20230125.3")
self.requires("protobuf/3.21.12")
self.requires("date/3.0.1")
self.requires("re2/20230801")
self.requires("re2/20230901")
self.requires(f"onnx/{self._onnx_version}")
self.requires("flatbuffers/1.12.0")
self.requires("boost/1.83.0", headers=True, libs=False, run=False) # for mp11, header only, no need for libraries to link/run
self.requires("safeint/3.0.28")
self.requires("nlohmann_json/3.11.2")
self.requires("eigen/3.4.0")
self.requires("ms-gsl/4.0.0")
self.requires("cpuinfo/cci.20220618")
self.requires("cpuinfo/cci.20230118")
if self.settings.os != "Windows":
self.requires("nsync/1.26.0")
else:
self.requires("wil/1.0.230824.2")
if self.options.with_xnnpack:
self.requires("xnnpack/cci.20220801")
self.requires("xnnpack/cci.20230715")
# FIXME: remove the override
self.requires("zlib/1.3", override=True)

def validate(self):
if self.settings.compiler.get_safe("cppstd"):
Expand Down Expand Up @@ -130,7 +133,7 @@ def generate(self):
# disable downloading dependencies to ensure conan ones are used
tc.variables["FETCHCONTENT_FULLY_DISCONNECTED"] = True
if self.version >= Version("1.15.0") and self.options.shared:
# Need to replace windows path seperators with linux path seperators to keep CMake from crashing
# Need to replace windows path separators with linux path separators to keep CMake from crashing
tc.variables["Python_EXECUTABLE"] = sys.executable.replace("\\", "/")

tc.variables["onnxruntime_BUILD_SHARED_LIB"] = self.options.shared
Expand Down Expand Up @@ -211,22 +214,31 @@ def generate(self):
tc.variables["onnxruntime_ENABLE_ROCM_PROFILING"] = False
tc.variables["onnxruntime_USE_CANN"] = False
tc.generate()
deps = CMakeDeps(self)

if self.dependencies["flatbuffers"].options.shared:
deps.set_property("flatbuffers", "cmake_target_name", "flatbuffers::flatbuffers")

deps = CMakeDeps(self)
deps.set_property("boost::headers", "cmake_target_name", "Boost::mp11")
deps.set_property("cpuinfo", "cmake_target_name", "cpuinfo")
deps.set_property("date", "cmake_target_name", "date_interface")
deps.set_property("flatbuffers", "cmake_target_name", "flatbuffers::flatbuffers")
deps.set_property("safeint", "cmake_target_name", "safeint_interface")
deps.set_property("xnnpack", "cmake_target_name", "XNNPACK")

deps.generate()

vbe = VirtualBuildEnv(self)
vbe.generate(scope="build")

def build(self):
def _patch_sources(self):
apply_conandata_patches(self)
copy(self, "onnxruntime_external_deps.cmake",
src=os.path.join(self.export_sources_folder, "cmake"),
dst=os.path.join(self.source_folder, "cmake", "external"))
# Avoid parsing of git commit info
if Version(self.version) >= "15.0":
replace_in_file(self, os.path.join(self.source_folder, "cmake", "CMakeLists.txt"),
"if (Git_FOUND)", "if (FALSE)")

def build(self):
self._patch_sources()
cmake = CMake(self)
# /~https://github.com/microsoft/onnxruntime/blob/v1.14.1/cmake/CMakeLists.txt#L792
# onnxruntime is builds its targets with COMPILE_WARNING_AS_ERROR ON
Expand Down
Loading

0 comments on commit ee980e5

Please sign in to comment.