From ef0dde9e0e5cbc97a9c685fcca1d7300b2afc351 Mon Sep 17 00:00:00 2001 From: hariharans29 Date: Wed, 27 Feb 2019 19:35:58 -0800 Subject: [PATCH 01/13] Initial commit --- cmake/CMakeLists.txt | 35 +++++++++++++++++++++++++++++++++-- tools/ci_build/build.py | 30 ++++++++++++++++++++++++++---- 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 9c42264a91c1a..10201a8d6e3da 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -65,11 +65,13 @@ option(onnxruntime_ENABLE_MICROSOFT_INTERNAL "Use this option to enable/disable option(onnxruntime_USE_NUPHAR "Build with Nupha" OFF) option(onnxruntime_USE_BRAINSLICE "Build with BrainSlice" OFF) option(onnxruntime_USE_TRT "Build with TensorRT support" OFF) +option(onnxruntime_CROSS_COMPILING "Cross compiling onnx runtime" OFF) set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE) #nsync tests failed on Mac Build set(NSYNC_ENABLE_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE) set(ONNX_ML 1) +set(ONNX_CUSTOM_PROTOC_EXECUTABLE "F:/onnxruntime/build/Windows/Debug/external/protobuf/cmake/Debug/protoc.exe") set(REPO_ROOT ${PROJECT_SOURCE_DIR}/..) set(ONNXRUNTIME_ROOT ${PROJECT_SOURCE_DIR}/../onnxruntime) @@ -84,6 +86,21 @@ if(onnxruntime_USE_OPENMP) endif() endif() +if(onnxruntime_CROSS_COMPILING) + set(CMAKE_CROSSCOMPILING ON) +endif() + +# Dummy code +message(${CMAKE_SYSTEM_NAME}) +message(${CMAKE_HOST_SYSTEM_NAME}) +set (CMAKE_SYSTEM_PROCESSOR ARM64) +message(${CMAKE_SYSTEM_PROCESSOR}) +message(${CMAKE_HOST_SYSTEM_PROCESSOR}) +#set(CMAKE_SYSTEM_PROCESSOR ARM64) +#message(${CMAKE_SYSTEM_PROCESSOR}) +message(${CMAKE_CROSSCOMPILING}) +# End dummy + #must after OpenMP settings find_package(Threads) @@ -182,7 +199,8 @@ list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/external) #3. both onnxruntime_USE_PREBUILT_PB and ONNX_CUSTOM_PROTOC_EXECUTABLE are not set # Compile everything from source code. Slowest option. -if(onnxruntime_USE_PREBUILT_PB) +# option 1 +if(onnxruntime_USE_PREBUILT_PB AND NOT "${ONNX_CUSTOM_PROTOC_EXECUTABLE}" STREQUAL "") get_filename_component( _PROTOBUF_INSTALL_PREFIX ${ONNX_CUSTOM_PROTOC_EXECUTABLE} @@ -197,6 +215,19 @@ if(onnxruntime_USE_PREBUILT_PB) include(${_PROTOBUF_INSTALL_PREFIX}/lib64/cmake/protobuf/protobuf-config.cmake) endif() include(protobuf_function.cmake) + +# option 2 +elseif(NOT "${ONNX_CUSTOM_PROTOC_EXECUTABLE}" STREQUAL "") + add_subdirectory(${PROJECT_SOURCE_DIR}/external/protobuf/cmake EXCLUDE_FROM_ALL) + set_target_properties(libprotobuf PROPERTIES FOLDER "External/Protobuf") + set_target_properties(libprotobuf-lite PROPERTIES FOLDER "External/Protobuf") + set_target_properties(libprotoc PROPERTIES FOLDER "External/Protobuf") + set_target_properties(protoc PROPERTIES FOLDER "External/Protobuf") + add_library(protobuf::libprotobuf ALIAS libprotobuf) + #add_executable(protobuf::protoc ALIAS protoc) + include(protobuf_function.cmake) + +# option 3 else() # use protobuf as a submodule add_subdirectory(${PROJECT_SOURCE_DIR}/external/protobuf/cmake EXCLUDE_FROM_ALL) @@ -340,7 +371,7 @@ if (WIN32) endif() # treat warning as error only on x64 platform. For x86, there are too many warnings to fix. - if (CMAKE_SIZEOF_VOID_P EQUAL 8 AND onnxruntime_DEV_MODE) + if (CMAKE_SIZEOF_VOID_P EQUAL 8 AND onnxruntime_DEV_MODE AND NOT CMAKE_CROSSCOMPILING) # treat warnings as errors string(APPEND CMAKE_CXX_FLAGS " /WX") foreach(type EXE STATIC SHARED) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index f3ff664d2d1a7..c1053d2d971c4 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -38,7 +38,8 @@ def __init__(self, message): def parse_arguments(): parser = argparse.ArgumentParser(description="ONNXRuntime CI build driver.", usage=''' -Default behavior is --update --build --test. +Default behavior is --update --build --test for native architecture builds. +Default behavior is --update --build for cross-compiled builds. The Update phase will update git submodules, and run cmake to generate makefiles. The Build phase will build all projects. @@ -93,6 +94,8 @@ def parse_arguments(): "These are just CMake -D options without the leading -D.") parser.add_argument("--x86", action='store_true', help="Create x86 makefiles. Requires --update and no existing cache CMake setup. Delete CMakeCache.txt if needed") + parser.add_argument("--arm64", action='store_true', + help="Create ARM64 makefiles. Requires --update and no existing cache CMake setup. Delete CMakeCache.txt if needed") parser.add_argument("--msvc_toolset", help="MSVC toolset to use. e.g. 14.11") # Arguments needed by CI @@ -136,6 +139,9 @@ def is_windows(): def is_ubuntu_1604(): return platform.linux_distribution()[0] == 'Ubuntu' and platform.linux_distribution()[1] == '16.04' +def is_processor_64_bit(): + return platform.machine().endswith('64') + def get_config_build_dir(build_dir, config): # build directory per configuration return os.path.join(build_dir, config) @@ -300,6 +306,8 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_USE_NUPHAR=" + ("ON" if args.use_nuphar else "OFF"), "-Donnxruntime_USE_EIGEN_THREADPOOL=" + ("ON" if args.use_eigenthreadpool else "OFF"), "-Donnxruntime_USE_TRT=" + ("ON" if args.use_trt else "OFF"), + #By default, only support cross-comipling for ARM64 + "-Donnxruntime_CROSS_COMPILING=" + ("ON" if args.arm64 else "OFF"), ] if args.use_brainslice: bs_pkg_name = args.brain_slice_package_name.split('.', 1) @@ -518,12 +526,15 @@ def main(): cmake_extra_defines = args.cmake_extra_defines if args.cmake_extra_defines else [] - # if there was no explicit argument saying what to do, default to update, build and test. + # if there was no explicit argument saying what to do, default to update, build and test (for native builds). if (args.update == False and args.clean == False and args.build == False and args.test == False): - log.debug("Defaulting to running update, build and test.") + log.debug("Defaulting to running update, build [and test for native builds].") args.update = True args.build = True - args.test = True + if args.arm64: + args.test = False + else: + args.test = True if args.build_wheel: args.enable_pybind = True @@ -552,6 +563,15 @@ def main(): if(is_windows()): if (args.x86): cmake_extra_args = ['-A','Win32','-G', 'Visual Studio 15 2017'] + elif (args.arm64): + # Cross-compiling for ARM64 architecture + if is_processor_64_bit(): + cmake_extra_args = ['-A','ARM64', '-T', 'host=x64', '-G', 'Visual Studio 15 2017'] + else: + cmake_extra_args = ['-A','ARM64', '-G', 'Visual Studio 15 2017'] + # Cannot test on host build machine for cross-compiled builds (Override any user-defined behaviour for test if any) + if args.test: + args.test = False else: toolset = 'host=x64' if (args.msvc_toolset): @@ -561,6 +581,8 @@ def main(): cmake_extra_args = ['-A','x64','-T', toolset, '-G', 'Visual Studio 15 2017'] if is_ubuntu_1604(): + if (args.arm64): + raise BuildError("Only Windows ARM64 builds supported currently") install_ubuntu_deps(args) if not is_docker(): install_python_deps() From 98f6b05678c48d060130c73289b333dcf6b9702e Mon Sep 17 00:00:00 2001 From: hariharans29 Date: Thu, 28 Feb 2019 12:37:35 -0800 Subject: [PATCH 02/13] More changes --- cmake/CMakeLists.txt | 13 ++++++++----- tools/ci_build/build.py | 16 ++++++++-------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 10201a8d6e3da..dcfdd1523b547 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -71,7 +71,6 @@ set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE) #nsync tests failed on Mac Build set(NSYNC_ENABLE_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE) set(ONNX_ML 1) -set(ONNX_CUSTOM_PROTOC_EXECUTABLE "F:/onnxruntime/build/Windows/Debug/external/protobuf/cmake/Debug/protoc.exe") set(REPO_ROOT ${PROJECT_SOURCE_DIR}/..) set(ONNXRUNTIME_ROOT ${PROJECT_SOURCE_DIR}/../onnxruntime) @@ -93,9 +92,10 @@ endif() # Dummy code message(${CMAKE_SYSTEM_NAME}) message(${CMAKE_HOST_SYSTEM_NAME}) -set (CMAKE_SYSTEM_PROCESSOR ARM64) message(${CMAKE_SYSTEM_PROCESSOR}) message(${CMAKE_HOST_SYSTEM_PROCESSOR}) +message(${CMAKE_GENERATOR_PLATFORM}) +message(${CMAKE_VS_PLATFORM_NAME}) #set(CMAKE_SYSTEM_PROCESSOR ARM64) #message(${CMAKE_SYSTEM_PROCESSOR}) message(${CMAKE_CROSSCOMPILING}) @@ -218,18 +218,21 @@ if(onnxruntime_USE_PREBUILT_PB AND NOT "${ONNX_CUSTOM_PROTOC_EXECUTABLE}" STREQU # option 2 elseif(NOT "${ONNX_CUSTOM_PROTOC_EXECUTABLE}" STREQUAL "") + message("option 2") add_subdirectory(${PROJECT_SOURCE_DIR}/external/protobuf/cmake EXCLUDE_FROM_ALL) set_target_properties(libprotobuf PROPERTIES FOLDER "External/Protobuf") set_target_properties(libprotobuf-lite PROPERTIES FOLDER "External/Protobuf") - set_target_properties(libprotoc PROPERTIES FOLDER "External/Protobuf") - set_target_properties(protoc PROPERTIES FOLDER "External/Protobuf") add_library(protobuf::libprotobuf ALIAS libprotobuf) - #add_executable(protobuf::protoc ALIAS protoc) + get_filename_component( + _PROTOBUF_INSTALL_PREFIX + ${ONNX_CUSTOM_PROTOC_EXECUTABLE} + DIRECTORY) include(protobuf_function.cmake) # option 3 else() # use protobuf as a submodule + message("option 3") add_subdirectory(${PROJECT_SOURCE_DIR}/external/protobuf/cmake EXCLUDE_FROM_ALL) set_target_properties(libprotobuf PROPERTIES FOLDER "External/Protobuf") set_target_properties(libprotobuf-lite PROPERTIES FOLDER "External/Protobuf") diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index c1053d2d971c4..ec47cf379d0d3 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -65,6 +65,7 @@ def parse_arguments(): parser.add_argument("--enable_onnx_tests", action='store_true', help='''When running the Test phase, run onnx_test_running against available test data directories.''') parser.add_argument("--pb_home", help="Path to protobuf installation") + parser.add_argument("--path_to_protoc_exe", help="Path to protoc exe. Will be overridden by pb_home if that is set.") parser.add_argument("--download_test_data", action="store_true", help='''Downloads test data without running the tests''') parser.add_argument("--test_data_url", help="Test data URL.") @@ -124,6 +125,7 @@ def parse_arguments(): parser.add_argument("--use_nuphar", action='store_true', help="Build with nuphar") parser.add_argument("--use_trt", action='store_true', help="Build with trt") parser.add_argument("--trt_path", action='store_true', help="Path to trt dir") + return parser.parse_args() def resolve_executable_path(command_or_path): @@ -139,9 +141,6 @@ def is_windows(): def is_ubuntu_1604(): return platform.linux_distribution()[0] == 'Ubuntu' and platform.linux_distribution()[1] == '16.04' -def is_processor_64_bit(): - return platform.machine().endswith('64') - def get_config_build_dir(build_dir, config): # build directory per configuration return os.path.join(build_dir, config) @@ -306,9 +305,10 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_USE_NUPHAR=" + ("ON" if args.use_nuphar else "OFF"), "-Donnxruntime_USE_EIGEN_THREADPOOL=" + ("ON" if args.use_eigenthreadpool else "OFF"), "-Donnxruntime_USE_TRT=" + ("ON" if args.use_trt else "OFF"), - #By default, only support cross-comipling for ARM64 + # By default - we currently support only cross compiling for ARM64 (no native compilation supported through this script) "-Donnxruntime_CROSS_COMPILING=" + ("ON" if args.arm64 else "OFF"), ] + if args.use_brainslice: bs_pkg_name = args.brain_slice_package_name.split('.', 1) bs_shared_lib_name = '.'.join((bs_pkg_name[0], 'redist', bs_pkg_name[1])) @@ -334,6 +334,9 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home if pb_home: cmake_args += ["-DONNX_CUSTOM_PROTOC_EXECUTABLE=" + os.path.join(pb_home,'bin','protoc'), '-Donnxruntime_USE_PREBUILT_PB=ON'] + if not pb_home and args.path_to_protoc_exe: + cmake_args += ["-DONNX_CUSTOM_PROTOC_EXECUTABLE=%s" % args.path_to_protoc_exe] + cmake_args += ["-D{}".format(define) for define in cmake_extra_defines] if is_windows(): @@ -565,10 +568,7 @@ def main(): cmake_extra_args = ['-A','Win32','-G', 'Visual Studio 15 2017'] elif (args.arm64): # Cross-compiling for ARM64 architecture - if is_processor_64_bit(): - cmake_extra_args = ['-A','ARM64', '-T', 'host=x64', '-G', 'Visual Studio 15 2017'] - else: - cmake_extra_args = ['-A','ARM64', '-G', 'Visual Studio 15 2017'] + cmake_extra_args = ['-A','ARM64', '-G', 'Visual Studio 15 2017'] # Cannot test on host build machine for cross-compiled builds (Override any user-defined behaviour for test if any) if args.test: args.test = False From 3565efcf25427dee7787535fec2f334a4bbdd4c5 Mon Sep 17 00:00:00 2001 From: hariharans29 Date: Tue, 5 Mar 2019 19:15:23 -0800 Subject: [PATCH 03/13] More changes --- cmake/CMakeLists.txt | 25 ++++++------------------- tools/ci_build/build.py | 39 +++++++++++++++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index dcfdd1523b547..ad284e9f5d76b 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -89,18 +89,6 @@ if(onnxruntime_CROSS_COMPILING) set(CMAKE_CROSSCOMPILING ON) endif() -# Dummy code -message(${CMAKE_SYSTEM_NAME}) -message(${CMAKE_HOST_SYSTEM_NAME}) -message(${CMAKE_SYSTEM_PROCESSOR}) -message(${CMAKE_HOST_SYSTEM_PROCESSOR}) -message(${CMAKE_GENERATOR_PLATFORM}) -message(${CMAKE_VS_PLATFORM_NAME}) -#set(CMAKE_SYSTEM_PROCESSOR ARM64) -#message(${CMAKE_SYSTEM_PROCESSOR}) -message(${CMAKE_CROSSCOMPILING}) -# End dummy - #must after OpenMP settings find_package(Threads) @@ -199,7 +187,7 @@ list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/external) #3. both onnxruntime_USE_PREBUILT_PB and ONNX_CUSTOM_PROTOC_EXECUTABLE are not set # Compile everything from source code. Slowest option. -# option 1 +# mode 1 if(onnxruntime_USE_PREBUILT_PB AND NOT "${ONNX_CUSTOM_PROTOC_EXECUTABLE}" STREQUAL "") get_filename_component( _PROTOBUF_INSTALL_PREFIX @@ -216,9 +204,8 @@ if(onnxruntime_USE_PREBUILT_PB AND NOT "${ONNX_CUSTOM_PROTOC_EXECUTABLE}" STREQU endif() include(protobuf_function.cmake) -# option 2 +# mode 2 elseif(NOT "${ONNX_CUSTOM_PROTOC_EXECUTABLE}" STREQUAL "") - message("option 2") add_subdirectory(${PROJECT_SOURCE_DIR}/external/protobuf/cmake EXCLUDE_FROM_ALL) set_target_properties(libprotobuf PROPERTIES FOLDER "External/Protobuf") set_target_properties(libprotobuf-lite PROPERTIES FOLDER "External/Protobuf") @@ -229,10 +216,9 @@ elseif(NOT "${ONNX_CUSTOM_PROTOC_EXECUTABLE}" STREQUAL "") DIRECTORY) include(protobuf_function.cmake) -# option 3 +# mode 3 else() # use protobuf as a submodule - message("option 3") add_subdirectory(${PROJECT_SOURCE_DIR}/external/protobuf/cmake EXCLUDE_FROM_ALL) set_target_properties(libprotobuf PROPERTIES FOLDER "External/Protobuf") set_target_properties(libprotobuf-lite PROPERTIES FOLDER "External/Protobuf") @@ -338,7 +324,7 @@ if (MSVC) endif() if (onnxruntime_RUN_ONNX_TESTS) - add_definitions(-DONNXRUNTIME_RUN_EXTERNAL_ONNX_TESTS) + add_definitions(-DORT_RUN_EXTERNAL_ONNX_TESTS) endif() if (onnxruntime_USE_MLAS) @@ -373,7 +359,8 @@ if (WIN32) string(APPEND CMAKE_CXX_FLAGS " /W4") endif() - # treat warning as error only on x64 platform. For x86, there are too many warnings to fix. + # treat warning as error only on x64 platform. + # For x86 and cross-compiled ARM64 binaries, there are too many warnings to fix, hence ignore warnings for now if (CMAKE_SIZEOF_VOID_P EQUAL 8 AND onnxruntime_DEV_MODE AND NOT CMAKE_CROSSCOMPILING) # treat warnings as errors string(APPEND CMAKE_CXX_FLAGS " /WX") diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index ec47cf379d0d3..eaa544d4ed43e 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -276,7 +276,7 @@ def setup_test_data(build_dir, configs, test_data_url, test_data_checksum, azure log.debug("creating shortcut %s -> %s" % (src_model_dir, dest_model_dir)) run_subprocess(['mklink', '/D', '/J', dest_model_dir, src_model_dir], shell=True) -def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, pb_home, configs, cmake_extra_defines, args, cmake_extra_args): +def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, pb_home, path_to_protoc_exe, configs, cmake_extra_defines, args, cmake_extra_args): log.info("Generating CMake build tree") cmake_dir = os.path.join(source_dir, "cmake") # TODO: fix jemalloc build so it does not conflict with onnxruntime shared lib builds. (e.g. onnxuntime_pybind) @@ -334,8 +334,8 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home if pb_home: cmake_args += ["-DONNX_CUSTOM_PROTOC_EXECUTABLE=" + os.path.join(pb_home,'bin','protoc'), '-Donnxruntime_USE_PREBUILT_PB=ON'] - if not pb_home and args.path_to_protoc_exe: - cmake_args += ["-DONNX_CUSTOM_PROTOC_EXECUTABLE=%s" % args.path_to_protoc_exe] + if not pb_home and path_to_protoc_exe: + cmake_args += ["-DONNX_CUSTOM_PROTOC_EXECUTABLE=%s" % path_to_protoc_exe] cmake_args += ["-D{}".format(define) for define in cmake_extra_defines] @@ -524,6 +524,28 @@ def build_python_wheel(source_dir, build_dir, configs, use_cuda): if is_ubuntu_1604(): run_subprocess([os.path.join(source_dir, 'rename_manylinux.sh')], cwd=cwd+'/dist') +def build_protobuf_for_windows_host(cmake_path, source_dir, build_dir): + if not is_windows(): + raise BuildError('Currently only support building protoc for Windows host while cross-compiling for ARM64 arch') + log.info("Building protoc for host to be used in cross-compiled build process") + protoc_build_dir = os.path.join(build_dir, 'host_protoc') + os.makedirs(protoc_build_dir, exist_ok=True) + # Generate step + cmd_args = [cmake_path, + os.path.join(source_dir, 'cmake\external\protobuf\cmake'), + '-G', + 'Visual Studio 15 2017', + '-Dprotobuf_BUILD_TESTS=OFF', + '-Dprotobuf_WITH_ZLIB_DEFAULT=OFF', + '-Dprotobuf_BUILD_SHARED_LIBS=OFF'] + run_subprocess(cmd_args, cwd= protoc_build_dir) + # Build step + cmd_args = [cmake_path, + "--build", protoc_build_dir, + "--config", "Release", + "--target", "protoc"] + run_subprocess(cmd_args) + def main(): args = parse_arguments() @@ -568,6 +590,8 @@ def main(): cmake_extra_args = ['-A','Win32','-G', 'Visual Studio 15 2017'] elif (args.arm64): # Cross-compiling for ARM64 architecture + # First build protobuf for host + build_protobuf_for_windows_host(cmake_path, source_dir, build_dir) cmake_extra_args = ['-A','ARM64', '-G', 'Visual Studio 15 2017'] # Cannot test on host build machine for cross-compiled builds (Override any user-defined behaviour for test if any) if args.test: @@ -596,7 +620,14 @@ def main(): raise UsageError("The test_data_url and test_data_checksum arguments are required.") setup_test_data(build_dir, configs, args.test_data_url, args.test_data_checksum, args.azure_sas_key) - generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, args.pb_home, configs, cmake_extra_defines, + path_to_protoc_exe = None + if args.path_to_protoc_exe: + path_to_protoc_exe = args.path_to_protoc_exe + # Need to provide path to protoc.exe built for host to be used in the cross-compiled build process + elif args.arm64: + path_to_protoc_exe = os.path.join(build_dir, 'host_protoc', 'Release', 'protoc.exe') + + generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, args.pb_home, path_to_protoc_exe, configs, cmake_extra_defines, args, cmake_extra_args) if (args.clean): From 39ede29996970a5bd24afa02c07e0a39c4fe64b6 Mon Sep 17 00:00:00 2001 From: hariharans29 Date: Tue, 5 Mar 2019 19:22:27 -0800 Subject: [PATCH 04/13] More changes --- tools/ci_build/build.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index eaa544d4ed43e..3f4221e1fc869 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -65,7 +65,7 @@ def parse_arguments(): parser.add_argument("--enable_onnx_tests", action='store_true', help='''When running the Test phase, run onnx_test_running against available test data directories.''') parser.add_argument("--pb_home", help="Path to protobuf installation") - parser.add_argument("--path_to_protoc_exe", help="Path to protoc exe. Will be overridden by pb_home if that is set.") + parser.add_argument("--path_to_protoc_exe", help="Path to protoc exe. Will be overridden by {pb_home}/bin/protoc.exe if {pb_home} is set.") parser.add_argument("--download_test_data", action="store_true", help='''Downloads test data without running the tests''') parser.add_argument("--test_data_url", help="Test data URL.") @@ -125,7 +125,6 @@ def parse_arguments(): parser.add_argument("--use_nuphar", action='store_true', help="Build with nuphar") parser.add_argument("--use_trt", action='store_true', help="Build with trt") parser.add_argument("--trt_path", action='store_true', help="Path to trt dir") - return parser.parse_args() def resolve_executable_path(command_or_path): @@ -308,7 +307,6 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home # By default - we currently support only cross compiling for ARM64 (no native compilation supported through this script) "-Donnxruntime_CROSS_COMPILING=" + ("ON" if args.arm64 else "OFF"), ] - if args.use_brainslice: bs_pkg_name = args.brain_slice_package_name.split('.', 1) bs_shared_lib_name = '.'.join((bs_pkg_name[0], 'redist', bs_pkg_name[1])) @@ -524,9 +522,10 @@ def build_python_wheel(source_dir, build_dir, configs, use_cuda): if is_ubuntu_1604(): run_subprocess([os.path.join(source_dir, 'rename_manylinux.sh')], cwd=cwd+'/dist') -def build_protobuf_for_windows_host(cmake_path, source_dir, build_dir): +def build_protoc_for_windows_host(cmake_path, source_dir, build_dir): if not is_windows(): raise BuildError('Currently only support building protoc for Windows host while cross-compiling for ARM64 arch') + log.info("Building protoc for host to be used in cross-compiled build process") protoc_build_dir = os.path.join(build_dir, 'host_protoc') os.makedirs(protoc_build_dir, exist_ok=True) @@ -591,7 +590,7 @@ def main(): elif (args.arm64): # Cross-compiling for ARM64 architecture # First build protobuf for host - build_protobuf_for_windows_host(cmake_path, source_dir, build_dir) + build_protoc_for_windows_host(cmake_path, source_dir, build_dir) cmake_extra_args = ['-A','ARM64', '-G', 'Visual Studio 15 2017'] # Cannot test on host build machine for cross-compiled builds (Override any user-defined behaviour for test if any) if args.test: From a0bfc28f7af296c12e34346132419b26aba06aaa Mon Sep 17 00:00:00 2001 From: hariharans29 Date: Tue, 5 Mar 2019 19:30:14 -0800 Subject: [PATCH 05/13] More changes --- tools/ci_build/build.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 3f4221e1fc869..9c16bca1f16e8 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -545,6 +545,9 @@ def build_protoc_for_windows_host(cmake_path, source_dir, build_dir): "--target", "protoc"] run_subprocess(cmd_args) + if not os.path.exists(os.path.join(build_dir, 'host_protoc', 'Release', 'protoc.exe')): + raise BuildError("Couldn't build protoc.exe for host. Failing build.") + def main(): args = parse_arguments() From 69a45d5fde2582522a06cff722bea8222c9f037e Mon Sep 17 00:00:00 2001 From: hariharans29 Date: Wed, 6 Mar 2019 18:03:00 -0800 Subject: [PATCH 06/13] PR feedback --- BUILD.md | 11 +++++++++-- cmake/CMakeLists.txt | 17 +---------------- tools/ci_build/build.py | 29 ++++++++++++++++++----------- 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/BUILD.md b/BUILD.md index 5029d06a98559..a525ffcbb350f 100644 --- a/BUILD.md +++ b/BUILD.md @@ -52,7 +52,7 @@ ONNX Runtime python binding only supports Python 3.x. Please use python 3.5+. ``` 5. Run `./build.sh --config RelWithDebInfo --build_wheel` for Linux (or `build.bat --config RelWithDebInfo --build_wheel` for Windows) -The build script runs all unit tests by default. +The build script runs all unit tests by default (for native builds and skips tests by default for cross-compiled builds). The complete list of build options can be found by running `./build.sh (or ./build.bat) --help` @@ -197,4 +197,11 @@ Please see [ARM docker file](dockerfiles/Dockerfile.arm32v7). Docker build runs By doing this, you could avoid hit the ACR-Tasks build timeout (8 hours) ### Cross compiling on Windows -(TODO) +#### Using Visual C++ compilers +1. Download and install Visual C++ compilers and libraries for ARM(64). + If you have Visual Studio installed, please use the Visual Studio Installer (look under the section `Individual components` after choosing to `modify` Visual Studio) to download and install the corresponding ARM(64) compilers and libraries. + +2. Use `build.bat` and specify `--arm` or `--arm64` as the build option to start building. Preferably use `Developer Command Prompt for VS` or make sure all the installed cross-compilers are findable from the command prompt being used to build using the PATH environmant variable. + +### Using other compilers +(TODO) \ No newline at end of file diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index ad284e9f5d76b..5ac1b4ba65618 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -187,8 +187,7 @@ list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/external) #3. both onnxruntime_USE_PREBUILT_PB and ONNX_CUSTOM_PROTOC_EXECUTABLE are not set # Compile everything from source code. Slowest option. -# mode 1 -if(onnxruntime_USE_PREBUILT_PB AND NOT "${ONNX_CUSTOM_PROTOC_EXECUTABLE}" STREQUAL "") +if(onnxruntime_USE_PREBUILT_PB) get_filename_component( _PROTOBUF_INSTALL_PREFIX ${ONNX_CUSTOM_PROTOC_EXECUTABLE} @@ -203,20 +202,6 @@ if(onnxruntime_USE_PREBUILT_PB AND NOT "${ONNX_CUSTOM_PROTOC_EXECUTABLE}" STREQU include(${_PROTOBUF_INSTALL_PREFIX}/lib64/cmake/protobuf/protobuf-config.cmake) endif() include(protobuf_function.cmake) - -# mode 2 -elseif(NOT "${ONNX_CUSTOM_PROTOC_EXECUTABLE}" STREQUAL "") - add_subdirectory(${PROJECT_SOURCE_DIR}/external/protobuf/cmake EXCLUDE_FROM_ALL) - set_target_properties(libprotobuf PROPERTIES FOLDER "External/Protobuf") - set_target_properties(libprotobuf-lite PROPERTIES FOLDER "External/Protobuf") - add_library(protobuf::libprotobuf ALIAS libprotobuf) - get_filename_component( - _PROTOBUF_INSTALL_PREFIX - ${ONNX_CUSTOM_PROTOC_EXECUTABLE} - DIRECTORY) - include(protobuf_function.cmake) - -# mode 3 else() # use protobuf as a submodule add_subdirectory(${PROJECT_SOURCE_DIR}/external/protobuf/cmake EXCLUDE_FROM_ALL) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index e57b0c3de7191..edb17c36f9756 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -95,6 +95,8 @@ def parse_arguments(): "These are just CMake -D options without the leading -D.") parser.add_argument("--x86", action='store_true', help="Create x86 makefiles. Requires --update and no existing cache CMake setup. Delete CMakeCache.txt if needed") + parser.add_argument("--arm", action='store_true', + help="Create ARM makefiles. Requires --update and no existing cache CMake setup. Delete CMakeCache.txt if needed") parser.add_argument("--arm64", action='store_true', help="Create ARM64 makefiles. Requires --update and no existing cache CMake setup. Delete CMakeCache.txt if needed") parser.add_argument("--msvc_toolset", help="MSVC toolset to use. e.g. 14.11") @@ -304,8 +306,8 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_USE_NUPHAR=" + ("ON" if args.use_nuphar else "OFF"), "-Donnxruntime_USE_EIGEN_THREADPOOL=" + ("ON" if args.use_eigenthreadpool else "OFF"), "-Donnxruntime_USE_TRT=" + ("ON" if args.use_trt else "OFF"), - # By default - we currently support only cross compiling for ARM64 (no native compilation supported through this script) - "-Donnxruntime_CROSS_COMPILING=" + ("ON" if args.arm64 else "OFF"), + # By default - we currently support only cross compiling for ARM/ARM64 (no native compilation supported through this script) + "-Donnxruntime_CROSS_COMPILING=" + ("ON" if args.arm64 or args.arm else "OFF"), ] if args.use_brainslice: bs_pkg_name = args.brain_slice_package_name.split('.', 1) @@ -529,7 +531,7 @@ def build_python_wheel(source_dir, build_dir, configs, use_cuda): def build_protoc_for_windows_host(cmake_path, source_dir, build_dir): if not is_windows(): - raise BuildError('Currently only support building protoc for Windows host while cross-compiling for ARM64 arch') + raise BuildError('Currently only support building protoc for Windows host while cross-compiling for ARM/ARM64 arch') log.info("Building protoc for host to be used in cross-compiled build process") protoc_build_dir = os.path.join(build_dir, 'host_protoc') @@ -563,7 +565,7 @@ def main(): log.debug("Defaulting to running update, build [and test for native builds].") args.update = True args.build = True - if args.arm64: + if args.arm or args.arm64: args.test = False else: args.test = True @@ -595,13 +597,18 @@ def main(): if(is_windows()): if (args.x86): cmake_extra_args = ['-A','Win32','-G', 'Visual Studio 15 2017'] - elif (args.arm64): - # Cross-compiling for ARM64 architecture - # First build protobuf for host + elif (args.arm or args.arm64): + # Cross-compiling for ARM(64) architecture + # First build protoc for host to use during cross-compilation build_protoc_for_windows_host(cmake_path, source_dir, build_dir) - cmake_extra_args = ['-A','ARM64', '-G', 'Visual Studio 15 2017'] + if args.arm: + cmake_extra_args = ['-A', 'ARM'] + else: + cmake_extra_args = ['-A', 'ARM64'] + cmake_extra_args += ['-G', 'Visual Studio 15 2017'] # Cannot test on host build machine for cross-compiled builds (Override any user-defined behaviour for test if any) if args.test: + log.info("Cannot test on host build machine for cross-compiled ARM(64) builds. Will skip test running after build.") args.test = False else: toolset = 'host=x64' @@ -612,8 +619,8 @@ def main(): cmake_extra_args = ['-A','x64','-T', toolset, '-G', 'Visual Studio 15 2017'] if is_ubuntu_1604(): - if (args.arm64): - raise BuildError("Only Windows ARM64 builds supported currently") + if (args.arm or args.arm64): + raise BuildError("Only Windows ARM(64) cross-compiled builds supported currently through this script") install_ubuntu_deps(args) if not is_docker(): install_python_deps() @@ -631,7 +638,7 @@ def main(): if args.path_to_protoc_exe: path_to_protoc_exe = args.path_to_protoc_exe # Need to provide path to protoc.exe built for host to be used in the cross-compiled build process - elif args.arm64: + elif args.arm or args.arm64: path_to_protoc_exe = os.path.join(build_dir, 'host_protoc', 'Release', 'protoc.exe') generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, args.pb_home, path_to_protoc_exe, configs, cmake_extra_defines, From c04f07605056bf6266b405f23cf5f560d42b5803 Mon Sep 17 00:00:00 2001 From: hariharans29 Date: Thu, 7 Mar 2019 15:16:35 -0800 Subject: [PATCH 07/13] Commiting Azure build config file --- .../win-arm-crosscompile-ci-pipeline.yml | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml diff --git a/tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml new file mode 100644 index 0000000000000..9d41887278c7f --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml @@ -0,0 +1,55 @@ +jobs: +- job: Windows_ARM_CrossCompile_CI_Dev + variables: + buildDirectory: '$(Build.BinariesDirectory)' + steps: + - template: templates/set-test-data-variables-step.yml + displayName: 'NuGet restore' + inputs: + restoreSolution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' + feedsToUse: config + nugetConfigPath: '$(Build.SourcesDirectory)\csharp\Nuget.CSharp.config' + restoreDirectory: '$(Build.SourcesDirectory)\csharp' + - task: UniversalPackages@0 + displayName: 'Download python' + inputs: + command: download + vstsFeed: '$(System.TeamProject)' + vstsFeedPackage: 'miniconda3_win64' + vstsPackageVersion: '4.5.11' + downloadDirectory: '$(Build.BinariesDirectory)\python' + - task: CmdLine@1 + displayName: 'Run python installer' + inputs: + filename: '$(Build.BinariesDirectory)\python\installer.exe' + arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python' + timeoutInMinutes: 10 + - task: BatchScript@1 + displayName: 'setup env' + inputs: + filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env.bat' + modifyEnvironment: true + workingFolder: '$(Build.BinariesDirectory)' + - task: CmdLine@1 + displayName: 'Install conda modules' + inputs: + filename: '$(Build.BinariesDirectory)\packages\python\scripts\conda.exe' + arguments: 'install -q --insecure -y pyopenssl setuptools wheel numpy' + timeoutInMinutes: 10 + - task: CmdLine@1 + displayName: 'Download cmake' + inputs: + filename: '$(Build.BinariesDirectory)\packages\python\python.exe' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\download_cmake.py --build_dir $(Build.BinariesDirectory)' + - task: CmdLine@1 + displayName: 'Generate cmake config and build Debug' + inputs: + filename: '$(Build.BinariesDirectory)\packages\python\python.exe' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --arm' + workingDirectory: "$(Build.BinariesDirectory)" + - task: CmdLine@1 + displayName: 'Generate cmake config and build Release' + inputs: + filename: '$(Build.BinariesDirectory)\packages\python\python.exe' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --arm' + workingDirectory: "$(Build.BinariesDirectory)" \ No newline at end of file From 9cd01a39239c11c9ffec4222c63b335c01eaaec1 Mon Sep 17 00:00:00 2001 From: hariharans29 Date: Thu, 7 Mar 2019 15:33:08 -0800 Subject: [PATCH 08/13] Fix build pipeline --- .../azure-pipelines/win-arm-crosscompile-ci-pipeline.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml index 9d41887278c7f..97e1b3a8060bb 100644 --- a/tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml @@ -4,12 +4,6 @@ jobs: buildDirectory: '$(Build.BinariesDirectory)' steps: - template: templates/set-test-data-variables-step.yml - displayName: 'NuGet restore' - inputs: - restoreSolution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' - feedsToUse: config - nugetConfigPath: '$(Build.SourcesDirectory)\csharp\Nuget.CSharp.config' - restoreDirectory: '$(Build.SourcesDirectory)\csharp' - task: UniversalPackages@0 displayName: 'Download python' inputs: From 1f50d09c4e5825d22898ef31985d3df684d57013 Mon Sep 17 00:00:00 2001 From: hariharans29 Date: Thu, 7 Mar 2019 15:52:23 -0800 Subject: [PATCH 09/13] Cleanup build dir template addition --- .../azure-pipelines/win-arm-crosscompile-ci-pipeline.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml index 97e1b3a8060bb..fb8cc7ae7e16d 100644 --- a/tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml @@ -46,4 +46,5 @@ jobs: inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --arm' - workingDirectory: "$(Build.BinariesDirectory)" \ No newline at end of file + workingDirectory: "$(Build.BinariesDirectory)" + - template: templates/clean-agent-build-directory-step.yml \ No newline at end of file From d37c4cbbfe765a1fc70bdecb67792e54c6d10777 Mon Sep 17 00:00:00 2001 From: hariharans29 Date: Thu, 7 Mar 2019 18:43:05 -0800 Subject: [PATCH 10/13] Remove conda modules download step --- .../win-arm-crosscompile-ci-pipeline.yml | 6 --- .../win-arm64-crosscompile-ci-pipeline.yml | 44 +++++++++++++++++++ 2 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 tools/ci_build/github/azure-pipelines/win-arm64-crosscompile-ci-pipeline.yml diff --git a/tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml index fb8cc7ae7e16d..38719698937ef 100644 --- a/tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml @@ -24,12 +24,6 @@ jobs: filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env.bat' modifyEnvironment: true workingFolder: '$(Build.BinariesDirectory)' - - task: CmdLine@1 - displayName: 'Install conda modules' - inputs: - filename: '$(Build.BinariesDirectory)\packages\python\scripts\conda.exe' - arguments: 'install -q --insecure -y pyopenssl setuptools wheel numpy' - timeoutInMinutes: 10 - task: CmdLine@1 displayName: 'Download cmake' inputs: diff --git a/tools/ci_build/github/azure-pipelines/win-arm64-crosscompile-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-arm64-crosscompile-ci-pipeline.yml new file mode 100644 index 0000000000000..76029b333009c --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/win-arm64-crosscompile-ci-pipeline.yml @@ -0,0 +1,44 @@ +jobs: +- job: Windows_ARM_CrossCompile_CI_Dev + variables: + buildDirectory: '$(Build.BinariesDirectory)' + steps: + - template: templates/set-test-data-variables-step.yml + - task: UniversalPackages@0 + displayName: 'Download python' + inputs: + command: download + vstsFeed: '$(System.TeamProject)' + vstsFeedPackage: 'miniconda3_win64' + vstsPackageVersion: '4.5.11' + downloadDirectory: '$(Build.BinariesDirectory)\python' + - task: CmdLine@1 + displayName: 'Run python installer' + inputs: + filename: '$(Build.BinariesDirectory)\python\installer.exe' + arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python' + timeoutInMinutes: 10 + - task: BatchScript@1 + displayName: 'setup env' + inputs: + filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env.bat' + modifyEnvironment: true + workingFolder: '$(Build.BinariesDirectory)' + - task: CmdLine@1 + displayName: 'Download cmake' + inputs: + filename: '$(Build.BinariesDirectory)\packages\python\python.exe' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\download_cmake.py --build_dir $(Build.BinariesDirectory)' + - task: CmdLine@1 + displayName: 'Generate cmake config and build Debug' + inputs: + filename: '$(Build.BinariesDirectory)\packages\python\python.exe' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --arm64' + workingDirectory: "$(Build.BinariesDirectory)" + - task: CmdLine@1 + displayName: 'Generate cmake config and build Release' + inputs: + filename: '$(Build.BinariesDirectory)\packages\python\python.exe' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --arm64' + workingDirectory: "$(Build.BinariesDirectory)" + - template: templates/clean-agent-build-directory-step.yml \ No newline at end of file From 16217e0fa3119fce8f7a6c2b7cd68d099dc2755a Mon Sep 17 00:00:00 2001 From: hariharans29 Date: Fri, 8 Mar 2019 12:29:10 -0800 Subject: [PATCH 11/13] PR feedback --- tools/ci_build/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index edb17c36f9756..b79f3ae00e81e 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -334,7 +334,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home if pb_home: cmake_args += ["-DONNX_CUSTOM_PROTOC_EXECUTABLE=" + os.path.join(pb_home,'bin','protoc'), '-Donnxruntime_USE_PREBUILT_PB=ON'] - if not pb_home and path_to_protoc_exe: + elif path_to_protoc_exe: cmake_args += ["-DONNX_CUSTOM_PROTOC_EXECUTABLE=%s" % path_to_protoc_exe] cmake_args += ["-D{}".format(define) for define in cmake_extra_defines] From e839792a17035672f0608ff3041f0919699f8974 Mon Sep 17 00:00:00 2001 From: hariharans29 Date: Fri, 8 Mar 2019 12:43:01 -0800 Subject: [PATCH 12/13] Revert x86 arguments to as they are currently --- tools/ci_build/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index b79f3ae00e81e..f841ca5d22539 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -596,7 +596,7 @@ def main(): cmake_extra_args = [] if(is_windows()): if (args.x86): - cmake_extra_args = ['-A','Win32','-G', 'Visual Studio 15 2017'] + cmake_extra_args = ['-A','Win32','-T','host=x64','-G', 'Visual Studio 15 2017'] elif (args.arm or args.arm64): # Cross-compiling for ARM(64) architecture # First build protoc for host to use during cross-compilation From 4747e2865ca900f90162d01ca798bc2cb30a4264 Mon Sep 17 00:00:00 2001 From: hariharans29 Date: Fri, 8 Mar 2019 13:42:40 -0800 Subject: [PATCH 13/13] More changes --- tools/ci_build/build.py | 2 ++ .../azure-pipelines/win-arm-crosscompile-ci-pipeline.yml | 3 +++ .../azure-pipelines/win-arm64-crosscompile-ci-pipeline.yml | 3 +++ 3 files changed, 8 insertions(+) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index f841ca5d22539..eafeba66ec025 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -539,6 +539,8 @@ def build_protoc_for_windows_host(cmake_path, source_dir, build_dir): # Generate step cmd_args = [cmake_path, os.path.join(source_dir, 'cmake\external\protobuf\cmake'), + '-T', + 'host=x64', '-G', 'Visual Studio 15 2017', '-Dprotobuf_BUILD_TESTS=OFF', diff --git a/tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml index 38719698937ef..644acf7dfa9a0 100644 --- a/tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-arm-crosscompile-ci-pipeline.yml @@ -41,4 +41,7 @@ jobs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --arm' workingDirectory: "$(Build.BinariesDirectory)" + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) - template: templates/clean-agent-build-directory-step.yml \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/win-arm64-crosscompile-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-arm64-crosscompile-ci-pipeline.yml index 76029b333009c..b70c419a49a96 100644 --- a/tools/ci_build/github/azure-pipelines/win-arm64-crosscompile-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-arm64-crosscompile-ci-pipeline.yml @@ -41,4 +41,7 @@ jobs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --arm64' workingDirectory: "$(Build.BinariesDirectory)" + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) - template: templates/clean-agent-build-directory-step.yml \ No newline at end of file