From df6d5c46ac38b05efb082ec615fc36b09401c902 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 24 Jul 2022 21:45:21 -0400 Subject: [PATCH] fix build and running issues on Windows --- deepmd/env.py | 8 +++++--- deepmd/train/trainer.py | 7 ++++++- pyproject.toml | 10 +++++++++- source/CMakeLists.txt | 13 ++++++++++++- source/cmake/Findtensorflow.cmake | 12 +++++++++++- source/op/optimizer/parallel.cc | 3 +++ 6 files changed, 46 insertions(+), 7 deletions(-) diff --git a/deepmd/env.py b/deepmd/env.py index 5415ff75a5..a864787411 100644 --- a/deepmd/env.py +++ b/deepmd/env.py @@ -224,13 +224,15 @@ def get_module(module_name: str) -> "ModuleType": """ if platform.system() == "Windows": ext = ".dll" + prefix = "" #elif platform.system() == "Darwin": # ext = ".dylib" else: ext = ".so" + prefix = "lib" module_file = ( - (Path(__file__).parent / SHARED_LIB_MODULE / module_name) + (Path(__file__).parent / SHARED_LIB_MODULE / (prefix + module_name)) .with_suffix(ext) .resolve() ) @@ -324,8 +326,8 @@ def _get_package_constants( TF_VERSION = GLOBAL_CONFIG["tf_version"] TF_CXX11_ABI_FLAG = int(GLOBAL_CONFIG["tf_cxx11_abi_flag"]) -op_module = get_module("libop_abi") -op_grads_module = get_module("libop_grads") +op_module = get_module("op_abi") +op_grads_module = get_module("op_grads") # FLOAT_PREC dp_float_prec = os.environ.get("DP_INTERFACE_PREC", "high").lower() diff --git a/deepmd/train/trainer.py b/deepmd/train/trainer.py index 6f1476c82d..86edb8db78 100644 --- a/deepmd/train/trainer.py +++ b/deepmd/train/trainer.py @@ -3,6 +3,7 @@ import logging import os import glob +import platform import time import shutil import google.protobuf.message @@ -574,7 +575,11 @@ def save_checkpoint(self, cur_batch: int): os.remove(new_ff) except OSError: pass - os.symlink(ori_ff, new_ff) + if platform.system() != 'Windows': + # by default one does not have access to create symlink on Windows + os.symlink(ori_ff, new_ff) + else: + shutil.copyfile(ori_ff, new_ff) log.info("saved checkpoint %s" % self.save_ckpt) def get_feed_dict(self, batch, is_training): diff --git a/pyproject.toml b/pyproject.toml index 50fe237581..8d4bfad3eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,11 @@ [build-system] -requires = ["setuptools", "setuptools_scm", "wheel", "scikit-build", "cmake", "ninja"] +requires = [ + "setuptools", + "setuptools_scm", + "wheel", + "scikit-build", + "cmake", + # see https://github.com/scikit-build/scikit-build/releases/tag/0.13.1 + "ninja; platform_system!='Windows'", +] diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index f068649fd4..9f6351a911 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -90,6 +90,16 @@ else() set (CMAKE_CXX_STANDARD 11) endif() +if (MSVC) + # see TF .bazelrc + add_compile_options(/W0 /Zc:__cplusplus /D_USE_MATH_DEFINES /d2ReducedOptimizeHugeFunctions) + set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS TRUE) +endif() +if (TENSORFLOW_VERSION GREATER_EQUAL 2.4 AND MSVC) + # see TF 2.4 release notes + add_compile_options(/Zc:preprocessor) +endif() + # find threads find_package(Threads) @@ -97,7 +107,8 @@ find_package(Threads) if (NOT DEFINED OP_CXX_ABI) if (BUILD_PY_IF) if (DEFINED TENSORFLOW_ROOT) - set(FIND_ABI_CMD "import sys,os; sys.path.insert(0, os.path.join('${TENSORFLOW_ROOT}', '..')); import tensorflow; print(tensorflow.CXX11_ABI_FLAG if 'CXX11_ABI_FLAG' in tensorflow.__dict__ else tensorflow.sysconfig.CXX11_ABI_FLAG, end = '')" ) + # note \ in windows path + set(FIND_ABI_CMD "import sys,os; sys.path.insert(0, os.path.join(r'${TENSORFLOW_ROOT}', '..')); import tensorflow; print(tensorflow.CXX11_ABI_FLAG if 'CXX11_ABI_FLAG' in tensorflow.__dict__ else tensorflow.sysconfig.CXX11_ABI_FLAG, end = '')" ) else() set(FIND_ABI_CMD "import tensorflow; print(tensorflow.CXX11_ABI_FLAG if 'CXX11_ABI_FLAG' in tensorflow.__dict__ else tensorflow.sysconfig.CXX11_ABI_FLAG, end = '')") endif() diff --git a/source/cmake/Findtensorflow.cmake b/source/cmake/Findtensorflow.cmake index 7c8de58e45..d6ef593968 100644 --- a/source/cmake/Findtensorflow.cmake +++ b/source/cmake/Findtensorflow.cmake @@ -114,16 +114,26 @@ endif (BUILD_CPP_IF) # tensorflow_framework if (NOT TensorFlowFramework_FIND_COMPONENTS) + if (WIN32) + set(TensorFlowFramework_FIND_COMPONENTS _pywrap_tensorflow_internal) + set(TF_SUFFIX "") + else () set(TensorFlowFramework_FIND_COMPONENTS tensorflow_framework) + set(TF_SUFFIX lib) + endif () endif () # the lib +if (WIN32) + list(APPEND TensorFlow_search_PATHS ${TENSORFLOW_ROOT}/python) +else () list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES .so.1) list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES .so.2) +endif() set (TensorFlowFramework_LIBRARY_PATH "") foreach (module ${TensorFlowFramework_FIND_COMPONENTS}) find_library(TensorFlowFramework_LIBRARY_${module} NAMES ${module} - PATHS ${TensorFlow_search_PATHS} PATH_SUFFIXES lib NO_DEFAULT_PATH + PATHS ${TensorFlow_search_PATHS} PATH_SUFFIXES ${TF_SUFFIX} NO_DEFAULT_PATH ) if (TensorFlowFramework_LIBRARY_${module}) list(APPEND TensorFlowFramework_LIBRARY ${TensorFlowFramework_LIBRARY_${module}}) diff --git a/source/op/optimizer/parallel.cc b/source/op/optimizer/parallel.cc index 317de4b11f..c3919f5e5d 100644 --- a/source/op/optimizer/parallel.cc +++ b/source/op/optimizer/parallel.cc @@ -1,5 +1,7 @@ // only support v1.15 or v2 #include "tensorflow/core/public/version.h" +// skip windows +#ifndef _WIN32 #if TF_MAJOR_VERSION >= 2 || (TF_MAJOR_VERSION == 1 && TF_MINOR_VERSION >= 15) #if TF_MAJOR_VERSION >= 2 && TF_MINOR_VERSION >= 7 @@ -157,3 +159,4 @@ Status DPParallel::Optimize(Cluster *cluster, const GrapplerItem &item, REGISTER_GRAPH_OPTIMIZER_AS(DPParallel, "dpparallel"); #endif +#endif