diff --git a/devchat/workflow/env_manager.py b/devchat/workflow/env_manager.py index 36e5223d..41728bbc 100644 --- a/devchat/workflow/env_manager.py +++ b/devchat/workflow/env_manager.py @@ -1,10 +1,13 @@ +import hashlib import os import subprocess import sys -from typing import Dict, Optional +from typing import Dict, Optional, Tuple + +from devchat.utils import get_logger from .envs import MAMBA_BIN_PATH -from .path import MAMBA_PY_ENVS, MAMBA_ROOT +from .path import ENV_CACHE_DIR, MAMBA_PY_ENVS, MAMBA_ROOT from .schema import ExternalPyConf from .user_setting import USER_SETTINGS @@ -15,6 +18,8 @@ CONDA_FORGE_TUNA = "https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/" PYPI_TUNA = "https://pypi.tuna.tsinghua.edu.cn/simple" +logger = get_logger(__name__) + def _get_external_envs() -> Dict[str, ExternalPyConf]: """ @@ -43,9 +48,7 @@ def get_py_version(py: str) -> Optional[str]: Get the version of the python executable. """ py_version_cmd = [py, "--version"] - with subprocess.Popen( - py_version_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) as proc: + with subprocess.Popen(py_version_cmd, stdout=subprocess.PIPE, stderr=None) as proc: proc.wait() if proc.returncode != 0: @@ -54,7 +57,91 @@ def get_py_version(py: str) -> Optional[str]: out = proc.stdout.read().decode("utf-8") return out.split()[1] - def install(self, env_name: str, requirements_file: str) -> bool: + @staticmethod + def get_dep_hash(reqirements_file: str) -> str: + """ + Get the hash of the requirements file content. + + Used to check if the requirements file has been changed. + """ + with open(reqirements_file, "r", encoding="utf-8") as f: + content = f.read() + return hashlib.md5(content.encode("utf-8")).hexdigest() + + def ensure( + self, env_name: str, py_version: str, reqirements_file: Optional[str] = None + ) -> Optional[str]: + """ + Ensure the python environment exists with the given name and version. + And install the requirements if provided. + + return the python executable path. + """ + py = self.get_py(env_name) + + should_remove_old = False + + if py: + # check the version of the python executable + current_version = self.get_py_version(py) + + if current_version != py_version: + should_remove_old = True + + if reqirements_file and self.should_reinstall(env_name, reqirements_file): + should_remove_old = True + + if not should_remove_old: + return py + + print("\n```Step\n# Setting up workflow environment\n", flush=True) + if should_remove_old: + print(f"- Dependencies of {env_name} have been changed.", flush=True) + print(f"- Removing the old {env_name}...", flush=True) + self.remove(env_name) + + # create the environment + print(f"- Creating {env_name} with {py_version}...", flush=True) + create_ok, msg = self.create(env_name, py_version) + if not create_ok: + print(f"- Failed to create {env_name}", flush=True) + print("\n```", flush=True) + print( + f"\n\nFailed to create {env_name}, the workflow will not run." + f"\n\nPlease try again later.", + flush=True, + ) + logger.warning(f"Failed to create {env_name}: {msg}") + sys.exit(0) + # return None + + # install the requirements + if reqirements_file: + filename = os.path.basename(reqirements_file) + print(f"- Installing dependencies from {filename}...", flush=True) + install_ok, msg = self.install(env_name, reqirements_file) + if not install_ok: + print(f"- Failed to install dependencies from {filename}", flush=True) + print("\n```", flush=True) + print( + "\n\nFailed to install dependencies, the workflow will not run." + "\n\nPlease try again later.", + flush=True, + ) + logger.warning(f"Failed to install dependencies: {msg}") + sys.exit(0) + # return None + + # save the hash of the requirements file content + dep_hash = self.get_dep_hash(reqirements_file) + cache_file = os.path.join(ENV_CACHE_DIR, f"{env_name}") + with open(cache_file, "w", encoding="utf-8") as f: + f.write(dep_hash) + + print("\n```", flush=True) + return self.get_py(env_name) + + def install(self, env_name: str, requirements_file: str) -> Tuple[bool, str]: """ Install requirements into the python environment. @@ -63,12 +150,10 @@ def install(self, env_name: str, requirements_file: str) -> bool: """ py = self.get_py(env_name) if not py: - # TODO: raise error? - return False + return False, "Python executable not found." if not os.path.exists(requirements_file): - # TODO: raise error? - return False + return False, "Dependencies file not found." cmd = [ py, @@ -79,52 +164,35 @@ def install(self, env_name: str, requirements_file: str) -> bool: requirements_file, "-i", PYPI_TUNA, + "--no-warn-script-location", ] env = os.environ.copy() env.pop("PYTHONPATH") - with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) as proc: - proc.wait() + with subprocess.Popen( + cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, env=env + ) as proc: + _, err = proc.communicate() if proc.returncode != 0: - print(f"Failed to install requirements: {requirements_file}", flush=True) - return False + return False, err.decode("utf-8") - return True + return True, "" - def ensure(self, env_name: str, py_version: str) -> Optional[str]: + def should_reinstall(self, env_name: str, requirements_file: str) -> bool: """ - Ensure the python environment exists with the given name and version. - - return the python executable path. + Check if the requirements file has been changed. """ - py = self.get_py(env_name) - should_remove = False - - if py: - # check the version of the python executable - current_version = self.get_py_version(py) - - if current_version == py_version: - return py - - should_remove = True - - print("\n```Step\n# Setting up workflow environment", flush=True) - print(f"\nenv_name: {env_name}") - print(f"python: {py_version}", flush=True) + cache_file = os.path.join(ENV_CACHE_DIR, f"{env_name}") + if not os.path.exists(cache_file): + return True - if should_remove: - self.remove(env_name) + dep_hash = self.get_dep_hash(requirements_file) + with open(cache_file, "r", encoding="utf-8") as f: + cache_hash = f.read() - # create the environment - create_ok = self.create(env_name, py_version) - print("\n```", flush=True) + return dep_hash != cache_hash - if not create_ok: - return None - return self.get_py(env_name) - - def create(self, env_name: str, py_version: str) -> bool: + def create(self, env_name: str, py_version: str) -> Tuple[bool, str]: """ Create a new python environment using mamba. """ @@ -146,12 +214,12 @@ def create(self, env_name: str, py_version: str) -> bool: "-y", ] with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as proc: - proc.wait() + out, err = proc.communicate() + msg = f"err: {err.decode()}\n-----\nout: {out.decode()}" if proc.returncode != 0: - return False - - return True + return False, msg + return True, "" def remove(self, env_name: str) -> bool: """ @@ -172,7 +240,7 @@ def remove(self, env_name: str) -> bool: self.mamba_root, "-y", ] - with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as proc: + with subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=None) as proc: proc.wait() if proc.returncode != 0: diff --git a/devchat/workflow/path.py b/devchat/workflow/path.py index 49dc6a0e..9343dc44 100644 --- a/devchat/workflow/path.py +++ b/devchat/workflow/path.py @@ -20,6 +20,14 @@ COMMAND_FILENAMES = ["command.yml", "command.yaml"] +# ------------------------------- +# workflow related cache data +# ------------------------------- +CACHE_DIR = os.path.join(WORKFLOWS_BASE, "cache") +ENV_CACHE_DIR = os.path.join(CACHE_DIR, "env_cache") +os.makedirs(ENV_CACHE_DIR, exist_ok=True) + + # ------------------------------- # config & settings files paths # ------------------------------- diff --git a/devchat/workflow/workflow.py b/devchat/workflow/workflow.py index d9f8925e..40a1d95b 100644 --- a/devchat/workflow/workflow.py +++ b/devchat/workflow/workflow.py @@ -139,14 +139,8 @@ def setup( print("\n```", flush=True) else: - # TODO: 有没有更好的时机判断方法？既保证运行时一定安装了依赖、又不用每次都检查？ - # TODO: 只在插件(IDE)启动后workflow第一次使用时ensure环境和依赖？ - # Create workflow python env manager = PyEnvManager() - workflow_py = manager.ensure(pyconf.env_name, pyconf.version) - - r_file = pyconf.dependencies - _ = manager.install(pyconf.env_name, r_file) + workflow_py = manager.ensure(pyconf.env_name, pyconf.version, pyconf.dependencies) runtime_param = { # from user interaction