Network mode develop

config/runtime_policy.py

@@ -72,6 +72,7 @@ def apply_runtime_environment(policy: RuntimePolicy) -> None:
     os.environ.setdefault("HF_HUB_OFFLINE", "1")
     os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
     os.environ.setdefault("HF_DATASETS_OFFLINE", "1")
+    os.environ.setdefault("DATAELF_OFFLINE_MODE", "1")
 
 
 def run_global_preflight(config: Any, policy: RuntimePolicy) -> list[PreflightIssue]:
@@ -103,7 +104,7 @@ def run_global_preflight(config: Any, policy: RuntimePolicy) -> list[PreflightIs
         return issues
 
 
-    # TODO: (network_mode) !!!!Re-enable strict offline endpoint checks after local
+    # TODO: (network_mode) Re-enable strict offline endpoint checks after local
     # LLM deployment is ready. Development still uses an external relay LLM.
     # issues.extend(_validate_offline_llm_endpoint(config, "agent", required=_agent_requires_llm(config)))
     # issues.extend(_validate_offline_llm_endpoint(config, "tool_llm", required=_tool_llm_configured(config)))

runtime/executor.py

@@ -382,7 +382,7 @@ def execute(self, job_id: str, pipeline: str) -> dict[str, Any]:
             # Save pipeline to file for reference
             pipeline_dir = Path("pipelines")
             pipeline_dir.mkdir(exist_ok=True)
-            pipeline_file = pipeline_dir / f"{job_id}.py"
+            pipeline_file = pipeline_dir / f"{job_id}.dsl"
             with open(pipeline_file, "w") as f:
                 f.write(pipeline)
 

tools/security_audit/checker/heuristic/graceful_backdoor_detection.py

@@ -34,6 +34,7 @@ def __init__(
         min_batch_size: int = _MIN_BATCH_SIZE,
         dct_keep_ratio: float = 0.125,
         device: str = "auto",
+        local_files_only: bool = False,
         **kwargs,
     ):
         """
@@ -45,6 +46,7 @@ def __init__(
             min_batch_size: Minimum samples required for clustering.
             dct_keep_ratio: Keep top-left ratio for DCT (default 1/8).
             device: "auto" | "cuda" | "cpu".
+            local_files_only: only load local model files; used by offline mode.
         """
         super().__init__(**kwargs)
         self.victim_config = victim_config or {}
@@ -54,6 +56,7 @@ def __init__(
         self.min_batch_size = min_batch_size
         self.dct_keep_ratio = dct_keep_ratio
         self.device = device
+        self.local_files_only = local_files_only
         self._victim: Optional[CasualLLMVictim] = None
 
     def check(self, sample: DataSample) -> CheckResult:
@@ -142,6 +145,7 @@ def _load_victim(self) -> CasualLLMVictim:
             config["device"] = "gpu"
         elif self.device == "cpu":
             config["device"] = "cpu"
+        config["local_files_only"] = self.local_files_only
         self._victim = load_victim(config)
         return self._victim
 

tools/security_audit/checker/heuristic/graceful_helpers.py

@@ -59,6 +59,7 @@ def __init__(
         model: Optional[str] = "llama",
         path: Optional[str] = None,
         max_len: Optional[int] = 4096,
+        local_files_only: bool = False,
         **kwargs,
     ):
         if not path:
@@ -67,16 +68,18 @@ def __init__(
             "cuda" if device in {"gpu", "cuda"} and torch.cuda.is_available() else "cpu"
         )
         self.model_type = model
-        self.model_config = AutoConfig.from_pretrained(path)
+        self.local_files_only = local_files_only
+        self.model_config = AutoConfig.from_pretrained(path, local_files_only=local_files_only)
         self.llm = AutoModelForCausalLM.from_pretrained(
             path,
             config=self.model_config,
             trust_remote_code=True,
             device_map="auto" if self.device.type == "cuda" else None,
+            local_files_only=local_files_only,
         )
         if self.device.type != "cuda":
             self.llm = self.llm.to(self.device)
-        self.tokenizer = AutoTokenizer.from_pretrained(path)
+        self.tokenizer = AutoTokenizer.from_pretrained(path, local_files_only=local_files_only)
         if self.model_type == "llama":
             pad_token = self.tokenizer.unk_token or self.tokenizer.eos_token
             self.llm.config.pad_token_id = self.tokenizer.convert_tokens_to_ids(pad_token)

tools/security_audit/checker/model_based/bias_classifier.py

@@ -31,26 +31,29 @@ def __init__(
         threshold: float = 0.5,
         max_length: int = 512,
         device: str = "auto",
+        local_files_only: bool = False,
     ):
         """
         Args:
             model_name_or_path: HuggingFace model ID or local path.
             threshold: per-category score above which text is flagged as biased.
             max_length: max token length for truncation.
             device: "auto", "cuda", or "cpu".
+            local_files_only: only load local model files; used by offline mode.
         """
         super().__init__()
         self.model_name_or_path = model_name_or_path
         self.threshold = threshold
         self.max_length = max_length
         self._device = device
+        self.local_files_only = local_files_only
         self.model = None
 
     def load_model(self):
         if self.model is not None:
             return
         try:
-            from transformers import pipeline
+            from transformers import AutoModelForSequenceClassification, AutoTokenizer, pipeline
             import torch
 
             if self._device == "auto":
@@ -61,9 +64,18 @@ def load_model(self):
                 device = -1
 
             self._log.info(f"BiasClassifier: loading model '{self.model_name_or_path}' ...")
+            tokenizer = AutoTokenizer.from_pretrained(
+                self.model_name_or_path,
+                local_files_only=self.local_files_only,
+            )
+            model = AutoModelForSequenceClassification.from_pretrained(
+                self.model_name_or_path,
+                local_files_only=self.local_files_only,
+            )
             self.model = pipeline(
                 "text-classification",
-                model=self.model_name_or_path,
+                model=model,
+                tokenizer=tokenizer,
                 top_k=None,  # return all label scores
                 device=device,
                 truncation=True,

tools/security_audit/checker/model_based/harmful_classifier.py

@@ -44,19 +44,22 @@ def __init__(
         device: str = "auto",
         dtype: str = "bfloat16",
         threshold: float = 0.5,
+        local_files_only: bool = False,
     ):
         """
         Args:
             model_name_or_path: HuggingFace model ID or local path to Llama Guard.
             device: "auto", "cuda", "cpu".
             dtype: torch dtype string, "bfloat16" or "float16".
             threshold: unsafe probability above which the sample is flagged.
+            local_files_only: only load local model files; used by offline mode.
         """
         super().__init__()
         self.model_name_or_path = model_name_or_path
         self.device = device
         self.dtype = getattr(torch, dtype, torch.bfloat16)
         self.threshold = threshold
+        self.local_files_only = local_files_only
         self._tokenizer = None
         self.model = None
         self._safe_token_id = None
@@ -67,19 +70,24 @@ def load_model(self):
             return
         try:
             self._log.info("HarmfulContentClassifier: loading Llama Guard model ...")
-            self._tokenizer = AutoTokenizer.from_pretrained(self.model_name_or_path)
+            self._tokenizer = AutoTokenizer.from_pretrained(
+                self.model_name_or_path,
+                local_files_only=self.local_files_only,
+            )
             if self.device in ("auto", "cuda"):
                 # Use device_map for GPU (handles multi-GPU sharding)
                 self.model = AutoModelForCausalLM.from_pretrained(
                     self.model_name_or_path,
                     torch_dtype=self.dtype,
                     device_map=self.device,
+                    local_files_only=self.local_files_only,
                 )
             else:
                 # CPU: load without device_map to avoid meta tensor issues
                 self.model = AutoModelForCausalLM.from_pretrained(
                     self.model_name_or_path,
                     torch_dtype=self.dtype,
+                    local_files_only=self.local_files_only,
                 ).to(self.device)
             self.model.eval()
             # Pre-compute token IDs for "safe" and "unsafe"

tools/security_audit/checker/model_based/jailbreak_classifier.py

@@ -94,17 +94,20 @@ def __init__(
         model_name_or_path: str = "allenai/wildguard",
         max_new_tokens: int = 32,
         device: str = "auto",
+        local_files_only: bool = False,
     ):
         """
         Args:
             model_name_or_path: HuggingFace model ID or local path.
             max_new_tokens: token budget for WildGuard's generated verdict.
             device: "auto", "cuda", or "cpu".
+            local_files_only: only load local model files; used by offline mode.
         """
         super().__init__()
         self.model_name_or_path = model_name_or_path
         self.max_new_tokens = max_new_tokens
         self._device = device
+        self.local_files_only = local_files_only
         self.model = None
         self._tokenizer = None
 
@@ -123,11 +126,16 @@ def load_model(self):
             dtype = torch.float16 if device == "cuda" else torch.float32
 
             self._log.info(f"JailbreakClassifier: loading model '{self.model_name_or_path}' on {device} ...")
-            self._tokenizer = AutoTokenizer.from_pretrained(self.model_name_or_path, use_fast=False)
+            self._tokenizer = AutoTokenizer.from_pretrained(
+                self.model_name_or_path,
+                use_fast=False,
+                local_files_only=self.local_files_only,
+            )
             self.model = AutoModelForCausalLM.from_pretrained(
                 self.model_name_or_path,
                 torch_dtype=dtype,
                 low_cpu_mem_usage=True,
+                local_files_only=self.local_files_only,
             ).to(device)
             self.model.eval()
             self._device_obj = device

tools/security_audit/checker/model_based/pii_ner.py

@@ -1,7 +1,8 @@
 # Model: Microsoft Presidio (regex + spaCy NER)
 # https://github.com/microsoft/presidio
 
-from typing import Dict, List, Optional, Tuple
+import os
+from typing import Dict, List, Optional, Tuple
 
 from ..base import ModelBasedChecker
 from ..registry import CheckerRegistry
@@ -38,10 +39,15 @@ def __init__(
 
         super().__init__()
         try:
-            spacy_model = "en_core_web_lg" if language == "en" else f"{language}_core_web_sm"
-            if not spacy.util.is_package(spacy_model):
-                self._log.info(f"spaCy model '{spacy_model}' not found, downloading ...")
-                spacy.cli.download(spacy_model)
+            spacy_model = "en_core_web_lg" if language == "en" else f"{language}_core_web_sm"
+            if not spacy.util.is_package(spacy_model):
+                if os.environ.get("DATAELF_OFFLINE_MODE") == "1":
+                    raise RuntimeError(
+                        f"spaCy model '{spacy_model}' is required in offline mode; "
+                        "install it before running PIINERDetector."
+                    )
+                self._log.info(f"spaCy model '{spacy_model}' not found, downloading ...")
+                spacy.cli.download(spacy_model)
 
             self.model = AnalyzerEngine()
             self.anonymizer = AnonymizerEngine()

tools/security_audit/checker/model_based/prompt_injection_classifier.py

@@ -23,19 +23,22 @@ def __init__(
         threshold: float = 0.5,
         max_length: int = 512,
         device: str = "auto",
+        local_files_only: bool = False,
     ):
         """
         Args:
             model_name_or_path: HuggingFace model ID or local path.
             threshold: injection probability score above which input is flagged.
             max_length: max token length passed to the tokenizer (PIGuard supports up to 2048).
             device: "auto", "cuda", or "cpu".
+            local_files_only: only load local model files; used by offline mode.
         """
         super().__init__()
         self.model_name_or_path = model_name_or_path
         self.threshold = threshold
         self.max_length = max_length
         self._device = device
+        self.local_files_only = local_files_only
         self.model = None
 
     def load_model(self):
@@ -56,10 +59,12 @@ def load_model(self):
             tokenizer = AutoTokenizer.from_pretrained(
                 self.model_name_or_path,
                 model_max_length=self.max_length,
+                local_files_only=self.local_files_only,
             )
             model = AutoModelForSequenceClassification.from_pretrained(
                 self.model_name_or_path,
                 trust_remote_code=True,
+                local_files_only=self.local_files_only,
             )
             self.model = pipeline(
                 "text-classification",