Adding MOCK_TIME env variable

cdx_toolkit/commoncrawl.py

@@ -11,6 +11,7 @@
 
 from .myrequests import myrequests_get
 from .timeutils import time_to_timestamp, timestamp_to_time, pad_timestamp, pad_timestamp_up, cc_index_to_time, cc_index_to_time_special
+from .settings import MOCK_TIME
 
 LOGGER = logging.getLogger(__name__)
 
@@ -70,7 +71,7 @@ def get_cc_endpoints(cc_mirror):
         url = cc_mirror.rstrip('/') + '/collinfo.json'
         r = myrequests_get(url)
         if r.status_code != 200:
-            raise RuntimeError('error {} getting list of cc indices from {}'.format(r.status_code, collinfo))  # pragma: no cover
+            raise RuntimeError('error {} getting list of cc indices from {}'.format(r.status_code, url))  # pragma: no cover
         set_collinfo_cache(cc_mirror, r.text)
         col = r.json()
 
@@ -119,9 +120,12 @@ def apply_cc_defaults(params, crawl_present=False, now=None):
                 LOGGER.info('to but no from_ts, setting from_ts=%s', params['from_ts'])
         else:
             if not now:
-                # now is passed in by tests. if not set, use actual now.
-                # XXX could be changed to mock
-                now = time.time()
+                # Check for test/override time first
+                if MOCK_TIME:
+                    now = float(MOCK_TIME)
+                else:
+                    # now is passed in by tests. if not set, use actual now.
+                    now = time.time()
             params['from_ts'] = time_to_timestamp(now - year)
             LOGGER.info('no from or to, setting default 1 year ago from_ts=%s', params['from_ts'])
     else:

cdx_toolkit/settings.py

@@ -7,3 +7,5 @@
 CC_INDEX_MIN_RETRY_INTERVAL = float(os.environ.get("CDXT_CC_INDEX_MIN_RETRY_INTERVAL", 1.0))
 CC_DATA_MIN_RETRY_INTERVAL = float(os.environ.get("CDXT_CC_DATA_MIN_RETRY_INTERVAL", 0.55))
 IA_MIN_RETRY_INTERVAL = float(os.environ.get("CDXT_IA_MIN_RETRY_INTERVAL", 6.0))
+
+MOCK_TIME = os.environ.get("CDXT_MOCK_TIME")

tests/conftest.py

@@ -31,6 +31,14 @@ def cleanup_cache():
         shutil.rmtree(cache_dir)
 
 
+@pytest.fixture(scope="session", autouse=True)
+def set_mock_time():
+    """Set CDXT_MOCK_TIME environment variable for consistent test results"""
+    # August 15, 2025 - ensures tests use CC-MAIN-2025-33 which exists in mock data
+    if 'CDXT_MOCK_TIME' not in os.environ:
+        os.environ['CDXT_MOCK_TIME'] = '1755259200'
+
+
 # Cache for AWS S3 access check to avoid repeated network calls
 _aws_s3_access_cache = None