diff --git a/README.md b/README.md index 26519a296..448d2c1b3 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,21 @@ It provides deep visibility into infrastructure costs, automated optimization re +
+
+ +
+
+ + + 🔖 NEW 🔖
Cloud costs under control? Time to go beyond – into AI. + +
OptScale AI covers it all – cost control, security and guardrails, and full visibility over every AI prompt, model, and agent. + + Try free at [optscale.ai](https://optscale.ai) +
+ +

diff --git a/diworker/diworker/importers/base.py b/diworker/diworker/importers/base.py index 6bdd4156b..ea231b14c 100644 --- a/diworker/diworker/importers/base.py +++ b/diworker/diworker/importers/base.py @@ -4,7 +4,9 @@ import requests import gzip import shutil +import threading import uuid +from contextlib import nullcontext from functools import cached_property from collections import defaultdict @@ -24,16 +26,30 @@ LOG = logging.getLogger(__name__) CHUNK_SIZE = 200 + +_THROTTLE_SEMAPHORES: dict[str, threading.Semaphore] = {} +_THROTTLE_LOCK = threading.Lock() + CSV_REWRITE_DAYS = 5 GZIP_ENDING = '.gz' REPORTS_PATH_PREFIX = 'reports' +def _get_throttle_semaphore(parent_id: str, + max_concurrent: int) -> threading.Semaphore: + with _THROTTLE_LOCK: + if parent_id not in _THROTTLE_SEMAPHORES: + _THROTTLE_SEMAPHORES[parent_id] = threading.Semaphore(max_concurrent) + return _THROTTLE_SEMAPHORES[parent_id] + + class BaseReportImporter: def __init__(self, cloud_account_id, rest_cl, config_cl, mongo_raw, mongo_resources, clickhouse_cl, import_file=None, - recalculate=False, detect_period_start=True): + recalculate=False, detect_period_start=True, + max_tenant_concurrent=1): self.cloud_acc_id = cloud_account_id + self.max_tenant_concurrent = max_tenant_concurrent self.rest_cl = rest_cl self.config_cl = config_cl self.mongo_raw = mongo_raw @@ -429,6 +445,15 @@ def data_import(self): self.generate_clean_records(regeneration=regeneration) def import_report(self): + parent_id = self.cloud_acc.get('parent_id') + throttle = ( + _get_throttle_semaphore(parent_id, self.max_tenant_concurrent) + if parent_id else nullcontext() + ) + with throttle: + self._run_import() + + def _run_import(self): LOG.info('Started import for %s', self.cloud_acc_id) self.prepare() try: diff --git a/diworker/diworker/main.py b/diworker/diworker/main.py index dcb017e8d..9bf990e53 100755 --- a/diworker/diworker/main.py +++ b/diworker/diworker/main.py @@ -40,6 +40,14 @@ ENVIRONMENT_CLOUD_TYPE = 'environment' HEARTBEAT_INTERVAL = 300 DEFAULT_MAX_WORKERS = 4 +DEFAULT_MAX_TENANT_WORKERS = 1 + + +def _is_rate_limit_exc(exc): + if getattr(exc, 'status_code', None) == 429: + return True + msg = str(exc).lower() + return '429' in msg or 'toomanyrequests' in msg or 'too many requests' in msg class DIWorker(ConsumerMixin): @@ -168,7 +176,9 @@ def report_import(self, task, config_cl, rest_cl, mongo_cl, clickhouse_cl): 'mongo_resources': mongo_cl.restapi['resources'], 'clickhouse_cl': clickhouse_cl, 'import_file': import_dict.get('import_file'), - 'recalculate': is_recalculation} + 'recalculate': is_recalculation, + 'max_tenant_concurrent': int(self.diworker_settings.get( + 'max_tenant_import_workers', DEFAULT_MAX_TENANT_WORKERS))} importer = None ca = None previous_attempt_ts = 0 @@ -220,11 +230,13 @@ def report_import(self, task, config_cl, rest_cl, mongo_cl, clickhouse_cl): if not importer: importer = BaseReportImporter(**importer_params) importer.update_cloud_import_attempt(now, reason) - self.send_report_failed_email(ca, previous_attempt_ts, now) + self.send_report_failed_email( + ca, previous_attempt_ts, now, + is_throttled=_is_rate_limit_exc(exc)) raise def send_report_failed_email(self, cloud_account, previous_attempt_ts, - now): + now, is_throttled=False): last_import_at = cloud_account['last_import_at'] if not last_import_at: last_import_at = cloud_account['created_at'] @@ -236,9 +248,11 @@ def send_report_failed_email(self, cloud_account, previous_attempt_ts, utcfromtimestamp(now)): # email already sent today during previous report import fails return + action = ('report_import_throttled' if is_throttled + else 'report_import_failed') self.publish_activities_task( cloud_account['organization_id'], cloud_account['id'], - 'cloud_account', 'report_import_failed', + 'cloud_account', action, 'organization.report_import.failed') def process_task(self, body, message): diff --git a/docker_images/keeper_executor/events.py b/docker_images/keeper_executor/events.py index 00affdac4..a29cfc525 100644 --- a/docker_images/keeper_executor/events.py +++ b/docker_images/keeper_executor/events.py @@ -391,3 +391,10 @@ class Events(enum.Enum): ['object_name', 'object_id'], "INFO" ] + N0163 = [ + "Billing data import for cloud account {object_name} " + "({cloud_account_id}) was throttled by the cloud provider: " + "{error_reason}", + ["object_name", "cloud_account_id", "error_reason"], + "WARNING" + ] diff --git a/docker_images/keeper_executor/executors/main_events.py b/docker_images/keeper_executor/executors/main_events.py index c3e665840..c45d87995 100644 --- a/docker_images/keeper_executor/executors/main_events.py +++ b/docker_images/keeper_executor/executors/main_events.py @@ -24,6 +24,7 @@ def action_event_map(self): 'cloud_account_deleted': Events.N0068, 'report_import_completed': Events.N0069, 'report_import_failed': Events.N0070, + 'report_import_throttled': Events.N0163, 'assignment_request_accepted': Events.N0071, 'assignment_request_declined': Events.N0072, 'root_assigned_resource': Events.N0076, diff --git a/documentation/images/OptScale-AI-description-ReadMe-GitHub.png b/documentation/images/OptScale-AI-description-ReadMe-GitHub.png new file mode 100644 index 000000000..867ce40f0 Binary files /dev/null and b/documentation/images/OptScale-AI-description-ReadMe-GitHub.png differ diff --git a/herald/Dockerfile_tests b/herald/Dockerfile_tests index c1de70ce4..ca3a4b668 100644 --- a/herald/Dockerfile_tests +++ b/herald/Dockerfile_tests @@ -12,4 +12,5 @@ COPY optscale_client/herald_client optscale_client/herald_client RUN uv --project herald sync --locked COPY herald/herald_server/tests herald/herald_server/tests +COPY herald/modules/__init__.py herald/modules/__init__.py COPY herald/modules/tests herald/modules/tests diff --git a/herald/modules/__init__.py b/herald/modules/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/herald/modules/email_generator/context_generator.py b/herald/modules/email_generator/context_generator.py index 42f773b2f..f2a47c8b6 100644 --- a/herald/modules/email_generator/context_generator.py +++ b/herald/modules/email_generator/context_generator.py @@ -69,7 +69,6 @@ def get_default_context(): return { "images": { "logo": "https://cdn.hystax.com/OptScale/OptScale-logo-white.png", - "telegram": "https://cdn.hystax.com/OptScale/email-images/telegram.png", "optscale_ml_banner": "https://cdn.hystax.com/OptScale/email-images/optscale-ml-welcome-banner.png", "optscale_finops": "https://cdn.hystax.com/OptScale/email-images/optscale-finops-capabilities.png", }, @@ -87,7 +86,6 @@ def get_default_context(): "linkedin": "https://linkedin.com/company/hystax", "twitter": "https://twitter.com/hystaxcom", "facebook": "https://facebook.com/hystax", - "telegram": "https://t.me/hystax", "terms_of_use": "https://hystax.com/terms-of-use/", "privacy_policy": "https://hystax.com/privacy-policy/", "documentation": "https://hystax.com/documentation/optscale", diff --git a/herald/modules/email_generator/templates/alert.html b/herald/modules/email_generator/templates/alert.html index ad2152dce..655213975 100644 --- a/herald/modules/email_generator/templates/alert.html +++ b/herald/modules/email_generator/templates/alert.html @@ -211,7 +211,7 @@
- + {% if links.linkedin|length %} {% endif %} - {% if links.telegram|length %} - - {% endif %}
diff --git a/herald/modules/email_generator/templates/anomaly_detection_alert.html b/herald/modules/email_generator/templates/anomaly_detection_alert.html index d47f8c416..f88f0a509 100644 --- a/herald/modules/email_generator/templates/anomaly_detection_alert.html +++ b/herald/modules/email_generator/templates/anomaly_detection_alert.html @@ -321,7 +321,7 @@
- {% if links.linkedin|length %} @@ -339,11 +339,6 @@ Facebook {% endif %} - {% if links.telegram|length %} - - {% endif %}
diff --git a/herald/modules/email_generator/templates/bumi_module_execution_failed.html b/herald/modules/email_generator/templates/bumi_module_execution_failed.html index 380b11a88..1e39d7c1d 100644 --- a/herald/modules/email_generator/templates/bumi_module_execution_failed.html +++ b/herald/modules/email_generator/templates/bumi_module_execution_failed.html @@ -383,7 +383,7 @@
- + {% if links.linkedin|length %} {% endif %} - {% if links.telegram|length %} - - {% endif %}
diff --git a/herald/modules/email_generator/templates/bumi_task_execution_failed.html b/herald/modules/email_generator/templates/bumi_task_execution_failed.html index 0938db887..7c4218095 100644 --- a/herald/modules/email_generator/templates/bumi_task_execution_failed.html +++ b/herald/modules/email_generator/templates/bumi_task_execution_failed.html @@ -427,7 +427,7 @@
- + {% if links.linkedin|length %} {% endif %} - {% if links.telegram|length %} - - {% endif %}
diff --git a/herald/modules/email_generator/templates/cloud_account_deleted.html b/herald/modules/email_generator/templates/cloud_account_deleted.html index d6aa77560..4b6230ae0 100644 --- a/herald/modules/email_generator/templates/cloud_account_deleted.html +++ b/herald/modules/email_generator/templates/cloud_account_deleted.html @@ -412,7 +412,7 @@
- + {% if links.linkedin|length %} {% endif %} - {% if links.telegram|length %} - - {% endif %}
diff --git a/herald/modules/email_generator/templates/disconnect_survey.html b/herald/modules/email_generator/templates/disconnect_survey.html index c4b6e3349..52ab0f01d 100644 --- a/herald/modules/email_generator/templates/disconnect_survey.html +++ b/herald/modules/email_generator/templates/disconnect_survey.html @@ -336,7 +336,7 @@
- + {% if links.linkedin|length %} {% endif %} - {% if links.telegram|length %} - - {% endif %}
diff --git a/herald/modules/email_generator/templates/employee_greetings.html b/herald/modules/email_generator/templates/employee_greetings.html index d13685f56..281c3f3f8 100644 --- a/herald/modules/email_generator/templates/employee_greetings.html +++ b/herald/modules/email_generator/templates/employee_greetings.html @@ -740,27 +740,6 @@ - {% if links.telegram|length %} - - - - - - -
- - - - - - -
- - - -
-
- {% endif %}