Sync upstream/integration (f850d9f) -> release/5 2026-06-23

README.md

@@ -33,6 +33,21 @@ It provides deep visibility into infrastructure costs, automated optimization re
 
 </div>
 
+<br>
+<br>
+
+<div>
+<br>  
+  <img src="documentation/images/OptScale-AI-description-ReadMe-GitHub.png" width="92" align="left" style="border-radius: 50%; margin-right: 15px">
+  <b>
+    🔖 NEW 🔖 <br>Cloud costs under control? Time to go beyond – into AI.
+  </b>
+  <br> OptScale AI covers it all – cost control, security and guardrails, and full visibility over every AI prompt, model, and agent.
+
+  <b>Try free at [optscale.ai](https://optscale.ai) </b>
+</div>
+
+<br>
 <br>
 
 <div>

diworker/diworker/importers/base.py

@@ -4,7 +4,9 @@
 import requests
 import gzip
 import shutil
+import threading
 import uuid
+from contextlib import nullcontext
 from functools import cached_property
 
 from collections import defaultdict
@@ -24,16 +26,30 @@
 
 LOG = logging.getLogger(__name__)
 CHUNK_SIZE = 200
+
+_THROTTLE_SEMAPHORES: dict[str, threading.Semaphore] = {}
+_THROTTLE_LOCK = threading.Lock()
+
 CSV_REWRITE_DAYS = 5
 GZIP_ENDING = '.gz'
 REPORTS_PATH_PREFIX = 'reports'
 
 
+def _get_throttle_semaphore(parent_id: str,
+                            max_concurrent: int) -> threading.Semaphore:
+    with _THROTTLE_LOCK:
+        if parent_id not in _THROTTLE_SEMAPHORES:
+            _THROTTLE_SEMAPHORES[parent_id] = threading.Semaphore(max_concurrent)
+        return _THROTTLE_SEMAPHORES[parent_id]
+
+
 class BaseReportImporter:
     def __init__(self, cloud_account_id, rest_cl, config_cl, mongo_raw,
                  mongo_resources, clickhouse_cl, import_file=None,
-                 recalculate=False, detect_period_start=True):
+                 recalculate=False, detect_period_start=True,
+                 max_tenant_concurrent=1):
         self.cloud_acc_id = cloud_account_id
+        self.max_tenant_concurrent = max_tenant_concurrent
         self.rest_cl = rest_cl
         self.config_cl = config_cl
         self.mongo_raw = mongo_raw
@@ -429,6 +445,15 @@ def data_import(self):
         self.generate_clean_records(regeneration=regeneration)
 
     def import_report(self):
+        parent_id = self.cloud_acc.get('parent_id')
+        throttle = (
+            _get_throttle_semaphore(parent_id, self.max_tenant_concurrent)
+            if parent_id else nullcontext()
+        )
+        with throttle:
+            self._run_import()
+
+    def _run_import(self):
         LOG.info('Started import for %s', self.cloud_acc_id)
         self.prepare()
         try:

diworker/diworker/main.py

@@ -40,6 +40,14 @@
 ENVIRONMENT_CLOUD_TYPE = 'environment'
 HEARTBEAT_INTERVAL = 300
 DEFAULT_MAX_WORKERS = 4
+DEFAULT_MAX_TENANT_WORKERS = 1
+
+
+def _is_rate_limit_exc(exc):
+    if getattr(exc, 'status_code', None) == 429:
+        return True
+    msg = str(exc).lower()
+    return '429' in msg or 'toomanyrequests' in msg or 'too many requests' in msg
 
 
 class DIWorker(ConsumerMixin):
@@ -168,7 +176,9 @@ def report_import(self, task, config_cl, rest_cl, mongo_cl, clickhouse_cl):
             'mongo_resources': mongo_cl.restapi['resources'],
             'clickhouse_cl': clickhouse_cl,
             'import_file': import_dict.get('import_file'),
-            'recalculate': is_recalculation}
+            'recalculate': is_recalculation,
+            'max_tenant_concurrent': int(self.diworker_settings.get(
+                'max_tenant_import_workers', DEFAULT_MAX_TENANT_WORKERS))}
         importer = None
         ca = None
         previous_attempt_ts = 0
@@ -220,11 +230,13 @@ def report_import(self, task, config_cl, rest_cl, mongo_cl, clickhouse_cl):
             if not importer:
                 importer = BaseReportImporter(**importer_params)
             importer.update_cloud_import_attempt(now, reason)
-            self.send_report_failed_email(ca, previous_attempt_ts, now)
+            self.send_report_failed_email(
+                ca, previous_attempt_ts, now,
+                is_throttled=_is_rate_limit_exc(exc))
             raise
 
     def send_report_failed_email(self, cloud_account, previous_attempt_ts,
-                                 now):
+                                 now, is_throttled=False):
         last_import_at = cloud_account['last_import_at']
         if not last_import_at:
             last_import_at = cloud_account['created_at']
@@ -236,9 +248,11 @@ def send_report_failed_email(self, cloud_account, previous_attempt_ts,
                     utcfromtimestamp(now)):
                 # email already sent today during previous report import fails
                 return
+        action = ('report_import_throttled' if is_throttled
+                  else 'report_import_failed')
         self.publish_activities_task(
             cloud_account['organization_id'], cloud_account['id'],
-            'cloud_account', 'report_import_failed',
+            'cloud_account', action,
             'organization.report_import.failed')
 
     def process_task(self, body, message):

docker_images/keeper_executor/events.py

@@ -391,3 +391,10 @@ class Events(enum.Enum):
         ['object_name', 'object_id'],
         "INFO"
     ]
+    N0163 = [
+        "Billing data import for cloud account {object_name} "
+        "({cloud_account_id}) was throttled by the cloud provider: "
+        "{error_reason}",
+        ["object_name", "cloud_account_id", "error_reason"],
+        "WARNING"
+    ]

docker_images/keeper_executor/executors/main_events.py

@@ -24,6 +24,7 @@ def action_event_map(self):
             'cloud_account_deleted': Events.N0068,
             'report_import_completed': Events.N0069,
             'report_import_failed': Events.N0070,
+            'report_import_throttled': Events.N0163,
             'assignment_request_accepted': Events.N0071,
             'assignment_request_declined': Events.N0072,
             'root_assigned_resource': Events.N0076,

herald/Dockerfile_tests

@@ -12,4 +12,5 @@ COPY optscale_client/herald_client optscale_client/herald_client
 RUN uv --project herald sync --locked
 
 COPY herald/herald_server/tests herald/herald_server/tests
+COPY herald/modules/__init__.py herald/modules/__init__.py
 COPY herald/modules/tests herald/modules/tests

herald/modules/email_generator/context_generator.py

@@ -69,7 +69,6 @@ def get_default_context():
     return {
         "images": {
             "logo": "https://cdn.hystax.com/OptScale/OptScale-logo-white.png",
-            "telegram": "https://cdn.hystax.com/OptScale/email-images/telegram.png",
             "optscale_ml_banner": "https://cdn.hystax.com/OptScale/email-images/optscale-ml-welcome-banner.png",
             "optscale_finops": "https://cdn.hystax.com/OptScale/email-images/optscale-finops-capabilities.png",
         },
@@ -87,7 +86,6 @@ def get_default_context():
             "linkedin": "https://linkedin.com/company/hystax",
             "twitter": "https://twitter.com/hystaxcom",
             "facebook": "https://facebook.com/hystax",
-            "telegram": "https://t.me/hystax",
             "terms_of_use": "https://hystax.com/terms-of-use/",
             "privacy_policy": "https://hystax.com/privacy-policy/",
             "documentation": "https://hystax.com/documentation/optscale",

herald/modules/email_generator/templates/alert.html

@@ -211,7 +211,7 @@
                               <tr>
                                 <td class="pad">
                                   <div class="alignment" style="text-align: center">
-                                    <table class="social-table" width="208px" border="0" cellpadding="0" cellspacing="0" role="presentation" style="mso-table-lspace: 0pt; mso-table-rspace: 0pt; display: inline-block">
+                                    <table class="social-table" width="156px" border="0" cellpadding="0" cellspacing="0" role="presentation" style="mso-table-lspace: 0pt; mso-table-rspace: 0pt; display: inline-block">
                                       <tr>
                                         {% if links.linkedin|length %}
                                         <td style="padding: 0 10px 0 10px">
@@ -228,11 +228,6 @@
                                           <a href="{{links.facebook}}" target="_blank" rel="noopener"><img src="https://app-rsrc.getbee.io/public/resources/social-networks-icon-sets/t-only-logo-color/facebook@2x.png" width="32" height="32" alt="Facebook" title="Facebook" style="display: block; height: auto; border: 0" /></a>
                                         </td>
                                         {% endif %}
-                                        {% if links.telegram|length %}
-                                        <td style="padding: 0 10px 0 10px">
-                                          <a href="{{links.telegram}}" target="_blank" rel="noopener"><img src="https://app-rsrc.getbee.io/public/resources/social-networks-icon-sets/t-only-logo-color/telegram@2x.png" width="32" height="32" alt="Telegram" title="Telegram" style="display: block; height: auto; border: 0" /></a>
-                                        </td>
-                                        {% endif %}
                                       </tr>
                                     </table>
                                   </div>

herald/modules/email_generator/templates/anomaly_detection_alert.html

@@ -321,7 +321,7 @@
                     <tr>
                       <td class="pad">
                         <div class="alignment" style="text-align:center">
-                          <table class="social-table" width="208px" border="0" cellpadding="0" cellspacing="0"
+                          <table class="social-table" width="156px" border="0" cellpadding="0" cellspacing="0"
                                  role="presentation" style="mso-table-lspace:0;mso-table-rspace:0;display:inline-block">
                             <tr>
                               {% if links.linkedin|length %}
@@ -339,11 +339,6 @@
                                 <a href="{{links.facebook}}" target="_blank" rel="noopener"><img src="https://app-rsrc.getbee.io/public/resources/social-networks-icon-sets/t-only-logo-color/facebook@2x.png" width="32" height="32" alt="Facebook" title="Facebook" style="display: block; height: auto; border: 0" /></a>
                               </td>
                               {% endif %}
-                              {% if links.telegram|length %}
-                              <td style="padding: 0 10px 0 10px">
-                                <a href="{{links.telegram}}" target="_blank" rel="noopener"><img src="https://app-rsrc.getbee.io/public/resources/social-networks-icon-sets/t-only-logo-color/telegram@2x.png" width="32" height="32" alt="Telegram" title="Telegram" style="display: block; height: auto; border: 0" /></a>
-                              </td>
-                              {% endif %}
                             </tr>
                           </table>
                         </div>

herald/modules/email_generator/templates/bumi_module_execution_failed.html

@@ -383,7 +383,7 @@
                               <tr>
                                 <td class="pad">
                                   <div class="alignment" style="text-align: center">
-                                    <table class="social-table" width="208px" border="0" cellpadding="0" cellspacing="0" role="presentation" style="mso-table-lspace: 0pt; mso-table-rspace: 0pt; display: inline-block">
+                                    <table class="social-table" width="156px" border="0" cellpadding="0" cellspacing="0" role="presentation" style="mso-table-lspace: 0pt; mso-table-rspace: 0pt; display: inline-block">
                                       <tr>
                                         {% if links.linkedin|length %}
                                         <td style="padding: 0 10px 0 10px">
@@ -400,11 +400,6 @@
                                           <a href="{{links.facebook}}" target="_blank" rel="noopener"><img src="https://app-rsrc.getbee.io/public/resources/social-networks-icon-sets/t-only-logo-color/facebook@2x.png" width="32" height="32" alt="Facebook" title="Facebook" style="display: block; height: auto; border: 0" /></a>
                                         </td>
                                         {% endif %}
-                                        {% if links.telegram|length %}
-                                        <td style="padding: 0 10px 0 10px">
-                                          <a href="{{links.telegram}}" target="_blank" rel="noopener"><img src="https://app-rsrc.getbee.io/public/resources/social-networks-icon-sets/t-only-logo-color/telegram@2x.png" width="32" height="32" alt="Telegram" title="Telegram" style="display: block; height: auto; border: 0" /></a>
-                                        </td>
-                                        {% endif %}
                                       </tr>
                                     </table>
                                   </div>

herald/modules/email_generator/templates/bumi_task_execution_failed.html

@@ -427,7 +427,7 @@
                               <tr>
                                 <td class="pad">
                                   <div class="alignment" style="text-align: center">
-                                    <table class="social-table" width="208px" border="0" cellpadding="0" cellspacing="0" role="presentation" style="mso-table-lspace: 0pt; mso-table-rspace: 0pt; display: inline-block">
+                                    <table class="social-table" width="156px" border="0" cellpadding="0" cellspacing="0" role="presentation" style="mso-table-lspace: 0pt; mso-table-rspace: 0pt; display: inline-block">
                                       <tr>
                                         {% if links.linkedin|length %}
                                         <td style="padding: 0 10px 0 10px">
@@ -444,11 +444,6 @@
                                           <a href="{{links.facebook}}" target="_blank" rel="noopener"><img src="https://app-rsrc.getbee.io/public/resources/social-networks-icon-sets/t-only-logo-color/facebook@2x.png" width="32" height="32" alt="Facebook" title="Facebook" style="display: block; height: auto; border: 0" /></a>
                                         </td>
                                         {% endif %}
-                                        {% if links.telegram|length %}
-                                        <td style="padding: 0 10px 0 10px">
-                                          <a href="{{links.telegram}}" target="_blank" rel="noopener"><img src="https://app-rsrc.getbee.io/public/resources/social-networks-icon-sets/t-only-logo-color/telegram@2x.png" width="32" height="32" alt="Telegram" title="Telegram" style="display: block; height: auto; border: 0" /></a>
-                                        </td>
-                                        {% endif %}
                                       </tr>
                                     </table>
                                   </div>

herald/modules/email_generator/templates/cloud_account_deleted.html

@@ -412,7 +412,7 @@
                               <tr>
                                 <td class="pad">
                                   <div class="alignment" style="text-align: center">
-                                    <table class="social-table" width="208px" border="0" cellpadding="0" cellspacing="0" role="presentation" style="mso-table-lspace: 0pt; mso-table-rspace: 0pt; display: inline-block">
+                                    <table class="social-table" width="156px" border="0" cellpadding="0" cellspacing="0" role="presentation" style="mso-table-lspace: 0pt; mso-table-rspace: 0pt; display: inline-block">
                                       <tr>
                                         {% if links.linkedin|length %}
                                         <td style="padding: 0 10px 0 10px">
@@ -429,11 +429,6 @@
                                           <a href="{{links.facebook}}" target="_blank" rel="noopener"><img src="https://app-rsrc.getbee.io/public/resources/social-networks-icon-sets/t-only-logo-color/facebook@2x.png" width="32" height="32" alt="Facebook" title="Facebook" style="display: block; height: auto; border: 0" /></a>
                                         </td>
                                         {% endif %}
-                                        {% if links.telegram|length %}
-                                        <td style="padding: 0 10px 0 10px">
-                                          <a href="{{links.telegram}}" target="_blank" rel="noopener"><img src="https://app-rsrc.getbee.io/public/resources/social-networks-icon-sets/t-only-logo-color/telegram@2x.png" width="32" height="32" alt="Telegram" title="Telegram" style="display: block; height: auto; border: 0" /></a>
-                                        </td>
-                                        {% endif %}
                                       </tr>
                                     </table>
                                   </div>

herald/modules/email_generator/templates/disconnect_survey.html

@@ -336,7 +336,7 @@
                               <tr>
                                 <td class="pad">
                                   <div class="alignment" style="text-align: center">
-                                    <table class="social-table" width="208px" border="0" cellpadding="0" cellspacing="0" role="presentation" style="mso-table-lspace: 0pt; mso-table-rspace: 0pt; display: inline-block">
+                                    <table class="social-table" width="156px" border="0" cellpadding="0" cellspacing="0" role="presentation" style="mso-table-lspace: 0pt; mso-table-rspace: 0pt; display: inline-block">
                                       <tr>
                                         {% if links.linkedin|length %}
                                         <td style="padding: 0 10px 0 10px">
@@ -353,11 +353,6 @@
                                           <a href="{{links.facebook}}" target="_blank" rel="noopener"><img src="https://app-rsrc.getbee.io/public/resources/social-networks-icon-sets/t-only-logo-color/facebook@2x.png" width="32" height="32" alt="Facebook" title="Facebook" style="display: block; height: auto; border: 0" /></a>
                                         </td>
                                         {% endif %}
-                                        {% if links.telegram|length %}
-                                        <td style="padding: 0 10px 0 10px">
-                                          <a href="{{links.telegram}}" target="_blank" rel="noopener"><img src="https://app-rsrc.getbee.io/public/resources/social-networks-icon-sets/t-only-logo-color/telegram@2x.png" width="32" height="32" alt="Telegram" title="Telegram" style="display: block; height: auto; border: 0" /></a>
-                                        </td>
-                                        {% endif %}
                                       </tr>
                                     </table>
                                   </div>