diff --git a/providers/elasticsearch/src/airflow/providers/elasticsearch/log/es_task_handler.py b/providers/elasticsearch/src/airflow/providers/elasticsearch/log/es_task_handler.py index 2fb610186e9d9..9b4261072dcd9 100644 --- a/providers/elasticsearch/src/airflow/providers/elasticsearch/log/es_task_handler.py +++ b/providers/elasticsearch/src/airflow/providers/elasticsearch/log/es_task_handler.py @@ -123,11 +123,11 @@ def _format_error_detail(error_detail: Any) -> str | None: def _build_log_fields(hit_dict: dict[str, Any]) -> dict[str, Any]: """Filter an ES hit to ``TASK_LOG_FIELDS`` and ensure compatibility with StructuredLogMessage.""" - fields = {k: v for k, v in hit_dict.items() if k.lower() in TASK_LOG_FIELDS or k == "@timestamp"} + fields = {k: v for k, v in hit_dict.items() if k.lower() in TASK_LOG_FIELDS} - # Map @timestamp to timestamp - if "@timestamp" in fields and "timestamp" not in fields: - fields["timestamp"] = fields.pop("@timestamp") + # Map @timestamp to timestamp but not include `@timestamp` in log fields + if "@timestamp" in hit_dict and "timestamp" not in fields: + fields["timestamp"] = hit_dict["@timestamp"] # Map levelname to level if "levelname" in fields and "level" not in fields: diff --git a/providers/elasticsearch/tests/unit/elasticsearch/log/test_es_task_handler.py b/providers/elasticsearch/tests/unit/elasticsearch/log/test_es_task_handler.py index a2cb4c315d892..589174c0c71c7 100644 --- a/providers/elasticsearch/tests/unit/elasticsearch/log/test_es_task_handler.py +++ b/providers/elasticsearch/tests/unit/elasticsearch/log/test_es_task_handler.py @@ -975,12 +975,18 @@ def test_levelname_mapped_to_level(self): assert result["level"] == "ERROR" assert "levelname" not in result - def test_at_timestamp_mapped_to_timestamp(self): + def test_at_timestamp_mapped_to_timestamp_if_no_timestamp_present(self): hit = {"event": "msg", "@timestamp": "2024-01-01T00:00:00Z"} result = _build_log_fields(hit) assert result["timestamp"] == "2024-01-01T00:00:00Z" assert "@timestamp" not in result + def test_at_timestamp_not_included_if_timestamp_present(self): + hit = {"event": "msg", "@timestamp": "2024-01-01T00:00:00Z", "timestamp": "2024-01-01T00:00:00Z"} + result = _build_log_fields(hit) + assert result["timestamp"] == "2024-01-01T00:00:00Z" + assert "@timestamp" not in result + def test_error_detail_is_kept_as_list(self): error_detail = [ { diff --git a/providers/opensearch/src/airflow/providers/opensearch/log/os_task_handler.py b/providers/opensearch/src/airflow/providers/opensearch/log/os_task_handler.py index a6fee7ba60124..6d9722478a5db 100644 --- a/providers/opensearch/src/airflow/providers/opensearch/log/os_task_handler.py +++ b/providers/opensearch/src/airflow/providers/opensearch/log/os_task_handler.py @@ -104,11 +104,11 @@ def _format_error_detail(error_detail: Any) -> str | None: def _build_log_fields(hit_dict: dict[str, Any]) -> dict[str, Any]: """Filter an OpenSearch hit to ``TASK_LOG_FIELDS`` and ensure compatibility with StructuredLogMessage.""" - fields = {k: v for k, v in hit_dict.items() if k.lower() in TASK_LOG_FIELDS or k == "@timestamp"} + fields = {k: v for k, v in hit_dict.items() if k.lower() in TASK_LOG_FIELDS} - # Map @timestamp to timestamp - if "@timestamp" in fields and "timestamp" not in fields: - fields["timestamp"] = fields.pop("@timestamp") + # Map @timestamp to timestamp but not include `@timestamp` in log fields + if "@timestamp" in hit_dict and "timestamp" not in fields: + fields["timestamp"] = hit_dict["@timestamp"] # Map levelname to level if "levelname" in fields and "level" not in fields: diff --git a/providers/opensearch/tests/unit/opensearch/log/test_os_task_handler.py b/providers/opensearch/tests/unit/opensearch/log/test_os_task_handler.py index 7ec7281f03902..ca49103b8a5e5 100644 --- a/providers/opensearch/tests/unit/opensearch/log/test_os_task_handler.py +++ b/providers/opensearch/tests/unit/opensearch/log/test_os_task_handler.py @@ -879,12 +879,18 @@ def test_levelname_mapped_to_level(self): assert result["level"] == "ERROR" assert "levelname" not in result - def test_at_timestamp_mapped_to_timestamp(self): + def test_at_timestamp_mapped_to_timestamp_if_no_timestamp_present(self): hit = {"event": "msg", "@timestamp": "2024-01-01T00:00:00Z"} result = _build_log_fields(hit) assert result["timestamp"] == "2024-01-01T00:00:00Z" assert "@timestamp" not in result + def test_at_timestamp_not_included_if_timestamp_present(self): + hit = {"event": "msg", "@timestamp": "2024-01-01T00:00:00Z", "timestamp": "2024-01-01T00:00:00Z"} + result = _build_log_fields(hit) + assert result["timestamp"] == "2024-01-01T00:00:00Z" + assert "@timestamp" not in result + def test_error_detail_is_kept_as_list(self): error_detail = [ {