Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions devel-common/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -118,8 +118,7 @@ dependencies = [
"types-certifi>=2021.10.8.3",
"types-croniter>=2.0.0.20240423",
"types-docutils>=0.21.0.20240704",
# TODO: Bump to >= 4.0.0 once https://github.com/apache/airflow/issues/54079
"types-paramiko>=3.4.0.20240423,<4.0.0",
"types-paramiko>=4.0.0.20260402,<5.0.0",
"types-protobuf>=5.26.0.20240422",
"types-python-dateutil>=2.9.0.20240316",
"types-python-slugify>=8.0.2.20240310",
Expand Down
2 changes: 1 addition & 1 deletion providers/sftp/README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ PIP package Version required
``apache-airflow`` ``>=2.11.0``
``apache-airflow-providers-ssh`` ``>=4.0.0``
``apache-airflow-providers-common-compat`` ``>=1.12.0``
``paramiko`` ``>=3.5.1,<4.0.0``
``paramiko`` ``>=4.0.0,<5.0.0``
``asyncssh`` ``>=2.12.0; python_version < "3.14"``
``asyncssh`` ``>=2.22.0; python_version >= "3.14"``
========================================== ======================================
Expand Down
7 changes: 7 additions & 0 deletions providers/sftp/docs/changelog.rst
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,13 @@
Changelog
---------

Breaking changes
~~~~~~~~~~~~~~~~

* ``Bump minimum paramiko to 4.0.0; DSA/DSS keys are no longer supported (#54079)``

This provider depends on paramiko 4.0+, which dropped ``DSS``/``DSA`` keys; see `paramiko changelog <https://www.paramiko.org/changelog.html>`__. To migrate: create a key pair that does not use ``DSA`` (Ed25519 or RSA are typical, e.g. ``ssh-keygen -t ed25519``), add the public key to the SFTP server, then update your Airflow SFTP (or shared SSH) connection so ``key_file`` or the ``private_key`` extra uses the new key, and ensure any ``host_key`` extra is not in ``ssh-dss`` form. If you are not ready to migrate keys, stay on a provider release that still pins ``paramiko<4`` until you can switch.

5.8.2
.....

Expand Down
2 changes: 1 addition & 1 deletion providers/sftp/docs/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ PIP package Version required
``apache-airflow`` ``>=2.11.0``
``apache-airflow-providers-ssh`` ``>=4.0.0``
``apache-airflow-providers-common-compat`` ``>=1.12.0``
``paramiko`` ``>=3.5.1,<4.0.0``
``paramiko`` ``>=4.0.0,<5.0.0``
``asyncssh`` ``>=2.12.0; python_version < "3.14"``
``asyncssh`` ``>=2.22.0; python_version >= "3.14"``
========================================== ======================================
Expand Down
5 changes: 2 additions & 3 deletions providers/sftp/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,9 @@ requires-python = ">=3.10"
dependencies = [
"aiofiles>=23.2.0",
"apache-airflow>=2.11.0",
"apache-airflow-providers-ssh>=4.0.0",
"apache-airflow-providers-ssh>=4.0.0", # use next version
"apache-airflow-providers-common-compat>=1.12.0",
# TODO: Bump to >= 4.0.0 once https://github.com/apache/airflow/issues/54079 is handled
"paramiko>=3.5.1,<4.0.0",
"paramiko>=4.0.0,<5.0.0",
"asyncssh>=2.12.0; python_version < '3.14'",
"asyncssh>=2.22.0; python_version >= '3.14'",
]
Expand Down
9 changes: 9 additions & 0 deletions providers/sftp/src/airflow/providers/sftp/hooks/sftp.py
Original file line number Diff line number Diff line change
Expand Up @@ -802,6 +802,15 @@ def _parse_extras(self, conn: Connection) -> None:
self.log.warning("No Host Key Verification. This won't protect against Man-In-The-Middle attacks")
self.known_hosts = "none"
elif host_key is not None:
host_key = host_key.strip()
host_key_parts = host_key.split()
if host_key_parts and host_key_parts[0] == "ssh-dss":
raise ValueError(
"DSA/DSS host keys are not supported. Paramiko 4.0 removed DSS support; "
"use an RSA, ECDSA, or Ed25519 host key and update the connection `host_key`."
)
if len(host_key_parts) >= 2:
host_key = " ".join(host_key_parts[:2])
self.known_hosts = f"{conn.host} {host_key}".encode()

async def _get_conn(self) -> asyncssh.SSHClientConnection:
Expand Down
21 changes: 20 additions & 1 deletion providers/sftp/tests/unit/sftp/hooks/test_sftp.py
Original file line number Diff line number Diff line change
Expand Up @@ -817,6 +817,10 @@ async def test_extra_dejson_fields_for_connection_building_known_hosts_none(
("mock_port", "mock_host_key"),
[
(22, "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFe8P8lk5HFfL/rMlcCMHQhw1cg+uZtlK5rXQk2C4pOY"),
(
22,
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFe8P8lk5HFfL/rMlcCMHQhw1cg+uZtlK5rXQk2C4pOY user@host",
),
(2222, "AAAAC3NzaC1lZDI1NTE5AAAAIFe8P8lk5HFfL/rMlcCMHQhw1cg+uZtlK5rXQk2C4pOY"),
(
2222,
Expand Down Expand Up @@ -848,7 +852,22 @@ async def test_extra_dejson_fields_for_connection_with_host_key(
hook = SFTPHookAsync()
await hook._get_conn()

assert hook.known_hosts == f"localhost {mock_host_key}".encode()
host_key_parts = mock_host_key.split()
expected_host_key = " ".join(host_key_parts[:2]) if len(host_key_parts) >= 2 else mock_host_key
assert hook.known_hosts == f"localhost {expected_host_key}".encode()

@patch("asyncssh.connect", new_callable=AsyncMock)
@patch("airflow.providers.sftp.hooks.sftp.get_async_connection")
@pytest.mark.asyncio
async def test_parse_extras_dss_host_key_raises(self, mock_get_connection, mock_connect):
"""Test that ssh-dss host_key is rejected."""
mock_get_connection.return_value = MockAirflowConnectionWithHostKey(
host_key="ssh-dss\tAAAAB3...", no_host_key_check=False
)

hook = SFTPHookAsync()
with pytest.raises(ValueError, match="DSA/DSS host keys"):
await hook._get_conn()

@patch("asyncssh.connect", new_callable=AsyncMock)
@patch("airflow.providers.sftp.hooks.sftp.get_async_connection")
Expand Down
2 changes: 1 addition & 1 deletion providers/ssh/README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ PIP package Version required
``apache-airflow`` ``>=2.11.0``
``apache-airflow-providers-common-compat`` ``>=1.12.0``
``asyncssh`` ``>=2.12.0``
``paramiko`` ``>=3.5.1,<4.0.0``
``paramiko`` ``>=4.0.0,<5.0.0``
========================================== ==================

The changelog for the provider package can be found in the
Expand Down
7 changes: 7 additions & 0 deletions providers/ssh/docs/changelog.rst
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,13 @@
Changelog
---------

Breaking changes
~~~~~~~~~~~~~~~~

* ``Bump minimum paramiko to 4.0.0; DSA/DSS private keys and ssh-dss host keys are no longer supported (#54079)``

Paramiko 4.0 removed DSS/DSA support; see `paramiko changelog <https://www.paramiko.org/changelog.html>`__ for upstream details. If you use a DSA private key in an SSH connection, generate a new key (for example ``ssh-keygen -t ed25519`` or ``-t rsa``), install the public key on the server, and point your Airflow connection at the new key file or ``private_key`` extra. If you pin the remote host with a ``host_key`` extra in ``ssh-dss`` form, obtain the server's current RSA, ECDSA, or Ed25519 host key and replace the value. The same constraints apply to SFTP connections that rely on paramiko via the SSH provider.

5.0.4
.....

Expand Down
2 changes: 1 addition & 1 deletion providers/ssh/docs/connections/ssh.rst
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ Extra (optional)
* ``no_host_key_check`` - Set to ``false`` to restrict connecting to hosts with no entries in ``~/.ssh/known_hosts`` (Hosts file). This provides maximum protection against trojan horse attacks, but can be troublesome when the ``/etc/ssh/ssh_known_hosts`` file is poorly maintained or connections to new hosts are frequently made. This option forces the user to manually add all new hosts. Default is ``true``, ssh will automatically add new host keys to the user known hosts files.
* ``allow_host_key_change`` - Set to ``true`` if you want to allow connecting to hosts that has host key changed or when you get 'REMOTE HOST IDENTIFICATION HAS CHANGED' error. This won't protect against Man-In-The-Middle attacks. Other possible solution is to remove the host entry from ``~/.ssh/known_hosts`` file. Default is ``false``.
* ``look_for_keys`` - Set to ``false`` if you want to disable searching for discoverable private key files in ``~/.ssh/``
* ``host_key`` - The base64 encoded ssh-rsa public key of the host or "ssh-<key type> <key data>" (as you would find in the ``known_hosts`` file). Specifying this allows making the connection if and only if the public key of the endpoint matches this value.
* ``host_key`` - The base64 encoded ssh-rsa public key of the host or ``"<key type> <key data>"`` (as you would find in the ``known_hosts`` file). Specifying this allows making the connection if and only if the public key of the endpoint matches this value. Supported key type strings are ``ssh-rsa``, ``ecdsa-sha2-nistp256``, ``ecdsa-sha2-nistp384``, ``ecdsa-sha2-nistp521``, and ``ssh-ed25519``; the legacy ``ssh-ecdsa`` form is also accepted for compatibility. DSA/DSS (``ssh-dss``) host keys are not supported because `paramiko 4.0 removed DSS support <https://www.paramiko.org/changelog.html>`__; generate a new host key and update this field.
* ``disabled_algorithms`` - A dictionary mapping algorithm type to an iterable of algorithm identifiers, which will be disabled for the lifetime of the transport.
* ``ciphers`` - A list of ciphers to use in order of preference.

Expand Down
2 changes: 1 addition & 1 deletion providers/ssh/docs/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ PIP package Version required
``apache-airflow`` ``>=2.11.0``
``apache-airflow-providers-common-compat`` ``>=1.12.0``
``asyncssh`` ``>=2.12.0``
``paramiko`` ``>=3.5.1,<4.0.0``
``paramiko`` ``>=4.0.0,<5.0.0``
========================================== ==================

Downloading official packages
Expand Down
3 changes: 1 addition & 2 deletions providers/ssh/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,7 @@ dependencies = [
"apache-airflow>=2.11.0",
"apache-airflow-providers-common-compat>=1.12.0",
"asyncssh>=2.12.0",
# TODO: Bump to >= 4.0.0 once https://github.com/apache/airflow/issues/54079 is handled
"paramiko>=3.5.1,<4.0.0",
"paramiko>=4.0.0,<5.0.0",

]

Expand Down
65 changes: 50 additions & 15 deletions providers/ssh/src/airflow/providers/ssh/hooks/ssh.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,16 @@ def is_arg_set(value): # type: ignore[misc,no-redef]

CMD_TIMEOUT = 10

_HostKeyConstructor = type[paramiko.RSAKey] | type[paramiko.ECDSAKey] | type[paramiko.Ed25519Key]
_SUPPORTED_HOST_KEY_TYPES = (
"ssh-rsa",
"ssh-ecdsa",
"ecdsa-sha2-nistp256",
"ecdsa-sha2-nistp384",
"ecdsa-sha2-nistp521",
"ssh-ed25519",
)


class SSHHook(BaseHook):
"""
Expand Down Expand Up @@ -87,19 +97,21 @@ class SSHHook(BaseHook):
once and some connections are transiently refused (e.g. ``sshd`` ``MaxStartups`` throttling).
"""

# List of classes to try loading private keys as, ordered (roughly) by most common to least common
# List of classes to try loading private keys as, ordered (roughly) by most common to least common.
# DSA/DSS keys are not supported (removed in paramiko 4.0).
_pkey_loaders: Sequence[type[paramiko.PKey]] = (
paramiko.RSAKey,
paramiko.ECDSAKey,
paramiko.Ed25519Key,
paramiko.DSSKey,
)

_host_key_mappings = {
"rsa": paramiko.RSAKey,
"dss": paramiko.DSSKey,
"ecdsa": paramiko.ECDSAKey,
"ed25519": paramiko.Ed25519Key,
_host_key_mappings: dict[str, _HostKeyConstructor] = {
"ssh-rsa": paramiko.RSAKey,
"ssh-ecdsa": paramiko.ECDSAKey,
"ecdsa-sha2-nistp256": paramiko.ECDSAKey,
"ecdsa-sha2-nistp384": paramiko.ECDSAKey,
"ecdsa-sha2-nistp521": paramiko.ECDSAKey,
"ssh-ed25519": paramiko.Ed25519Key,
}

conn_name_attr = "ssh_conn_id"
Expand Down Expand Up @@ -227,11 +239,25 @@ def __init__(
self.ciphers = extra_options.get("ciphers")

if host_key is not None:
if host_key.startswith("ssh-"):
key_type, host_key = host_key.split(None)[:2]
key_constructor = self._host_key_mappings[key_type[4:]]
else:
key_constructor = paramiko.RSAKey
host_key = host_key.strip()
host_key_parts = host_key.split()
key_constructor: _HostKeyConstructor = paramiko.RSAKey
if len(host_key_parts) >= 2:
key_type, host_key = host_key_parts[:2]
if key_type == "ssh-dss":
raise ValueError(
"DSA/DSS host keys are not supported. Paramiko 4.0 removed DSS support; "
"use an RSA, ECDSA, or Ed25519 host key and update the connection `host_key`."
)
key_constructor_for_type = self._host_key_mappings.get(key_type)
if key_constructor_for_type is None:
raise ValueError(
f"Unsupported SSH host key algorithm {key_type!r}. "
f"Supported types are: {', '.join(_SUPPORTED_HOST_KEY_TYPES)}."
)
key_constructor = key_constructor_for_type
elif host_key in self._host_key_mappings or host_key == "ssh-dss":
raise ValueError(f"SSH host key {host_key!r} is missing key data.")
decoded_host_key = decodebytes(host_key.encode("utf-8"))
self.host_key = key_constructor(data=decoded_host_key)
self.no_host_key_check = False
Expand Down Expand Up @@ -423,9 +449,9 @@ def _pkey_from_private_key(self, private_key: str, passphrase: str | None = None
except (paramiko.ssh_exception.SSHException, ValueError):
continue
raise AirflowException(
"Private key provided cannot be read by paramiko."
"Ensure key provided is valid for one of the following"
"key formats: RSA, DSS, ECDSA, or Ed25519"
"Private key provided cannot be read by paramiko. "
"Ensure key provided is valid for one of the following "
"key formats: RSA, ECDSA, or Ed25519."
)

def exec_ssh_client_command(
Expand Down Expand Up @@ -599,6 +625,15 @@ def _parse_extras(self, conn: Any) -> None:
self.log.warning("No Host Key Verification. This won't protect against Man-In-The-Middle attacks")
self.known_hosts = "none"
elif host_key is not None:
host_key = host_key.strip()
host_key_parts = host_key.split()
if host_key_parts and host_key_parts[0] == "ssh-dss":
raise ValueError(
"DSA/DSS host keys are not supported. Paramiko 4.0 removed DSS support; "
"use an RSA, ECDSA, or Ed25519 host key and update the connection `host_key`."
)
if len(host_key_parts) >= 2:
host_key = " ".join(host_key_parts[:2])
self.known_hosts = f"{conn.host} {host_key}".encode()

async def _get_conn(self):
Expand Down
Loading
Loading