diff --git a/dvc/scm/git/backend/dulwich/asyncssh_vendor.py b/dvc/scm/git/backend/dulwich/asyncssh_vendor.py
index 3264c5c39..0fc529797 100644
--- a/dvc/scm/git/backend/dulwich/asyncssh_vendor.py
+++ b/dvc/scm/git/backend/dulwich/asyncssh_vendor.py
@@ -79,10 +79,11 @@ class AsyncSSHVendor(BaseAsyncObject, SSHVendor):
 
         conn = await asyncssh.connect(
             host,
-            port=port,
+            port=port if port is not None else (),
             username=username,
             password=password,
-            client_keys=[key_filename] if key_filename else [],
+            client_keys=[key_filename] if key_filename else (),
+            ignore_encrypted=not key_filename,
             known_hosts=None,
             encoding=None,
         )