diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c
--- a/sys/geom/eli/g_eli.c
+++ b/sys/geom/eli/g_eli.c
@@ -1325,6 +1325,7 @@
 	u_char key[G_ELI_USERKEYLEN], mkey[G_ELI_DATAIVKEYLEN];
 	u_int i, nkey, nkeyfiles, tries, showpass;
 	int error;
+	bool skippassphrase;
         struct keybuf *keybuf;
 
 	g_trace(G_T_TOPOLOGY, "%s(%s, %s)", __func__, mp->name, pp->name);
@@ -1408,8 +1409,14 @@
                         return (NULL);
                 }
 
+		skippassphrase = false;
+		if (i == 0 && nkeyfiles > 0 && md.md_iterations != -1) {
+			tries += 1;
+			skippassphrase = true;
+		}
+
                 /* Ask for the passphrase if defined. */
-                if (md.md_iterations >= 0) {
+                if (md.md_iterations >= 0 && !skippassphrase) {
                         /* Try first with cached passphrase. */
                         if (i == 0) {
                                 if (!g_eli_boot_passcache)
@@ -1431,13 +1438,13 @@
                 /*
                  * Prepare Derived-Key from the user passphrase.
                  */
-                if (md.md_iterations == 0) {
+                if (md.md_iterations == 0 && !skippassphrase) {
                         g_eli_crypto_hmac_update(&ctx, md.md_salt,
                             sizeof(md.md_salt));
                         g_eli_crypto_hmac_update(&ctx, passphrase,
                             strlen(passphrase));
                         explicit_bzero(passphrase, sizeof(passphrase));
-                } else if (md.md_iterations > 0) {
+                } else if (md.md_iterations > 0 && !skippassphrase) {
                         u_char dkey[G_ELI_USERKEYLEN];
 
                         pkcs5v2_genkey(dkey, sizeof(dkey), md.md_salt,
@@ -1462,7 +1469,7 @@
                                 g_eli_keyfiles_clear(pp->name);
                                 return (NULL);
                         }
-                        if (i > 0) {
+                        if (i > 0 && !skippassphrase) {
                                 G_ELI_DEBUG(0,
                                     "Wrong key for %s. Tries left: %u.",
                                     pp->name, tries - i);