diff --git a/provisioner/ansible/provisioner.go b/provisioner/ansible/provisioner.go
index 50e0702cb..378fd7beb 100644
--- a/provisioner/ansible/provisioner.go
+++ b/provisioner/ansible/provisioner.go
@@ -241,6 +241,7 @@ func (p *Provisioner) Provision(ctx context.Context, ui packer.Ui, comm packer.C
 
 			return nil, nil
 		},
+		IsUserAuthority: func(k ssh.PublicKey) bool { return true },
 	}
 
 	config := &ssh.ServerConfig{
diff --git a/provisioner/inspec/provisioner.go b/provisioner/inspec/provisioner.go
index 15c975c20..c90b66d49 100644
--- a/provisioner/inspec/provisioner.go
+++ b/provisioner/inspec/provisioner.go
@@ -234,6 +234,7 @@ func (p *Provisioner) Provision(ctx context.Context, ui packer.Ui, comm packer.C
 
 			return nil, nil
 		},
+		IsUserAuthority: func(k ssh.PublicKey) bool { return true },
 	}
 
 	config := &ssh.ServerConfig{