diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c
index e5b48a331c58..c2ad58b19e7b 100644
--- a/drivers/thunderbolt/switch.c
+++ b/drivers/thunderbolt/switch.c
@@ -347,7 +347,7 @@ static int nvm_write(void *priv, unsigned int offset, void *val, size_t bytes)
 	return ret;
 }
 
-static int tb_switch_nvm_add(struct tb_switch *sw)
+static int tb_switch_nvm_init(struct tb_switch *sw)
 {
 	struct tb_nvm *nvm;
 	int ret;
@@ -365,6 +365,26 @@ static int tb_switch_nvm_add(struct tb_switch *sw)
 	if (ret)
 		goto err_nvm;
 
+	sw->nvm = nvm;
+	return 0;
+
+err_nvm:
+	tb_sw_dbg(sw, "NVM upgrade disabled\n");
+	sw->no_nvm_upgrade = true;
+	if (!IS_ERR(nvm))
+		tb_nvm_free(nvm);
+
+	return ret;
+}
+
+static int tb_switch_nvm_add(struct tb_switch *sw)
+{
+	struct tb_nvm *nvm = sw->nvm;
+	int ret;
+
+	if (!nvm)
+		return 0;
+
 	/*
 	 * If the switch is in safe-mode the only accessible portion of
 	 * the NVM is the non-active one where userspace is expected to
@@ -383,14 +403,12 @@ static int tb_switch_nvm_add(struct tb_switch *sw)
 			goto err_nvm;
 	}
 
-	sw->nvm = nvm;
 	return 0;
 
 err_nvm:
 	tb_sw_dbg(sw, "NVM upgrade disabled\n");
 	sw->no_nvm_upgrade = true;
-	if (!IS_ERR(nvm))
-		tb_nvm_free(nvm);
+	tb_nvm_free(nvm);
 
 	return ret;
 }
@@ -3311,6 +3329,10 @@ int tb_switch_add(struct tb_switch *sw)
 		return ret;
 	}
 
+	ret = tb_switch_nvm_init(sw);
+	if (ret)
+		return ret;
+
 	if (!sw->safe_mode) {
 		tb_switch_credits_init(sw);