diff --git a/sound/soc/sof/sof-audio.c b/sound/soc/sof/sof-audio.c
index f17d405a9da9..d242a3076ed0 100644
--- a/sound/soc/sof/sof-audio.c
+++ b/sound/soc/sof/sof-audio.c
@@ -357,19 +357,16 @@ static int sof_free_widgets_in_path(struct snd_sof_dev *sdev, struct snd_soc_dap
 	int err;
 	int ret = 0;
 
-	/* free all widgets even in case of error to keep use counts balanced */
-	snd_soc_dapm_widget_for_each_sink_path(widget, p) {
-		if (!p->walking && p->sink->dobj.private && widget->dobj.private) {
-			p->walking = true;
-			if (WIDGET_IS_AIF_OR_DAI(widget->id)) {
-				err = sof_widget_free(sdev, widget->dobj.private);
-				if (err < 0)
-					ret = err;
-			}
+	if (widget->dobj.private) {
+		err = sof_widget_free(sdev, widget->dobj.private);
+		if (err < 0)
+			ret = err;
+	}
 
-			err = sof_widget_free(sdev, p->sink->dobj.private);
-			if (err < 0)
-				ret = err;
+	/* free all widgets in the sink paths even in case of error to keep use counts balanced */
+	snd_soc_dapm_widget_for_each_sink_path(widget, p) {
+		if (!p->walking) {
+			p->walking = true;
 
 			err = sof_free_widgets_in_path(sdev, p->sink, dir);
 			if (err < 0)
@@ -392,32 +389,23 @@ static int sof_set_up_widgets_in_path(struct snd_sof_dev *sdev, struct snd_soc_d
 	struct snd_soc_dapm_path *p;
 	int ret;
 
-	snd_soc_dapm_widget_for_each_sink_path(widget, p) {
-		if (!p->walking && p->sink->dobj.private && widget->dobj.private) {
-			p->walking = true;
-			if (WIDGET_IS_AIF_OR_DAI(widget->id)) {
-				ret = sof_widget_setup(sdev, widget->dobj.private);
-				if (ret < 0)
-					goto out;
-			}
+	if (widget->dobj.private) {
+		ret = sof_widget_setup(sdev, widget->dobj.private);
+		if (ret < 0)
+			return ret;
+	}
 
-			ret = sof_widget_setup(sdev, p->sink->dobj.private);
-			if (ret < 0) {
-				if (WIDGET_IS_AIF_OR_DAI(widget->id))
-					sof_widget_free(sdev, widget->dobj.private);
-				goto out;
-			}
+	snd_soc_dapm_widget_for_each_sink_path(widget, p) {
+		if (!p->walking) {
+			p->walking = true;
 
 			ret = sof_set_up_widgets_in_path(sdev, p->sink, dir);
-			if (ret < 0) {
-				if (WIDGET_IS_AIF_OR_DAI(widget->id))
-					sof_widget_free(sdev, widget->dobj.private);
-				sof_widget_free(sdev, p->sink->dobj.private);
-			}
-out:
 			p->walking = false;
-			if (ret < 0)
+			if (ret < 0) {
+				if (widget->dobj.private)
+					sof_widget_free(sdev, widget->dobj.private);
 				return ret;
+			}
 		}
 	}