--- a/src/drivers/driver_nl80211.c
+++ b/src/drivers/driver_nl80211.c
@@ -8480,12 +8480,7 @@ static int wpa_driver_nl80211_set_mode(s
 			/* Try to set the mode again while the interface is
 			 * down */
 			ret = nl80211_set_mode(drv, drv->ifindex, nlmode);
-			if (ret == -EACCES)
-				break;
-			res = i802_set_iface_flags(bss, 1);
-			if (res && !ret)
-				ret = -1;
-			else if (ret != -EBUSY)
+			if (ret != -EBUSY)
 				break;
 		} else
 			wpa_printf(MSG_DEBUG, "nl80211: Failed to set "
@@ -8498,6 +8493,8 @@ static int wpa_driver_nl80211_set_mode(s
 			   "interface is down");
 		drv->nlmode = nlmode;
 		drv->ignore_if_down_event = 1;
+		if (i802_set_iface_flags(bss, 1))
+			ret = -1;
 	}
 
 done: