diff --git a/drivers/phy/tegra/Kconfig b/drivers/phy/tegra/Kconfig
index 342fb736da4b..f0734415fc94 100644
--- a/drivers/phy/tegra/Kconfig
+++ b/drivers/phy/tegra/Kconfig
@@ -3,7 +3,7 @@ config PHY_TEGRA_XUSB
 	tristate "NVIDIA Tegra XUSB pad controller driver"
 	depends on ARCH_TEGRA && USB_SUPPORT
 	select USB_COMMON
-	select USB_CONN_GPIO
+	select USB_CONN_GPIO if GPIOLIB
 	select USB_PHY
 	help
 	  Choose this option if you have an NVIDIA Tegra SoC.