diff --git a/arch/xtensa/kernel/align.S b/arch/xtensa/kernel/align.S
index ee97edce2300..f2d0d831ce43 100644
--- a/arch/xtensa/kernel/align.S
+++ b/arch/xtensa/kernel/align.S
@@ -21,6 +21,9 @@
 #include <asm/asm-offsets.h>
 #include <asm/asmmacro.h>
 #include <asm/processor.h>
+#ifdef CONFIG_MMU
+#include <asm/asm-uaccess.h>
+#endif
 
 #if XCHAL_UNALIGNED_LOAD_EXCEPTION || defined CONFIG_XTENSA_LOAD_STORE
 #define LOAD_EXCEPTION_HANDLER
@@ -185,8 +188,11 @@ ENTRY(fast_load_store)
 
 #ifdef CONFIG_MMU
 	/* l32e can't be used here even when it's available. */
-	/* TODO access_ok(a3) could be used here */
-	j	.Linvalid_instruction
+	rsr	a6, ps
+	bbci.l	a6, PS_UM_BIT, 1f # kernel mode
+	movi	a5, 8
+	access_ok a3, a5, a6, a2, .Linvalid_instruction
+1:
 #endif
 	l32i	a5, a3, 0
 	l32i	a6, a3, 4