diff --git a/drivers/media/platform/rockchip/isp/capture.h b/drivers/media/platform/rockchip/isp/capture.h
index b76c075d90fc..bbdcf1e6df6e 100644
--- a/drivers/media/platform/rockchip/isp/capture.h
+++ b/drivers/media/platform/rockchip/isp/capture.h
@@ -287,6 +287,7 @@ struct rkisp_stream {
 	struct frame_debug_info dbg;
 	int conn_id;
 	u32 memory;
+	u32 skip_frame;
 	union {
 		struct rkisp_stream_sp sp;
 		struct rkisp_stream_mp mp;
diff --git a/drivers/media/platform/rockchip/isp/capture_v32.c b/drivers/media/platform/rockchip/isp/capture_v32.c
index a87ef1a878c5..05b5356841d2 100644
--- a/drivers/media/platform/rockchip/isp/capture_v32.c
+++ b/drivers/media/platform/rockchip/isp/capture_v32.c
@@ -1399,10 +1399,12 @@ static int mi_frame_end(struct rkisp_stream *stream, u32 state)
 		struct rkisp_stream *vir = &dev->cap_dev.stream[RKISP_STREAM_VIR];
 		u64 ns = 0;
 
-		if (dev->skip_frame) {
+		if (dev->skip_frame || stream->skip_frame) {
 			spin_lock_irqsave(&stream->vbq_lock, lock_flags);
 			list_add_tail(&buf->queue, &stream->buf_queue);
 			spin_unlock_irqrestore(&stream->vbq_lock, lock_flags);
+			if (stream->skip_frame)
+				stream->skip_frame--;
 			goto end;
 		}
 
@@ -1540,6 +1542,7 @@ static int rkisp_start(struct rkisp_stream *stream)
 		stream->ops->enable_mi(stream);
 
 	stream->streaming = true;
+	stream->skip_frame = 0;
 	return 0;
 }
 
diff --git a/drivers/media/platform/rockchip/isp/isp_rockit.c b/drivers/media/platform/rockchip/isp/isp_rockit.c
index b969592a5a2e..0ba72aad0c73 100644
--- a/drivers/media/platform/rockchip/isp/isp_rockit.c
+++ b/drivers/media/platform/rockchip/isp/isp_rockit.c
@@ -591,9 +591,11 @@ bool rkisp_rockit_ctrl_fps(struct rkisp_stream *stream)
 			*fps_cnt = fps_in - dst_fps;
 		*fps_cnt += dst_fps;
 
-		if (*fps_cnt < fps_in)
+		if (*fps_cnt < fps_in) {
 			*is_discard = true;
-		else {
+			if (stream->next_buf || !list_empty(&stream->buf_queue))
+				stream->skip_frame = 1;
+		} else {
 			*fps_cnt -= fps_in;
 			*is_discard = false;
 			++cur_fps[stream->id];