diff --git a/libavcodec/pcm_rechunk_bsf.c b/libavcodec/pcm_rechunk_bsf.c
index 032f914916..28b5722ac9 100644
--- a/libavcodec/pcm_rechunk_bsf.c
+++ b/libavcodec/pcm_rechunk_bsf.c
@@ -139,6 +139,7 @@ static int rechunk_filter(AVBSFContext *ctx, AVPacket *pkt)
                     av_packet_move_ref(pkt, s->out_pkt);
                     return send_packet(s, nb_samples, pkt);
                 }
+                av_assert0(!s->in_pkt->size);
             } else if (s->in_pkt->size > data_size) {
                 ret = av_packet_ref(pkt, s->in_pkt);
                 if (ret < 0)