diff --git a/libavformat/mpegenc.c b/libavformat/mpegenc.c
index 9bd0a555d4..111a9f3a46 100644
--- a/libavformat/mpegenc.c
+++ b/libavformat/mpegenc.c
@@ -976,9 +976,9 @@ static int remove_decoded_packets(AVFormatContext *ctx, int64_t scr)
         PacketDesc *pkt_desc;
 
         while ((pkt_desc = stream->predecode_packet) &&
+               pkt_desc != stream->premux_packet &&
                scr > pkt_desc->dts) { // FIXME: > vs >=
-            if (stream->buffer_index < pkt_desc->size ||
-                stream->predecode_packet == stream->premux_packet) {
+            if (stream->buffer_index < pkt_desc->size) {
                 av_log(ctx, AV_LOG_ERROR,
                        "buffer underflow st=%d bufi=%d size=%d\n",
                        i, stream->buffer_index, pkt_desc->size);