From: bcoudurier <bcoudurier@9553f0bf-9b14-0410-a0b8-cfaf0461ba5b>
Date: Sat, 23 May 2009 05:55:29 +0000 (+0000)
Subject: fix qtrle encoding when previous frame linesize differs, fix #998
X-Git-Url: https://rtime.felk.cvut.cz/gitweb/frescor/ffmpeg.git/commitdiff_plain/8b25492b6176859557142bf2be6a01b8f85792df

fix qtrle encoding when previous frame linesize differs, fix #998

git-svn-id: file:///var/local/repositories/ffmpeg/trunk@18908 9553f0bf-9b14-0410-a0b8-cfaf0461ba5b
---

diff --git a/libavcodec/qtrleenc.c b/libavcodec/qtrleenc.c
index 3f0cc8735..b302964b9 100644
--- a/libavcodec/qtrleenc.c
+++ b/libavcodec/qtrleenc.c
@@ -126,8 +126,10 @@ static void qtrle_encode_line(QtrleEncContext *s, AVFrame *p, int line, uint8_t
     int temp_cost;
     int j;
 
-    uint8_t *this_line = p->               data[0] + line*p->linesize[0] + (width - 1)*s->pixel_size;
-    uint8_t *prev_line = s->previous_frame.data[0] + line*p->linesize[0] + (width - 1)*s->pixel_size;
+    uint8_t *this_line = p->               data[0] + line*p->               linesize[0] +
+        (width - 1)*s->pixel_size;
+    uint8_t *prev_line = s->previous_frame.data[0] + line*s->previous_frame.linesize[0] +
+        (width - 1)*s->pixel_size;
 
     s->length_table[width] = 0;
     skipcount = 0;
@@ -239,16 +241,17 @@ static int encode_frame(QtrleEncContext *s, AVFrame *p, uint8_t *buf)
     uint8_t *orig_buf = buf;
 
     if (!s->frame.key_frame) {
+        unsigned line_size = s->avctx->width * s->pixel_size;
         for (start_line = 0; start_line < s->avctx->height; start_line++)
             if (memcmp(p->data[0] + start_line*p->linesize[0],
-                       s->previous_frame.data[0] + start_line*p->linesize[0],
-                       p->linesize[0]))
+                       s->previous_frame.data[0] + start_line*s->previous_frame.linesize[0],
+                       line_size))
                 break;
 
         for (end_line=s->avctx->height; end_line > start_line; end_line--)
             if (memcmp(p->data[0] + (end_line - 1)*p->linesize[0],
-                       s->previous_frame.data[0] + (end_line - 1)*p->linesize[0],
-                       p->linesize[0]))
+                       s->previous_frame.data[0] + (end_line - 1)*s->previous_frame.linesize[0],
+                       line_size))
                 break;
     }