fix qtrle encoding when previous frame linesize differs, fix #998

Originally committed as revision 18908 to svn://svn.ffmpeg.org/ffmpeg/trunk

libavcodec/qtrleenc.c

@@ -126,8 +126,10 @@ static void qtrle_encode_line(QtrleEncContext *s, AVFrame *p, int line, uint8_t
     int temp_cost;
     int j;
 
-    uint8_t *this_line = p->               data[0] + line*p->linesize[0] + (width - 1)*s->pixel_size;
-    uint8_t *prev_line = s->previous_frame.data[0] + line*p->linesize[0] + (width - 1)*s->pixel_size;
+    uint8_t *this_line = p->               data[0] + line*p->               linesize[0] +
+        (width - 1)*s->pixel_size;
+    uint8_t *prev_line = s->previous_frame.data[0] + line*s->previous_frame.linesize[0] +
+        (width - 1)*s->pixel_size;
 
     s->length_table[width] = 0;
     skipcount = 0;
@@ -239,16 +241,17 @@ static int encode_frame(QtrleEncContext *s, AVFrame *p, uint8_t *buf)
     uint8_t *orig_buf = buf;
 
     if (!s->frame.key_frame) {
+        unsigned line_size = s->avctx->width * s->pixel_size;
         for (start_line = 0; start_line < s->avctx->height; start_line++)
             if (memcmp(p->data[0] + start_line*p->linesize[0],
-                       s->previous_frame.data[0] + start_line*p->linesize[0],
-                       p->linesize[0]))
+                       s->previous_frame.data[0] + start_line*s->previous_frame.linesize[0],
+                       line_size))
                 break;
 
         for (end_line=s->avctx->height; end_line > start_line; end_line--)
             if (memcmp(p->data[0] + (end_line - 1)*p->linesize[0],
-                       s->previous_frame.data[0] + (end_line - 1)*p->linesize[0],
-                       p->linesize[0]))
+                       s->previous_frame.data[0] + (end_line - 1)*s->previous_frame.linesize[0],
+                       line_size))
                 break;
     }