annotate msrledec.c @ 12530:63edd10ad4bc libavcodec tip

Try to fix crashes introduced by r25218 r25218 made assumptions about the existence of past reference frames that weren't necessarily true.
author darkshikari
date Tue, 28 Sep 2010 09:06:22 +0000
parents 7dd2a45249a9
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
1 /*
9684
c1ed557cd3b8 cosmetics: Fix some typos.
diego
parents: 9657
diff changeset
2 * Microsoft RLE decoder
7885
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
3 * Copyright (C) 2008 Konstantin Shishkov
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
4 *
3947
c8c591fe26f8 Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents: 3800
diff changeset
5 * This file is part of FFmpeg.
c8c591fe26f8 Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents: 3800
diff changeset
6 *
c8c591fe26f8 Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents: 3800
diff changeset
7 * FFmpeg is free software; you can redistribute it and/or
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
8 * modify it under the terms of the GNU Lesser General Public
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
9 * License as published by the Free Software Foundation; either
3947
c8c591fe26f8 Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents: 3800
diff changeset
10 * version 2.1 of the License, or (at your option) any later version.
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
11 *
3947
c8c591fe26f8 Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents: 3800
diff changeset
12 * FFmpeg is distributed in the hope that it will be useful,
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
15 * Lesser General Public License for more details.
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
16 *
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
17 * You should have received a copy of the GNU Lesser General Public
3947
c8c591fe26f8 Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents: 3800
diff changeset
18 * License along with FFmpeg; if not, write to the Free Software
3036
0b546eab515d Update licensing information: The FSF changed postal address.
diego
parents: 2979
diff changeset
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
20 */
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
21
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
22 /**
11644
7dd2a45249a9 Remove explicit filename from Doxygen @file commands.
diego
parents: 11317
diff changeset
23 * @file
9684
c1ed557cd3b8 cosmetics: Fix some typos.
diego
parents: 9657
diff changeset
24 * MS RLE decoder based on decoder by Mike Melanson and my own for TSCC
7885
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
25 * For more information about the MS RLE format, visit:
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
26 * http://www.multimedia.cx/msrle.txt
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
27 */
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
28
8573
2acf0ae7b041 Fix build: Add intreadwrite.h and bswap.h #includes where necessary.
diego
parents: 8266
diff changeset
29 #include "libavutil/intreadwrite.h"
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
30 #include "avcodec.h"
11317
35d05a2f4361 Include msrledec.h: It contains the prototype for ff_msrle_decode().
cehoyos
parents: 10334
diff changeset
31 #include "msrledec.h"
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
32
7885
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
33 #define FETCH_NEXT_STREAM_BYTE() \
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
34 if (stream_ptr >= data_size) \
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
35 { \
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
36 av_log(avctx, AV_LOG_ERROR, " MS RLE: stream ptr just went out of bounds (1)\n"); \
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
37 return -1; \
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
38 } \
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
39 stream_byte = data[stream_ptr++];
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
40
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
41 static int msrle_decode_pal4(AVCodecContext *avctx, AVPicture *pic,
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
42 const uint8_t *data, int data_size)
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
43 {
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
44 int stream_ptr = 0;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
45 unsigned char rle_code;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
46 unsigned char extra_byte, odd_pixel;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
47 unsigned char stream_byte;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
48 int pixel_ptr = 0;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
49 int row_dec = pic->linesize[0];
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
50 int row_ptr = (avctx->height - 1) * row_dec;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
51 int frame_size = row_dec * avctx->height;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
52 int i;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
53
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
54 while (row_ptr >= 0) {
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
55 FETCH_NEXT_STREAM_BYTE();
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
56 rle_code = stream_byte;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
57 if (rle_code == 0) {
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
58 /* fetch the next byte to see how to handle escape code */
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
59 FETCH_NEXT_STREAM_BYTE();
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
60 if (stream_byte == 0) {
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
61 /* line is done, goto the next one */
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
62 row_ptr -= row_dec;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
63 pixel_ptr = 0;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
64 } else if (stream_byte == 1) {
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
65 /* decode is done */
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
66 return 0;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
67 } else if (stream_byte == 2) {
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
68 /* reposition frame decode coordinates */
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
69 FETCH_NEXT_STREAM_BYTE();
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
70 pixel_ptr += stream_byte;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
71 FETCH_NEXT_STREAM_BYTE();
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
72 row_ptr -= stream_byte * row_dec;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
73 } else {
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
74 // copy pixels from encoded stream
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
75 odd_pixel = stream_byte & 1;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
76 rle_code = (stream_byte + 1) / 2;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
77 extra_byte = rle_code & 0x01;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
78 if ((row_ptr + pixel_ptr + stream_byte > frame_size) ||
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
79 (row_ptr < 0)) {
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
80 av_log(avctx, AV_LOG_ERROR, " MS RLE: frame ptr just went out of bounds (1)\n");
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
81 return -1;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
82 }
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
83
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
84 for (i = 0; i < rle_code; i++) {
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
85 if (pixel_ptr >= avctx->width)
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
86 break;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
87 FETCH_NEXT_STREAM_BYTE();
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
88 pic->data[0][row_ptr + pixel_ptr] = stream_byte >> 4;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
89 pixel_ptr++;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
90 if (i + 1 == rle_code && odd_pixel)
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
91 break;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
92 if (pixel_ptr >= avctx->width)
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
93 break;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
94 pic->data[0][row_ptr + pixel_ptr] = stream_byte & 0x0F;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
95 pixel_ptr++;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
96 }
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
97
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
98 // if the RLE code is odd, skip a byte in the stream
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
99 if (extra_byte)
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
100 stream_ptr++;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
101 }
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
102 } else {
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
103 // decode a run of data
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
104 if ((row_ptr + pixel_ptr + stream_byte > frame_size) ||
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
105 (row_ptr < 0)) {
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
106 av_log(avctx, AV_LOG_ERROR, " MS RLE: frame ptr just went out of bounds (1)\n");
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
107 return -1;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
108 }
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
109 FETCH_NEXT_STREAM_BYTE();
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
110 for (i = 0; i < rle_code; i++) {
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
111 if (pixel_ptr >= avctx->width)
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
112 break;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
113 if ((i & 1) == 0)
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
114 pic->data[0][row_ptr + pixel_ptr] = stream_byte >> 4;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
115 else
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
116 pic->data[0][row_ptr + pixel_ptr] = stream_byte & 0x0F;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
117 pixel_ptr++;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
118 }
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
119 }
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
120 }
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
121
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
122 /* one last sanity check on the way out */
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
123 if (stream_ptr < data_size) {
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
124 av_log(avctx, AV_LOG_ERROR, " MS RLE: ended frame decode with bytes left over (%d < %d)\n",
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
125 stream_ptr, data_size);
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
126 return -1;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
127 }
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
128
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
129 return 0;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
130 }
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
131
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
132
7885
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
133 static int msrle_decode_8_16_24_32(AVCodecContext *avctx, AVPicture *pic, int depth,
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
134 const uint8_t *data, int srcsize)
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
135 {
7885
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
136 uint8_t *output, *output_end;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
137 const uint8_t* src = data;
9657
8a65ae8929fb Fix off-by-one error in MS RLE decoder which may result into writing past
kostya
parents: 8803
diff changeset
138 int p1, p2, line=avctx->height - 1, pos=0, i;
8803
096ab16191a9 Add av_uninit macro to variables to avoid false positive warnings:
diego
parents: 8718
diff changeset
139 uint16_t av_uninit(pix16);
096ab16191a9 Add av_uninit macro to variables to avoid false positive warnings:
diego
parents: 8718
diff changeset
140 uint32_t av_uninit(pix32);
2967
ef2149182f1c COSMETICS: Remove all trailing whitespace.
diego
parents: 2510
diff changeset
141
7885
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
142 output = pic->data[0] + (avctx->height - 1) * pic->linesize[0];
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
143 output_end = pic->data[0] + (avctx->height) * pic->linesize[0];
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
144 while(src < data + srcsize) {
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
145 p1 = *src++;
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
146 if(p1 == 0) { //Escape code
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
147 p2 = *src++;
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
148 if(p2 == 0) { //End-of-line
7885
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
149 output = pic->data[0] + (--line) * pic->linesize[0];
9719
e7032c44d4a7 do not print false error if eof follows eol
bcoudurier
parents: 9684
diff changeset
150 if (line < 0 && !(src+1 < data + srcsize && AV_RB16(src) == 1)) {
8265
bb310197d59f Print out some errors found during MS RLE decoding
kostya
parents: 7885
diff changeset
151 av_log(avctx, AV_LOG_ERROR, "Next line is beyond picture bounds\n");
2455
d74d342cabb9 Check pointers before writing to memory
rtognimp
parents: 2453
diff changeset
152 return -1;
8265
bb310197d59f Print out some errors found during MS RLE decoding
kostya
parents: 7885
diff changeset
153 }
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
154 pos = 0;
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
155 continue;
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
156 } else if(p2 == 1) { //End-of-picture
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
157 return 0;
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
158 } else if(p2 == 2) { //Skip
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
159 p1 = *src++;
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
160 p2 = *src++;
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
161 line -= p2;
8265
bb310197d59f Print out some errors found during MS RLE decoding
kostya
parents: 7885
diff changeset
162 if (line < 0){
bb310197d59f Print out some errors found during MS RLE decoding
kostya
parents: 7885
diff changeset
163 av_log(avctx, AV_LOG_ERROR, "Skip beyond picture bounds\n");
2455
d74d342cabb9 Check pointers before writing to memory
rtognimp
parents: 2453
diff changeset
164 return -1;
8265
bb310197d59f Print out some errors found during MS RLE decoding
kostya
parents: 7885
diff changeset
165 }
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
166 pos += p1;
7885
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
167 output = pic->data[0] + line * pic->linesize[0] + pos * (depth >> 3);
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
168 continue;
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
169 }
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
170 // Copy data
10334
71cf44ecaa70 Make MS RLE decoder produce both bottom-up and top-down pictures
kostya
parents: 9719
diff changeset
171 if ((pic->linesize[0] > 0 && output + p2 * (depth >> 3) > output_end)
71cf44ecaa70 Make MS RLE decoder produce both bottom-up and top-down pictures
kostya
parents: 9719
diff changeset
172 ||(pic->linesize[0] < 0 && output + p2 * (depth >> 3) < output_end)) {
7885
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
173 src += p2 * (depth >> 3);
2455
d74d342cabb9 Check pointers before writing to memory
rtognimp
parents: 2453
diff changeset
174 continue;
d74d342cabb9 Check pointers before writing to memory
rtognimp
parents: 2453
diff changeset
175 }
7885
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
176 if ((depth == 8) || (depth == 24)) {
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
177 for(i = 0; i < p2 * (depth >> 3); i++) {
3121
99cbff5f8038 make TSCC endian-safe, PPC testing courtesy of Diego B.
melanson
parents: 3036
diff changeset
178 *output++ = *src++;
99cbff5f8038 make TSCC endian-safe, PPC testing courtesy of Diego B.
melanson
parents: 3036
diff changeset
179 }
99cbff5f8038 make TSCC endian-safe, PPC testing courtesy of Diego B.
melanson
parents: 3036
diff changeset
180 // RLE8 copy is actually padded - and runs are not!
7885
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
181 if(depth == 8 && (p2 & 1)) {
3121
99cbff5f8038 make TSCC endian-safe, PPC testing courtesy of Diego B.
melanson
parents: 3036
diff changeset
182 src++;
99cbff5f8038 make TSCC endian-safe, PPC testing courtesy of Diego B.
melanson
parents: 3036
diff changeset
183 }
7885
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
184 } else if (depth == 16) {
3121
99cbff5f8038 make TSCC endian-safe, PPC testing courtesy of Diego B.
melanson
parents: 3036
diff changeset
185 for(i = 0; i < p2; i++) {
4364
05e932ddaaa9 rename BE/LE_8/16/32 to AV_RL/B_8/16/32
alex
parents: 3947
diff changeset
186 pix16 = AV_RL16(src);
3121
99cbff5f8038 make TSCC endian-safe, PPC testing courtesy of Diego B.
melanson
parents: 3036
diff changeset
187 src += 2;
99cbff5f8038 make TSCC endian-safe, PPC testing courtesy of Diego B.
melanson
parents: 3036
diff changeset
188 *(uint16_t*)output = pix16;
99cbff5f8038 make TSCC endian-safe, PPC testing courtesy of Diego B.
melanson
parents: 3036
diff changeset
189 output += 2;
99cbff5f8038 make TSCC endian-safe, PPC testing courtesy of Diego B.
melanson
parents: 3036
diff changeset
190 }
7885
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
191 } else if (depth == 32) {
3121
99cbff5f8038 make TSCC endian-safe, PPC testing courtesy of Diego B.
melanson
parents: 3036
diff changeset
192 for(i = 0; i < p2; i++) {
4364
05e932ddaaa9 rename BE/LE_8/16/32 to AV_RL/B_8/16/32
alex
parents: 3947
diff changeset
193 pix32 = AV_RL32(src);
3121
99cbff5f8038 make TSCC endian-safe, PPC testing courtesy of Diego B.
melanson
parents: 3036
diff changeset
194 src += 4;
99cbff5f8038 make TSCC endian-safe, PPC testing courtesy of Diego B.
melanson
parents: 3036
diff changeset
195 *(uint32_t*)output = pix32;
99cbff5f8038 make TSCC endian-safe, PPC testing courtesy of Diego B.
melanson
parents: 3036
diff changeset
196 output += 4;
99cbff5f8038 make TSCC endian-safe, PPC testing courtesy of Diego B.
melanson
parents: 3036
diff changeset
197 }
2979
bfabfdf9ce55 COSMETICS: tabs --> spaces, some prettyprinting
diego
parents: 2967
diff changeset
198 }
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
199 pos += p2;
9684
c1ed557cd3b8 cosmetics: Fix some typos.
diego
parents: 9657
diff changeset
200 } else { //run of pixels
8266
ce2bb0ae8587 Make decoding run of pixels in MS RLE saner.
kostya
parents: 8265
diff changeset
201 uint8_t pix[3]; //original pixel
7885
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
202 switch(depth){
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
203 case 8: pix[0] = *src++;
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
204 break;
4364
05e932ddaaa9 rename BE/LE_8/16/32 to AV_RL/B_8/16/32
alex
parents: 3947
diff changeset
205 case 16: pix16 = AV_RL16(src);
3121
99cbff5f8038 make TSCC endian-safe, PPC testing courtesy of Diego B.
melanson
parents: 3036
diff changeset
206 src += 2;
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
207 break;
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
208 case 24: pix[0] = *src++;
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
209 pix[1] = *src++;
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
210 pix[2] = *src++;
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
211 break;
4364
05e932ddaaa9 rename BE/LE_8/16/32 to AV_RL/B_8/16/32
alex
parents: 3947
diff changeset
212 case 32: pix32 = AV_RL32(src);
3121
99cbff5f8038 make TSCC endian-safe, PPC testing courtesy of Diego B.
melanson
parents: 3036
diff changeset
213 src += 4;
2481
0803adcb3ec3 Add 32bit RGB support
rtognimp
parents: 2455
diff changeset
214 break;
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
215 }
10334
71cf44ecaa70 Make MS RLE decoder produce both bottom-up and top-down pictures
kostya
parents: 9719
diff changeset
216 if ((pic->linesize[0] > 0 && output + p1 * (depth >> 3) > output_end)
71cf44ecaa70 Make MS RLE decoder produce both bottom-up and top-down pictures
kostya
parents: 9719
diff changeset
217 ||(pic->linesize[0] < 0 && output + p1 * (depth >> 3) < output_end))
2455
d74d342cabb9 Check pointers before writing to memory
rtognimp
parents: 2453
diff changeset
218 continue;
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
219 for(i = 0; i < p1; i++) {
7885
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
220 switch(depth){
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
221 case 8: *output++ = pix[0];
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
222 break;
3121
99cbff5f8038 make TSCC endian-safe, PPC testing courtesy of Diego B.
melanson
parents: 3036
diff changeset
223 case 16: *(uint16_t*)output = pix16;
99cbff5f8038 make TSCC endian-safe, PPC testing courtesy of Diego B.
melanson
parents: 3036
diff changeset
224 output += 2;
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
225 break;
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
226 case 24: *output++ = pix[0];
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
227 *output++ = pix[1];
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
228 *output++ = pix[2];
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
229 break;
3121
99cbff5f8038 make TSCC endian-safe, PPC testing courtesy of Diego B.
melanson
parents: 3036
diff changeset
230 case 32: *(uint32_t*)output = pix32;
99cbff5f8038 make TSCC endian-safe, PPC testing courtesy of Diego B.
melanson
parents: 3036
diff changeset
231 output += 4;
2481
0803adcb3ec3 Add 32bit RGB support
rtognimp
parents: 2455
diff changeset
232 break;
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
233 }
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
234 }
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
235 pos += p1;
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
236 }
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
237 }
2967
ef2149182f1c COSMETICS: Remove all trailing whitespace.
diego
parents: 2510
diff changeset
238
9684
c1ed557cd3b8 cosmetics: Fix some typos.
diego
parents: 9657
diff changeset
239 av_log(avctx, AV_LOG_WARNING, "MS RLE warning: no end-of-picture code\n");
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
240 return 0;
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
241 }
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
242
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
243
7885
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
244 int ff_msrle_decode(AVCodecContext *avctx, AVPicture *pic, int depth,
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
245 const uint8_t* data, int data_size)
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
246 {
7885
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
247 switch(depth){
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
248 case 4:
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
249 return msrle_decode_pal4(avctx, pic, data, data_size);
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
250 case 8:
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
251 case 16:
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
252 case 24:
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
253 case 32:
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
254 return msrle_decode_8_16_24_32(avctx, pic, depth, data, data_size);
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
255 default:
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
256 av_log(avctx, AV_LOG_ERROR, "Unknown depth %d\n", depth);
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
257 return -1;
f874e1d5cf07 Factorize out code used for MS RLE format decoding in different decoders.
kostya
parents: 7823
diff changeset
258 }
2170
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
259 }
51da590b31a3 TechSmith Camtasia (TSCC) video decoder, courtesy of Konstantin Shishkov
melanson
parents:
diff changeset
260