annotate sunrast.c @ 9370:25fa07ef8e2b libavcodec

Make sure the block array is of the correct size. This might have been exploitable.
author michael
date Thu, 09 Apr 2009 18:47:50 +0000
parents 54bc8a2727b0
children 6cd9e789dd9e
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
6084
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
1 /*
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
2 * Sun Rasterfile (.sun/.ras/im{1,8,24}/.sunras) image decoder
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
3 * Copyright (c) 2007, 2008 Ivo van Poorten
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
4 *
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
5 * This file is part of FFmpeg.
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
6 *
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
7 * FFmpeg is free software; you can redistribute it and/or
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
8 * modify it under the terms of the GNU Lesser General Public
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
9 * License as published by the Free Software Foundation; either
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
10 * version 2.1 of the License, or (at your option) any later version.
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
11 *
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
12 * FFmpeg is distributed in the hope that it will be useful,
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
15 * Lesser General Public License for more details.
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
16 *
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
17 * You should have received a copy of the GNU Lesser General Public
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
18 * License along with FFmpeg; if not, write to the Free Software
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
20 */
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
21
8573
2acf0ae7b041 Fix build: Add intreadwrite.h and bswap.h #includes where necessary.
diego
parents: 7040
diff changeset
22 #include "libavutil/intreadwrite.h"
6084
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
23 #include "avcodec.h"
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
24
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
25 #define RT_OLD 0
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
26 #define RT_STANDARD 1
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
27 #define RT_BYTE_ENCODED 2
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
28 #define RT_FORMAT_RGB 3
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
29 #define RT_FORMAT_TIFF 4
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
30 #define RT_FORMAT_IFF 5
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
31
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
32 typedef struct SUNRASTContext {
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
33 AVFrame picture;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
34 } SUNRASTContext;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
35
6517
48759bfbd073 Apply 'cold' attribute to init/uninit functions in libavcodec
zuxy
parents: 6254
diff changeset
36 static av_cold int sunrast_init(AVCodecContext *avctx) {
6084
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
37 SUNRASTContext *s = avctx->priv_data;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
38
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
39 avcodec_get_frame_defaults(&s->picture);
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
40 avctx->coded_frame= &s->picture;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
41
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
42 return 0;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
43 }
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
44
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
45 static int sunrast_decode_frame(AVCodecContext *avctx, void *data,
9355
54bc8a2727b0 Implement avcodec_decode_video2(), _audio3() and _subtitle2() which takes an
rbultje
parents: 8573
diff changeset
46 int *data_size, AVPacket *avpkt) {
54bc8a2727b0 Implement avcodec_decode_video2(), _audio3() and _subtitle2() which takes an
rbultje
parents: 8573
diff changeset
47 const uint8_t *buf = avpkt->data;
54bc8a2727b0 Implement avcodec_decode_video2(), _audio3() and _subtitle2() which takes an
rbultje
parents: 8573
diff changeset
48 int buf_size = avpkt->size;
6084
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
49 SUNRASTContext * const s = avctx->priv_data;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
50 AVFrame *picture = data;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
51 AVFrame * const p = &s->picture;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
52 unsigned int w, h, depth, type, maptype, maplength, stride, x, y, len, alen;
6254
michael
parents: 6086
diff changeset
53 uint8_t *ptr;
michael
parents: 6086
diff changeset
54 const uint8_t *bufstart = buf;
6084
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
55
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
56 if (AV_RB32(buf) != 0x59a66a95) {
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
57 av_log(avctx, AV_LOG_ERROR, "this is not sunras encoded data\n");
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
58 return -1;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
59 }
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
60
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
61 w = AV_RB32(buf+4);
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
62 h = AV_RB32(buf+8);
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
63 depth = AV_RB32(buf+12);
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
64 type = AV_RB32(buf+20);
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
65 maptype = AV_RB32(buf+24);
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
66 maplength = AV_RB32(buf+28);
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
67
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
68 if (type > RT_BYTE_ENCODED && type <= RT_FORMAT_IFF) {
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
69 av_log(avctx, AV_LOG_ERROR, "unsupported (compression) type\n");
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
70 return -1;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
71 }
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
72 if (type > RT_FORMAT_IFF) {
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
73 av_log(avctx, AV_LOG_ERROR, "invalid (compression) type\n");
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
74 return -1;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
75 }
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
76 if (maptype & ~1) {
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
77 av_log(avctx, AV_LOG_ERROR, "invalid colormap type\n");
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
78 return -1;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
79 }
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
80
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
81 buf += 32;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
82
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
83 switch (depth) {
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
84 case 1:
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
85 avctx->pix_fmt = PIX_FMT_MONOWHITE;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
86 break;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
87 case 8:
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
88 avctx->pix_fmt = PIX_FMT_PAL8;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
89 break;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
90 case 24:
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
91 avctx->pix_fmt = PIX_FMT_BGR24;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
92 break;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
93 default:
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
94 av_log(avctx, AV_LOG_ERROR, "invalid depth\n");
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
95 return -1;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
96 }
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
97
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
98 if (p->data[0])
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
99 avctx->release_buffer(avctx, p);
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
100
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
101 if (avcodec_check_dimensions(avctx, w, h))
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
102 return -1;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
103 if (w != avctx->width || h != avctx->height)
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
104 avcodec_set_dimensions(avctx, w, h);
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
105 if (avctx->get_buffer(avctx, p) < 0) {
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
106 av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n");
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
107 return -1;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
108 }
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
109
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
110 p->pict_type = FF_I_TYPE;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
111
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
112 if (depth != 8 && maplength) {
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
113 av_log(avctx, AV_LOG_WARNING, "useless colormap found or file is corrupted, trying to recover\n");
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
114
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
115 } else if (depth == 8) {
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
116 unsigned int len = maplength / 3;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
117
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
118 if (!maplength) {
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
119 av_log(avctx, AV_LOG_ERROR, "colormap expected\n");
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
120 return -1;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
121 }
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
122 if (maplength % 3 || maplength > 768) {
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
123 av_log(avctx, AV_LOG_WARNING, "invalid colormap length\n");
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
124 return -1;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
125 }
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
126
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
127 ptr = p->data[1];
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
128 for (x=0; x<len; x++, ptr+=4)
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
129 *(uint32_t *)ptr = (buf[x]<<16) + (buf[len+x]<<8) + buf[len+len+x];
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
130 }
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
131
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
132 buf += maplength;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
133
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
134 ptr = p->data[0];
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
135 stride = p->linesize[0];
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
136
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
137 /* scanlines are aligned on 16 bit boundaries */
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
138 len = (depth * w + 7) >> 3;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
139 alen = len + (len&1);
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
140
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
141 if (type == RT_BYTE_ENCODED) {
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
142 int value, run;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
143 uint8_t *end = ptr + h*stride;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
144
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
145 x = 0;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
146 while (ptr != end) {
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
147 run = 1;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
148 if ((value = *buf++) == 0x80) {
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
149 run = *buf++ + 1;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
150 if (run != 1)
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
151 value = *buf++;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
152 }
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
153 while (run--) {
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
154 if (x < len)
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
155 ptr[x] = value;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
156 if (++x >= alen) {
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
157 x = 0;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
158 ptr += stride;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
159 if (ptr == end)
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
160 break;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
161 }
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
162 }
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
163 }
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
164 } else {
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
165 for (y=0; y<h; y++) {
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
166 memcpy(ptr, buf, len);
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
167 ptr += stride;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
168 buf += alen;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
169 }
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
170 }
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
171
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
172 *picture = s->picture;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
173 *data_size = sizeof(AVFrame);
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
174
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
175 return buf - bufstart;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
176 }
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
177
6517
48759bfbd073 Apply 'cold' attribute to init/uninit functions in libavcodec
zuxy
parents: 6254
diff changeset
178 static av_cold int sunrast_end(AVCodecContext *avctx) {
6084
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
179 SUNRASTContext *s = avctx->priv_data;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
180
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
181 if(s->picture.data[0])
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
182 avctx->release_buffer(avctx, &s->picture);
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
183
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
184 return 0;
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
185 }
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
186
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
187 AVCodec sunrast_decoder = {
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
188 "sunrast",
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
189 CODEC_TYPE_VIDEO,
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
190 CODEC_ID_SUNRAST,
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
191 sizeof(SUNRASTContext),
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
192 sunrast_init,
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
193 NULL,
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
194 sunrast_end,
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
195 sunrast_decode_frame,
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
196 0,
6722
6eeb19edcee3 Add long names to some AVCodec declarations.
diego
parents: 6517
diff changeset
197 NULL,
7040
e943e1409077 Make AVCodec long_names definition conditional depending on CONFIG_SMALL.
stefano
parents: 6722
diff changeset
198 .long_name = NULL_IF_CONFIG_SMALL("Sun Rasterfile image"),
6084
d33520d1ca92 Sun Rasterfile decoder
ivo
parents:
diff changeset
199 };