Mercurial > libavcodec.hg
annotate dnxhddec.c @ 4733:507d08212e36 libavcodec
check input validity, this prevents a few variables from reachin odd values which might have lead to out of array writes and thus might have been exploitable
author | michael |
---|---|
date | Sun, 25 Mar 2007 23:37:38 +0000 |
parents | 3e15e577ca32 |
children | 521465ec12e0 |
rev | line source |
---|---|
4687 | 1 /* |
2 * VC3/DNxHD decoder. | |
3 * Copyright (c) 2007 SmartJog S.A., Baptiste Coudurier <baptiste dot coudurier at smartjog dot com>. | |
4 * | |
5 * This file is part of FFmpeg. | |
6 * | |
7 * FFmpeg is free software; you can redistribute it and/or | |
8 * modify it under the terms of the GNU Lesser General Public | |
9 * License as published by the Free Software Foundation; either | |
10 * version 2.1 of the License, or (at your option) any later version. | |
11 * | |
12 * FFmpeg is distributed in the hope that it will be useful, | |
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
15 * Lesser General Public License for more details. | |
16 * | |
17 * You should have received a copy of the GNU Lesser General Public | |
18 * License along with FFmpeg; if not, write to the Free Software | |
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | |
20 */ | |
21 | |
22 //#define TRACE | |
23 //#define DEBUG | |
24 | |
25 #include "avcodec.h" | |
26 #include "bitstream.h" | |
27 #include "dnxhddata.h" | |
28 #include "dsputil.h" | |
29 #include "mpegvideo.h" | |
30 | |
31 typedef struct { | |
32 int cid; | |
33 unsigned int width, height; | |
34 int interlaced; | |
35 unsigned int frame_size; | |
36 int index_bits; | |
37 int bit_depth; | |
38 const uint8_t *luma_weigth, *chroma_weigth; | |
39 const uint8_t *dc_codes, *dc_bits; | |
40 const uint16_t *ac_codes; | |
41 const uint8_t *ac_bits, *ac_level; | |
42 const uint8_t *ac_run_flag, *ac_index_flag; | |
43 const uint16_t *run_codes; | |
44 const uint8_t *run_bits, *run; | |
45 } CIDEntry; | |
46 | |
47 typedef struct { | |
48 AVCodecContext *avctx; | |
49 AVFrame picture; | |
50 GetBitContext gb; | |
51 int cid; ///< compression id | |
52 unsigned int width, height; | |
53 unsigned int mb_width, mb_height; | |
54 uint32_t mb_scan_index[68]; /* max for 1080p */ | |
55 int cur_field; ///< current interlaced field | |
56 int index_bits; ///< length of index value | |
57 VLC ac_vlc, dc_vlc, run_vlc; | |
58 const uint8_t *ac_level, *run; | |
59 const uint8_t *ac_run_flag, *ac_index_flag; | |
60 const uint8_t *luma_weigth, *chroma_weigth; | |
61 int last_dc[3]; | |
62 DSPContext dsp; | |
63 DECLARE_ALIGNED_16(DCTELEM, blocks[8][64]); | |
64 DECLARE_ALIGNED_8(ScanTable, scantable); | |
65 const CIDEntry *cid_table; | |
66 } DNXHDContext; | |
67 | |
68 static const CIDEntry cid_table[] = { | |
69 { 1238, 1920, 1080, 0, 917504, 4, 8, | |
70 dnxhd_1238_luma_weigth, dnxhd_1238_chroma_weigth, | |
71 dnxhd_1238_dc_codes, dnxhd_1238_dc_bits, | |
72 dnxhd_1238_ac_codes, dnxhd_1238_ac_bits, dnxhd_1238_ac_level, | |
73 dnxhd_1238_ac_run_flag, dnxhd_1238_ac_index_flag, | |
74 dnxhd_1238_run_codes, dnxhd_1238_run_bits, dnxhd_1238_run }, | |
75 /* { 1243, 1920, 1080, 1, 917504, 4, 8, */ | |
76 /* dnxhd_1243_luma_weigth, dnxhd_1243_chroma_weigth, */ | |
77 /* dnxhd_1238_dc_codes, dnxhd_1238_dc_bits, */ | |
78 /* dnxhd_1238_ac_codes, dnxhd_1238_ac_bits, dnxhd_1238_ac_level, */ | |
79 /* dnxhd_1238_ac_run_flag, dnxhd_1238_ac_index_flag, */ | |
80 /* dnxhd_1238_run_codes, dnxhd_1238_run_bits, dnxhd_1238_run }, */ | |
81 }; | |
82 | |
83 static int dnxhd_get_cid_table(int cid) | |
84 { | |
85 int i; | |
86 for (i = 0; i < sizeof(cid_table)/sizeof(CIDEntry); i++) | |
87 if (cid_table[i].cid == cid) | |
88 return i; | |
89 return -1; | |
90 } | |
91 | |
92 #define DNXHD_VLC_BITS 9 | |
93 #define DNXHD_DC_VLC_BITS 6 | |
94 | |
95 static int dnxhd_decode_init(AVCodecContext *avctx) | |
96 { | |
97 DNXHDContext *ctx = avctx->priv_data; | |
98 | |
99 ctx->avctx = avctx; | |
100 dsputil_init(&ctx->dsp, avctx); | |
101 avctx->coded_frame = &ctx->picture; | |
102 ctx->picture.type = FF_I_TYPE; | |
103 return 0; | |
104 } | |
105 | |
106 static int dnxhd_init_vlc(DNXHDContext *ctx, int cid) | |
107 { | |
108 if (!ctx->cid_table) { | |
109 int index; | |
110 | |
111 if ((index = dnxhd_get_cid_table(cid)) < 0) { | |
112 av_log(ctx->avctx, AV_LOG_ERROR, "unsupported cid %d\n", cid); | |
113 return -1; | |
114 } | |
115 ctx->cid_table = &cid_table[index]; | |
116 init_vlc(&ctx->ac_vlc, DNXHD_VLC_BITS, 257, | |
117 cid_table->ac_bits, 1, 1, | |
118 cid_table->ac_codes, 2, 2, 0); | |
119 init_vlc(&ctx->dc_vlc, DNXHD_DC_VLC_BITS, 12, | |
120 cid_table->dc_bits, 1, 1, | |
121 cid_table->dc_codes, 1, 1, 0); | |
122 init_vlc(&ctx->run_vlc, DNXHD_VLC_BITS, 62, | |
123 cid_table->run_bits, 1, 1, | |
124 cid_table->run_codes, 2, 2, 0); | |
125 | |
126 ctx->run = cid_table->run; | |
127 ctx->ac_level = cid_table->ac_level; | |
128 ctx->ac_run_flag = cid_table->ac_run_flag; | |
129 ctx->ac_index_flag = cid_table->ac_index_flag; | |
130 ctx->luma_weigth = cid_table->luma_weigth; | |
131 ctx->chroma_weigth = cid_table->chroma_weigth; | |
132 | |
133 ctx->index_bits = cid_table->index_bits; | |
134 | |
135 ff_init_scantable(ctx->dsp.idct_permutation, &ctx->scantable, ff_zigzag_direct); | |
136 } | |
137 return 0; | |
138 } | |
139 | |
140 static int dnxhd_decode_header(DNXHDContext *ctx, uint8_t *buf, int buf_size) | |
141 { | |
142 static const uint8_t header_prefix[] = { 0x00, 0x00, 0x02, 0x80, 0x01 }; | |
143 int i; | |
144 | |
145 if (buf_size < 0x280) | |
146 return -1; | |
147 | |
148 if (memcmp(buf, header_prefix, 5)) { | |
149 av_log(ctx->avctx, AV_LOG_ERROR, "error in header\n"); | |
150 return -1; | |
151 } | |
152 if (buf[5] & 2) {/* interlaced FIXME top or bottom */ | |
153 ctx->picture.interlaced_frame = 1; | |
154 av_log(ctx->avctx, AV_LOG_DEBUG, "interlaced %d\n", buf[5] & 3); | |
155 } | |
156 | |
157 ctx->height = AV_RB16(buf + 0x18); | |
158 ctx->width = AV_RB16(buf + 0x1a); | |
159 | |
4688
1cefcd7da878
10l, fix debug, dprintf does not have log level
bcoudurier
parents:
4687
diff
changeset
|
160 dprintf(ctx->avctx, "width %d, heigth %d\n", ctx->width, ctx->height); |
4687 | 161 |
162 if (buf[0x21] & 0x80) { | |
163 av_log(ctx->avctx, AV_LOG_ERROR, "10 bit per component\n"); | |
164 return -1; | |
165 } | |
166 | |
167 ctx->cid = AV_RB32(buf + 0x28); | |
4688
1cefcd7da878
10l, fix debug, dprintf does not have log level
bcoudurier
parents:
4687
diff
changeset
|
168 dprintf(ctx->avctx, "compression id %d\n", ctx->cid); |
4687 | 169 |
170 if (dnxhd_init_vlc(ctx, ctx->cid) < 0) | |
171 return -1; | |
172 | |
173 if (buf_size < ctx->cid_table->frame_size) { | |
174 av_log(ctx->avctx, AV_LOG_ERROR, "incorrect frame size\n"); | |
175 return -1; | |
176 } | |
177 | |
178 ctx->mb_width = ctx->width>>4; | |
179 ctx->mb_height = buf[0x16d]; | |
180 | |
181 if (ctx->mb_height > 68) { | |
182 av_log(ctx->avctx, AV_LOG_ERROR, "mb height too big\n"); | |
183 return -1; | |
184 } | |
185 | |
4688
1cefcd7da878
10l, fix debug, dprintf does not have log level
bcoudurier
parents:
4687
diff
changeset
|
186 dprintf(ctx->avctx, "mb width %d, mb height %d\n", ctx->mb_width, ctx->mb_height); |
4687 | 187 for (i = 0; i < ctx->mb_height; i++) { |
188 ctx->mb_scan_index[i] = AV_RB32(buf + 0x170 + (i<<2)); | |
4688
1cefcd7da878
10l, fix debug, dprintf does not have log level
bcoudurier
parents:
4687
diff
changeset
|
189 dprintf(ctx->avctx, "mb scan index %d\n", ctx->mb_scan_index[i]); |
4687 | 190 if (buf_size < ctx->mb_scan_index[i] + 0x280) { |
191 av_log(ctx->avctx, AV_LOG_ERROR, "invalid mb scan index\n"); | |
192 return -1; | |
193 } | |
194 } | |
195 | |
196 return 0; | |
197 } | |
198 | |
199 static int dnxhd_decode_dc(DNXHDContext *ctx) | |
200 { | |
201 int len; | |
202 | |
203 len = get_vlc2(&ctx->gb, ctx->dc_vlc.table, DNXHD_DC_VLC_BITS, 1); | |
204 return len ? get_xbits(&ctx->gb, len) : 0; | |
205 } | |
206 | |
207 static void dnxhd_decode_dct_block(DNXHDContext *ctx, DCTELEM *block, int n, int qscale) | |
208 { | |
209 int i, j, index, index2; | |
210 int level, component, sign; | |
211 const uint8_t *weigth_matrix; | |
212 | |
213 if (n&2) { | |
214 component = 1 + (n&1); | |
215 weigth_matrix = ctx->chroma_weigth; | |
216 } else { | |
217 component = 0; | |
218 weigth_matrix = ctx->luma_weigth; | |
219 } | |
220 | |
221 ctx->last_dc[component] += dnxhd_decode_dc(ctx); | |
222 block[0] = ctx->last_dc[component]; | |
223 //av_log(ctx->avctx, AV_LOG_DEBUG, "dc %d\n", block[0]); | |
224 for (i = 1; ; i++) { | |
225 index = get_vlc2(&ctx->gb, ctx->ac_vlc.table, DNXHD_VLC_BITS, 2); | |
226 //av_log(ctx->avctx, AV_LOG_DEBUG, "index %d\n", index); | |
227 level = ctx->ac_level[index]; | |
228 if (!level) { /* EOB */ | |
229 //av_log(ctx->avctx, AV_LOG_DEBUG, "EOB\n"); | |
230 return; | |
231 } | |
232 sign = get_sbits(&ctx->gb, 1); | |
233 | |
234 if (ctx->ac_index_flag[index]) { | |
235 level += get_bits(&ctx->gb, ctx->index_bits)<<6; | |
236 } | |
237 | |
238 if (ctx->ac_run_flag[index]) { | |
239 index2 = get_vlc2(&ctx->gb, ctx->run_vlc.table, DNXHD_VLC_BITS, 2); | |
240 i += ctx->run[index2]; | |
241 } | |
242 | |
243 j = ctx->scantable.permutated[i]; | |
244 //av_log(ctx->avctx, AV_LOG_DEBUG, "j %d\n", j); | |
245 //av_log(ctx->avctx, AV_LOG_DEBUG, "level %d, weigth %d\n", level, weigth_matrix[i]); | |
246 level = (2*level+1) * qscale * weigth_matrix[i]; | |
247 if (weigth_matrix[i] != 32) // FIXME 10bit | |
248 level += 32; | |
249 level >>= 6; | |
250 level = (level^sign) - sign; | |
251 | |
252 if (i > 63) { | |
253 av_log(ctx->avctx, AV_LOG_ERROR, "ac tex damaged %d, %d\n", n, i); | |
254 return; | |
255 } | |
256 | |
257 //av_log(NULL, AV_LOG_DEBUG, "i %d, j %d, end level %d\n", i, j, level); | |
258 block[j] = level; | |
259 } | |
260 } | |
261 | |
262 static int dnxhd_decode_macroblock(DNXHDContext *ctx, int x, int y) | |
263 { | |
264 int dct_linesize_luma = ctx->picture.linesize[0]; | |
265 int dct_linesize_chroma = ctx->picture.linesize[1]; | |
266 uint8_t *dest_y, *dest_u, *dest_v; | |
267 int dct_offset; | |
268 int qscale, i; | |
269 | |
270 ctx->dsp.clear_blocks(ctx->blocks[0]); | |
271 ctx->dsp.clear_blocks(ctx->blocks[2]); // FIXME change clear blocks to take block amount | |
272 | |
273 qscale = get_bits(&ctx->gb, 11); | |
274 skip_bits1(&ctx->gb); | |
275 //av_log(ctx->avctx, AV_LOG_DEBUG, "qscale %d\n", qscale); | |
276 | |
277 for (i = 0; i < 8; i++) { | |
278 dnxhd_decode_dct_block(ctx, ctx->blocks[i], i, qscale); | |
279 } | |
280 dest_y = ctx->picture.data[0] + ((y * dct_linesize_luma) << 4) + (x << 4); | |
281 dest_u = ctx->picture.data[1] + ((y * dct_linesize_chroma) << 4) + (x << 3); | |
282 dest_v = ctx->picture.data[2] + ((y * dct_linesize_chroma) << 4) + (x << 3); | |
283 | |
284 dct_offset = dct_linesize_luma << 3; | |
285 ctx->dsp.idct_put(dest_y, dct_linesize_luma, ctx->blocks[0]); | |
286 ctx->dsp.idct_put(dest_y + 8, dct_linesize_luma, ctx->blocks[1]); | |
287 ctx->dsp.idct_put(dest_y + dct_offset, dct_linesize_luma, ctx->blocks[4]); | |
288 ctx->dsp.idct_put(dest_y + dct_offset + 8, dct_linesize_luma, ctx->blocks[5]); | |
289 | |
290 if (!(ctx->avctx->flags & CODEC_FLAG_GRAY)) { | |
291 dct_offset = dct_linesize_chroma << 3; | |
292 ctx->dsp.idct_put(dest_u, dct_linesize_chroma, ctx->blocks[2]); | |
293 ctx->dsp.idct_put(dest_v, dct_linesize_chroma, ctx->blocks[3]); | |
294 ctx->dsp.idct_put(dest_u + dct_offset, dct_linesize_chroma, ctx->blocks[6]); | |
295 ctx->dsp.idct_put(dest_v + dct_offset, dct_linesize_chroma, ctx->blocks[7]); | |
296 } | |
297 | |
298 return 0; | |
299 } | |
300 | |
301 static int dnxhd_decode_macroblocks(DNXHDContext *ctx, uint8_t *buf, int buf_size) | |
302 { | |
303 int x, y; | |
304 for (y = 0; y < ctx->mb_height; y++) { | |
305 memset(ctx->last_dc, 4, sizeof(ctx->last_dc)); // 4 for levels +128 | |
306 init_get_bits(&ctx->gb, buf + ctx->mb_scan_index[y], (buf_size - ctx->mb_scan_index[y]) << 3); | |
307 for (x = 0; x < ctx->mb_width; x++) { | |
308 //START_TIMER; | |
309 dnxhd_decode_macroblock(ctx, x, y); | |
310 //STOP_TIMER("decode macroblock"); | |
311 } | |
312 } | |
313 return 0; | |
314 } | |
315 | |
316 static int dnxhd_decode_frame(AVCodecContext *avctx, void *data, int *data_size, | |
317 uint8_t *buf, int buf_size) | |
318 { | |
319 DNXHDContext *ctx = avctx->priv_data; | |
320 AVFrame *picture = data; | |
321 | |
4688
1cefcd7da878
10l, fix debug, dprintf does not have log level
bcoudurier
parents:
4687
diff
changeset
|
322 dprintf(avctx, "frame size %d\n", buf_size); |
4687 | 323 |
324 if (dnxhd_decode_header(ctx, buf, buf_size) < 0) | |
325 return -1; | |
326 | |
327 avctx->pix_fmt = PIX_FMT_YUV422P; | |
328 if (avcodec_check_dimensions(avctx, ctx->width, ctx->height)) | |
329 return -1; | |
330 avcodec_set_dimensions(avctx, ctx->width, ctx->height); | |
331 | |
332 if (ctx->picture.data[0]) | |
333 avctx->release_buffer(avctx, &ctx->picture); | |
334 if (avctx->get_buffer(avctx, &ctx->picture) < 0) { | |
335 av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n"); | |
336 return -1; | |
337 } | |
338 | |
339 dnxhd_decode_macroblocks(ctx, buf + 0x280, buf_size - 0x280); | |
340 | |
341 *picture = ctx->picture; | |
342 *data_size = sizeof(AVPicture); | |
4721 | 343 return buf_size; |
4687 | 344 } |
345 | |
346 static int dnxhd_decode_close(AVCodecContext *avctx) | |
347 { | |
348 DNXHDContext *ctx = avctx->priv_data; | |
349 | |
350 if(ctx->picture.data[0]) | |
351 avctx->release_buffer(avctx, &ctx->picture); | |
352 free_vlc(&ctx->ac_vlc); | |
353 free_vlc(&ctx->dc_vlc); | |
354 free_vlc(&ctx->run_vlc); | |
355 return 0; | |
356 } | |
357 | |
358 AVCodec dnxhd_decoder = { | |
359 "dnxhd", | |
360 CODEC_TYPE_VIDEO, | |
361 CODEC_ID_DNXHD, | |
362 sizeof(DNXHDContext), | |
363 dnxhd_decode_init, | |
364 NULL, | |
365 dnxhd_decode_close, | |
366 dnxhd_decode_frame, | |
367 CODEC_CAP_DR1, | |
368 }; |