Mercurial > libavcodec.hg
annotate vp5.c @ 4733:507d08212e36 libavcodec
check input validity, this prevents a few variables from reachin odd values which might have lead to out of array writes and thus might have been exploitable
author | michael |
---|---|
date | Sun, 25 Mar 2007 23:37:38 +0000 |
parents | 340c876320eb |
children | 143b89ab8187 |
rev | line source |
---|---|
3695 | 1 /** |
2 * @file vp5.c | |
3 * VP5 compatible video decoder | |
4 * | |
5 * Copyright (C) 2006 Aurelien Jacobs <aurel@gnuage.org> | |
6 * | |
3947
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3697
diff
changeset
|
7 * This file is part of FFmpeg. |
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3697
diff
changeset
|
8 * |
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3697
diff
changeset
|
9 * FFmpeg is free software; you can redistribute it and/or |
3695 | 10 * modify it under the terms of the GNU Lesser General Public |
11 * License as published by the Free Software Foundation; either | |
12 * version 2.1 of the License, or (at your option) any later version. | |
13 * | |
3947
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3697
diff
changeset
|
14 * FFmpeg is distributed in the hope that it will be useful, |
3695 | 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of |
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
17 * Lesser General Public License for more details. | |
18 * | |
19 * You should have received a copy of the GNU Lesser General Public | |
3947
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3697
diff
changeset
|
20 * License along with FFmpeg; if not, write to the Free Software |
3695 | 21 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA |
22 */ | |
23 | |
24 #include <stdlib.h> | |
25 #include <string.h> | |
26 | |
27 #include "avcodec.h" | |
28 #include "dsputil.h" | |
29 #include "bitstream.h" | |
30 #include "mpegvideo.h" | |
31 | |
32 #include "vp56.h" | |
33 #include "vp56data.h" | |
34 #include "vp5data.h" | |
35 | |
36 | |
37 static int vp5_parse_header(vp56_context_t *s, uint8_t *buf, int buf_size, | |
38 int *golden_frame) | |
39 { | |
40 vp56_range_coder_t *c = &s->c; | |
41 int rows, cols; | |
42 | |
43 vp56_init_range_decoder(&s->c, buf, buf_size); | |
4595 | 44 s->framep[VP56_FRAME_CURRENT]->key_frame = !vp56_rac_get(c); |
3695 | 45 vp56_rac_get(c); |
46 vp56_init_dequant(s, vp56_rac_gets(c, 6)); | |
4595 | 47 if (s->framep[VP56_FRAME_CURRENT]->key_frame) |
3695 | 48 { |
49 vp56_rac_gets(c, 8); | |
50 if(vp56_rac_gets(c, 5) > 5) | |
51 return 0; | |
52 vp56_rac_gets(c, 2); | |
53 if (vp56_rac_get(c)) { | |
54 av_log(s->avctx, AV_LOG_ERROR, "interlacing not supported\n"); | |
55 return 0; | |
56 } | |
57 rows = vp56_rac_gets(c, 8); /* number of stored macroblock rows */ | |
58 cols = vp56_rac_gets(c, 8); /* number of stored macroblock cols */ | |
59 vp56_rac_gets(c, 8); /* number of displayed macroblock rows */ | |
60 vp56_rac_gets(c, 8); /* number of displayed macroblock cols */ | |
61 vp56_rac_gets(c, 2); | |
62 if (16*cols != s->avctx->coded_width || | |
63 16*rows != s->avctx->coded_height) { | |
64 avcodec_set_dimensions(s->avctx, 16*cols, 16*rows); | |
65 return 2; | |
66 } | |
67 } | |
68 return 1; | |
69 } | |
70 | |
71 /* Gives very similar result than the vp6 version except in a few cases */ | |
72 static int vp5_adjust(int v, int t) | |
73 { | |
74 int s2, s1 = v >> 31; | |
75 v ^= s1; | |
76 v -= s1; | |
77 v *= v < 2*t; | |
78 v -= t; | |
79 s2 = v >> 31; | |
80 v ^= s2; | |
81 v -= s2; | |
82 v = t - v; | |
83 v += s1; | |
84 v ^= s1; | |
85 return v; | |
86 } | |
87 | |
3697 | 88 static void vp5_parse_vector_adjustment(vp56_context_t *s, vp56_mv_t *vect) |
3695 | 89 { |
90 vp56_range_coder_t *c = &s->c; | |
91 int comp, di; | |
92 | |
93 for (comp=0; comp<2; comp++) { | |
94 int delta = 0; | |
95 if (vp56_rac_get_prob(c, s->vector_model_dct[comp])) { | |
96 int sign = vp56_rac_get_prob(c, s->vector_model_sig[comp]); | |
97 di = vp56_rac_get_prob(c, s->vector_model_pdi[comp][0]); | |
98 di |= vp56_rac_get_prob(c, s->vector_model_pdi[comp][1]) << 1; | |
99 delta = vp56_rac_get_tree(c, vp56_pva_tree, | |
100 s->vector_model_pdv[comp]); | |
101 delta = di | (delta << 2); | |
102 delta = (delta ^ -sign) + sign; | |
103 } | |
104 if (!comp) | |
3697 | 105 vect->x = delta; |
3695 | 106 else |
3697 | 107 vect->y = delta; |
3695 | 108 } |
109 } | |
110 | |
111 static void vp5_parse_vector_models(vp56_context_t *s) | |
112 { | |
113 vp56_range_coder_t *c = &s->c; | |
114 int comp, node; | |
115 | |
116 for (comp=0; comp<2; comp++) { | |
117 if (vp56_rac_get_prob(c, vp5_vmc_pct[comp][0])) | |
118 s->vector_model_dct[comp] = vp56_rac_gets_nn(c, 7); | |
119 if (vp56_rac_get_prob(c, vp5_vmc_pct[comp][1])) | |
120 s->vector_model_sig[comp] = vp56_rac_gets_nn(c, 7); | |
121 if (vp56_rac_get_prob(c, vp5_vmc_pct[comp][2])) | |
122 s->vector_model_pdi[comp][0] = vp56_rac_gets_nn(c, 7); | |
123 if (vp56_rac_get_prob(c, vp5_vmc_pct[comp][3])) | |
124 s->vector_model_pdi[comp][1] = vp56_rac_gets_nn(c, 7); | |
125 } | |
126 | |
127 for (comp=0; comp<2; comp++) | |
128 for (node=0; node<7; node++) | |
129 if (vp56_rac_get_prob(c, vp5_vmc_pct[comp][4 + node])) | |
130 s->vector_model_pdv[comp][node] = vp56_rac_gets_nn(c, 7); | |
131 } | |
132 | |
133 static void vp5_parse_coeff_models(vp56_context_t *s) | |
134 { | |
135 vp56_range_coder_t *c = &s->c; | |
136 uint8_t def_prob[11]; | |
137 int node, cg, ctx; | |
138 int ct; /* code type */ | |
139 int pt; /* plane type (0 for Y, 1 for U or V) */ | |
140 | |
141 memset(def_prob, 0x80, sizeof(def_prob)); | |
142 | |
143 for (pt=0; pt<2; pt++) | |
144 for (node=0; node<11; node++) | |
145 if (vp56_rac_get_prob(c, vp5_dccv_pct[pt][node])) { | |
146 def_prob[node] = vp56_rac_gets_nn(c, 7); | |
147 s->coeff_model_dccv[pt][node] = def_prob[node]; | |
4595 | 148 } else if (s->framep[VP56_FRAME_CURRENT]->key_frame) { |
3695 | 149 s->coeff_model_dccv[pt][node] = def_prob[node]; |
150 } | |
151 | |
152 for (ct=0; ct<3; ct++) | |
153 for (pt=0; pt<2; pt++) | |
154 for (cg=0; cg<6; cg++) | |
155 for (node=0; node<11; node++) | |
156 if (vp56_rac_get_prob(c, vp5_ract_pct[ct][pt][cg][node])) { | |
157 def_prob[node] = vp56_rac_gets_nn(c, 7); | |
158 s->coeff_model_ract[pt][ct][cg][node] = def_prob[node]; | |
4595 | 159 } else if (s->framep[VP56_FRAME_CURRENT]->key_frame) { |
3695 | 160 s->coeff_model_ract[pt][ct][cg][node] = def_prob[node]; |
161 } | |
162 | |
163 /* coeff_model_dcct is a linear combination of coeff_model_dccv */ | |
164 for (pt=0; pt<2; pt++) | |
165 for (ctx=0; ctx<36; ctx++) | |
166 for (node=0; node<5; node++) | |
4594 | 167 s->coeff_model_dcct[pt][ctx][node] = av_clip(((s->coeff_model_dccv[pt][node] * vp5_dccv_lc[node][ctx][0] + 128) >> 8) + vp5_dccv_lc[node][ctx][1], 1, 254); |
3695 | 168 |
169 /* coeff_model_acct is a linear combination of coeff_model_ract */ | |
170 for (ct=0; ct<3; ct++) | |
171 for (pt=0; pt<2; pt++) | |
172 for (cg=0; cg<3; cg++) | |
173 for (ctx=0; ctx<6; ctx++) | |
174 for (node=0; node<5; node++) | |
4594 | 175 s->coeff_model_acct[pt][ct][cg][ctx][node] = av_clip(((s->coeff_model_ract[pt][ct][cg][node] * vp5_ract_lc[ct][cg][node][ctx][0] + 128) >> 8) + vp5_ract_lc[ct][cg][node][ctx][1], 1, 254); |
3695 | 176 } |
177 | |
178 static void vp5_parse_coeff(vp56_context_t *s) | |
179 { | |
180 vp56_range_coder_t *c = &s->c; | |
181 uint8_t *permute = s->scantable.permutated; | |
182 uint8_t *model, *model2; | |
183 int coeff, sign, coeff_idx; | |
184 int b, i, cg, idx, ctx, ctx_last; | |
185 int pt = 0; /* plane type (0 for Y, 1 for U or V) */ | |
186 | |
187 for (b=0; b<6; b++) { | |
188 int ct = 1; /* code type */ | |
189 | |
190 if (b > 3) pt = 1; | |
191 | |
192 ctx = 6*s->coeff_ctx[vp56_b6to4[b]][0] | |
193 + s->above_blocks[s->above_block_idx[b]].not_null_dc; | |
194 model = s->coeff_model_dccv[pt]; | |
195 model2 = s->coeff_model_dcct[pt][ctx]; | |
196 | |
197 for (coeff_idx=0; coeff_idx<64; ) { | |
198 if (vp56_rac_get_prob(c, model2[0])) { | |
199 if (vp56_rac_get_prob(c, model2[2])) { | |
200 if (vp56_rac_get_prob(c, model2[3])) { | |
201 s->coeff_ctx[vp56_b6to4[b]][coeff_idx] = 4; | |
202 idx = vp56_rac_get_tree(c, vp56_pc_tree, model); | |
203 sign = vp56_rac_get(c); | |
204 coeff = vp56_coeff_bias[idx]; | |
205 for (i=vp56_coeff_bit_length[idx]; i>=0; i--) | |
206 coeff += vp56_rac_get_prob(c, vp56_coeff_parse_table[idx][i]) << i; | |
207 } else { | |
208 if (vp56_rac_get_prob(c, model2[4])) { | |
209 coeff = 3 + vp56_rac_get_prob(c, model[5]); | |
210 s->coeff_ctx[vp56_b6to4[b]][coeff_idx] = 3; | |
211 } else { | |
212 coeff = 2; | |
213 s->coeff_ctx[vp56_b6to4[b]][coeff_idx] = 2; | |
214 } | |
215 sign = vp56_rac_get(c); | |
216 } | |
217 ct = 2; | |
218 } else { | |
219 ct = 1; | |
220 s->coeff_ctx[vp56_b6to4[b]][coeff_idx] = 1; | |
221 sign = vp56_rac_get(c); | |
222 coeff = 1; | |
223 } | |
224 coeff = (coeff ^ -sign) + sign; | |
225 if (coeff_idx) | |
226 coeff *= s->dequant_ac; | |
227 s->block_coeff[b][permute[coeff_idx]] = coeff; | |
228 } else { | |
229 if (ct && !vp56_rac_get_prob(c, model2[1])) | |
230 break; | |
231 ct = 0; | |
232 s->coeff_ctx[vp56_b6to4[b]][coeff_idx] = 0; | |
233 } | |
234 | |
235 cg = vp5_coeff_groups[++coeff_idx]; | |
236 ctx = s->coeff_ctx[vp56_b6to4[b]][coeff_idx]; | |
237 model = s->coeff_model_ract[pt][ct][cg]; | |
238 model2 = cg > 2 ? model : s->coeff_model_acct[pt][ct][cg][ctx]; | |
239 } | |
240 | |
241 ctx_last = FFMIN(s->coeff_ctx_last[vp56_b6to4[b]], 24); | |
242 s->coeff_ctx_last[vp56_b6to4[b]] = coeff_idx; | |
243 if (coeff_idx < ctx_last) | |
244 for (i=coeff_idx; i<=ctx_last; i++) | |
245 s->coeff_ctx[vp56_b6to4[b]][i] = 5; | |
246 s->above_blocks[s->above_block_idx[b]].not_null_dc = s->coeff_ctx[vp56_b6to4[b]][0]; | |
247 } | |
248 } | |
249 | |
250 static void vp5_default_models_init(vp56_context_t *s) | |
251 { | |
252 int i; | |
253 | |
254 for (i=0; i<2; i++) { | |
255 s->vector_model_sig[i] = 0x80; | |
256 s->vector_model_dct[i] = 0x80; | |
257 s->vector_model_pdi[i][0] = 0x55; | |
258 s->vector_model_pdi[i][1] = 0x80; | |
259 } | |
260 memcpy(s->mb_types_stats, vp56_def_mb_types_stats, sizeof(s->mb_types_stats)); | |
261 memset(s->vector_model_pdv, 0x80, sizeof(s->vector_model_pdv)); | |
262 } | |
263 | |
264 static int vp5_decode_init(AVCodecContext *avctx) | |
265 { | |
266 vp56_context_t *s = avctx->priv_data; | |
267 | |
268 vp56_init(s, avctx, 1); | |
269 s->vp56_coord_div = vp5_coord_div; | |
270 s->parse_vector_adjustment = vp5_parse_vector_adjustment; | |
271 s->adjust = vp5_adjust; | |
272 s->parse_coeff = vp5_parse_coeff; | |
273 s->default_models_init = vp5_default_models_init; | |
274 s->parse_vector_models = vp5_parse_vector_models; | |
275 s->parse_coeff_models = vp5_parse_coeff_models; | |
276 s->parse_header = vp5_parse_header; | |
277 | |
278 return 0; | |
279 } | |
280 | |
281 AVCodec vp5_decoder = { | |
282 "vp5", | |
283 CODEC_TYPE_VIDEO, | |
284 CODEC_ID_VP5, | |
285 sizeof(vp56_context_t), | |
286 vp5_decode_init, | |
287 NULL, | |
288 vp56_free, | |
289 vp56_decode_frame, | |
290 }; |