Mercurial > libavcodec.hg
annotate h264_mvpred.h @ 10992:9aae10c862f7 libavcodec
Fix crash in MLP decoder due to integer overflow.
Probably only DoS, init_get_bits sets buffer to NULL, thus causing a
NULL-dereference directly after.
author | reimar |
---|---|
date | Sun, 24 Jan 2010 18:07:29 +0000 |
parents | 3a7fa91fa168 |
children | 10c06a9bd3d9 |
rev | line source |
---|---|
1168 | 1 /* |
10864 | 2 * H.26L/H.264/AVC/JVT/14496-10/... motion vector predicion |
1168 | 3 * Copyright (c) 2003 Michael Niedermayer <michaelni@gmx.at> |
4 * | |
3947
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3927
diff
changeset
|
5 * This file is part of FFmpeg. |
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3927
diff
changeset
|
6 * |
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3927
diff
changeset
|
7 * FFmpeg is free software; you can redistribute it and/or |
1168 | 8 * modify it under the terms of the GNU Lesser General Public |
9 * License as published by the Free Software Foundation; either | |
3947
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3927
diff
changeset
|
10 * version 2.1 of the License, or (at your option) any later version. |
1168 | 11 * |
3947
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3927
diff
changeset
|
12 * FFmpeg is distributed in the hope that it will be useful, |
1168 | 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of |
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
15 * Lesser General Public License for more details. | |
16 * | |
17 * You should have received a copy of the GNU Lesser General Public | |
3947
c8c591fe26f8
Change license headers to say 'FFmpeg' instead of 'this program/this library'
diego
parents:
3927
diff
changeset
|
18 * License along with FFmpeg; if not, write to the Free Software |
3036
0b546eab515d
Update licensing information: The FSF changed postal address.
diego
parents:
3029
diff
changeset
|
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
1168 | 20 */ |
2967 | 21 |
1168 | 22 /** |
10864 | 23 * @file libavcodec/h264_mvpred.h |
24 * H.264 / AVC / MPEG4 part10 motion vector predicion. | |
1168 | 25 * @author Michael Niedermayer <michaelni@gmx.at> |
26 */ | |
27 | |
10882
5d34ab807e91
Add forgotton multiple inclusion guards to h264_mvpred.h.
michael
parents:
10864
diff
changeset
|
28 #ifndef AVCODEC_H264_MVPRED_H |
5d34ab807e91
Add forgotton multiple inclusion guards to h264_mvpred.h.
michael
parents:
10864
diff
changeset
|
29 #define AVCODEC_H264_MVPRED_H |
5d34ab807e91
Add forgotton multiple inclusion guards to h264_mvpred.h.
michael
parents:
10864
diff
changeset
|
30 |
9012
15a3df8c01fd
More approved hunks for VAAPI & our new and cleaner hwaccel API.
michael
parents:
9004
diff
changeset
|
31 #include "internal.h" |
1168 | 32 #include "avcodec.h" |
4975 | 33 #include "h264.h" |
1908
e20fd60b215c
h264 - progressive I frame CABAC support patch by (Laurent Aimar <fenrir at via dot ecp dot fr>)
michael
parents:
1899
diff
changeset
|
34 |
3284
a224d9752912
don't force asserts in release builds. 2% faster h264.
lorenm
parents:
3219
diff
changeset
|
35 //#undef NDEBUG |
1168 | 36 #include <assert.h> |
37 | |
10864 | 38 static inline int fetch_diagonal_mv(H264Context *h, const int16_t **C, int i, int list, int part_width){ |
39 const int topright_ref= h->ref_cache[list][ i - 8 + part_width ]; | |
40 MpegEncContext *s = &h->s; | |
7481 | 41 |
10864 | 42 /* there is no consistent mapping of mvs to neighboring locations that will |
43 * make mbaff happy, so we can't move all this logic to fill_caches */ | |
44 if(FRAME_MBAFF){ | |
8443 | 45 |
10864 | 46 #define SET_DIAG_MV(MV_OP, REF_OP, X4, Y4)\ |
47 const int x4 = X4, y4 = Y4;\ | |
48 const int mb_type = mb_types[(x4>>2)+(y4>>2)*s->mb_stride];\ | |
49 if(!USES_LIST(mb_type,list))\ | |
50 return LIST_NOT_USED;\ | |
51 mv = s->current_picture_ptr->motion_val[list][x4 + y4*h->b_stride];\ | |
52 h->mv_cache[list][scan8[0]-2][0] = mv[0];\ | |
53 h->mv_cache[list][scan8[0]-2][1] = mv[1] MV_OP;\ | |
54 return s->current_picture_ptr->ref_index[list][(x4>>1) + (y4>>1)*h->b8_stride] REF_OP; | |
2967 | 55 |
10864 | 56 if(topright_ref == PART_NOT_AVAILABLE |
10933
3a7fa91fa168
Remove 2 checks from fetch_diagonal_mv() that apparently serve no purpose.
michael
parents:
10932
diff
changeset
|
57 && i >= scan8[0]+8 && (i&7)==4 |
10864 | 58 && h->ref_cache[list][scan8[0]-1] != PART_NOT_AVAILABLE){ |
10932
5f0ef177bbac
Move a few things into a deeper if() in fetch_diagonal_mv().
michael
parents:
10931
diff
changeset
|
59 const uint32_t *mb_types = s->current_picture_ptr->mb_type; |
5f0ef177bbac
Move a few things into a deeper if() in fetch_diagonal_mv().
michael
parents:
10931
diff
changeset
|
60 const int16_t *mv; |
5f0ef177bbac
Move a few things into a deeper if() in fetch_diagonal_mv().
michael
parents:
10931
diff
changeset
|
61 *(uint32_t*)h->mv_cache[list][scan8[0]-2] = 0; |
5f0ef177bbac
Move a few things into a deeper if() in fetch_diagonal_mv().
michael
parents:
10931
diff
changeset
|
62 *C = h->mv_cache[list][scan8[0]-2]; |
5f0ef177bbac
Move a few things into a deeper if() in fetch_diagonal_mv().
michael
parents:
10931
diff
changeset
|
63 |
10864 | 64 if(!MB_FIELD |
65 && IS_INTERLACED(mb_types[h->left_mb_xy[0]])){ | |
66 SET_DIAG_MV(*2, >>1, s->mb_x*4-1, (s->mb_y|1)*4+(s->mb_y&1)*2+(i>>4)-1); | |
67 } | |
68 if(MB_FIELD | |
10933
3a7fa91fa168
Remove 2 checks from fetch_diagonal_mv() that apparently serve no purpose.
michael
parents:
10932
diff
changeset
|
69 && !IS_INTERLACED(mb_types[h->left_mb_xy[0]])){ |
10864 | 70 // left shift will turn LIST_NOT_USED into PART_NOT_AVAILABLE, but that's OK. |
71 SET_DIAG_MV(/2, <<1, s->mb_x*4-1, (s->mb_y&~1)*4 - 1 + ((i-scan8[0])>>3)*2); | |
2551
615995277bc5
MBAFF I slice no deblocking patch by (Loic >>lll+ffmpeg m4x org<<)
michael
parents:
2548
diff
changeset
|
72 } |
615995277bc5
MBAFF I slice no deblocking patch by (Loic >>lll+ffmpeg m4x org<<)
michael
parents:
2548
diff
changeset
|
73 } |
10864 | 74 #undef SET_DIAG_MV |
1168 | 75 } |
76 | |
10864 | 77 if(topright_ref != PART_NOT_AVAILABLE){ |
78 *C= h->mv_cache[list][ i - 8 + part_width ]; | |
79 return topright_ref; | |
80 }else{ | |
81 tprintf(s->avctx, "topright MV not available\n"); | |
82 | |
83 *C= h->mv_cache[list][ i - 8 - 1 ]; | |
84 return h->ref_cache[list][ i - 8 - 1 ]; | |
85 } | |
86 } | |
87 | |
88 /** | |
89 * gets the predicted MV. | |
90 * @param n the block index | |
91 * @param part_width the width of the partition (4, 8,16) -> (1, 2, 4) | |
92 * @param mx the x component of the predicted motion vector | |
93 * @param my the y component of the predicted motion vector | |
94 */ | |
95 static inline void pred_motion(H264Context * const h, int n, int part_width, int list, int ref, int * const mx, int * const my){ | |
96 const int index8= scan8[n]; | |
97 const int top_ref= h->ref_cache[list][ index8 - 8 ]; | |
98 const int left_ref= h->ref_cache[list][ index8 - 1 ]; | |
99 const int16_t * const A= h->mv_cache[list][ index8 - 1 ]; | |
100 const int16_t * const B= h->mv_cache[list][ index8 - 8 ]; | |
101 const int16_t * C; | |
102 int diagonal_ref, match_count; | |
3316 | 103 |
10864 | 104 assert(part_width==1 || part_width==2 || part_width==4); |
105 | |
106 /* mv_cache | |
107 B . . A T T T T | |
108 U . . L . . , . | |
109 U . . L . . . . | |
110 U . . L . . , . | |
111 . . . L . . . . | |
112 */ | |
113 | |
114 diagonal_ref= fetch_diagonal_mv(h, &C, index8, list, part_width); | |
115 match_count= (diagonal_ref==ref) + (top_ref==ref) + (left_ref==ref); | |
116 tprintf(h->s.avctx, "pred_motion match_count=%d\n", match_count); | |
117 if(match_count > 1){ //most common | |
118 *mx= mid_pred(A[0], B[0], C[0]); | |
119 *my= mid_pred(A[1], B[1], C[1]); | |
120 }else if(match_count==1){ | |
121 if(left_ref==ref){ | |
122 *mx= A[0]; | |
123 *my= A[1]; | |
124 }else if(top_ref==ref){ | |
125 *mx= B[0]; | |
126 *my= B[1]; | |
127 }else{ | |
128 *mx= C[0]; | |
129 *my= C[1]; | |
3316 | 130 } |
2449 | 131 }else{ |
10864 | 132 if(top_ref == PART_NOT_AVAILABLE && diagonal_ref == PART_NOT_AVAILABLE && left_ref != PART_NOT_AVAILABLE){ |
133 *mx= A[0]; | |
134 *my= A[1]; | |
7532 | 135 }else{ |
10864 | 136 *mx= mid_pred(A[0], B[0], C[0]); |
137 *my= mid_pred(A[1], B[1], C[1]); | |
2551
615995277bc5
MBAFF I slice no deblocking patch by (Loic >>lll+ffmpeg m4x org<<)
michael
parents:
2548
diff
changeset
|
138 } |
615995277bc5
MBAFF I slice no deblocking patch by (Loic >>lll+ffmpeg m4x org<<)
michael
parents:
2548
diff
changeset
|
139 } |
615995277bc5
MBAFF I slice no deblocking patch by (Loic >>lll+ffmpeg m4x org<<)
michael
parents:
2548
diff
changeset
|
140 |
10864 | 141 tprintf(h->s.avctx, "pred_motion (%2d %2d %2d) (%2d %2d %2d) (%2d %2d %2d) -> (%2d %2d %2d) at %2d %2d %d list %d\n", top_ref, B[0], B[1], diagonal_ref, C[0], C[1], left_ref, A[0], A[1], ref, *mx, *my, h->s.mb_x, h->s.mb_y, n, list); |
1168 | 142 } |
143 | |
144 /** | |
145 * gets the directionally predicted 16x8 MV. | |
146 * @param n the block index | |
147 * @param mx the x component of the predicted motion vector | |
148 * @param my the y component of the predicted motion vector | |
149 */ | |
150 static inline void pred_16x8_motion(H264Context * const h, int n, int list, int ref, int * const mx, int * const my){ | |
151 if(n==0){ | |
152 const int top_ref= h->ref_cache[list][ scan8[0] - 8 ]; | |
153 const int16_t * const B= h->mv_cache[list][ scan8[0] - 8 ]; | |
154 | |
4600 | 155 tprintf(h->s.avctx, "pred_16x8: (%2d %2d %2d) at %2d %2d %d list %d\n", top_ref, B[0], B[1], h->s.mb_x, h->s.mb_y, n, list); |
2967 | 156 |
1168 | 157 if(top_ref == ref){ |
158 *mx= B[0]; | |
159 *my= B[1]; | |
160 return; | |
161 } | |
162 }else{ | |
163 const int left_ref= h->ref_cache[list][ scan8[8] - 1 ]; | |
164 const int16_t * const A= h->mv_cache[list][ scan8[8] - 1 ]; | |
2967 | 165 |
4600 | 166 tprintf(h->s.avctx, "pred_16x8: (%2d %2d %2d) at %2d %2d %d list %d\n", left_ref, A[0], A[1], h->s.mb_x, h->s.mb_y, n, list); |
1168 | 167 |
168 if(left_ref == ref){ | |
169 *mx= A[0]; | |
170 *my= A[1]; | |
171 return; | |
172 } | |
173 } | |
174 | |
175 //RARE | |
176 pred_motion(h, n, 4, list, ref, mx, my); | |
177 } | |
178 | |
179 /** | |
180 * gets the directionally predicted 8x16 MV. | |
181 * @param n the block index | |
182 * @param mx the x component of the predicted motion vector | |
183 * @param my the y component of the predicted motion vector | |
184 */ | |
185 static inline void pred_8x16_motion(H264Context * const h, int n, int list, int ref, int * const mx, int * const my){ | |
186 if(n==0){ | |
187 const int left_ref= h->ref_cache[list][ scan8[0] - 1 ]; | |
188 const int16_t * const A= h->mv_cache[list][ scan8[0] - 1 ]; | |
2967 | 189 |
4600 | 190 tprintf(h->s.avctx, "pred_8x16: (%2d %2d %2d) at %2d %2d %d list %d\n", left_ref, A[0], A[1], h->s.mb_x, h->s.mb_y, n, list); |
1168 | 191 |
192 if(left_ref == ref){ | |
193 *mx= A[0]; | |
194 *my= A[1]; | |
195 return; | |
196 } | |
197 }else{ | |
1169 | 198 const int16_t * C; |
199 int diagonal_ref; | |
200 | |
201 diagonal_ref= fetch_diagonal_mv(h, &C, scan8[4], list, 2); | |
2967 | 202 |
4600 | 203 tprintf(h->s.avctx, "pred_8x16: (%2d %2d %2d) at %2d %2d %d list %d\n", diagonal_ref, C[0], C[1], h->s.mb_x, h->s.mb_y, n, list); |
1168 | 204 |
2967 | 205 if(diagonal_ref == ref){ |
1168 | 206 *mx= C[0]; |
207 *my= C[1]; | |
208 return; | |
209 } | |
210 } | |
211 | |
212 //RARE | |
213 pred_motion(h, n, 2, list, ref, mx, my); | |
214 } | |
215 | |
216 static inline void pred_pskip_motion(H264Context * const h, int * const mx, int * const my){ | |
217 const int top_ref = h->ref_cache[0][ scan8[0] - 8 ]; | |
218 const int left_ref= h->ref_cache[0][ scan8[0] - 1 ]; | |
219 | |
4600 | 220 tprintf(h->s.avctx, "pred_pskip: (%d) (%d) at %2d %2d\n", top_ref, left_ref, h->s.mb_x, h->s.mb_y); |
1168 | 221 |
222 if(top_ref == PART_NOT_AVAILABLE || left_ref == PART_NOT_AVAILABLE | |
8454 | 223 || !( top_ref | *(uint32_t*)h->mv_cache[0][ scan8[0] - 8 ]) |
224 || !(left_ref | *(uint32_t*)h->mv_cache[0][ scan8[0] - 1 ])){ | |
2967 | 225 |
1168 | 226 *mx = *my = 0; |
227 return; | |
228 } | |
2967 | 229 |
1168 | 230 pred_motion(h, 0, 4, 0, 0, mx, my); |
231 | |
232 return; | |
233 } | |
10882
5d34ab807e91
Add forgotton multiple inclusion guards to h264_mvpred.h.
michael
parents:
10864
diff
changeset
|
234 |
5d34ab807e91
Add forgotton multiple inclusion guards to h264_mvpred.h.
michael
parents:
10864
diff
changeset
|
235 #endif /* AVCODEC_H264_MVPRED_H */ |