libavcodec.hg: lcl.c comparison

comparison lcl.c @ 2418:82af834636c2 libavcodec

Check pointers before writing to memory, fix possible integer overflows Force alignement for mszh and zlib decoders

author	rtognimp
date	Sun, 09 Jan 2005 23:39:32 +0000
parents	582e635cfa08
children	4b350cc506a7

comparison

equal deleted inserted replaced

-:991f39305057
+:82af834636c2
 return fix((yq << 20) + rq * 1470103);
 }
-static int mszh_decomp(unsigned char * srcptr, int srclen, unsigned char * destptr)
+static unsigned int mszh_decomp(unsigned char * srcptr, int srclen, unsigned char * destptr, unsigned int destsize)
 {
 unsigned char *destptr_bak = destptr;
+unsigned char *destptr_end = destptr + destsize;
 unsigned char mask = 0;
 unsigned char maskbit = 0;
 unsigned int ofs, cnt;
-while (srclen > 0) {
+while ((srclen > 0) && (destptr < destptr_end)) {
 if (maskbit == 0) {
 mask = *(srcptr++);
 maskbit = 8;
 srclen--;
 continue;
 }
 if ((mask & (1 << (--maskbit))) == 0) {
+if (destptr + 4 > destptr_end)
+break;
 *(int*)destptr = *(int*)srcptr;
 srclen -= 4;
 destptr += 4;
 srcptr += 4;
 } else {
 ofs += cnt * 256;;
 cnt = ((cnt >> 3) & 0x1f) + 1;
 ofs &= 0x7ff;
 srclen -= 2;
 cnt *= 4;
+if (destptr + cnt > destptr_end) {
+cnt =  destptr_end - destptr;
+}
 for (; cnt > 0; cnt--) {
 *(destptr) = *(destptr - ofs);
 destptr++;
 }
 }
 */
 static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, uint8_t *buf, int buf_size)
 {
 	LclContext * const c = (LclContext *)avctx->priv_data;
 	unsigned char *encoded = (unsigned char *)buf;
-int pixel_ptr;
+unsigned int pixel_ptr;
 int row, col;
 unsigned char *outptr;
 unsigned int width = avctx->width; // Real image width
 unsigned int height = avctx->height; // Real image height
 unsigned int mszh_dlen;
 int uqvq;
 unsigned int mthread_inlen, mthread_outlen;
 #ifdef CONFIG_ZLIB
 int zret; // Zlib return code
 #endif
-int len = buf_size;
+unsigned int len = buf_size;
 	/* no supplementary picture */
 	if (buf_size == 0)
 		return 0;
 switch (c->compression) {
 case COMP_MSZH:
 if (c->flags & FLAG_MULTITHREAD) {
 mthread_inlen = *((unsigned int*)encoded);
 mthread_outlen = *((unsigned int*)(encoded+4));
-mszh_dlen = mszh_decomp(encoded + 8, mthread_inlen, c->decomp_buf);
+if (mthread_outlen > c->decomp_size) // this should not happen
+mthread_outlen = c->decomp_size;
+mszh_dlen = mszh_decomp(encoded + 8, mthread_inlen, c->decomp_buf, c->decomp_size);
 if (mthread_outlen != mszh_dlen) {
 av_log(avctx, AV_LOG_ERROR, "Mthread1 decoded size differs (%d != %d)\n",
 mthread_outlen, mszh_dlen);
+return -1;
 }
 mszh_dlen = mszh_decomp(encoded + 8 + mthread_inlen, len - mthread_inlen,
-c->decomp_buf + mthread_outlen);
+c->decomp_buf + mthread_outlen, c->decomp_size - mthread_outlen);
-if ((c->decomp_size - mthread_outlen) != mszh_dlen) {
+if (mthread_outlen != mszh_dlen) {
 av_log(avctx, AV_LOG_ERROR, "Mthread2 decoded size differs (%d != %d)\n",
-c->decomp_size - mthread_outlen, mszh_dlen);
+mthread_outlen, mszh_dlen);
+return -1;
 }
 encoded = c->decomp_buf;
 len = c->decomp_size;
 } else {
-mszh_dlen = mszh_decomp(encoded, len, c->decomp_buf);
+mszh_dlen = mszh_decomp(encoded, len, c->decomp_buf, c->decomp_size);
 if (c->decomp_size != mszh_dlen) {
 av_log(avctx, AV_LOG_ERROR, "Decoded size differs (%d != %d)\n",
 c->decomp_size, mszh_dlen);
+return -1;
 }
 encoded = c->decomp_buf;
 len = mszh_dlen;
 }
 break;
 return -1;
 }
 if (c->flags & FLAG_MULTITHREAD) {
 mthread_inlen = *((unsigned int*)encoded);
 mthread_outlen = *((unsigned int*)(encoded+4));
+if (mthread_outlen > c->decomp_size)
+mthread_outlen = c->decomp_size;
 c->zstream.next_in = encoded + 8;
 c->zstream.avail_in = mthread_inlen;
 c->zstream.next_out = c->decomp_buf;
-c->zstream.avail_out = mthread_outlen;
+c->zstream.avail_out = c->decomp_size;
 zret = inflate(&(c->zstream), Z_FINISH);
 if ((zret != Z_OK) && (zret != Z_STREAM_END)) {
 av_log(avctx, AV_LOG_ERROR, "Mthread1 inflate error: %d\n", zret);
 return -1;
 }
 if (mthread_outlen != (unsigned int)(c->zstream.total_out)) {
 av_log(avctx, AV_LOG_ERROR, "Mthread1 decoded size differs (%u != %lu)\n",
 mthread_outlen, c->zstream.total_out);
+return -1;
 }
 zret = inflateReset(&(c->zstream));
 if (zret != Z_OK) {
 av_log(avctx, AV_LOG_ERROR, "Mthread2 inflate reset error: %d\n", zret);
 return -1;
 }
 c->zstream.next_in = encoded + 8 + mthread_inlen;
 c->zstream.avail_in = len - mthread_inlen;
 c->zstream.next_out = c->decomp_buf + mthread_outlen;
-c->zstream.avail_out = mthread_outlen;
+c->zstream.avail_out = c->decomp_size - mthread_outlen;
 zret = inflate(&(c->zstream), Z_FINISH);
 if ((zret != Z_OK) && (zret != Z_STREAM_END)) {
 av_log(avctx, AV_LOG_ERROR, "Mthread2 inflate error: %d\n", zret);
 return -1;
 }
-if ((c->decomp_size - mthread_outlen) != (unsigned int)(c->zstream.total_out)) {
+if (mthread_outlen != (unsigned int)(c->zstream.total_out)) {
 av_log(avctx, AV_LOG_ERROR, "Mthread2 decoded size differs (%d != %lu)\n",
-c->decomp_size - mthread_outlen, c->zstream.total_out);
+mthread_outlen, c->zstream.total_out);
+return -1;
 }
 } else {
 c->zstream.next_in = encoded;
 c->zstream.avail_in = len;
 c->zstream.next_out = c->decomp_buf;
 return -1;
 }
 if (c->decomp_size != (unsigned int)(c->zstream.total_out)) {
 av_log(avctx, AV_LOG_ERROR, "Decoded size differs (%d != %lu)\n",
 c->decomp_size, c->zstream.total_out);
+return -1;
 }
 }
 encoded = c->decomp_buf;
 len = c->decomp_size;;
 #else
 *
 */
 static int decode_init(AVCodecContext *avctx)
 {
 LclContext * const c = (LclContext *)avctx->priv_data;
-int basesize = avctx->width * avctx->height;
+unsigned int basesize = avctx->width * avctx->height;
+unsigned int max_basesize = ((avctx->width + 3) & ~3) * ((avctx->height + 3) & ~3);
+unsigned int max_decomp_size;
 int zret; // Zlib return code
 c->avctx = avctx;
 avctx->has_b_frames = 0;
 memset(&(c->zstream), 0, sizeof(z_stream));
 #endif
 if (avctx->extradata_size < 8) {
 av_log(avctx, AV_LOG_ERROR, "Extradata size too small.\n");
+return 1;
+}
+// FIXME: find a better way to prevent integer overflow
+if (((unsigned int)avctx->width > 32000) || ((unsigned int)avctx->height > 32000)) {
+av_log(avctx, AV_LOG_ERROR, "Bad image size (w = %d, h = %d).\n", avctx->width, avctx->height);
 return 1;
 }
 /* Check codec type */
 if (((avctx->codec_id == CODEC_ID_MSZH)  && (*((char *)avctx->extradata + 7) != CODEC_MSZH)) ||
 /* Detect image type */
 switch (c->imgtype = *((char *)avctx->extradata + 4)) {
 case IMGTYPE_YUV111:
 c->decomp_size = basesize * 3;
+max_decomp_size = max_basesize * 3;
 av_log(avctx, AV_LOG_INFO, "Image type is YUV 1:1:1.\n");
 break;
 case IMGTYPE_YUV422:
 c->decomp_size = basesize * 2;
+max_decomp_size = max_basesize * 2;
 av_log(avctx, AV_LOG_INFO, "Image type is YUV 4:2:2.\n");
 break;
 case IMGTYPE_RGB24:
 c->decomp_size = basesize * 3;
+max_decomp_size = max_basesize * 3;
 av_log(avctx, AV_LOG_INFO, "Image type is RGB 24.\n");
 break;
 case IMGTYPE_YUV411:
 c->decomp_size = basesize / 2 * 3;
+max_decomp_size = max_basesize / 2 * 3;
 av_log(avctx, AV_LOG_INFO, "Image type is YUV 4:1:1.\n");
 break;
 case IMGTYPE_YUV211:
 c->decomp_size = basesize * 2;
+max_decomp_size = max_basesize * 2;
 av_log(avctx, AV_LOG_INFO, "Image type is YUV 2:1:1.\n");
 break;
 case IMGTYPE_YUV420:
 c->decomp_size = basesize / 2 * 3;
+max_decomp_size = max_basesize / 2 * 3;
 av_log(avctx, AV_LOG_INFO, "Image type is YUV 4:2:0.\n");
 break;
 default:
 av_log(avctx, AV_LOG_ERROR, "Unsupported image format %d.\n", c->imgtype);
 return 1;
 av_log(avctx, AV_LOG_ERROR, "BUG! Unknown codec in compression switch.\n");
 return 1;
 }
 /* Allocate decompression buffer */
-/* 4*8 max overflow space for mszh decomp algorithm */
 if (c->decomp_size) {
-if ((c->decomp_buf = av_malloc(c->decomp_size+4*8)) == NULL) {
+if ((c->decomp_buf = av_malloc(max_decomp_size)) == NULL) {
 av_log(avctx, AV_LOG_ERROR, "Can't allocate decompression buffer.\n");
 return 1;
 }
 }

Mercurial > libavcodec.hg

comparison lcl.c @ 2418:82af834636c2 libavcodec