# HG changeset patch
# User michael
# Date 1233925190 0
# Node ID eaa08ce79f9abc8674fa2da92d2d7ba93e9645c0
# Parent  a261e3271d1f3e6c4bbe2e289449ec54acc477ea
Ensure that the palette is set in data[1] for all 8bit formats.
Also document it.

diff -r a261e3271d1f -r eaa08ce79f9a imgconvert.c
--- a/imgconvert.c	Fri Feb 06 11:49:31 2009 +0000
+++ b/imgconvert.c	Fri Feb 06 12:59:50 2009 +0000
@@ -443,6 +443,43 @@
     }
 }
 
+int ff_set_systematic_pal(uint32_t pal[256], enum PixelFormat pix_fmt){
+    int i;
+
+    for(i=0; i<256; i++){
+        int r,g,b;
+
+        switch(pix_fmt) {
+        case PIX_FMT_RGB8:
+            r= (i>>5    )*36;
+            g= ((i>>2)&7)*36;
+            b= (i&3     )*85;
+            break;
+        case PIX_FMT_BGR8:
+            b= (i>>6    )*85;
+            g= ((i>>3)&7)*36;
+            r= (i&7     )*36;
+            break;
+        case PIX_FMT_RGB4_BYTE:
+            r= (i>>3    )*255;
+            g= ((i>>1)&3)*85;
+            b= (i&1     )*255;
+            break;
+        case PIX_FMT_BGR4_BYTE:
+            b= (i>>3    )*255;
+            g= ((i>>1)&3)*85;
+            r= (i&1     )*255;
+            break;
+        case PIX_FMT_GRAY8:
+            r=b=g= i;
+            break;
+        }
+        pal[i] =  b + (g<<8) + (r<<16);
+    }
+
+    return 0;
+}
+
 int ff_fill_linesize(AVPicture *picture, int pix_fmt, int width)
 {
     int w2;
@@ -505,13 +542,6 @@
     case PIX_FMT_UYYVYY411:
         picture->linesize[0] = width + width/2;
         break;
-    case PIX_FMT_RGB8:
-    case PIX_FMT_BGR8:
-    case PIX_FMT_RGB4_BYTE:
-    case PIX_FMT_BGR4_BYTE:
-    case PIX_FMT_GRAY8:
-        picture->linesize[0] = width;
-        break;
     case PIX_FMT_RGB4:
     case PIX_FMT_BGR4:
         picture->linesize[0] = width / 2;
@@ -521,6 +551,11 @@
         picture->linesize[0] = (width + 7) >> 3;
         break;
     case PIX_FMT_PAL8:
+    case PIX_FMT_RGB8:
+    case PIX_FMT_BGR8:
+    case PIX_FMT_RGB4_BYTE:
+    case PIX_FMT_BGR4_BYTE:
+    case PIX_FMT_GRAY8:
         picture->linesize[0] = width;
         picture->linesize[1] = 4;
         break;
@@ -588,11 +623,6 @@
     case PIX_FMT_YUYV422:
     case PIX_FMT_UYVY422:
     case PIX_FMT_UYYVYY411:
-    case PIX_FMT_RGB8:
-    case PIX_FMT_BGR8:
-    case PIX_FMT_RGB4_BYTE:
-    case PIX_FMT_BGR4_BYTE:
-    case PIX_FMT_GRAY8:
     case PIX_FMT_RGB4:
     case PIX_FMT_BGR4:
     case PIX_FMT_MONOWHITE:
@@ -603,6 +633,11 @@
         picture->data[3] = NULL;
         return size;
     case PIX_FMT_PAL8:
+    case PIX_FMT_RGB8:
+    case PIX_FMT_BGR8:
+    case PIX_FMT_RGB4_BYTE:
+    case PIX_FMT_BGR4_BYTE:
+    case PIX_FMT_GRAY8:
         size2 = (size + 3) & ~3;
         picture->data[0] = ptr;
         picture->data[1] = ptr + size2; /* palette is stored here as 256 32 bit words */
@@ -2000,6 +2035,9 @@
     if (!ptr)
         goto fail;
     avpicture_fill(picture, ptr, pix_fmt, width, height);
+    if(picture->data[1] && !picture->data[2])
+        ff_set_systematic_pal((uint32_t*)picture->data[1], pix_fmt);
+
     return 0;
  fail:
     memset(picture, 0, sizeof(AVPicture));
diff -r a261e3271d1f -r eaa08ce79f9a imgconvert.h
--- a/imgconvert.h	Fri Feb 06 11:49:31 2009 +0000
+++ b/imgconvert.h	Fri Feb 06 12:59:50 2009 +0000
@@ -33,6 +33,8 @@
 
 int ff_get_plane_bytewidth(enum PixelFormat pix_fmt, int width, int plane);
 
+int ff_set_systematic_pal(uint32_t pal[256], enum PixelFormat pix_fmt);
+
 int img_convert(AVPicture *dst, int dst_pix_fmt, const AVPicture *src,
                 int src_pix_fmt, int src_width, int src_height);
 
diff -r a261e3271d1f -r eaa08ce79f9a utils.c
--- a/utils.c	Fri Feb 06 11:49:31 2009 +0000
+++ b/utils.c	Fri Feb 06 12:59:50 2009 +0000
@@ -154,6 +154,8 @@
             h_align=4;
         }
     case PIX_FMT_PAL8:
+    case PIX_FMT_BGR8:
+    case PIX_FMT_RGB8:
         if(s->codec_id == CODEC_ID_SMC){
             w_align=4;
             h_align=4;
@@ -281,12 +283,14 @@
             if(buf->base[i]==NULL) return -1;
             memset(buf->base[i], 128, size[i]);
 
-            // no edge if EDEG EMU or not planar YUV, we check for PAL8 redundantly to protect against a exploitable bug regression ...
-            if((s->flags&CODEC_FLAG_EMU_EDGE) || (s->pix_fmt == PIX_FMT_PAL8) || !size[2])
+            // no edge if EDEG EMU or not planar YUV
+            if((s->flags&CODEC_FLAG_EMU_EDGE) || !size[2])
                 buf->data[i] = buf->base[i];
             else
                 buf->data[i] = buf->base[i] + ALIGN((buf->linesize[i]*EDGE_WIDTH>>v_shift) + (EDGE_WIDTH>>h_shift), stride_align[i]);
         }
+        if(size[1] && !size[2])
+            ff_set_systematic_pal((uint32_t*)buf->data[1], s->pix_fmt);
         buf->width  = s->width;
         buf->height = s->height;
         buf->pix_fmt= s->pix_fmt;