comparison vp3.c @ 1272:777d4145cdfb libavcodec

fix subtle logic problem in block unpacker that leads to incorrect token decoding which leads to segfaults

1271:42ea05e4a391

    int bit = 0;
    int current_superblock = 0;
    int current_run = 0;
    int decode_fully_flags = 0;
    int decode_partial_blocks = 0;

    int i, j;
    int current_fragment;

    debug_vp3("  vp3: unpacking superblock coding\n");

    /* figure out which fragments are coded; iterate through each
     * superblock (all planes) */
    s->coded_fragment_list_index = 0;
    s->first_coded_y_fragment = s->first_coded_c_fragment = 0;
    s->last_coded_y_fragment = s->last_coded_c_fragment = -1;
    memset(s->macroblock_coding, MODE_COPY, s->macroblock_count);
    for (i = 0; i < s->superblock_count; i++) {

        /* iterate through all 16 fragments in a superblock */
        for (j = 0; j < 16; j++) {

                        bit ^= 1;
                        current_run = get_fragment_run_length(gb);
                    }

                    if (bit) {
                        /* mode will be decoded in the next phase */
                        s->all_fragments[current_fragment].coding_method = 
                            MODE_INTER_NO_MV;
                        s->coded_fragment_list[s->coded_fragment_list_index] = 
                            current_fragment;
                        if ((current_fragment >= s->u_fragment_start) &&
                            (s->last_coded_y_fragment == -1)) {
                            s->first_coded_c_fragment = s->coded_fragment_list_index;
                            s->last_coded_y_fragment = s->first_coded_c_fragment - 1;
                        }
                        s->coded_fragment_list_index++;
                        s->macroblock_coding[s->all_fragments[current_fragment].macroblock] = MODE_INTER_NO_MV;
                        debug_block_coding("      superblock %d is partially coded, fragment %d is coded\n",
                            i, current_fragment);

                    s->all_fragments[current_fragment].coding_method = 
                        MODE_INTER_NO_MV;
                    s->coded_fragment_list[s->coded_fragment_list_index] = 
                        current_fragment;
                    if ((current_fragment >= s->u_fragment_start) &&
                        (s->last_coded_y_fragment == -1)) {
                        s->first_coded_c_fragment = s->coded_fragment_list_index;
                        s->last_coded_y_fragment = s->first_coded_c_fragment - 1;
                    }
                    s->coded_fragment_list_index++;
                    s->macroblock_coding[s->all_fragments[current_fragment].macroblock] = MODE_INTER_NO_MV;
                    debug_block_coding("      superblock %d is fully coded, fragment %d is coded\n",
                        i, current_fragment);
                }
            }
        }
    }

    if (s->first_coded_c_fragment == 0)
        /* no C fragments coded */
        s->last_coded_y_fragment = s->coded_fragment_list_index - 1;
    else
        s->last_coded_c_fragment = s->coded_fragment_list_index - 1;
    debug_block_coding("    %d total coded fragments, y: %d -> %d, c: %d -> %d\n",
        s->coded_fragment_list_index,
        s->first_coded_y_fragment,
        s->last_coded_y_fragment,
        s->first_coded_c_fragment,

    int token;
    int zero_run;
    DCTELEM coeff;
    Vp3Fragment *fragment;

    if ((first_fragment < 0) ||
        (first_fragment >= s->fragment_count) ||
        (last_fragment < 0) ||
        (last_fragment >= s->fragment_count)) {

        printf ("  vp3:unpack_vlcs(): bad fragment number (%d -> %d ?)\n",
            first_fragment, last_fragment);
        return 1;
    }

    for (i = first_fragment; i <= last_fragment; i++) {

        fragment = &s->all_fragments[s->coded_fragment_list[i]];

    debug_vp3(" VP3 frame #%d: Q index = %d", counter, s->quality_index);
    counter++;

    if (s->keyframe) {
        if (s->last_frame.data[0] == s->golden_frame.data[0]) {
            if (s->golden_frame.data[0])
                avctx->release_buffer(avctx, &s->golden_frame);
        } else {
            if (s->golden_frame.data[0])

        if (!s->pixel_addresses_inited)
            vp3_calculate_pixel_addresses(s);

    } else {

        /* allocate a new current frame */
        s->current_frame.reference = 0;
        if(avctx->get_buffer(avctx, &s->current_frame) < 0) {
            printf("vp3: get_buffer() failed\n");
            return -1;
        }

    }

    if (s->keyframe) {
      debug_vp3(", keyframe\n");
      /* skip the other 2 header bytes for now */
      skip_bits(&gb, 16);
    } else
      debug_vp3("\n");

    init_frame(s, &gb);

#if KEYFRAMES_ONLY
if (!s->keyframe) {

1272:777d4145cdfb

    int bit = 0;
    int current_superblock = 0;
    int current_run = 0;
    int decode_fully_flags = 0;
    int decode_partial_blocks = 0;
    int first_c_fragment_seen;

    int i, j;
    int current_fragment;

    debug_vp3("  vp3: unpacking superblock coding\n");

    /* figure out which fragments are coded; iterate through each
     * superblock (all planes) */
    s->coded_fragment_list_index = 0;
    s->first_coded_y_fragment = s->first_coded_c_fragment = 0;
    s->last_coded_y_fragment = s->last_coded_c_fragment = -1;
    first_c_fragment_seen = 0;
    memset(s->macroblock_coding, MODE_COPY, s->macroblock_count);
    for (i = 0; i < s->superblock_count; i++) {

        /* iterate through all 16 fragments in a superblock */
        for (j = 0; j < 16; j++) {

                        bit ^= 1;
                        current_run = get_fragment_run_length(gb);
                    }

                    if (bit) {
                        /* default mode; actual mode will be decoded in 
                         * the next phase */
                        s->all_fragments[current_fragment].coding_method = 
                            MODE_INTER_NO_MV;
                        s->coded_fragment_list[s->coded_fragment_list_index] = 
                            current_fragment;
                        if ((current_fragment >= s->u_fragment_start) &&
                            (s->last_coded_y_fragment == -1) &&
                            (!first_c_fragment_seen)) {
                            s->first_coded_c_fragment = s->coded_fragment_list_index;
                            s->last_coded_y_fragment = s->first_coded_c_fragment - 1;
                            first_c_fragment_seen = 1;
                        }
                        s->coded_fragment_list_index++;
                        s->macroblock_coding[s->all_fragments[current_fragment].macroblock] = MODE_INTER_NO_MV;
                        debug_block_coding("      superblock %d is partially coded, fragment %d is coded\n",
                            i, current_fragment);

                    s->all_fragments[current_fragment].coding_method = 
                        MODE_INTER_NO_MV;
                    s->coded_fragment_list[s->coded_fragment_list_index] = 
                        current_fragment;
                    if ((current_fragment >= s->u_fragment_start) &&
                        (s->last_coded_y_fragment == -1) &&
                        (!first_c_fragment_seen)) {
                        s->first_coded_c_fragment = s->coded_fragment_list_index;
                        s->last_coded_y_fragment = s->first_coded_c_fragment - 1;
                        first_c_fragment_seen = 1;
                    }
                    s->coded_fragment_list_index++;
                    s->macroblock_coding[s->all_fragments[current_fragment].macroblock] = MODE_INTER_NO_MV;
                    debug_block_coding("      superblock %d is fully coded, fragment %d is coded\n",
                        i, current_fragment);
                }
            }
        }
    }

    if (!first_c_fragment_seen)
        /* only Y fragments coded in this frame */
        s->last_coded_y_fragment = s->coded_fragment_list_index - 1;
    else 
        /* end the list of coded fragments */
        s->last_coded_c_fragment = s->coded_fragment_list_index - 1;

    debug_block_coding("    %d total coded fragments, y: %d -> %d, c: %d -> %d\n",
        s->coded_fragment_list_index,
        s->first_coded_y_fragment,
        s->last_coded_y_fragment,
        s->first_coded_c_fragment,

    int token;
    int zero_run;
    DCTELEM coeff;
    Vp3Fragment *fragment;

    if ((first_fragment >= s->fragment_count) ||
        (last_fragment >= s->fragment_count)) {

        printf ("  vp3:unpack_vlcs(): bad fragment number (%d -> %d ?)\n",
            first_fragment, last_fragment);
        return 0;
    }

    for (i = first_fragment; i <= last_fragment; i++) {

        fragment = &s->all_fragments[s->coded_fragment_list[i]];

    debug_vp3(" VP3 frame #%d: Q index = %d", counter, s->quality_index);
    counter++;

    if (s->keyframe) {

        debug_vp3(", keyframe\n");
        /* skip the other 2 header bytes for now */
        skip_bits(&gb, 16);

        if (s->last_frame.data[0] == s->golden_frame.data[0]) {
            if (s->golden_frame.data[0])
                avctx->release_buffer(avctx, &s->golden_frame);
        } else {
            if (s->golden_frame.data[0])

        if (!s->pixel_addresses_inited)
            vp3_calculate_pixel_addresses(s);

    } else {

        debug_vp3("\n");

        /* allocate a new current frame */
        s->current_frame.reference = 0;
        if(avctx->get_buffer(avctx, &s->current_frame) < 0) {
            printf("vp3: get_buffer() failed\n");
            return -1;
        }
    }

    init_frame(s, &gb);

#if KEYFRAMES_ONLY
if (!s->keyframe) {