comparison libaf/af_format.c @ 33736:a5a54c7a15ce

Fix the precision loss in float -> 32bit conversion case, introduced by my earlier commit. Instead use method proposed by Reimar. Also, avoid using ldexp, it is slower than multiply with constant.

33735:84defaf2e7f8

    for(i=0;i<len;i++)
      ((int16_t*)out)[i] = av_clip_int16(lrintf(32768.0 * in[i]));
    break;
  case(3):
    for(i=0;i<len;i++){
      f = ldexp(in[i], 23);
      store24bit(out, i,   av_clip(lrintf(f), -1*(1<<23), (1<<23)-1) << 8);
    }
    break;
  case(4):
    for(i=0;i<len;i++){
      f = ldexp(in[i], 23);
      //The mantissa is only 23 bit, that's all the precision there is.
      ((int32_t*)out)[i] = av_clip(lrintf(f), -1*(1<<23), (1<<23)-1) << 8;
    }
    break;
  }
}

33736:a5a54c7a15ce

    for(i=0;i<len;i++)
      ((int16_t*)out)[i] = av_clip_int16(lrintf(32768.0 * in[i]));
    break;
  case(3):
    for(i=0;i<len;i++){
      f = in[i] * 8388608;
      store24bit(out, i,   av_clip(lrintf(f), -1*(1<<23), (1<<23)-1) << 8);
    }
    break;
  case(4):
    for(i=0;i<len;i++){
      f = in[i];
      if (f <= -1.0)
        ((int32_t*)out)[i] = INT_MIN;
      else
      if (f >=  1.0)//no need to use corrected constant, rounding won't cause overflow
        ((int32_t*)out)[i] = INT_MAX;
      else
        ((int32_t*)out)[i] = lrintf(f*2147483648.0);

    }
    break;
  }
}