vp6: ensure that huffman decoding table is sorted with descending symbol order

Previous code worked by pure luck with glibc qsort implementation.
patch by Peter Ross   pross _at_ xvid _dot_ org

git-svn-id: file:///var/local/repositories/ffmpeg/trunk@13887 9553f0bf-9b14-0410-a0b8-cfaf0461ba5b

diff --git a/libavcodec/vp6.c b/libavcodec/vp6.c
index 74c62ff66d918bd46fd20c1d648bdacc50e303cb..c1cd6fe64bcf3e3c0a4e5c8cc742f619bb88a989 100644 (file)
--- a/libavcodec/vp6.c
+++ b/libavcodec/vp6.c
@@ -202,10 +202,11 @@ static void vp6_parse_vector_models(vp56_context_t *s)
                 model->vector_fdv[comp][node] = vp56_rac_gets_nn(c, 7);
 }
 
+/* nodes must ascend by count, but with descending symbol order */
 static int vp6_huff_cmp(const void *va, const void *vb)
 {
     const Node *a = va, *b = vb;
-    return a->count >= b->count;
+    return (a->count - b->count)*16 + (b->sym - a->sym);
 }
 
 static void vp6_build_huff_tree(vp56_context_t *s, uint8_t coeff_model[],