-#define BeginIDCT() { \
- movq_m2r(*I(3), r2); \
- movq_m2r(*C(3), r6); \
- movq_r2r(r2, r4); \
- movq_m2r(*J(5), r7); \
- pmulhw_r2r(r6, r4); /* r4 = c3*i3 - i3 */ \
- movq_m2r(*C(5), r1); \
- pmulhw_r2r(r7, r6); /* r6 = c3*i5 - i5 */ \
- movq_r2r(r1, r5); \
- pmulhw_r2r(r2, r1); /* r1 = c5*i3 - i3 */ \
- movq_m2r(*I(1), r3); \
- pmulhw_r2r(r7, r5); /* r5 = c5*i5 - i5 */ \
- movq_m2r(*C(1), r0); /* (all registers are in use) */ \
- paddw_r2r(r2, r4); /* r4 = c3*i3 */ \
- paddw_r2r(r7, r6); /* r6 = c3*i5 */ \
- paddw_r2r(r1, r2); /* r2 = c5*i3 */ \
- movq_m2r(*J(7), r1); \
- paddw_r2r(r5, r7); /* r7 = c5*i5 */ \
- movq_r2r(r0, r5); /* r5 = c1 */ \
- pmulhw_r2r(r3, r0); /* r0 = c1*i1 - i1 */ \
- paddsw_r2r(r7, r4); /* r4 = C = c3*i3 + c5*i5 */ \
- pmulhw_r2r(r1, r5); /* r5 = c1*i7 - i7 */ \
- movq_m2r(*C(7), r7); \
- psubsw_r2r(r2, r6); /* r6 = D = c3*i5 - c5*i3 */ \
- paddw_r2r(r3, r0); /* r0 = c1*i1 */ \
- pmulhw_r2r(r7, r3); /* r3 = c7*i1 */ \
- movq_m2r(*I(2), r2); \
- pmulhw_r2r(r1, r7); /* r7 = c7*i7 */ \
- paddw_r2r(r1, r5); /* r5 = c1*i7 */ \
- movq_r2r(r2, r1); /* r1 = i2 */ \
- pmulhw_m2r(*C(2), r2); /* r2 = c2*i2 - i2 */ \
- psubsw_r2r(r5, r3); /* r3 = B = c7*i1 - c1*i7 */ \
- movq_m2r(*J(6), r5); \
- paddsw_r2r(r7, r0); /* r0 = A = c1*i1 + c7*i7 */ \
- movq_r2r(r5, r7); /* r7 = i6 */ \
- psubsw_r2r(r4, r0); /* r0 = A - C */ \
- pmulhw_m2r(*C(2), r5); /* r5 = c2*i6 - i6 */ \
- paddw_r2r(r1, r2); /* r2 = c2*i2 */ \
- pmulhw_m2r(*C(6), r1); /* r1 = c6*i2 */ \
- paddsw_r2r(r4, r4); /* r4 = C + C */ \
- paddsw_r2r(r0, r4); /* r4 = C. = A + C */ \
- psubsw_r2r(r6, r3); /* r3 = B - D */ \
- paddw_r2r(r7, r5); /* r5 = c2*i6 */ \
- paddsw_r2r(r6, r6); /* r6 = D + D */ \
- pmulhw_m2r(*C(6), r7); /* r7 = c6*i6 */ \
- paddsw_r2r(r3, r6); /* r6 = D. = B + D */ \
- movq_r2m(r4, *I(1)); /* save C. at I(1) */ \
- psubsw_r2r(r5, r1); /* r1 = H = c6*i2 - c2*i6 */ \
- movq_m2r(*C(4), r4); \
- movq_r2r(r3, r5); /* r5 = B - D */ \
- pmulhw_r2r(r4, r3); /* r3 = (c4 - 1) * (B - D) */ \
- paddsw_r2r(r2, r7); /* r7 = G = c6*i6 + c2*i2 */ \
- movq_r2m(r6, *I(2)); /* save D. at I(2) */ \
- movq_r2r(r0, r2); /* r2 = A - C */ \
- movq_m2r(*I(0), r6); \
- pmulhw_r2r(r4, r0); /* r0 = (c4 - 1) * (A - C) */ \
- paddw_r2r(r3, r5); /* r5 = B. = c4 * (B - D) */ \
- movq_m2r(*J(4), r3); \
- psubsw_r2r(r1, r5); /* r5 = B.. = B. - H */ \
- paddw_r2r(r0, r2); /* r0 = A. = c4 * (A - C) */ \
- psubsw_r2r(r3, r6); /* r6 = i0 - i4 */ \
- movq_r2r(r6, r0); \
- pmulhw_r2r(r4, r6); /* r6 = (c4 - 1) * (i0 - i4) */ \
- paddsw_r2r(r3, r3); /* r3 = i4 + i4 */ \
- paddsw_r2r(r1, r1); /* r1 = H + H */ \
- paddsw_r2r(r0, r3); /* r3 = i0 + i4 */ \
- paddsw_r2r(r5, r1); /* r1 = H. = B + H */ \
- pmulhw_r2r(r3, r4); /* r4 = (c4 - 1) * (i0 + i4) */ \
- paddsw_r2r(r0, r6); /* r6 = F = c4 * (i0 - i4) */ \
- psubsw_r2r(r2, r6); /* r6 = F. = F - A. */ \
- paddsw_r2r(r2, r2); /* r2 = A. + A. */ \
- movq_m2r(*I(1), r0); /* r0 = C. */ \
- paddsw_r2r(r6, r2); /* r2 = A.. = F + A. */ \
- paddw_r2r(r3, r4); /* r4 = E = c4 * (i0 + i4) */ \
- psubsw_r2r(r1, r2); /* r2 = R2 = A.. - H. */ \
-}
+#define BeginIDCT() \
+ "movq "I(3)", %%mm2 \n\t" \
+ "movq "C(3)", %%mm6 \n\t" \
+ "movq %%mm2, %%mm4 \n\t" \
+ "movq "J(5)", %%mm7 \n\t" \
+ "pmulhw %%mm6, %%mm4 \n\t" /* r4 = c3*i3 - i3 */ \
+ "movq "C(5)", %%mm1 \n\t" \
+ "pmulhw %%mm7, %%mm6 \n\t" /* r6 = c3*i5 - i5 */ \
+ "movq %%mm1, %%mm5 \n\t" \
+ "pmulhw %%mm2, %%mm1 \n\t" /* r1 = c5*i3 - i3 */ \
+ "movq "I(1)", %%mm3 \n\t" \
+ "pmulhw %%mm7, %%mm5 \n\t" /* r5 = c5*i5 - i5 */ \
+ "movq "C(1)", %%mm0 \n\t" \
+ "paddw %%mm2, %%mm4 \n\t" /* r4 = c3*i3 */ \
+ "paddw %%mm7, %%mm6 \n\t" /* r6 = c3*i5 */ \
+ "paddw %%mm1, %%mm2 \n\t" /* r2 = c5*i3 */ \
+ "movq "J(7)", %%mm1 \n\t" \
+ "paddw %%mm5, %%mm7 \n\t" /* r7 = c5*i5 */ \
+ "movq %%mm0, %%mm5 \n\t" /* r5 = c1 */ \
+ "pmulhw %%mm3, %%mm0 \n\t" /* r0 = c1*i1 - i1 */ \
+ "paddsw %%mm7, %%mm4 \n\t" /* r4 = C = c3*i3 + c5*i5 */ \
+ "pmulhw %%mm1, %%mm5 \n\t" /* r5 = c1*i7 - i7 */ \
+ "movq "C(7)", %%mm7 \n\t" \
+ "psubsw %%mm2, %%mm6 \n\t" /* r6 = D = c3*i5 - c5*i3 */ \
+ "paddw %%mm3, %%mm0 \n\t" /* r0 = c1*i1 */ \
+ "pmulhw %%mm7, %%mm3 \n\t" /* r3 = c7*i1 */ \
+ "movq "I(2)", %%mm2 \n\t" \
+ "pmulhw %%mm1, %%mm7 \n\t" /* r7 = c7*i7 */ \
+ "paddw %%mm1, %%mm5 \n\t" /* r5 = c1*i7 */ \
+ "movq %%mm2, %%mm1 \n\t" /* r1 = i2 */ \
+ "pmulhw "C(2)", %%mm2 \n\t" /* r2 = c2*i2 - i2 */ \
+ "psubsw %%mm5, %%mm3 \n\t" /* r3 = B = c7*i1 - c1*i7 */ \
+ "movq "J(6)", %%mm5 \n\t" \
+ "paddsw %%mm7, %%mm0 \n\t" /* r0 = A = c1*i1 + c7*i7 */ \
+ "movq %%mm5, %%mm7 \n\t" /* r7 = i6 */ \
+ "psubsw %%mm4, %%mm0 \n\t" /* r0 = A - C */ \
+ "pmulhw "C(2)", %%mm5 \n\t" /* r5 = c2*i6 - i6 */ \
+ "paddw %%mm1, %%mm2 \n\t" /* r2 = c2*i2 */ \
+ "pmulhw "C(6)", %%mm1 \n\t" /* r1 = c6*i2 */ \
+ "paddsw %%mm4, %%mm4 \n\t" /* r4 = C + C */ \
+ "paddsw %%mm0, %%mm4 \n\t" /* r4 = C. = A + C */ \
+ "psubsw %%mm6, %%mm3 \n\t" /* r3 = B - D */ \
+ "paddw %%mm7, %%mm5 \n\t" /* r5 = c2*i6 */ \
+ "paddsw %%mm6, %%mm6 \n\t" /* r6 = D + D */ \
+ "pmulhw "C(6)", %%mm7 \n\t" /* r7 = c6*i6 */ \
+ "paddsw %%mm3, %%mm6 \n\t" /* r6 = D. = B + D */ \
+ "movq %%mm4, "I(1)"\n\t" /* save C. at I(1) */ \
+ "psubsw %%mm5, %%mm1 \n\t" /* r1 = H = c6*i2 - c2*i6 */ \
+ "movq "C(4)", %%mm4 \n\t" \
+ "movq %%mm3, %%mm5 \n\t" /* r5 = B - D */ \
+ "pmulhw %%mm4, %%mm3 \n\t" /* r3 = (c4 - 1) * (B - D) */ \
+ "paddsw %%mm2, %%mm7 \n\t" /* r3 = (c4 - 1) * (B - D) */ \
+ "movq %%mm6, "I(2)"\n\t" /* save D. at I(2) */ \
+ "movq %%mm0, %%mm2 \n\t" /* r2 = A - C */ \
+ "movq "I(0)", %%mm6 \n\t" \
+ "pmulhw %%mm4, %%mm0 \n\t" /* r0 = (c4 - 1) * (A - C) */ \
+ "paddw %%mm3, %%mm5 \n\t" /* r5 = B. = c4 * (B - D) */ \
+ "movq "J(4)", %%mm3 \n\t" \
+ "psubsw %%mm1, %%mm5 \n\t" /* r5 = B.. = B. - H */ \
+ "paddw %%mm0, %%mm2 \n\t" /* r0 = A. = c4 * (A - C) */ \
+ "psubsw %%mm3, %%mm6 \n\t" /* r6 = i0 - i4 */ \
+ "movq %%mm6, %%mm0 \n\t" \
+ "pmulhw %%mm4, %%mm6 \n\t" /* r6 = (c4 - 1) * (i0 - i4) */ \
+ "paddsw %%mm3, %%mm3 \n\t" /* r3 = i4 + i4 */ \
+ "paddsw %%mm1, %%mm1 \n\t" /* r1 = H + H */ \
+ "paddsw %%mm0, %%mm3 \n\t" /* r3 = i0 + i4 */ \
+ "paddsw %%mm5, %%mm1 \n\t" /* r1 = H. = B + H */ \
+ "pmulhw %%mm3, %%mm4 \n\t" /* r4 = (c4 - 1) * (i0 + i4) */ \
+ "paddsw %%mm0, %%mm6 \n\t" /* r6 = F = c4 * (i0 - i4) */ \
+ "psubsw %%mm2, %%mm6 \n\t" /* r6 = F. = F - A. */ \
+ "paddsw %%mm2, %%mm2 \n\t" /* r2 = A. + A. */ \
+ "movq "I(1)", %%mm0 \n\t" /* r0 = C. */ \
+ "paddsw %%mm6, %%mm2 \n\t" /* r2 = A.. = F + A. */ \
+ "paddw %%mm3, %%mm4 \n\t" /* r4 = E = c4 * (i0 + i4) */ \
+ "psubsw %%mm1, %%mm2 \n\t" /* r2 = R2 = A.. - H. */