l4/pkg/valgrind/src/valgrind-3.6.0-svn/include/valgrind.h

   1 /* -*- c -*-
   2    ----------------------------------------------------------------
   3
   4    Notice that the following BSD-style license applies to this one
   5    file (valgrind.h) only.  The rest of Valgrind is licensed under the
   6    terms of the GNU General Public License, version 2, unless
   7    otherwise indicated.  See the COPYING file in the source
   8    distribution for details.
   9
  10    ----------------------------------------------------------------
  11
  12    This file is part of Valgrind, a dynamic binary instrumentation
  13    framework.
  14
  15    Copyright (C) 2000-2010 Julian Seward.  All rights reserved.
  16
  17    Redistribution and use in source and binary forms, with or without
  18    modification, are permitted provided that the following conditions
  19    are met:
  20
  21    1. Redistributions of source code must retain the above copyright
  22       notice, this list of conditions and the following disclaimer.
  23
  24    2. The origin of this software must not be misrepresented; you must
  25       not claim that you wrote the original software.  If you use this
  26       software in a product, an acknowledgment in the product
  27       documentation would be appreciated but is not required.
  28
  29    3. Altered source versions must be plainly marked as such, and must
  30       not be misrepresented as being the original software.
  31
  32    4. The name of the author may not be used to endorse or promote
  33       products derived from this software without specific prior written
  34       permission.
  35
  36    THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
  37    OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  38    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  39    ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
  40    DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  41    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
  42    GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  43    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  44    WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  45    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  46    SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  47
  48    ----------------------------------------------------------------
  49
  50    Notice that the above BSD-style license applies to this one file
  51    (valgrind.h) only.  The entire rest of Valgrind is licensed under
  52    the terms of the GNU General Public License, version 2.  See the
  53    COPYING file in the source distribution for details.
  54
  55    ----------------------------------------------------------------
  56 */
  57
  58
  59 /* This file is for inclusion into client (your!) code.
  60
  61    You can use these macros to manipulate and query Valgrind's
  62    execution inside your own programs.
  63
  64    The resulting executables will still run without Valgrind, just a
  65    little bit more slowly than they otherwise would, but otherwise
  66    unchanged.  When not running on valgrind, each client request
  67    consumes very few (eg. 7) instructions, so the resulting performance
  68    loss is negligible unless you plan to execute client requests
  69    millions of times per second.  Nevertheless, if that is still a
  70    problem, you can compile with the NVALGRIND symbol defined (gcc
  71    -DNVALGRIND) so that client requests are not even compiled in.  */
  72
  73 #ifndef __VALGRIND_H
  74 #define __VALGRIND_H
  75
  76
  77 /* ------------------------------------------------------------------ */
  78 /* VERSION NUMBER OF VALGRIND                                         */
  79 /* ------------------------------------------------------------------ */
  80
  81 /* Specify Valgrind's version number, so that user code can
  82    conditionally compile based on our version number.  Note that these
  83    were introduced at version 3.6 and so do not exist in version 3.5
  84    or earlier.  The recommended way to use them to check for "version
  85    X.Y or later" is (eg)
  86
  87 #if defined(__VALGRIND_MAJOR__) && defined(__VALGRIND_MINOR__)   \
  88     && (__VALGRIND_MAJOR__ > 3                                   \
  89         || (__VALGRIND_MAJOR__ == 3 && __VALGRIND_MINOR__ >= 6))
  90 */
  91 #define __VALGRIND_MAJOR__    3
  92 #define __VALGRIND_MINOR__    6
  93
  94
  95 #include <stdarg.h>
  96
  97 /* Nb: this file might be included in a file compiled with -ansi.  So
  98    we can't use C++ style "//" comments nor the "asm" keyword (instead
  99    use "__asm__"). */
 100
 101 /* Derive some tags indicating what the target platform is.  Note
 102    that in this file we're using the compiler's CPP symbols for
 103    identifying architectures, which are different to the ones we use
 104    within the rest of Valgrind.  Note, __powerpc__ is active for both
 105    32 and 64-bit PPC, whereas __powerpc64__ is only active for the
 106    latter (on Linux, that is).
 107
 108    Misc note: how to find out what's predefined in gcc by default:
 109    gcc -Wp,-dM somefile.c
 110 */
 111 #undef PLAT_ppc64_aix5
 112 #undef PLAT_ppc32_aix5
 113 #undef PLAT_x86_darwin
 114 #undef PLAT_amd64_darwin
 115 #undef PLAT_x86_linux
 116 #undef PLAT_amd64_linux
 117 #undef PLAT_ppc32_linux
 118 #undef PLAT_ppc64_linux
 119 #undef PLAT_arm_linux
 120
 121 #if defined(_AIX) && defined(__64BIT__)
 122 #  define PLAT_ppc64_aix5 1
 123 #elif defined(_AIX) && !defined(__64BIT__)
 124 #  define PLAT_ppc32_aix5 1
 125 #elif defined(__APPLE__) && defined(__i386__)
 126 #  define PLAT_x86_darwin 1
 127 #elif defined(__APPLE__) && defined(__x86_64__)
 128 #  define PLAT_amd64_darwin 1
 129 #elif defined(__linux__) && defined(__i386__)
 130 #  define PLAT_x86_linux 1
 131 #elif defined(__linux__) && defined(__x86_64__)
 132 #  define PLAT_amd64_linux 1
 133 #elif defined(__linux__) && defined(__powerpc__) && !defined(__powerpc64__)
 134 #  define PLAT_ppc32_linux 1
 135 #elif defined(__linux__) && defined(__powerpc__) && defined(__powerpc64__)
 136 #  define PLAT_ppc64_linux 1
 137 #elif defined(__linux__) && defined(__arm__)
 138 #  define PLAT_arm_linux 1
 139 #else
 140 /* If we're not compiling for our target platform, don't generate
 141    any inline asms.  */
 142 #  if !defined(NVALGRIND)
 143 #    define NVALGRIND 1
 144 #  endif
 145 #endif
 146
 147
 148 /* ------------------------------------------------------------------ */
 149 /* ARCHITECTURE SPECIFICS for SPECIAL INSTRUCTIONS.  There is nothing */
 150 /* in here of use to end-users -- skip to the next section.           */
 151 /* ------------------------------------------------------------------ */
 152
 153 #if defined(NVALGRIND)
 154
 155 /* Define NVALGRIND to completely remove the Valgrind magic sequence
 156    from the compiled code (analogous to NDEBUG's effects on
 157    assert()) */
 158 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 159         _zzq_rlval, _zzq_default, _zzq_request,                   \
 160         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 161    {                                                              \
 162       (_zzq_rlval) = (_zzq_default);                              \
 163    }
 164
 165 #else  /* ! NVALGRIND */
 166
 167 /* The following defines the magic code sequences which the JITter
 168    spots and handles magically.  Don't look too closely at them as
 169    they will rot your brain.
 170
 171    The assembly code sequences for all architectures is in this one
 172    file.  This is because this file must be stand-alone, and we don't
 173    want to have multiple files.
 174
 175    For VALGRIND_DO_CLIENT_REQUEST, we must ensure that the default
 176    value gets put in the return slot, so that everything works when
 177    this is executed not under Valgrind.  Args are passed in a memory
 178    block, and so there's no intrinsic limit to the number that could
 179    be passed, but it's currently five.
 180
 181    The macro args are:
 182       _zzq_rlval    result lvalue
 183       _zzq_default  default value (result returned when running on real CPU)
 184       _zzq_request  request code
 185       _zzq_arg1..5  request params
 186
 187    The other two macros are used to support function wrapping, and are
 188    a lot simpler.  VALGRIND_GET_NR_CONTEXT returns the value of the
 189    guest's NRADDR pseudo-register and whatever other information is
 190    needed to safely run the call original from the wrapper: on
 191    ppc64-linux, the R2 value at the divert point is also needed.  This
 192    information is abstracted into a user-visible type, OrigFn.
 193
 194    VALGRIND_CALL_NOREDIR_* behaves the same as the following on the
 195    guest, but guarantees that the branch instruction will not be
 196    redirected: x86: call *%eax, amd64: call *%rax, ppc32/ppc64:
 197    branch-and-link-to-r11.  VALGRIND_CALL_NOREDIR is just text, not a
 198    complete inline asm, since it needs to be combined with more magic
 199    inline asm stuff to be useful.
 200 */
 201
 202 /* ------------------------- x86-{linux,darwin} ---------------- */
 203
 204 #if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin)
 205
 206 typedef
 207    struct {
 208       unsigned int nraddr; /* where's the code? */
 209    }
 210    OrigFn;
 211
 212 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 213                      "roll $3,  %%edi ; roll $13, %%edi\n\t"      \
 214                      "roll $29, %%edi ; roll $19, %%edi\n\t"
 215
 216 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 217         _zzq_rlval, _zzq_default, _zzq_request,                   \
 218         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 219   { volatile unsigned int _zzq_args[6];                           \
 220     volatile unsigned int _zzq_result;                            \
 221     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 222     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 223     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 224     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 225     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 226     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 227     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 228                      /* %EDX = client_request ( %EAX ) */         \
 229                      "xchgl %%ebx,%%ebx"                          \
 230                      : "=d" (_zzq_result)                         \
 231                      : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
 232                      : "cc", "memory"                             \
 233                     );                                            \
 234     _zzq_rlval = _zzq_result;                                     \
 235   }
 236
 237 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 238   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 239     volatile unsigned int __addr;                                 \
 240     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 241                      /* %EAX = guest_NRADDR */                    \
 242                      "xchgl %%ecx,%%ecx"                          \
 243                      : "=a" (__addr)                              \
 244                      :                                            \
 245                      : "cc", "memory"                             \
 246                     );                                            \
 247     _zzq_orig->nraddr = __addr;                                   \
 248   }
 249
 250 #define VALGRIND_CALL_NOREDIR_EAX                                 \
 251                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 252                      /* call-noredir *%EAX */                     \
 253                      "xchgl %%edx,%%edx\n\t"
 254 #endif /* PLAT_x86_linux || PLAT_x86_darwin */
 255
 256 /* ------------------------ amd64-{linux,darwin} --------------- */
 257
 258 #if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin)
 259
 260 typedef
 261    struct {
 262       unsigned long long int nraddr; /* where's the code? */
 263    }
 264    OrigFn;
 265
 266 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 267                      "rolq $3,  %%rdi ; rolq $13, %%rdi\n\t"      \
 268                      "rolq $61, %%rdi ; rolq $51, %%rdi\n\t"
 269
 270 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 271         _zzq_rlval, _zzq_default, _zzq_request,                   \
 272         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 273   { volatile unsigned long long int _zzq_args[6];                 \
 274     volatile unsigned long long int _zzq_result;                  \
 275     _zzq_args[0] = (unsigned long long int)(_zzq_request);        \
 276     _zzq_args[1] = (unsigned long long int)(_zzq_arg1);           \
 277     _zzq_args[2] = (unsigned long long int)(_zzq_arg2);           \
 278     _zzq_args[3] = (unsigned long long int)(_zzq_arg3);           \
 279     _zzq_args[4] = (unsigned long long int)(_zzq_arg4);           \
 280     _zzq_args[5] = (unsigned long long int)(_zzq_arg5);           \
 281     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 282                      /* %RDX = client_request ( %RAX ) */         \
 283                      "xchgq %%rbx,%%rbx"                          \
 284                      : "=d" (_zzq_result)                         \
 285                      : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
 286                      : "cc", "memory"                             \
 287                     );                                            \
 288     _zzq_rlval = _zzq_result;                                     \
 289   }
 290
 291 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 292   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 293     volatile unsigned long long int __addr;                       \
 294     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 295                      /* %RAX = guest_NRADDR */                    \
 296                      "xchgq %%rcx,%%rcx"                          \
 297                      : "=a" (__addr)                              \
 298                      :                                            \
 299                      : "cc", "memory"                             \
 300                     );                                            \
 301     _zzq_orig->nraddr = __addr;                                   \
 302   }
 303
 304 #define VALGRIND_CALL_NOREDIR_RAX                                 \
 305                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 306                      /* call-noredir *%RAX */                     \
 307                      "xchgq %%rdx,%%rdx\n\t"
 308 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin */
 309
 310 /* ------------------------ ppc32-linux ------------------------ */
 311
 312 #if defined(PLAT_ppc32_linux)
 313
 314 typedef
 315    struct {
 316       unsigned int nraddr; /* where's the code? */
 317    }
 318    OrigFn;
 319
 320 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 321                      "rlwinm 0,0,3,0,0  ; rlwinm 0,0,13,0,0\n\t"  \
 322                      "rlwinm 0,0,29,0,0 ; rlwinm 0,0,19,0,0\n\t"
 323
 324 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 325         _zzq_rlval, _zzq_default, _zzq_request,                   \
 326         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 327                                                                   \
 328   {          unsigned int  _zzq_args[6];                          \
 329              unsigned int  _zzq_result;                           \
 330              unsigned int* _zzq_ptr;                              \
 331     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 332     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 333     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 334     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 335     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 336     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 337     _zzq_ptr = _zzq_args;                                         \
 338     __asm__ volatile("mr 3,%1\n\t" /*default*/                    \
 339                      "mr 4,%2\n\t" /*ptr*/                        \
 340                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 341                      /* %R3 = client_request ( %R4 ) */           \
 342                      "or 1,1,1\n\t"                               \
 343                      "mr %0,3"     /*result*/                     \
 344                      : "=b" (_zzq_result)                         \
 345                      : "b" (_zzq_default), "b" (_zzq_ptr)         \
 346                      : "cc", "memory", "r3", "r4");               \
 347     _zzq_rlval = _zzq_result;                                     \
 348   }
 349
 350 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 351   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 352     unsigned int __addr;                                          \
 353     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 354                      /* %R3 = guest_NRADDR */                     \
 355                      "or 2,2,2\n\t"                               \
 356                      "mr %0,3"                                    \
 357                      : "=b" (__addr)                              \
 358                      :                                            \
 359                      : "cc", "memory", "r3"                       \
 360                     );                                            \
 361     _zzq_orig->nraddr = __addr;                                   \
 362   }
 363
 364 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 365                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 366                      /* branch-and-link-to-noredir *%R11 */       \
 367                      "or 3,3,3\n\t"
 368 #endif /* PLAT_ppc32_linux */
 369
 370 /* ------------------------ ppc64-linux ------------------------ */
 371
 372 #if defined(PLAT_ppc64_linux)
 373
 374 typedef
 375    struct {
 376       unsigned long long int nraddr; /* where's the code? */
 377       unsigned long long int r2;  /* what tocptr do we need? */
 378    }
 379    OrigFn;
 380
 381 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 382                      "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
 383                      "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
 384
 385 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 386         _zzq_rlval, _zzq_default, _zzq_request,                   \
 387         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 388                                                                   \
 389   {          unsigned long long int  _zzq_args[6];                \
 390     register unsigned long long int  _zzq_result __asm__("r3");   \
 391     register unsigned long long int* _zzq_ptr __asm__("r4");      \
 392     _zzq_args[0] = (unsigned long long int)(_zzq_request);        \
 393     _zzq_args[1] = (unsigned long long int)(_zzq_arg1);           \
 394     _zzq_args[2] = (unsigned long long int)(_zzq_arg2);           \
 395     _zzq_args[3] = (unsigned long long int)(_zzq_arg3);           \
 396     _zzq_args[4] = (unsigned long long int)(_zzq_arg4);           \
 397     _zzq_args[5] = (unsigned long long int)(_zzq_arg5);           \
 398     _zzq_ptr = _zzq_args;                                         \
 399     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 400                      /* %R3 = client_request ( %R4 ) */           \
 401                      "or 1,1,1"                                   \
 402                      : "=r" (_zzq_result)                         \
 403                      : "0" (_zzq_default), "r" (_zzq_ptr)         \
 404                      : "cc", "memory");                           \
 405     _zzq_rlval = _zzq_result;                                     \
 406   }
 407
 408 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 409   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 410     register unsigned long long int __addr __asm__("r3");         \
 411     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 412                      /* %R3 = guest_NRADDR */                     \
 413                      "or 2,2,2"                                   \
 414                      : "=r" (__addr)                              \
 415                      :                                            \
 416                      : "cc", "memory"                             \
 417                     );                                            \
 418     _zzq_orig->nraddr = __addr;                                   \
 419     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 420                      /* %R3 = guest_NRADDR_GPR2 */                \
 421                      "or 4,4,4"                                   \
 422                      : "=r" (__addr)                              \
 423                      :                                            \
 424                      : "cc", "memory"                             \
 425                     );                                            \
 426     _zzq_orig->r2 = __addr;                                       \
 427   }
 428
 429 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 430                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 431                      /* branch-and-link-to-noredir *%R11 */       \
 432                      "or 3,3,3\n\t"
 433
 434 #endif /* PLAT_ppc64_linux */
 435
 436 /* ------------------------- arm-linux ------------------------- */
 437
 438 #if defined(PLAT_arm_linux)
 439
 440 typedef
 441    struct {
 442       unsigned int nraddr; /* where's the code? */
 443    }
 444    OrigFn;
 445
 446 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 447             "mov r12, r12, ror #3  ; mov r12, r12, ror #13 \n\t"  \
 448             "mov r12, r12, ror #29 ; mov r12, r12, ror #19 \n\t"
 449
 450 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 451         _zzq_rlval, _zzq_default, _zzq_request,                   \
 452         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 453                                                                   \
 454   { volatile unsigned int  _zzq_args[6];                          \
 455     volatile unsigned int  _zzq_result;                           \
 456     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 457     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 458     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 459     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 460     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 461     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 462     __asm__ volatile("mov r3, %1\n\t" /*default*/                 \
 463                      "mov r4, %2\n\t" /*ptr*/                     \
 464                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 465                      /* R3 = client_request ( R4 ) */             \
 466                      "orr r10, r10, r10\n\t"                      \
 467                      "mov %0, r3"     /*result*/                  \
 468                      : "=r" (_zzq_result)                         \
 469                      : "r" (_zzq_default), "r" (&_zzq_args[0])    \
 470                      : "cc","memory", "r3", "r4");                \
 471     _zzq_rlval = _zzq_result;                                     \
 472   }
 473
 474 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 475   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 476     unsigned int __addr;                                          \
 477     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 478                      /* R3 = guest_NRADDR */                      \
 479                      "orr r11, r11, r11\n\t"                      \
 480                      "mov %0, r3"                                 \
 481                      : "=r" (__addr)                              \
 482                      :                                            \
 483                      : "cc", "memory", "r3"                       \
 484                     );                                            \
 485     _zzq_orig->nraddr = __addr;                                   \
 486   }
 487
 488 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                    \
 489                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 490                      /* branch-and-link-to-noredir *%R4 */        \
 491                      "orr r12, r12, r12\n\t"
 492
 493 #endif /* PLAT_arm_linux */
 494
 495 /* ------------------------ ppc32-aix5 ------------------------- */
 496
 497 #if defined(PLAT_ppc32_aix5)
 498
 499 typedef
 500    struct {
 501       unsigned int nraddr; /* where's the code? */
 502       unsigned int r2;  /* what tocptr do we need? */
 503    }
 504    OrigFn;
 505
 506 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 507                      "rlwinm 0,0,3,0,0  ; rlwinm 0,0,13,0,0\n\t"  \
 508                      "rlwinm 0,0,29,0,0 ; rlwinm 0,0,19,0,0\n\t"
 509
 510 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 511         _zzq_rlval, _zzq_default, _zzq_request,                   \
 512         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 513                                                                   \
 514   {          unsigned int  _zzq_args[7];                          \
 515     register unsigned int  _zzq_result;                           \
 516     register unsigned int* _zzq_ptr;                              \
 517     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 518     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 519     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 520     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 521     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 522     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 523     _zzq_args[6] = (unsigned int)(_zzq_default);                  \
 524     _zzq_ptr = _zzq_args;                                         \
 525     __asm__ volatile("mr 4,%1\n\t"                                \
 526                      "lwz 3, 24(4)\n\t"                           \
 527                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 528                      /* %R3 = client_request ( %R4 ) */           \
 529                      "or 1,1,1\n\t"                               \
 530                      "mr %0,3"                                    \
 531                      : "=b" (_zzq_result)                         \
 532                      : "b" (_zzq_ptr)                             \
 533                      : "r3", "r4", "cc", "memory");               \
 534     _zzq_rlval = _zzq_result;                                     \
 535   }
 536
 537 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 538   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 539     register unsigned int __addr;                                 \
 540     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 541                      /* %R3 = guest_NRADDR */                     \
 542                      "or 2,2,2\n\t"                               \
 543                      "mr %0,3"                                    \
 544                      : "=b" (__addr)                              \
 545                      :                                            \
 546                      : "r3", "cc", "memory"                       \
 547                     );                                            \
 548     _zzq_orig->nraddr = __addr;                                   \
 549     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 550                      /* %R3 = guest_NRADDR_GPR2 */                \
 551                      "or 4,4,4\n\t"                               \
 552                      "mr %0,3"                                    \
 553                      : "=b" (__addr)                              \
 554                      :                                            \
 555                      : "r3", "cc", "memory"                       \
 556                     );                                            \
 557     _zzq_orig->r2 = __addr;                                       \
 558   }
 559
 560 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 561                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 562                      /* branch-and-link-to-noredir *%R11 */       \
 563                      "or 3,3,3\n\t"
 564
 565 #endif /* PLAT_ppc32_aix5 */
 566
 567 /* ------------------------ ppc64-aix5 ------------------------- */
 568
 569 #if defined(PLAT_ppc64_aix5)
 570
 571 typedef
 572    struct {
 573       unsigned long long int nraddr; /* where's the code? */
 574       unsigned long long int r2;  /* what tocptr do we need? */
 575    }
 576    OrigFn;
 577
 578 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 579                      "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
 580                      "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
 581
 582 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 583         _zzq_rlval, _zzq_default, _zzq_request,                   \
 584         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 585                                                                   \
 586   {          unsigned long long int  _zzq_args[7];                \
 587     register unsigned long long int  _zzq_result;                 \
 588     register unsigned long long int* _zzq_ptr;                    \
 589     _zzq_args[0] = (unsigned int long long)(_zzq_request);        \
 590     _zzq_args[1] = (unsigned int long long)(_zzq_arg1);           \
 591     _zzq_args[2] = (unsigned int long long)(_zzq_arg2);           \
 592     _zzq_args[3] = (unsigned int long long)(_zzq_arg3);           \
 593     _zzq_args[4] = (unsigned int long long)(_zzq_arg4);           \
 594     _zzq_args[5] = (unsigned int long long)(_zzq_arg5);           \
 595     _zzq_args[6] = (unsigned int long long)(_zzq_default);        \
 596     _zzq_ptr = _zzq_args;                                         \
 597     __asm__ volatile("mr 4,%1\n\t"                                \
 598                      "ld 3, 48(4)\n\t"                            \
 599                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 600                      /* %R3 = client_request ( %R4 ) */           \
 601                      "or 1,1,1\n\t"                               \
 602                      "mr %0,3"                                    \
 603                      : "=b" (_zzq_result)                         \
 604                      : "b" (_zzq_ptr)                             \
 605                      : "r3", "r4", "cc", "memory");               \
 606     _zzq_rlval = _zzq_result;                                     \
 607   }
 608
 609 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 610   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 611     register unsigned long long int __addr;                       \
 612     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 613                      /* %R3 = guest_NRADDR */                     \
 614                      "or 2,2,2\n\t"                               \
 615                      "mr %0,3"                                    \
 616                      : "=b" (__addr)                              \
 617                      :                                            \
 618                      : "r3", "cc", "memory"                       \
 619                     );                                            \
 620     _zzq_orig->nraddr = __addr;                                   \
 621     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 622                      /* %R3 = guest_NRADDR_GPR2 */                \
 623                      "or 4,4,4\n\t"                               \
 624                      "mr %0,3"                                    \
 625                      : "=b" (__addr)                              \
 626                      :                                            \
 627                      : "r3", "cc", "memory"                       \
 628                     );                                            \
 629     _zzq_orig->r2 = __addr;                                       \
 630   }
 631
 632 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 633                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 634                      /* branch-and-link-to-noredir *%R11 */       \
 635                      "or 3,3,3\n\t"
 636
 637 #endif /* PLAT_ppc64_aix5 */
 638
 639 /* Insert assembly code for other platforms here... */
 640
 641 #endif /* NVALGRIND */
 642
 643
 644 /* ------------------------------------------------------------------ */
 645 /* PLATFORM SPECIFICS for FUNCTION WRAPPING.  This is all very        */
 646 /* ugly.  It's the least-worst tradeoff I can think of.               */
 647 /* ------------------------------------------------------------------ */
 648
 649 /* This section defines magic (a.k.a appalling-hack) macros for doing
 650    guaranteed-no-redirection macros, so as to get from function
 651    wrappers to the functions they are wrapping.  The whole point is to
 652    construct standard call sequences, but to do the call itself with a
 653    special no-redirect call pseudo-instruction that the JIT
 654    understands and handles specially.  This section is long and
 655    repetitious, and I can't see a way to make it shorter.
 656
 657    The naming scheme is as follows:
 658
 659       CALL_FN_{W,v}_{v,W,WW,WWW,WWWW,5W,6W,7W,etc}
 660
 661    'W' stands for "word" and 'v' for "void".  Hence there are
 662    different macros for calling arity 0, 1, 2, 3, 4, etc, functions,
 663    and for each, the possibility of returning a word-typed result, or
 664    no result.
 665 */
 666
 667 /* Use these to write the name of your wrapper.  NOTE: duplicates
 668    VG_WRAP_FUNCTION_Z{U,Z} in pub_tool_redir.h. */
 669
 670 /* Use an extra level of macroisation so as to ensure the soname/fnname
 671    args are fully macro-expanded before pasting them together. */
 672 #define VG_CONCAT4(_aa,_bb,_cc,_dd) _aa##_bb##_cc##_dd
 673
 674 #define I_WRAP_SONAME_FNNAME_ZU(soname,fnname)                    \
 675    VG_CONCAT4(_vgwZU_,soname,_,fnname)
 676
 677 #define I_WRAP_SONAME_FNNAME_ZZ(soname,fnname)                    \
 678    VG_CONCAT4(_vgwZZ_,soname,_,fnname)
 679
 680 /* Use this macro from within a wrapper function to collect the
 681    context (address and possibly other info) of the original function.
 682    Once you have that you can then use it in one of the CALL_FN_
 683    macros.  The type of the argument _lval is OrigFn. */
 684 #define VALGRIND_GET_ORIG_FN(_lval)  VALGRIND_GET_NR_CONTEXT(_lval)
 685
 686 /* Derivatives of the main macros below, for calling functions
 687    returning void. */
 688
 689 #define CALL_FN_v_v(fnptr)                                        \
 690    do { volatile unsigned long _junk;                             \
 691         CALL_FN_W_v(_junk,fnptr); } while (0)
 692
 693 #define CALL_FN_v_W(fnptr, arg1)                                  \
 694    do { volatile unsigned long _junk;                             \
 695         CALL_FN_W_W(_junk,fnptr,arg1); } while (0)
 696
 697 #define CALL_FN_v_WW(fnptr, arg1,arg2)                            \
 698    do { volatile unsigned long _junk;                             \
 699         CALL_FN_W_WW(_junk,fnptr,arg1,arg2); } while (0)
 700
 701 #define CALL_FN_v_WWW(fnptr, arg1,arg2,arg3)                      \
 702    do { volatile unsigned long _junk;                             \
 703         CALL_FN_W_WWW(_junk,fnptr,arg1,arg2,arg3); } while (0)
 704
 705 #define CALL_FN_v_WWWW(fnptr, arg1,arg2,arg3,arg4)                \
 706    do { volatile unsigned long _junk;                             \
 707         CALL_FN_W_WWWW(_junk,fnptr,arg1,arg2,arg3,arg4); } while (0)
 708
 709 #define CALL_FN_v_5W(fnptr, arg1,arg2,arg3,arg4,arg5)             \
 710    do { volatile unsigned long _junk;                             \
 711         CALL_FN_W_5W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5); } while (0)
 712
 713 #define CALL_FN_v_6W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6)        \
 714    do { volatile unsigned long _junk;                             \
 715         CALL_FN_W_6W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6); } while (0)
 716
 717 #define CALL_FN_v_7W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6,arg7)   \
 718    do { volatile unsigned long _junk;                             \
 719         CALL_FN_W_7W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6,arg7); } while (0)
 720
 721 /* ------------------------- x86-{linux,darwin} ---------------- */
 722
 723 #if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin)
 724
 725 /* These regs are trashed by the hidden call.  No need to mention eax
 726    as gcc can already see that, plus causes gcc to bomb. */
 727 #define __CALLER_SAVED_REGS /*"eax"*/ "ecx", "edx"
 728
 729 /* These CALL_FN_ macros assume that on x86-linux, sizeof(unsigned
 730    long) == 4. */
 731
 732 #define CALL_FN_W_v(lval, orig)                                   \
 733    do {                                                           \
 734       volatile OrigFn        _orig = (orig);                      \
 735       volatile unsigned long _argvec[1];                          \
 736       volatile unsigned long _res;                                \
 737       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 738       __asm__ volatile(                                           \
 739          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 740          VALGRIND_CALL_NOREDIR_EAX                                \
 741          : /*out*/   "=a" (_res)                                  \
 742          : /*in*/    "a" (&_argvec[0])                            \
 743          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 744       );                                                          \
 745       lval = (__typeof__(lval)) _res;                             \
 746    } while (0)
 747
 748 #define CALL_FN_W_W(lval, orig, arg1)                             \
 749    do {                                                           \
 750       volatile OrigFn        _orig = (orig);                      \
 751       volatile unsigned long _argvec[2];                          \
 752       volatile unsigned long _res;                                \
 753       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 754       _argvec[1] = (unsigned long)(arg1);                         \
 755       __asm__ volatile(                                           \
 756          "pushl 4(%%eax)\n\t"                                     \
 757          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 758          VALGRIND_CALL_NOREDIR_EAX                                \
 759          "addl $4, %%esp\n"                                       \
 760          : /*out*/   "=a" (_res)                                  \
 761          : /*in*/    "a" (&_argvec[0])                            \
 762          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 763       );                                                          \
 764       lval = (__typeof__(lval)) _res;                             \
 765    } while (0)
 766
 767 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
 768    do {                                                           \
 769       volatile OrigFn        _orig = (orig);                      \
 770       volatile unsigned long _argvec[3];                          \
 771       volatile unsigned long _res;                                \
 772       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 773       _argvec[1] = (unsigned long)(arg1);                         \
 774       _argvec[2] = (unsigned long)(arg2);                         \
 775       __asm__ volatile(                                           \
 776          "pushl 8(%%eax)\n\t"                                     \
 777          "pushl 4(%%eax)\n\t"                                     \
 778          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 779          VALGRIND_CALL_NOREDIR_EAX                                \
 780          "addl $8, %%esp\n"                                       \
 781          : /*out*/   "=a" (_res)                                  \
 782          : /*in*/    "a" (&_argvec[0])                            \
 783          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 784       );                                                          \
 785       lval = (__typeof__(lval)) _res;                             \
 786    } while (0)
 787
 788 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
 789    do {                                                           \
 790       volatile OrigFn        _orig = (orig);                      \
 791       volatile unsigned long _argvec[4];                          \
 792       volatile unsigned long _res;                                \
 793       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 794       _argvec[1] = (unsigned long)(arg1);                         \
 795       _argvec[2] = (unsigned long)(arg2);                         \
 796       _argvec[3] = (unsigned long)(arg3);                         \
 797       __asm__ volatile(                                           \
 798          "pushl 12(%%eax)\n\t"                                    \
 799          "pushl 8(%%eax)\n\t"                                     \
 800          "pushl 4(%%eax)\n\t"                                     \
 801          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 802          VALGRIND_CALL_NOREDIR_EAX                                \
 803          "addl $12, %%esp\n"                                      \
 804          : /*out*/   "=a" (_res)                                  \
 805          : /*in*/    "a" (&_argvec[0])                            \
 806          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 807       );                                                          \
 808       lval = (__typeof__(lval)) _res;                             \
 809    } while (0)
 810
 811 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
 812    do {                                                           \
 813       volatile OrigFn        _orig = (orig);                      \
 814       volatile unsigned long _argvec[5];                          \
 815       volatile unsigned long _res;                                \
 816       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 817       _argvec[1] = (unsigned long)(arg1);                         \
 818       _argvec[2] = (unsigned long)(arg2);                         \
 819       _argvec[3] = (unsigned long)(arg3);                         \
 820       _argvec[4] = (unsigned long)(arg4);                         \
 821       __asm__ volatile(                                           \
 822          "pushl 16(%%eax)\n\t"                                    \
 823          "pushl 12(%%eax)\n\t"                                    \
 824          "pushl 8(%%eax)\n\t"                                     \
 825          "pushl 4(%%eax)\n\t"                                     \
 826          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 827          VALGRIND_CALL_NOREDIR_EAX                                \
 828          "addl $16, %%esp\n"                                      \
 829          : /*out*/   "=a" (_res)                                  \
 830          : /*in*/    "a" (&_argvec[0])                            \
 831          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 832       );                                                          \
 833       lval = (__typeof__(lval)) _res;                             \
 834    } while (0)
 835
 836 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
 837    do {                                                           \
 838       volatile OrigFn        _orig = (orig);                      \
 839       volatile unsigned long _argvec[6];                          \
 840       volatile unsigned long _res;                                \
 841       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 842       _argvec[1] = (unsigned long)(arg1);                         \
 843       _argvec[2] = (unsigned long)(arg2);                         \
 844       _argvec[3] = (unsigned long)(arg3);                         \
 845       _argvec[4] = (unsigned long)(arg4);                         \
 846       _argvec[5] = (unsigned long)(arg5);                         \
 847       __asm__ volatile(                                           \
 848          "pushl 20(%%eax)\n\t"                                    \
 849          "pushl 16(%%eax)\n\t"                                    \
 850          "pushl 12(%%eax)\n\t"                                    \
 851          "pushl 8(%%eax)\n\t"                                     \
 852          "pushl 4(%%eax)\n\t"                                     \
 853          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 854          VALGRIND_CALL_NOREDIR_EAX                                \
 855          "addl $20, %%esp\n"                                      \
 856          : /*out*/   "=a" (_res)                                  \
 857          : /*in*/    "a" (&_argvec[0])                            \
 858          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 859       );                                                          \
 860       lval = (__typeof__(lval)) _res;                             \
 861    } while (0)
 862
 863 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
 864    do {                                                           \
 865       volatile OrigFn        _orig = (orig);                      \
 866       volatile unsigned long _argvec[7];                          \
 867       volatile unsigned long _res;                                \
 868       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 869       _argvec[1] = (unsigned long)(arg1);                         \
 870       _argvec[2] = (unsigned long)(arg2);                         \
 871       _argvec[3] = (unsigned long)(arg3);                         \
 872       _argvec[4] = (unsigned long)(arg4);                         \
 873       _argvec[5] = (unsigned long)(arg5);                         \
 874       _argvec[6] = (unsigned long)(arg6);                         \
 875       __asm__ volatile(                                           \
 876          "pushl 24(%%eax)\n\t"                                    \
 877          "pushl 20(%%eax)\n\t"                                    \
 878          "pushl 16(%%eax)\n\t"                                    \
 879          "pushl 12(%%eax)\n\t"                                    \
 880          "pushl 8(%%eax)\n\t"                                     \
 881          "pushl 4(%%eax)\n\t"                                     \
 882          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 883          VALGRIND_CALL_NOREDIR_EAX                                \
 884          "addl $24, %%esp\n"                                      \
 885          : /*out*/   "=a" (_res)                                  \
 886          : /*in*/    "a" (&_argvec[0])                            \
 887          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 888       );                                                          \
 889       lval = (__typeof__(lval)) _res;                             \
 890    } while (0)
 891
 892 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
 893                                  arg7)                            \
 894    do {                                                           \
 895       volatile OrigFn        _orig = (orig);                      \
 896       volatile unsigned long _argvec[8];                          \
 897       volatile unsigned long _res;                                \
 898       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 899       _argvec[1] = (unsigned long)(arg1);                         \
 900       _argvec[2] = (unsigned long)(arg2);                         \
 901       _argvec[3] = (unsigned long)(arg3);                         \
 902       _argvec[4] = (unsigned long)(arg4);                         \
 903       _argvec[5] = (unsigned long)(arg5);                         \
 904       _argvec[6] = (unsigned long)(arg6);                         \
 905       _argvec[7] = (unsigned long)(arg7);                         \
 906       __asm__ volatile(                                           \
 907          "pushl 28(%%eax)\n\t"                                    \
 908          "pushl 24(%%eax)\n\t"                                    \
 909          "pushl 20(%%eax)\n\t"                                    \
 910          "pushl 16(%%eax)\n\t"                                    \
 911          "pushl 12(%%eax)\n\t"                                    \
 912          "pushl 8(%%eax)\n\t"                                     \
 913          "pushl 4(%%eax)\n\t"                                     \
 914          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 915          VALGRIND_CALL_NOREDIR_EAX                                \
 916          "addl $28, %%esp\n"                                      \
 917          : /*out*/   "=a" (_res)                                  \
 918          : /*in*/    "a" (&_argvec[0])                            \
 919          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 920       );                                                          \
 921       lval = (__typeof__(lval)) _res;                             \
 922    } while (0)
 923
 924 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
 925                                  arg7,arg8)                       \
 926    do {                                                           \
 927       volatile OrigFn        _orig = (orig);                      \
 928       volatile unsigned long _argvec[9];                          \
 929       volatile unsigned long _res;                                \
 930       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 931       _argvec[1] = (unsigned long)(arg1);                         \
 932       _argvec[2] = (unsigned long)(arg2);                         \
 933       _argvec[3] = (unsigned long)(arg3);                         \
 934       _argvec[4] = (unsigned long)(arg4);                         \
 935       _argvec[5] = (unsigned long)(arg5);                         \
 936       _argvec[6] = (unsigned long)(arg6);                         \
 937       _argvec[7] = (unsigned long)(arg7);                         \
 938       _argvec[8] = (unsigned long)(arg8);                         \
 939       __asm__ volatile(                                           \
 940          "pushl 32(%%eax)\n\t"                                    \
 941          "pushl 28(%%eax)\n\t"                                    \
 942          "pushl 24(%%eax)\n\t"                                    \
 943          "pushl 20(%%eax)\n\t"                                    \
 944          "pushl 16(%%eax)\n\t"                                    \
 945          "pushl 12(%%eax)\n\t"                                    \
 946          "pushl 8(%%eax)\n\t"                                     \
 947          "pushl 4(%%eax)\n\t"                                     \
 948          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 949          VALGRIND_CALL_NOREDIR_EAX                                \
 950          "addl $32, %%esp\n"                                      \
 951          : /*out*/   "=a" (_res)                                  \
 952          : /*in*/    "a" (&_argvec[0])                            \
 953          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 954       );                                                          \
 955       lval = (__typeof__(lval)) _res;                             \
 956    } while (0)
 957
 958 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
 959                                  arg7,arg8,arg9)                  \
 960    do {                                                           \
 961       volatile OrigFn        _orig = (orig);                      \
 962       volatile unsigned long _argvec[10];                         \
 963       volatile unsigned long _res;                                \
 964       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 965       _argvec[1] = (unsigned long)(arg1);                         \
 966       _argvec[2] = (unsigned long)(arg2);                         \
 967       _argvec[3] = (unsigned long)(arg3);                         \
 968       _argvec[4] = (unsigned long)(arg4);                         \
 969       _argvec[5] = (unsigned long)(arg5);                         \
 970       _argvec[6] = (unsigned long)(arg6);                         \
 971       _argvec[7] = (unsigned long)(arg7);                         \
 972       _argvec[8] = (unsigned long)(arg8);                         \
 973       _argvec[9] = (unsigned long)(arg9);                         \
 974       __asm__ volatile(                                           \
 975          "pushl 36(%%eax)\n\t"                                    \
 976          "pushl 32(%%eax)\n\t"                                    \
 977          "pushl 28(%%eax)\n\t"                                    \
 978          "pushl 24(%%eax)\n\t"                                    \
 979          "pushl 20(%%eax)\n\t"                                    \
 980          "pushl 16(%%eax)\n\t"                                    \
 981          "pushl 12(%%eax)\n\t"                                    \
 982          "pushl 8(%%eax)\n\t"                                     \
 983          "pushl 4(%%eax)\n\t"                                     \
 984          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 985          VALGRIND_CALL_NOREDIR_EAX                                \
 986          "addl $36, %%esp\n"                                      \
 987          : /*out*/   "=a" (_res)                                  \
 988          : /*in*/    "a" (&_argvec[0])                            \
 989          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 990       );                                                          \
 991       lval = (__typeof__(lval)) _res;                             \
 992    } while (0)
 993
 994 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
 995                                   arg7,arg8,arg9,arg10)           \
 996    do {                                                           \
 997       volatile OrigFn        _orig = (orig);                      \
 998       volatile unsigned long _argvec[11];                         \
 999       volatile unsigned long _res;                                \
1000       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1001       _argvec[1] = (unsigned long)(arg1);                         \
1002       _argvec[2] = (unsigned long)(arg2);                         \
1003       _argvec[3] = (unsigned long)(arg3);                         \
1004       _argvec[4] = (unsigned long)(arg4);                         \
1005       _argvec[5] = (unsigned long)(arg5);                         \
1006       _argvec[6] = (unsigned long)(arg6);                         \
1007       _argvec[7] = (unsigned long)(arg7);                         \
1008       _argvec[8] = (unsigned long)(arg8);                         \
1009       _argvec[9] = (unsigned long)(arg9);                         \
1010       _argvec[10] = (unsigned long)(arg10);                       \
1011       __asm__ volatile(                                           \
1012          "pushl 40(%%eax)\n\t"                                    \
1013          "pushl 36(%%eax)\n\t"                                    \
1014          "pushl 32(%%eax)\n\t"                                    \
1015          "pushl 28(%%eax)\n\t"                                    \
1016          "pushl 24(%%eax)\n\t"                                    \
1017          "pushl 20(%%eax)\n\t"                                    \
1018          "pushl 16(%%eax)\n\t"                                    \
1019          "pushl 12(%%eax)\n\t"                                    \
1020          "pushl 8(%%eax)\n\t"                                     \
1021          "pushl 4(%%eax)\n\t"                                     \
1022          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1023          VALGRIND_CALL_NOREDIR_EAX                                \
1024          "addl $40, %%esp\n"                                      \
1025          : /*out*/   "=a" (_res)                                  \
1026          : /*in*/    "a" (&_argvec[0])                            \
1027          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1028       );                                                          \
1029       lval = (__typeof__(lval)) _res;                             \
1030    } while (0)
1031
1032 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
1033                                   arg6,arg7,arg8,arg9,arg10,      \
1034                                   arg11)                          \
1035    do {                                                           \
1036       volatile OrigFn        _orig = (orig);                      \
1037       volatile unsigned long _argvec[12];                         \
1038       volatile unsigned long _res;                                \
1039       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1040       _argvec[1] = (unsigned long)(arg1);                         \
1041       _argvec[2] = (unsigned long)(arg2);                         \
1042       _argvec[3] = (unsigned long)(arg3);                         \
1043       _argvec[4] = (unsigned long)(arg4);                         \
1044       _argvec[5] = (unsigned long)(arg5);                         \
1045       _argvec[6] = (unsigned long)(arg6);                         \
1046       _argvec[7] = (unsigned long)(arg7);                         \
1047       _argvec[8] = (unsigned long)(arg8);                         \
1048       _argvec[9] = (unsigned long)(arg9);                         \
1049       _argvec[10] = (unsigned long)(arg10);                       \
1050       _argvec[11] = (unsigned long)(arg11);                       \
1051       __asm__ volatile(                                           \
1052          "pushl 44(%%eax)\n\t"                                    \
1053          "pushl 40(%%eax)\n\t"                                    \
1054          "pushl 36(%%eax)\n\t"                                    \
1055          "pushl 32(%%eax)\n\t"                                    \
1056          "pushl 28(%%eax)\n\t"                                    \
1057          "pushl 24(%%eax)\n\t"                                    \
1058          "pushl 20(%%eax)\n\t"                                    \
1059          "pushl 16(%%eax)\n\t"                                    \
1060          "pushl 12(%%eax)\n\t"                                    \
1061          "pushl 8(%%eax)\n\t"                                     \
1062          "pushl 4(%%eax)\n\t"                                     \
1063          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1064          VALGRIND_CALL_NOREDIR_EAX                                \
1065          "addl $44, %%esp\n"                                      \
1066          : /*out*/   "=a" (_res)                                  \
1067          : /*in*/    "a" (&_argvec[0])                            \
1068          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1069       );                                                          \
1070       lval = (__typeof__(lval)) _res;                             \
1071    } while (0)
1072
1073 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
1074                                   arg6,arg7,arg8,arg9,arg10,      \
1075                                   arg11,arg12)                    \
1076    do {                                                           \
1077       volatile OrigFn        _orig = (orig);                      \
1078       volatile unsigned long _argvec[13];                         \
1079       volatile unsigned long _res;                                \
1080       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1081       _argvec[1] = (unsigned long)(arg1);                         \
1082       _argvec[2] = (unsigned long)(arg2);                         \
1083       _argvec[3] = (unsigned long)(arg3);                         \
1084       _argvec[4] = (unsigned long)(arg4);                         \
1085       _argvec[5] = (unsigned long)(arg5);                         \
1086       _argvec[6] = (unsigned long)(arg6);                         \
1087       _argvec[7] = (unsigned long)(arg7);                         \
1088       _argvec[8] = (unsigned long)(arg8);                         \
1089       _argvec[9] = (unsigned long)(arg9);                         \
1090       _argvec[10] = (unsigned long)(arg10);                       \
1091       _argvec[11] = (unsigned long)(arg11);                       \
1092       _argvec[12] = (unsigned long)(arg12);                       \
1093       __asm__ volatile(                                           \
1094          "pushl 48(%%eax)\n\t"                                    \
1095          "pushl 44(%%eax)\n\t"                                    \
1096          "pushl 40(%%eax)\n\t"                                    \
1097          "pushl 36(%%eax)\n\t"                                    \
1098          "pushl 32(%%eax)\n\t"                                    \
1099          "pushl 28(%%eax)\n\t"                                    \
1100          "pushl 24(%%eax)\n\t"                                    \
1101          "pushl 20(%%eax)\n\t"                                    \
1102          "pushl 16(%%eax)\n\t"                                    \
1103          "pushl 12(%%eax)\n\t"                                    \
1104          "pushl 8(%%eax)\n\t"                                     \
1105          "pushl 4(%%eax)\n\t"                                     \
1106          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1107          VALGRIND_CALL_NOREDIR_EAX                                \
1108          "addl $48, %%esp\n"                                      \
1109          : /*out*/   "=a" (_res)                                  \
1110          : /*in*/    "a" (&_argvec[0])                            \
1111          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1112       );                                                          \
1113       lval = (__typeof__(lval)) _res;                             \
1114    } while (0)
1115
1116 #endif /* PLAT_x86_linux || PLAT_x86_darwin */
1117
1118 /* ------------------------ amd64-{linux,darwin} --------------- */
1119
1120 #if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin)
1121
1122 /* ARGREGS: rdi rsi rdx rcx r8 r9 (the rest on stack in R-to-L order) */
1123
1124 /* These regs are trashed by the hidden call. */
1125 #define __CALLER_SAVED_REGS /*"rax",*/ "rcx", "rdx", "rsi",       \
1126                             "rdi", "r8", "r9", "r10", "r11"
1127
1128 /* These CALL_FN_ macros assume that on amd64-linux, sizeof(unsigned
1129    long) == 8. */
1130
1131 /* NB 9 Sept 07.  There is a nasty kludge here in all these CALL_FN_
1132    macros.  In order not to trash the stack redzone, we need to drop
1133    %rsp by 128 before the hidden call, and restore afterwards.  The
1134    nastyness is that it is only by luck that the stack still appears
1135    to be unwindable during the hidden call - since then the behaviour
1136    of any routine using this macro does not match what the CFI data
1137    says.  Sigh.
1138
1139    Why is this important?  Imagine that a wrapper has a stack
1140    allocated local, and passes to the hidden call, a pointer to it.
1141    Because gcc does not know about the hidden call, it may allocate
1142    that local in the redzone.  Unfortunately the hidden call may then
1143    trash it before it comes to use it.  So we must step clear of the
1144    redzone, for the duration of the hidden call, to make it safe.
1145
1146    Probably the same problem afflicts the other redzone-style ABIs too
1147    (ppc64-linux, ppc32-aix5, ppc64-aix5); but for those, the stack is
1148    self describing (none of this CFI nonsense) so at least messing
1149    with the stack pointer doesn't give a danger of non-unwindable
1150    stack. */
1151
1152 #define CALL_FN_W_v(lval, orig)                                   \
1153    do {                                                           \
1154       volatile OrigFn        _orig = (orig);                      \
1155       volatile unsigned long _argvec[1];                          \
1156       volatile unsigned long _res;                                \
1157       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1158       __asm__ volatile(                                           \
1159          "subq $128,%%rsp\n\t"                                    \
1160          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1161          VALGRIND_CALL_NOREDIR_RAX                                \
1162          "addq $128,%%rsp\n\t"                                    \
1163          : /*out*/   "=a" (_res)                                  \
1164          : /*in*/    "a" (&_argvec[0])                            \
1165          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1166       );                                                          \
1167       lval = (__typeof__(lval)) _res;                             \
1168    } while (0)
1169
1170 #define CALL_FN_W_W(lval, orig, arg1)                             \
1171    do {                                                           \
1172       volatile OrigFn        _orig = (orig);                      \
1173       volatile unsigned long _argvec[2];                          \
1174       volatile unsigned long _res;                                \
1175       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1176       _argvec[1] = (unsigned long)(arg1);                         \
1177       __asm__ volatile(                                           \
1178          "subq $128,%%rsp\n\t"                                    \
1179          "movq 8(%%rax), %%rdi\n\t"                               \
1180          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1181          VALGRIND_CALL_NOREDIR_RAX                                \
1182          "addq $128,%%rsp\n\t"                                    \
1183          : /*out*/   "=a" (_res)                                  \
1184          : /*in*/    "a" (&_argvec[0])                            \
1185          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1186       );                                                          \
1187       lval = (__typeof__(lval)) _res;                             \
1188    } while (0)
1189
1190 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
1191    do {                                                           \
1192       volatile OrigFn        _orig = (orig);                      \
1193       volatile unsigned long _argvec[3];                          \
1194       volatile unsigned long _res;                                \
1195       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1196       _argvec[1] = (unsigned long)(arg1);                         \
1197       _argvec[2] = (unsigned long)(arg2);                         \
1198       __asm__ volatile(                                           \
1199          "subq $128,%%rsp\n\t"                                    \
1200          "movq 16(%%rax), %%rsi\n\t"                              \
1201          "movq 8(%%rax), %%rdi\n\t"                               \
1202          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1203          VALGRIND_CALL_NOREDIR_RAX                                \
1204          "addq $128,%%rsp\n\t"                                    \
1205          : /*out*/   "=a" (_res)                                  \
1206          : /*in*/    "a" (&_argvec[0])                            \
1207          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1208       );                                                          \
1209       lval = (__typeof__(lval)) _res;                             \
1210    } while (0)
1211
1212 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
1213    do {                                                           \
1214       volatile OrigFn        _orig = (orig);                      \
1215       volatile unsigned long _argvec[4];                          \
1216       volatile unsigned long _res;                                \
1217       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1218       _argvec[1] = (unsigned long)(arg1);                         \
1219       _argvec[2] = (unsigned long)(arg2);                         \
1220       _argvec[3] = (unsigned long)(arg3);                         \
1221       __asm__ volatile(                                           \
1222          "subq $128,%%rsp\n\t"                                    \
1223          "movq 24(%%rax), %%rdx\n\t"                              \
1224          "movq 16(%%rax), %%rsi\n\t"                              \
1225          "movq 8(%%rax), %%rdi\n\t"                               \
1226          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1227          VALGRIND_CALL_NOREDIR_RAX                                \
1228          "addq $128,%%rsp\n\t"                                    \
1229          : /*out*/   "=a" (_res)                                  \
1230          : /*in*/    "a" (&_argvec[0])                            \
1231          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1232       );                                                          \
1233       lval = (__typeof__(lval)) _res;                             \
1234    } while (0)
1235
1236 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
1237    do {                                                           \
1238       volatile OrigFn        _orig = (orig);                      \
1239       volatile unsigned long _argvec[5];                          \
1240       volatile unsigned long _res;                                \
1241       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1242       _argvec[1] = (unsigned long)(arg1);                         \
1243       _argvec[2] = (unsigned long)(arg2);                         \
1244       _argvec[3] = (unsigned long)(arg3);                         \
1245       _argvec[4] = (unsigned long)(arg4);                         \
1246       __asm__ volatile(                                           \
1247          "subq $128,%%rsp\n\t"                                    \
1248          "movq 32(%%rax), %%rcx\n\t"                              \
1249          "movq 24(%%rax), %%rdx\n\t"                              \
1250          "movq 16(%%rax), %%rsi\n\t"                              \
1251          "movq 8(%%rax), %%rdi\n\t"                               \
1252          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1253          VALGRIND_CALL_NOREDIR_RAX                                \
1254          "addq $128,%%rsp\n\t"                                    \
1255          : /*out*/   "=a" (_res)                                  \
1256          : /*in*/    "a" (&_argvec[0])                            \
1257          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1258       );                                                          \
1259       lval = (__typeof__(lval)) _res;                             \
1260    } while (0)
1261
1262 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
1263    do {                                                           \
1264       volatile OrigFn        _orig = (orig);                      \
1265       volatile unsigned long _argvec[6];                          \
1266       volatile unsigned long _res;                                \
1267       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1268       _argvec[1] = (unsigned long)(arg1);                         \
1269       _argvec[2] = (unsigned long)(arg2);                         \
1270       _argvec[3] = (unsigned long)(arg3);                         \
1271       _argvec[4] = (unsigned long)(arg4);                         \
1272       _argvec[5] = (unsigned long)(arg5);                         \
1273       __asm__ volatile(                                           \
1274          "subq $128,%%rsp\n\t"                                    \
1275          "movq 40(%%rax), %%r8\n\t"                               \
1276          "movq 32(%%rax), %%rcx\n\t"                              \
1277          "movq 24(%%rax), %%rdx\n\t"                              \
1278          "movq 16(%%rax), %%rsi\n\t"                              \
1279          "movq 8(%%rax), %%rdi\n\t"                               \
1280          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1281          VALGRIND_CALL_NOREDIR_RAX                                \
1282          "addq $128,%%rsp\n\t"                                    \
1283          : /*out*/   "=a" (_res)                                  \
1284          : /*in*/    "a" (&_argvec[0])                            \
1285          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1286       );                                                          \
1287       lval = (__typeof__(lval)) _res;                             \
1288    } while (0)
1289
1290 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
1291    do {                                                           \
1292       volatile OrigFn        _orig = (orig);                      \
1293       volatile unsigned long _argvec[7];                          \
1294       volatile unsigned long _res;                                \
1295       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1296       _argvec[1] = (unsigned long)(arg1);                         \
1297       _argvec[2] = (unsigned long)(arg2);                         \
1298       _argvec[3] = (unsigned long)(arg3);                         \
1299       _argvec[4] = (unsigned long)(arg4);                         \
1300       _argvec[5] = (unsigned long)(arg5);                         \
1301       _argvec[6] = (unsigned long)(arg6);                         \
1302       __asm__ volatile(                                           \
1303          "subq $128,%%rsp\n\t"                                    \
1304          "movq 48(%%rax), %%r9\n\t"                               \
1305          "movq 40(%%rax), %%r8\n\t"                               \
1306          "movq 32(%%rax), %%rcx\n\t"                              \
1307          "movq 24(%%rax), %%rdx\n\t"                              \
1308          "movq 16(%%rax), %%rsi\n\t"                              \
1309          "movq 8(%%rax), %%rdi\n\t"                               \
1310          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1311          "addq $128,%%rsp\n\t"                                    \
1312          VALGRIND_CALL_NOREDIR_RAX                                \
1313          : /*out*/   "=a" (_res)                                  \
1314          : /*in*/    "a" (&_argvec[0])                            \
1315          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1316       );                                                          \
1317       lval = (__typeof__(lval)) _res;                             \
1318    } while (0)
1319
1320 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1321                                  arg7)                            \
1322    do {                                                           \
1323       volatile OrigFn        _orig = (orig);                      \
1324       volatile unsigned long _argvec[8];                          \
1325       volatile unsigned long _res;                                \
1326       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1327       _argvec[1] = (unsigned long)(arg1);                         \
1328       _argvec[2] = (unsigned long)(arg2);                         \
1329       _argvec[3] = (unsigned long)(arg3);                         \
1330       _argvec[4] = (unsigned long)(arg4);                         \
1331       _argvec[5] = (unsigned long)(arg5);                         \
1332       _argvec[6] = (unsigned long)(arg6);                         \
1333       _argvec[7] = (unsigned long)(arg7);                         \
1334       __asm__ volatile(                                           \
1335          "subq $128,%%rsp\n\t"                                    \
1336          "pushq 56(%%rax)\n\t"                                    \
1337          "movq 48(%%rax), %%r9\n\t"                               \
1338          "movq 40(%%rax), %%r8\n\t"                               \
1339          "movq 32(%%rax), %%rcx\n\t"                              \
1340          "movq 24(%%rax), %%rdx\n\t"                              \
1341          "movq 16(%%rax), %%rsi\n\t"                              \
1342          "movq 8(%%rax), %%rdi\n\t"                               \
1343          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1344          VALGRIND_CALL_NOREDIR_RAX                                \
1345          "addq $8, %%rsp\n"                                       \
1346          "addq $128,%%rsp\n\t"                                    \
1347          : /*out*/   "=a" (_res)                                  \
1348          : /*in*/    "a" (&_argvec[0])                            \
1349          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1350       );                                                          \
1351       lval = (__typeof__(lval)) _res;                             \
1352    } while (0)
1353
1354 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1355                                  arg7,arg8)                       \
1356    do {                                                           \
1357       volatile OrigFn        _orig = (orig);                      \
1358       volatile unsigned long _argvec[9];                          \
1359       volatile unsigned long _res;                                \
1360       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1361       _argvec[1] = (unsigned long)(arg1);                         \
1362       _argvec[2] = (unsigned long)(arg2);                         \
1363       _argvec[3] = (unsigned long)(arg3);                         \
1364       _argvec[4] = (unsigned long)(arg4);                         \
1365       _argvec[5] = (unsigned long)(arg5);                         \
1366       _argvec[6] = (unsigned long)(arg6);                         \
1367       _argvec[7] = (unsigned long)(arg7);                         \
1368       _argvec[8] = (unsigned long)(arg8);                         \
1369       __asm__ volatile(                                           \
1370          "subq $128,%%rsp\n\t"                                    \
1371          "pushq 64(%%rax)\n\t"                                    \
1372          "pushq 56(%%rax)\n\t"                                    \
1373          "movq 48(%%rax), %%r9\n\t"                               \
1374          "movq 40(%%rax), %%r8\n\t"                               \
1375          "movq 32(%%rax), %%rcx\n\t"                              \
1376          "movq 24(%%rax), %%rdx\n\t"                              \
1377          "movq 16(%%rax), %%rsi\n\t"                              \
1378          "movq 8(%%rax), %%rdi\n\t"                               \
1379          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1380          VALGRIND_CALL_NOREDIR_RAX                                \
1381          "addq $16, %%rsp\n"                                      \
1382          "addq $128,%%rsp\n\t"                                    \
1383          : /*out*/   "=a" (_res)                                  \
1384          : /*in*/    "a" (&_argvec[0])                            \
1385          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1386       );                                                          \
1387       lval = (__typeof__(lval)) _res;                             \
1388    } while (0)
1389
1390 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1391                                  arg7,arg8,arg9)                  \
1392    do {                                                           \
1393       volatile OrigFn        _orig = (orig);                      \
1394       volatile unsigned long _argvec[10];                         \
1395       volatile unsigned long _res;                                \
1396       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1397       _argvec[1] = (unsigned long)(arg1);                         \
1398       _argvec[2] = (unsigned long)(arg2);                         \
1399       _argvec[3] = (unsigned long)(arg3);                         \
1400       _argvec[4] = (unsigned long)(arg4);                         \
1401       _argvec[5] = (unsigned long)(arg5);                         \
1402       _argvec[6] = (unsigned long)(arg6);                         \
1403       _argvec[7] = (unsigned long)(arg7);                         \
1404       _argvec[8] = (unsigned long)(arg8);                         \
1405       _argvec[9] = (unsigned long)(arg9);                         \
1406       __asm__ volatile(                                           \
1407          "subq $128,%%rsp\n\t"                                    \
1408          "pushq 72(%%rax)\n\t"                                    \
1409          "pushq 64(%%rax)\n\t"                                    \
1410          "pushq 56(%%rax)\n\t"                                    \
1411          "movq 48(%%rax), %%r9\n\t"                               \
1412          "movq 40(%%rax), %%r8\n\t"                               \
1413          "movq 32(%%rax), %%rcx\n\t"                              \
1414          "movq 24(%%rax), %%rdx\n\t"                              \
1415          "movq 16(%%rax), %%rsi\n\t"                              \
1416          "movq 8(%%rax), %%rdi\n\t"                               \
1417          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1418          VALGRIND_CALL_NOREDIR_RAX                                \
1419          "addq $24, %%rsp\n"                                      \
1420          "addq $128,%%rsp\n\t"                                    \
1421          : /*out*/   "=a" (_res)                                  \
1422          : /*in*/    "a" (&_argvec[0])                            \
1423          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1424       );                                                          \
1425       lval = (__typeof__(lval)) _res;                             \
1426    } while (0)
1427
1428 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1429                                   arg7,arg8,arg9,arg10)           \
1430    do {                                                           \
1431       volatile OrigFn        _orig = (orig);                      \
1432       volatile unsigned long _argvec[11];                         \
1433       volatile unsigned long _res;                                \
1434       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1435       _argvec[1] = (unsigned long)(arg1);                         \
1436       _argvec[2] = (unsigned long)(arg2);                         \
1437       _argvec[3] = (unsigned long)(arg3);                         \
1438       _argvec[4] = (unsigned long)(arg4);                         \
1439       _argvec[5] = (unsigned long)(arg5);                         \
1440       _argvec[6] = (unsigned long)(arg6);                         \
1441       _argvec[7] = (unsigned long)(arg7);                         \
1442       _argvec[8] = (unsigned long)(arg8);                         \
1443       _argvec[9] = (unsigned long)(arg9);                         \
1444       _argvec[10] = (unsigned long)(arg10);                       \
1445       __asm__ volatile(                                           \
1446          "subq $128,%%rsp\n\t"                                    \
1447          "pushq 80(%%rax)\n\t"                                    \
1448          "pushq 72(%%rax)\n\t"                                    \
1449          "pushq 64(%%rax)\n\t"                                    \
1450          "pushq 56(%%rax)\n\t"                                    \
1451          "movq 48(%%rax), %%r9\n\t"                               \
1452          "movq 40(%%rax), %%r8\n\t"                               \
1453          "movq 32(%%rax), %%rcx\n\t"                              \
1454          "movq 24(%%rax), %%rdx\n\t"                              \
1455          "movq 16(%%rax), %%rsi\n\t"                              \
1456          "movq 8(%%rax), %%rdi\n\t"                               \
1457          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1458          VALGRIND_CALL_NOREDIR_RAX                                \
1459          "addq $32, %%rsp\n"                                      \
1460          "addq $128,%%rsp\n\t"                                    \
1461          : /*out*/   "=a" (_res)                                  \
1462          : /*in*/    "a" (&_argvec[0])                            \
1463          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1464       );                                                          \
1465       lval = (__typeof__(lval)) _res;                             \
1466    } while (0)
1467
1468 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1469                                   arg7,arg8,arg9,arg10,arg11)     \
1470    do {                                                           \
1471       volatile OrigFn        _orig = (orig);                      \
1472       volatile unsigned long _argvec[12];                         \
1473       volatile unsigned long _res;                                \
1474       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1475       _argvec[1] = (unsigned long)(arg1);                         \
1476       _argvec[2] = (unsigned long)(arg2);                         \
1477       _argvec[3] = (unsigned long)(arg3);                         \
1478       _argvec[4] = (unsigned long)(arg4);                         \
1479       _argvec[5] = (unsigned long)(arg5);                         \
1480       _argvec[6] = (unsigned long)(arg6);                         \
1481       _argvec[7] = (unsigned long)(arg7);                         \
1482       _argvec[8] = (unsigned long)(arg8);                         \
1483       _argvec[9] = (unsigned long)(arg9);                         \
1484       _argvec[10] = (unsigned long)(arg10);                       \
1485       _argvec[11] = (unsigned long)(arg11);                       \
1486       __asm__ volatile(                                           \
1487          "subq $128,%%rsp\n\t"                                    \
1488          "pushq 88(%%rax)\n\t"                                    \
1489          "pushq 80(%%rax)\n\t"                                    \
1490          "pushq 72(%%rax)\n\t"                                    \
1491          "pushq 64(%%rax)\n\t"                                    \
1492          "pushq 56(%%rax)\n\t"                                    \
1493          "movq 48(%%rax), %%r9\n\t"                               \
1494          "movq 40(%%rax), %%r8\n\t"                               \
1495          "movq 32(%%rax), %%rcx\n\t"                              \
1496          "movq 24(%%rax), %%rdx\n\t"                              \
1497          "movq 16(%%rax), %%rsi\n\t"                              \
1498          "movq 8(%%rax), %%rdi\n\t"                               \
1499          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1500          VALGRIND_CALL_NOREDIR_RAX                                \
1501          "addq $40, %%rsp\n"                                      \
1502          "addq $128,%%rsp\n\t"                                    \
1503          : /*out*/   "=a" (_res)                                  \
1504          : /*in*/    "a" (&_argvec[0])                            \
1505          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1506       );                                                          \
1507       lval = (__typeof__(lval)) _res;                             \
1508    } while (0)
1509
1510 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1511                                 arg7,arg8,arg9,arg10,arg11,arg12) \
1512    do {                                                           \
1513       volatile OrigFn        _orig = (orig);                      \
1514       volatile unsigned long _argvec[13];                         \
1515       volatile unsigned long _res;                                \
1516       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1517       _argvec[1] = (unsigned long)(arg1);                         \
1518       _argvec[2] = (unsigned long)(arg2);                         \
1519       _argvec[3] = (unsigned long)(arg3);                         \
1520       _argvec[4] = (unsigned long)(arg4);                         \
1521       _argvec[5] = (unsigned long)(arg5);                         \
1522       _argvec[6] = (unsigned long)(arg6);                         \
1523       _argvec[7] = (unsigned long)(arg7);                         \
1524       _argvec[8] = (unsigned long)(arg8);                         \
1525       _argvec[9] = (unsigned long)(arg9);                         \
1526       _argvec[10] = (unsigned long)(arg10);                       \
1527       _argvec[11] = (unsigned long)(arg11);                       \
1528       _argvec[12] = (unsigned long)(arg12);                       \
1529       __asm__ volatile(                                           \
1530          "subq $128,%%rsp\n\t"                                    \
1531          "pushq 96(%%rax)\n\t"                                    \
1532          "pushq 88(%%rax)\n\t"                                    \
1533          "pushq 80(%%rax)\n\t"                                    \
1534          "pushq 72(%%rax)\n\t"                                    \
1535          "pushq 64(%%rax)\n\t"                                    \
1536          "pushq 56(%%rax)\n\t"                                    \
1537          "movq 48(%%rax), %%r9\n\t"                               \
1538          "movq 40(%%rax), %%r8\n\t"                               \
1539          "movq 32(%%rax), %%rcx\n\t"                              \
1540          "movq 24(%%rax), %%rdx\n\t"                              \
1541          "movq 16(%%rax), %%rsi\n\t"                              \
1542          "movq 8(%%rax), %%rdi\n\t"                               \
1543          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1544          VALGRIND_CALL_NOREDIR_RAX                                \
1545          "addq $48, %%rsp\n"                                      \
1546          "addq $128,%%rsp\n\t"                                    \
1547          : /*out*/   "=a" (_res)                                  \
1548          : /*in*/    "a" (&_argvec[0])                            \
1549          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1550       );                                                          \
1551       lval = (__typeof__(lval)) _res;                             \
1552    } while (0)
1553
1554 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin */
1555
1556 /* ------------------------ ppc32-linux ------------------------ */
1557
1558 #if defined(PLAT_ppc32_linux)
1559
1560 /* This is useful for finding out about the on-stack stuff:
1561
1562    extern int f9  ( int,int,int,int,int,int,int,int,int );
1563    extern int f10 ( int,int,int,int,int,int,int,int,int,int );
1564    extern int f11 ( int,int,int,int,int,int,int,int,int,int,int );
1565    extern int f12 ( int,int,int,int,int,int,int,int,int,int,int,int );
1566
1567    int g9 ( void ) {
1568       return f9(11,22,33,44,55,66,77,88,99);
1569    }
1570    int g10 ( void ) {
1571       return f10(11,22,33,44,55,66,77,88,99,110);
1572    }
1573    int g11 ( void ) {
1574       return f11(11,22,33,44,55,66,77,88,99,110,121);
1575    }
1576    int g12 ( void ) {
1577       return f12(11,22,33,44,55,66,77,88,99,110,121,132);
1578    }
1579 */
1580
1581 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
1582
1583 /* These regs are trashed by the hidden call. */
1584 #define __CALLER_SAVED_REGS                                       \
1585    "lr", "ctr", "xer",                                            \
1586    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
1587    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
1588    "r11", "r12", "r13"
1589
1590 /* These CALL_FN_ macros assume that on ppc32-linux,
1591    sizeof(unsigned long) == 4. */
1592
1593 #define CALL_FN_W_v(lval, orig)                                   \
1594    do {                                                           \
1595       volatile OrigFn        _orig = (orig);                      \
1596       volatile unsigned long _argvec[1];                          \
1597       volatile unsigned long _res;                                \
1598       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1599       __asm__ volatile(                                           \
1600          "mr 11,%1\n\t"                                           \
1601          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1602          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1603          "mr %0,3"                                                \
1604          : /*out*/   "=r" (_res)                                  \
1605          : /*in*/    "r" (&_argvec[0])                            \
1606          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1607       );                                                          \
1608       lval = (__typeof__(lval)) _res;                             \
1609    } while (0)
1610
1611 #define CALL_FN_W_W(lval, orig, arg1)                             \
1612    do {                                                           \
1613       volatile OrigFn        _orig = (orig);                      \
1614       volatile unsigned long _argvec[2];                          \
1615       volatile unsigned long _res;                                \
1616       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1617       _argvec[1] = (unsigned long)arg1;                           \
1618       __asm__ volatile(                                           \
1619          "mr 11,%1\n\t"                                           \
1620          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1621          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1622          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1623          "mr %0,3"                                                \
1624          : /*out*/   "=r" (_res)                                  \
1625          : /*in*/    "r" (&_argvec[0])                            \
1626          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1627       );                                                          \
1628       lval = (__typeof__(lval)) _res;                             \
1629    } while (0)
1630
1631 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
1632    do {                                                           \
1633       volatile OrigFn        _orig = (orig);                      \
1634       volatile unsigned long _argvec[3];                          \
1635       volatile unsigned long _res;                                \
1636       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1637       _argvec[1] = (unsigned long)arg1;                           \
1638       _argvec[2] = (unsigned long)arg2;                           \
1639       __asm__ volatile(                                           \
1640          "mr 11,%1\n\t"                                           \
1641          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1642          "lwz 4,8(11)\n\t"                                        \
1643          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1644          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1645          "mr %0,3"                                                \
1646          : /*out*/   "=r" (_res)                                  \
1647          : /*in*/    "r" (&_argvec[0])                            \
1648          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1649       );                                                          \
1650       lval = (__typeof__(lval)) _res;                             \
1651    } while (0)
1652
1653 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
1654    do {                                                           \
1655       volatile OrigFn        _orig = (orig);                      \
1656       volatile unsigned long _argvec[4];                          \
1657       volatile unsigned long _res;                                \
1658       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1659       _argvec[1] = (unsigned long)arg1;                           \
1660       _argvec[2] = (unsigned long)arg2;                           \
1661       _argvec[3] = (unsigned long)arg3;                           \
1662       __asm__ volatile(                                           \
1663          "mr 11,%1\n\t"                                           \
1664          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1665          "lwz 4,8(11)\n\t"                                        \
1666          "lwz 5,12(11)\n\t"                                       \
1667          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1668          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1669          "mr %0,3"                                                \
1670          : /*out*/   "=r" (_res)                                  \
1671          : /*in*/    "r" (&_argvec[0])                            \
1672          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1673       );                                                          \
1674       lval = (__typeof__(lval)) _res;                             \
1675    } while (0)
1676
1677 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
1678    do {                                                           \
1679       volatile OrigFn        _orig = (orig);                      \
1680       volatile unsigned long _argvec[5];                          \
1681       volatile unsigned long _res;                                \
1682       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1683       _argvec[1] = (unsigned long)arg1;                           \
1684       _argvec[2] = (unsigned long)arg2;                           \
1685       _argvec[3] = (unsigned long)arg3;                           \
1686       _argvec[4] = (unsigned long)arg4;                           \
1687       __asm__ volatile(                                           \
1688          "mr 11,%1\n\t"                                           \
1689          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1690          "lwz 4,8(11)\n\t"                                        \
1691          "lwz 5,12(11)\n\t"                                       \
1692          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1693          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1694          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1695          "mr %0,3"                                                \
1696          : /*out*/   "=r" (_res)                                  \
1697          : /*in*/    "r" (&_argvec[0])                            \
1698          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1699       );                                                          \
1700       lval = (__typeof__(lval)) _res;                             \
1701    } while (0)
1702
1703 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
1704    do {                                                           \
1705       volatile OrigFn        _orig = (orig);                      \
1706       volatile unsigned long _argvec[6];                          \
1707       volatile unsigned long _res;                                \
1708       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1709       _argvec[1] = (unsigned long)arg1;                           \
1710       _argvec[2] = (unsigned long)arg2;                           \
1711       _argvec[3] = (unsigned long)arg3;                           \
1712       _argvec[4] = (unsigned long)arg4;                           \
1713       _argvec[5] = (unsigned long)arg5;                           \
1714       __asm__ volatile(                                           \
1715          "mr 11,%1\n\t"                                           \
1716          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1717          "lwz 4,8(11)\n\t"                                        \
1718          "lwz 5,12(11)\n\t"                                       \
1719          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1720          "lwz 7,20(11)\n\t"                                       \
1721          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1722          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1723          "mr %0,3"                                                \
1724          : /*out*/   "=r" (_res)                                  \
1725          : /*in*/    "r" (&_argvec[0])                            \
1726          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1727       );                                                          \
1728       lval = (__typeof__(lval)) _res;                             \
1729    } while (0)
1730
1731 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
1732    do {                                                           \
1733       volatile OrigFn        _orig = (orig);                      \
1734       volatile unsigned long _argvec[7];                          \
1735       volatile unsigned long _res;                                \
1736       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1737       _argvec[1] = (unsigned long)arg1;                           \
1738       _argvec[2] = (unsigned long)arg2;                           \
1739       _argvec[3] = (unsigned long)arg3;                           \
1740       _argvec[4] = (unsigned long)arg4;                           \
1741       _argvec[5] = (unsigned long)arg5;                           \
1742       _argvec[6] = (unsigned long)arg6;                           \
1743       __asm__ volatile(                                           \
1744          "mr 11,%1\n\t"                                           \
1745          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1746          "lwz 4,8(11)\n\t"                                        \
1747          "lwz 5,12(11)\n\t"                                       \
1748          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1749          "lwz 7,20(11)\n\t"                                       \
1750          "lwz 8,24(11)\n\t"                                       \
1751          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1752          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1753          "mr %0,3"                                                \
1754          : /*out*/   "=r" (_res)                                  \
1755          : /*in*/    "r" (&_argvec[0])                            \
1756          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1757       );                                                          \
1758       lval = (__typeof__(lval)) _res;                             \
1759    } while (0)
1760
1761 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1762                                  arg7)                            \
1763    do {                                                           \
1764       volatile OrigFn        _orig = (orig);                      \
1765       volatile unsigned long _argvec[8];                          \
1766       volatile unsigned long _res;                                \
1767       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1768       _argvec[1] = (unsigned long)arg1;                           \
1769       _argvec[2] = (unsigned long)arg2;                           \
1770       _argvec[3] = (unsigned long)arg3;                           \
1771       _argvec[4] = (unsigned long)arg4;                           \
1772       _argvec[5] = (unsigned long)arg5;                           \
1773       _argvec[6] = (unsigned long)arg6;                           \
1774       _argvec[7] = (unsigned long)arg7;                           \
1775       __asm__ volatile(                                           \
1776          "mr 11,%1\n\t"                                           \
1777          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1778          "lwz 4,8(11)\n\t"                                        \
1779          "lwz 5,12(11)\n\t"                                       \
1780          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1781          "lwz 7,20(11)\n\t"                                       \
1782          "lwz 8,24(11)\n\t"                                       \
1783          "lwz 9,28(11)\n\t"                                       \
1784          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1785          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1786          "mr %0,3"                                                \
1787          : /*out*/   "=r" (_res)                                  \
1788          : /*in*/    "r" (&_argvec[0])                            \
1789          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1790       );                                                          \
1791       lval = (__typeof__(lval)) _res;                             \
1792    } while (0)
1793
1794 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1795                                  arg7,arg8)                       \
1796    do {                                                           \
1797       volatile OrigFn        _orig = (orig);                      \
1798       volatile unsigned long _argvec[9];                          \
1799       volatile unsigned long _res;                                \
1800       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1801       _argvec[1] = (unsigned long)arg1;                           \
1802       _argvec[2] = (unsigned long)arg2;                           \
1803       _argvec[3] = (unsigned long)arg3;                           \
1804       _argvec[4] = (unsigned long)arg4;                           \
1805       _argvec[5] = (unsigned long)arg5;                           \
1806       _argvec[6] = (unsigned long)arg6;                           \
1807       _argvec[7] = (unsigned long)arg7;                           \
1808       _argvec[8] = (unsigned long)arg8;                           \
1809       __asm__ volatile(                                           \
1810          "mr 11,%1\n\t"                                           \
1811          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1812          "lwz 4,8(11)\n\t"                                        \
1813          "lwz 5,12(11)\n\t"                                       \
1814          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1815          "lwz 7,20(11)\n\t"                                       \
1816          "lwz 8,24(11)\n\t"                                       \
1817          "lwz 9,28(11)\n\t"                                       \
1818          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
1819          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1820          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1821          "mr %0,3"                                                \
1822          : /*out*/   "=r" (_res)                                  \
1823          : /*in*/    "r" (&_argvec[0])                            \
1824          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1825       );                                                          \
1826       lval = (__typeof__(lval)) _res;                             \
1827    } while (0)
1828
1829 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1830                                  arg7,arg8,arg9)                  \
1831    do {                                                           \
1832       volatile OrigFn        _orig = (orig);                      \
1833       volatile unsigned long _argvec[10];                         \
1834       volatile unsigned long _res;                                \
1835       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1836       _argvec[1] = (unsigned long)arg1;                           \
1837       _argvec[2] = (unsigned long)arg2;                           \
1838       _argvec[3] = (unsigned long)arg3;                           \
1839       _argvec[4] = (unsigned long)arg4;                           \
1840       _argvec[5] = (unsigned long)arg5;                           \
1841       _argvec[6] = (unsigned long)arg6;                           \
1842       _argvec[7] = (unsigned long)arg7;                           \
1843       _argvec[8] = (unsigned long)arg8;                           \
1844       _argvec[9] = (unsigned long)arg9;                           \
1845       __asm__ volatile(                                           \
1846          "mr 11,%1\n\t"                                           \
1847          "addi 1,1,-16\n\t"                                       \
1848          /* arg9 */                                               \
1849          "lwz 3,36(11)\n\t"                                       \
1850          "stw 3,8(1)\n\t"                                         \
1851          /* args1-8 */                                            \
1852          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1853          "lwz 4,8(11)\n\t"                                        \
1854          "lwz 5,12(11)\n\t"                                       \
1855          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1856          "lwz 7,20(11)\n\t"                                       \
1857          "lwz 8,24(11)\n\t"                                       \
1858          "lwz 9,28(11)\n\t"                                       \
1859          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
1860          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1861          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1862          "addi 1,1,16\n\t"                                        \
1863          "mr %0,3"                                                \
1864          : /*out*/   "=r" (_res)                                  \
1865          : /*in*/    "r" (&_argvec[0])                            \
1866          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1867       );                                                          \
1868       lval = (__typeof__(lval)) _res;                             \
1869    } while (0)
1870
1871 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1872                                   arg7,arg8,arg9,arg10)           \
1873    do {                                                           \
1874       volatile OrigFn        _orig = (orig);                      \
1875       volatile unsigned long _argvec[11];                         \
1876       volatile unsigned long _res;                                \
1877       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1878       _argvec[1] = (unsigned long)arg1;                           \
1879       _argvec[2] = (unsigned long)arg2;                           \
1880       _argvec[3] = (unsigned long)arg3;                           \
1881       _argvec[4] = (unsigned long)arg4;                           \
1882       _argvec[5] = (unsigned long)arg5;                           \
1883       _argvec[6] = (unsigned long)arg6;                           \
1884       _argvec[7] = (unsigned long)arg7;                           \
1885       _argvec[8] = (unsigned long)arg8;                           \
1886       _argvec[9] = (unsigned long)arg9;                           \
1887       _argvec[10] = (unsigned long)arg10;                         \
1888       __asm__ volatile(                                           \
1889          "mr 11,%1\n\t"                                           \
1890          "addi 1,1,-16\n\t"                                       \
1891          /* arg10 */                                              \
1892          "lwz 3,40(11)\n\t"                                       \
1893          "stw 3,12(1)\n\t"                                        \
1894          /* arg9 */                                               \
1895          "lwz 3,36(11)\n\t"                                       \
1896          "stw 3,8(1)\n\t"                                         \
1897          /* args1-8 */                                            \
1898          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1899          "lwz 4,8(11)\n\t"                                        \
1900          "lwz 5,12(11)\n\t"                                       \
1901          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1902          "lwz 7,20(11)\n\t"                                       \
1903          "lwz 8,24(11)\n\t"                                       \
1904          "lwz 9,28(11)\n\t"                                       \
1905          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
1906          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1907          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1908          "addi 1,1,16\n\t"                                        \
1909          "mr %0,3"                                                \
1910          : /*out*/   "=r" (_res)                                  \
1911          : /*in*/    "r" (&_argvec[0])                            \
1912          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1913       );                                                          \
1914       lval = (__typeof__(lval)) _res;                             \
1915    } while (0)
1916
1917 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1918                                   arg7,arg8,arg9,arg10,arg11)     \
1919    do {                                                           \
1920       volatile OrigFn        _orig = (orig);                      \
1921       volatile unsigned long _argvec[12];                         \
1922       volatile unsigned long _res;                                \
1923       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1924       _argvec[1] = (unsigned long)arg1;                           \
1925       _argvec[2] = (unsigned long)arg2;                           \
1926       _argvec[3] = (unsigned long)arg3;                           \
1927       _argvec[4] = (unsigned long)arg4;                           \
1928       _argvec[5] = (unsigned long)arg5;                           \
1929       _argvec[6] = (unsigned long)arg6;                           \
1930       _argvec[7] = (unsigned long)arg7;                           \
1931       _argvec[8] = (unsigned long)arg8;                           \
1932       _argvec[9] = (unsigned long)arg9;                           \
1933       _argvec[10] = (unsigned long)arg10;                         \
1934       _argvec[11] = (unsigned long)arg11;                         \
1935       __asm__ volatile(                                           \
1936          "mr 11,%1\n\t"                                           \
1937          "addi 1,1,-32\n\t"                                       \
1938          /* arg11 */                                              \
1939          "lwz 3,44(11)\n\t"                                       \
1940          "stw 3,16(1)\n\t"                                        \
1941          /* arg10 */                                              \
1942          "lwz 3,40(11)\n\t"                                       \
1943          "stw 3,12(1)\n\t"                                        \
1944          /* arg9 */                                               \
1945          "lwz 3,36(11)\n\t"                                       \
1946          "stw 3,8(1)\n\t"                                         \
1947          /* args1-8 */                                            \
1948          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1949          "lwz 4,8(11)\n\t"                                        \
1950          "lwz 5,12(11)\n\t"                                       \
1951          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1952          "lwz 7,20(11)\n\t"                                       \
1953          "lwz 8,24(11)\n\t"                                       \
1954          "lwz 9,28(11)\n\t"                                       \
1955          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
1956          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1957          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1958          "addi 1,1,32\n\t"                                        \
1959          "mr %0,3"                                                \
1960          : /*out*/   "=r" (_res)                                  \
1961          : /*in*/    "r" (&_argvec[0])                            \
1962          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1963       );                                                          \
1964       lval = (__typeof__(lval)) _res;                             \
1965    } while (0)
1966
1967 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1968                                 arg7,arg8,arg9,arg10,arg11,arg12) \
1969    do {                                                           \
1970       volatile OrigFn        _orig = (orig);                      \
1971       volatile unsigned long _argvec[13];                         \
1972       volatile unsigned long _res;                                \
1973       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1974       _argvec[1] = (unsigned long)arg1;                           \
1975       _argvec[2] = (unsigned long)arg2;                           \
1976       _argvec[3] = (unsigned long)arg3;                           \
1977       _argvec[4] = (unsigned long)arg4;                           \
1978       _argvec[5] = (unsigned long)arg5;                           \
1979       _argvec[6] = (unsigned long)arg6;                           \
1980       _argvec[7] = (unsigned long)arg7;                           \
1981       _argvec[8] = (unsigned long)arg8;                           \
1982       _argvec[9] = (unsigned long)arg9;                           \
1983       _argvec[10] = (unsigned long)arg10;                         \
1984       _argvec[11] = (unsigned long)arg11;                         \
1985       _argvec[12] = (unsigned long)arg12;                         \
1986       __asm__ volatile(                                           \
1987          "mr 11,%1\n\t"                                           \
1988          "addi 1,1,-32\n\t"                                       \
1989          /* arg12 */                                              \
1990          "lwz 3,48(11)\n\t"                                       \
1991          "stw 3,20(1)\n\t"                                        \
1992          /* arg11 */                                              \
1993          "lwz 3,44(11)\n\t"                                       \
1994          "stw 3,16(1)\n\t"                                        \
1995          /* arg10 */                                              \
1996          "lwz 3,40(11)\n\t"                                       \
1997          "stw 3,12(1)\n\t"                                        \
1998          /* arg9 */                                               \
1999          "lwz 3,36(11)\n\t"                                       \
2000          "stw 3,8(1)\n\t"                                         \
2001          /* args1-8 */                                            \
2002          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2003          "lwz 4,8(11)\n\t"                                        \
2004          "lwz 5,12(11)\n\t"                                       \
2005          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2006          "lwz 7,20(11)\n\t"                                       \
2007          "lwz 8,24(11)\n\t"                                       \
2008          "lwz 9,28(11)\n\t"                                       \
2009          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2010          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2011          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2012          "addi 1,1,32\n\t"                                        \
2013          "mr %0,3"                                                \
2014          : /*out*/   "=r" (_res)                                  \
2015          : /*in*/    "r" (&_argvec[0])                            \
2016          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2017       );                                                          \
2018       lval = (__typeof__(lval)) _res;                             \
2019    } while (0)
2020
2021 #endif /* PLAT_ppc32_linux */
2022
2023 /* ------------------------ ppc64-linux ------------------------ */
2024
2025 #if defined(PLAT_ppc64_linux)
2026
2027 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
2028
2029 /* These regs are trashed by the hidden call. */
2030 #define __CALLER_SAVED_REGS                                       \
2031    "lr", "ctr", "xer",                                            \
2032    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
2033    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
2034    "r11", "r12", "r13"
2035
2036 /* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned
2037    long) == 8. */
2038
2039 #define CALL_FN_W_v(lval, orig)                                   \
2040    do {                                                           \
2041       volatile OrigFn        _orig = (orig);                      \
2042       volatile unsigned long _argvec[3+0];                        \
2043       volatile unsigned long _res;                                \
2044       /* _argvec[0] holds current r2 across the call */           \
2045       _argvec[1] = (unsigned long)_orig.r2;                       \
2046       _argvec[2] = (unsigned long)_orig.nraddr;                   \
2047       __asm__ volatile(                                           \
2048          "mr 11,%1\n\t"                                           \
2049          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2050          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2051          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2052          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2053          "mr 11,%1\n\t"                                           \
2054          "mr %0,3\n\t"                                            \
2055          "ld 2,-16(11)" /* restore tocptr */                      \
2056          : /*out*/   "=r" (_res)                                  \
2057          : /*in*/    "r" (&_argvec[2])                            \
2058          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2059       );                                                          \
2060       lval = (__typeof__(lval)) _res;                             \
2061    } while (0)
2062
2063 #define CALL_FN_W_W(lval, orig, arg1)                             \
2064    do {                                                           \
2065       volatile OrigFn        _orig = (orig);                      \
2066       volatile unsigned long _argvec[3+1];                        \
2067       volatile unsigned long _res;                                \
2068       /* _argvec[0] holds current r2 across the call */           \
2069       _argvec[1]   = (unsigned long)_orig.r2;                     \
2070       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2071       _argvec[2+1] = (unsigned long)arg1;                         \
2072       __asm__ volatile(                                           \
2073          "mr 11,%1\n\t"                                           \
2074          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2075          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2076          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2077          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2078          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2079          "mr 11,%1\n\t"                                           \
2080          "mr %0,3\n\t"                                            \
2081          "ld 2,-16(11)" /* restore tocptr */                      \
2082          : /*out*/   "=r" (_res)                                  \
2083          : /*in*/    "r" (&_argvec[2])                            \
2084          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2085       );                                                          \
2086       lval = (__typeof__(lval)) _res;                             \
2087    } while (0)
2088
2089 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
2090    do {                                                           \
2091       volatile OrigFn        _orig = (orig);                      \
2092       volatile unsigned long _argvec[3+2];                        \
2093       volatile unsigned long _res;                                \
2094       /* _argvec[0] holds current r2 across the call */           \
2095       _argvec[1]   = (unsigned long)_orig.r2;                     \
2096       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2097       _argvec[2+1] = (unsigned long)arg1;                         \
2098       _argvec[2+2] = (unsigned long)arg2;                         \
2099       __asm__ volatile(                                           \
2100          "mr 11,%1\n\t"                                           \
2101          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2102          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2103          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2104          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2105          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2106          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2107          "mr 11,%1\n\t"                                           \
2108          "mr %0,3\n\t"                                            \
2109          "ld 2,-16(11)" /* restore tocptr */                      \
2110          : /*out*/   "=r" (_res)                                  \
2111          : /*in*/    "r" (&_argvec[2])                            \
2112          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2113       );                                                          \
2114       lval = (__typeof__(lval)) _res;                             \
2115    } while (0)
2116
2117 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
2118    do {                                                           \
2119       volatile OrigFn        _orig = (orig);                      \
2120       volatile unsigned long _argvec[3+3];                        \
2121       volatile unsigned long _res;                                \
2122       /* _argvec[0] holds current r2 across the call */           \
2123       _argvec[1]   = (unsigned long)_orig.r2;                     \
2124       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2125       _argvec[2+1] = (unsigned long)arg1;                         \
2126       _argvec[2+2] = (unsigned long)arg2;                         \
2127       _argvec[2+3] = (unsigned long)arg3;                         \
2128       __asm__ volatile(                                           \
2129          "mr 11,%1\n\t"                                           \
2130          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2131          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2132          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2133          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2134          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2135          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2136          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2137          "mr 11,%1\n\t"                                           \
2138          "mr %0,3\n\t"                                            \
2139          "ld 2,-16(11)" /* restore tocptr */                      \
2140          : /*out*/   "=r" (_res)                                  \
2141          : /*in*/    "r" (&_argvec[2])                            \
2142          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2143       );                                                          \
2144       lval = (__typeof__(lval)) _res;                             \
2145    } while (0)
2146
2147 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
2148    do {                                                           \
2149       volatile OrigFn        _orig = (orig);                      \
2150       volatile unsigned long _argvec[3+4];                        \
2151       volatile unsigned long _res;                                \
2152       /* _argvec[0] holds current r2 across the call */           \
2153       _argvec[1]   = (unsigned long)_orig.r2;                     \
2154       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2155       _argvec[2+1] = (unsigned long)arg1;                         \
2156       _argvec[2+2] = (unsigned long)arg2;                         \
2157       _argvec[2+3] = (unsigned long)arg3;                         \
2158       _argvec[2+4] = (unsigned long)arg4;                         \
2159       __asm__ volatile(                                           \
2160          "mr 11,%1\n\t"                                           \
2161          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2162          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2163          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2164          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2165          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2166          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2167          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2168          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2169          "mr 11,%1\n\t"                                           \
2170          "mr %0,3\n\t"                                            \
2171          "ld 2,-16(11)" /* restore tocptr */                      \
2172          : /*out*/   "=r" (_res)                                  \
2173          : /*in*/    "r" (&_argvec[2])                            \
2174          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2175       );                                                          \
2176       lval = (__typeof__(lval)) _res;                             \
2177    } while (0)
2178
2179 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
2180    do {                                                           \
2181       volatile OrigFn        _orig = (orig);                      \
2182       volatile unsigned long _argvec[3+5];                        \
2183       volatile unsigned long _res;                                \
2184       /* _argvec[0] holds current r2 across the call */           \
2185       _argvec[1]   = (unsigned long)_orig.r2;                     \
2186       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2187       _argvec[2+1] = (unsigned long)arg1;                         \
2188       _argvec[2+2] = (unsigned long)arg2;                         \
2189       _argvec[2+3] = (unsigned long)arg3;                         \
2190       _argvec[2+4] = (unsigned long)arg4;                         \
2191       _argvec[2+5] = (unsigned long)arg5;                         \
2192       __asm__ volatile(                                           \
2193          "mr 11,%1\n\t"                                           \
2194          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2195          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2196          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2197          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2198          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2199          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2200          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2201          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2202          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2203          "mr 11,%1\n\t"                                           \
2204          "mr %0,3\n\t"                                            \
2205          "ld 2,-16(11)" /* restore tocptr */                      \
2206          : /*out*/   "=r" (_res)                                  \
2207          : /*in*/    "r" (&_argvec[2])                            \
2208          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2209       );                                                          \
2210       lval = (__typeof__(lval)) _res;                             \
2211    } while (0)
2212
2213 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
2214    do {                                                           \
2215       volatile OrigFn        _orig = (orig);                      \
2216       volatile unsigned long _argvec[3+6];                        \
2217       volatile unsigned long _res;                                \
2218       /* _argvec[0] holds current r2 across the call */           \
2219       _argvec[1]   = (unsigned long)_orig.r2;                     \
2220       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2221       _argvec[2+1] = (unsigned long)arg1;                         \
2222       _argvec[2+2] = (unsigned long)arg2;                         \
2223       _argvec[2+3] = (unsigned long)arg3;                         \
2224       _argvec[2+4] = (unsigned long)arg4;                         \
2225       _argvec[2+5] = (unsigned long)arg5;                         \
2226       _argvec[2+6] = (unsigned long)arg6;                         \
2227       __asm__ volatile(                                           \
2228          "mr 11,%1\n\t"                                           \
2229          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2230          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2231          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2232          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2233          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2234          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2235          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2236          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2237          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2238          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2239          "mr 11,%1\n\t"                                           \
2240          "mr %0,3\n\t"                                            \
2241          "ld 2,-16(11)" /* restore tocptr */                      \
2242          : /*out*/   "=r" (_res)                                  \
2243          : /*in*/    "r" (&_argvec[2])                            \
2244          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2245       );                                                          \
2246       lval = (__typeof__(lval)) _res;                             \
2247    } while (0)
2248
2249 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2250                                  arg7)                            \
2251    do {                                                           \
2252       volatile OrigFn        _orig = (orig);                      \
2253       volatile unsigned long _argvec[3+7];                        \
2254       volatile unsigned long _res;                                \
2255       /* _argvec[0] holds current r2 across the call */           \
2256       _argvec[1]   = (unsigned long)_orig.r2;                     \
2257       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2258       _argvec[2+1] = (unsigned long)arg1;                         \
2259       _argvec[2+2] = (unsigned long)arg2;                         \
2260       _argvec[2+3] = (unsigned long)arg3;                         \
2261       _argvec[2+4] = (unsigned long)arg4;                         \
2262       _argvec[2+5] = (unsigned long)arg5;                         \
2263       _argvec[2+6] = (unsigned long)arg6;                         \
2264       _argvec[2+7] = (unsigned long)arg7;                         \
2265       __asm__ volatile(                                           \
2266          "mr 11,%1\n\t"                                           \
2267          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2268          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2269          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2270          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2271          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2272          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2273          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2274          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2275          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2276          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2277          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2278          "mr 11,%1\n\t"                                           \
2279          "mr %0,3\n\t"                                            \
2280          "ld 2,-16(11)" /* restore tocptr */                      \
2281          : /*out*/   "=r" (_res)                                  \
2282          : /*in*/    "r" (&_argvec[2])                            \
2283          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2284       );                                                          \
2285       lval = (__typeof__(lval)) _res;                             \
2286    } while (0)
2287
2288 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2289                                  arg7,arg8)                       \
2290    do {                                                           \
2291       volatile OrigFn        _orig = (orig);                      \
2292       volatile unsigned long _argvec[3+8];                        \
2293       volatile unsigned long _res;                                \
2294       /* _argvec[0] holds current r2 across the call */           \
2295       _argvec[1]   = (unsigned long)_orig.r2;                     \
2296       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2297       _argvec[2+1] = (unsigned long)arg1;                         \
2298       _argvec[2+2] = (unsigned long)arg2;                         \
2299       _argvec[2+3] = (unsigned long)arg3;                         \
2300       _argvec[2+4] = (unsigned long)arg4;                         \
2301       _argvec[2+5] = (unsigned long)arg5;                         \
2302       _argvec[2+6] = (unsigned long)arg6;                         \
2303       _argvec[2+7] = (unsigned long)arg7;                         \
2304       _argvec[2+8] = (unsigned long)arg8;                         \
2305       __asm__ volatile(                                           \
2306          "mr 11,%1\n\t"                                           \
2307          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2308          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2309          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2310          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2311          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2312          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2313          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2314          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2315          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2316          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2317          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2318          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2319          "mr 11,%1\n\t"                                           \
2320          "mr %0,3\n\t"                                            \
2321          "ld 2,-16(11)" /* restore tocptr */                      \
2322          : /*out*/   "=r" (_res)                                  \
2323          : /*in*/    "r" (&_argvec[2])                            \
2324          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2325       );                                                          \
2326       lval = (__typeof__(lval)) _res;                             \
2327    } while (0)
2328
2329 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2330                                  arg7,arg8,arg9)                  \
2331    do {                                                           \
2332       volatile OrigFn        _orig = (orig);                      \
2333       volatile unsigned long _argvec[3+9];                        \
2334       volatile unsigned long _res;                                \
2335       /* _argvec[0] holds current r2 across the call */           \
2336       _argvec[1]   = (unsigned long)_orig.r2;                     \
2337       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2338       _argvec[2+1] = (unsigned long)arg1;                         \
2339       _argvec[2+2] = (unsigned long)arg2;                         \
2340       _argvec[2+3] = (unsigned long)arg3;                         \
2341       _argvec[2+4] = (unsigned long)arg4;                         \
2342       _argvec[2+5] = (unsigned long)arg5;                         \
2343       _argvec[2+6] = (unsigned long)arg6;                         \
2344       _argvec[2+7] = (unsigned long)arg7;                         \
2345       _argvec[2+8] = (unsigned long)arg8;                         \
2346       _argvec[2+9] = (unsigned long)arg9;                         \
2347       __asm__ volatile(                                           \
2348          "mr 11,%1\n\t"                                           \
2349          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2350          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2351          "addi 1,1,-128\n\t"  /* expand stack frame */            \
2352          /* arg9 */                                               \
2353          "ld  3,72(11)\n\t"                                       \
2354          "std 3,112(1)\n\t"                                       \
2355          /* args1-8 */                                            \
2356          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2357          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2358          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2359          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2360          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2361          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2362          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2363          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2364          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2365          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2366          "mr 11,%1\n\t"                                           \
2367          "mr %0,3\n\t"                                            \
2368          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2369          "addi 1,1,128"     /* restore frame */                   \
2370          : /*out*/   "=r" (_res)                                  \
2371          : /*in*/    "r" (&_argvec[2])                            \
2372          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2373       );                                                          \
2374       lval = (__typeof__(lval)) _res;                             \
2375    } while (0)
2376
2377 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2378                                   arg7,arg8,arg9,arg10)           \
2379    do {                                                           \
2380       volatile OrigFn        _orig = (orig);                      \
2381       volatile unsigned long _argvec[3+10];                       \
2382       volatile unsigned long _res;                                \
2383       /* _argvec[0] holds current r2 across the call */           \
2384       _argvec[1]   = (unsigned long)_orig.r2;                     \
2385       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2386       _argvec[2+1] = (unsigned long)arg1;                         \
2387       _argvec[2+2] = (unsigned long)arg2;                         \
2388       _argvec[2+3] = (unsigned long)arg3;                         \
2389       _argvec[2+4] = (unsigned long)arg4;                         \
2390       _argvec[2+5] = (unsigned long)arg5;                         \
2391       _argvec[2+6] = (unsigned long)arg6;                         \
2392       _argvec[2+7] = (unsigned long)arg7;                         \
2393       _argvec[2+8] = (unsigned long)arg8;                         \
2394       _argvec[2+9] = (unsigned long)arg9;                         \
2395       _argvec[2+10] = (unsigned long)arg10;                       \
2396       __asm__ volatile(                                           \
2397          "mr 11,%1\n\t"                                           \
2398          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2399          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2400          "addi 1,1,-128\n\t"  /* expand stack frame */            \
2401          /* arg10 */                                              \
2402          "ld  3,80(11)\n\t"                                       \
2403          "std 3,120(1)\n\t"                                       \
2404          /* arg9 */                                               \
2405          "ld  3,72(11)\n\t"                                       \
2406          "std 3,112(1)\n\t"                                       \
2407          /* args1-8 */                                            \
2408          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2409          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2410          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2411          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2412          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2413          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2414          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2415          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2416          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2417          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2418          "mr 11,%1\n\t"                                           \
2419          "mr %0,3\n\t"                                            \
2420          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2421          "addi 1,1,128"     /* restore frame */                   \
2422          : /*out*/   "=r" (_res)                                  \
2423          : /*in*/    "r" (&_argvec[2])                            \
2424          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2425       );                                                          \
2426       lval = (__typeof__(lval)) _res;                             \
2427    } while (0)
2428
2429 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2430                                   arg7,arg8,arg9,arg10,arg11)     \
2431    do {                                                           \
2432       volatile OrigFn        _orig = (orig);                      \
2433       volatile unsigned long _argvec[3+11];                       \
2434       volatile unsigned long _res;                                \
2435       /* _argvec[0] holds current r2 across the call */           \
2436       _argvec[1]   = (unsigned long)_orig.r2;                     \
2437       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2438       _argvec[2+1] = (unsigned long)arg1;                         \
2439       _argvec[2+2] = (unsigned long)arg2;                         \
2440       _argvec[2+3] = (unsigned long)arg3;                         \
2441       _argvec[2+4] = (unsigned long)arg4;                         \
2442       _argvec[2+5] = (unsigned long)arg5;                         \
2443       _argvec[2+6] = (unsigned long)arg6;                         \
2444       _argvec[2+7] = (unsigned long)arg7;                         \
2445       _argvec[2+8] = (unsigned long)arg8;                         \
2446       _argvec[2+9] = (unsigned long)arg9;                         \
2447       _argvec[2+10] = (unsigned long)arg10;                       \
2448       _argvec[2+11] = (unsigned long)arg11;                       \
2449       __asm__ volatile(                                           \
2450          "mr 11,%1\n\t"                                           \
2451          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2452          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2453          "addi 1,1,-144\n\t"  /* expand stack frame */            \
2454          /* arg11 */                                              \
2455          "ld  3,88(11)\n\t"                                       \
2456          "std 3,128(1)\n\t"                                       \
2457          /* arg10 */                                              \
2458          "ld  3,80(11)\n\t"                                       \
2459          "std 3,120(1)\n\t"                                       \
2460          /* arg9 */                                               \
2461          "ld  3,72(11)\n\t"                                       \
2462          "std 3,112(1)\n\t"                                       \
2463          /* args1-8 */                                            \
2464          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2465          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2466          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2467          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2468          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2469          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2470          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2471          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2472          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2473          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2474          "mr 11,%1\n\t"                                           \
2475          "mr %0,3\n\t"                                            \
2476          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2477          "addi 1,1,144"     /* restore frame */                   \
2478          : /*out*/   "=r" (_res)                                  \
2479          : /*in*/    "r" (&_argvec[2])                            \
2480          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2481       );                                                          \
2482       lval = (__typeof__(lval)) _res;                             \
2483    } while (0)
2484
2485 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2486                                 arg7,arg8,arg9,arg10,arg11,arg12) \
2487    do {                                                           \
2488       volatile OrigFn        _orig = (orig);                      \
2489       volatile unsigned long _argvec[3+12];                       \
2490       volatile unsigned long _res;                                \
2491       /* _argvec[0] holds current r2 across the call */           \
2492       _argvec[1]   = (unsigned long)_orig.r2;                     \
2493       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2494       _argvec[2+1] = (unsigned long)arg1;                         \
2495       _argvec[2+2] = (unsigned long)arg2;                         \
2496       _argvec[2+3] = (unsigned long)arg3;                         \
2497       _argvec[2+4] = (unsigned long)arg4;                         \
2498       _argvec[2+5] = (unsigned long)arg5;                         \
2499       _argvec[2+6] = (unsigned long)arg6;                         \
2500       _argvec[2+7] = (unsigned long)arg7;                         \
2501       _argvec[2+8] = (unsigned long)arg8;                         \
2502       _argvec[2+9] = (unsigned long)arg9;                         \
2503       _argvec[2+10] = (unsigned long)arg10;                       \
2504       _argvec[2+11] = (unsigned long)arg11;                       \
2505       _argvec[2+12] = (unsigned long)arg12;                       \
2506       __asm__ volatile(                                           \
2507          "mr 11,%1\n\t"                                           \
2508          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2509          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2510          "addi 1,1,-144\n\t"  /* expand stack frame */            \
2511          /* arg12 */                                              \
2512          "ld  3,96(11)\n\t"                                       \
2513          "std 3,136(1)\n\t"                                       \
2514          /* arg11 */                                              \
2515          "ld  3,88(11)\n\t"                                       \
2516          "std 3,128(1)\n\t"                                       \
2517          /* arg10 */                                              \
2518          "ld  3,80(11)\n\t"                                       \
2519          "std 3,120(1)\n\t"                                       \
2520          /* arg9 */                                               \
2521          "ld  3,72(11)\n\t"                                       \
2522          "std 3,112(1)\n\t"                                       \
2523          /* args1-8 */                                            \
2524          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2525          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2526          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2527          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2528          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2529          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2530          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2531          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2532          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2533          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2534          "mr 11,%1\n\t"                                           \
2535          "mr %0,3\n\t"                                            \
2536          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2537          "addi 1,1,144"     /* restore frame */                   \
2538          : /*out*/   "=r" (_res)                                  \
2539          : /*in*/    "r" (&_argvec[2])                            \
2540          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2541       );                                                          \
2542       lval = (__typeof__(lval)) _res;                             \
2543    } while (0)
2544
2545 #endif /* PLAT_ppc64_linux */
2546
2547 /* ------------------------- arm-linux ------------------------- */
2548
2549 #if defined(PLAT_arm_linux)
2550
2551 /* These regs are trashed by the hidden call. */
2552 #define __CALLER_SAVED_REGS "r0", "r1", "r2", "r3","r4","r14"
2553
2554 /* These CALL_FN_ macros assume that on arm-linux, sizeof(unsigned
2555    long) == 4. */
2556
2557 #define CALL_FN_W_v(lval, orig)                                   \
2558    do {                                                           \
2559       volatile OrigFn        _orig = (orig);                      \
2560       volatile unsigned long _argvec[1];                          \
2561       volatile unsigned long _res;                                \
2562       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2563       __asm__ volatile(                                           \
2564          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2565          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2566          "mov %0, r0\n"                                           \
2567          : /*out*/   "=r" (_res)                                  \
2568          : /*in*/    "0" (&_argvec[0])                            \
2569          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2570       );                                                          \
2571       lval = (__typeof__(lval)) _res;                             \
2572    } while (0)
2573
2574 #define CALL_FN_W_W(lval, orig, arg1)                             \
2575    do {                                                           \
2576       volatile OrigFn        _orig = (orig);                      \
2577       volatile unsigned long _argvec[2];                          \
2578       volatile unsigned long _res;                                \
2579       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2580       _argvec[1] = (unsigned long)(arg1);                         \
2581       __asm__ volatile(                                           \
2582          "ldr r0, [%1, #4] \n\t"                                  \
2583          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2584          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2585          "mov %0, r0\n"                                           \
2586          : /*out*/   "=r" (_res)                                  \
2587          : /*in*/    "0" (&_argvec[0])                            \
2588          : /*trash*/ "cc", "memory",  __CALLER_SAVED_REGS         \
2589       );                                                          \
2590       lval = (__typeof__(lval)) _res;                             \
2591    } while (0)
2592
2593 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
2594    do {                                                           \
2595       volatile OrigFn        _orig = (orig);                      \
2596       volatile unsigned long _argvec[3];                          \
2597       volatile unsigned long _res;                                \
2598       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2599       _argvec[1] = (unsigned long)(arg1);                         \
2600       _argvec[2] = (unsigned long)(arg2);                         \
2601       __asm__ volatile(                                           \
2602          "ldr r0, [%1, #4] \n\t"                                  \
2603          "ldr r1, [%1, #8] \n\t"                                  \
2604          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2605          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2606          "mov %0, r0\n"                                           \
2607          : /*out*/   "=r" (_res)                                  \
2608          : /*in*/    "0" (&_argvec[0])                            \
2609          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2610       );                                                          \
2611       lval = (__typeof__(lval)) _res;                             \
2612    } while (0)
2613
2614 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
2615    do {                                                           \
2616       volatile OrigFn        _orig = (orig);                      \
2617       volatile unsigned long _argvec[4];                          \
2618       volatile unsigned long _res;                                \
2619       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2620       _argvec[1] = (unsigned long)(arg1);                         \
2621       _argvec[2] = (unsigned long)(arg2);                         \
2622       _argvec[3] = (unsigned long)(arg3);                         \
2623       __asm__ volatile(                                           \
2624          "ldr r0, [%1, #4] \n\t"                                  \
2625          "ldr r1, [%1, #8] \n\t"                                  \
2626          "ldr r2, [%1, #12] \n\t"                                 \
2627          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2628          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2629          "mov %0, r0\n"                                           \
2630          : /*out*/   "=r" (_res)                                  \
2631          : /*in*/    "0" (&_argvec[0])                            \
2632          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2633       );                                                          \
2634       lval = (__typeof__(lval)) _res;                             \
2635    } while (0)
2636
2637 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
2638    do {                                                           \
2639       volatile OrigFn        _orig = (orig);                      \
2640       volatile unsigned long _argvec[5];                          \
2641       volatile unsigned long _res;                                \
2642       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2643       _argvec[1] = (unsigned long)(arg1);                         \
2644       _argvec[2] = (unsigned long)(arg2);                         \
2645       _argvec[3] = (unsigned long)(arg3);                         \
2646       _argvec[4] = (unsigned long)(arg4);                         \
2647       __asm__ volatile(                                           \
2648          "ldr r0, [%1, #4] \n\t"                                  \
2649          "ldr r1, [%1, #8] \n\t"                                  \
2650          "ldr r2, [%1, #12] \n\t"                                 \
2651          "ldr r3, [%1, #16] \n\t"                                 \
2652          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2653          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2654          "mov %0, r0"                                             \
2655          : /*out*/   "=r" (_res)                                  \
2656          : /*in*/    "0" (&_argvec[0])                            \
2657          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2658       );                                                          \
2659       lval = (__typeof__(lval)) _res;                             \
2660    } while (0)
2661
2662 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
2663    do {                                                           \
2664       volatile OrigFn        _orig = (orig);                      \
2665       volatile unsigned long _argvec[6];                          \
2666       volatile unsigned long _res;                                \
2667       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2668       _argvec[1] = (unsigned long)(arg1);                         \
2669       _argvec[2] = (unsigned long)(arg2);                         \
2670       _argvec[3] = (unsigned long)(arg3);                         \
2671       _argvec[4] = (unsigned long)(arg4);                         \
2672       _argvec[5] = (unsigned long)(arg5);                         \
2673       __asm__ volatile(                                           \
2674          "ldr r0, [%1, #20] \n\t"                                 \
2675          "push {r0} \n\t"                                         \
2676          "ldr r0, [%1, #4] \n\t"                                  \
2677          "ldr r1, [%1, #8] \n\t"                                  \
2678          "ldr r2, [%1, #12] \n\t"                                 \
2679          "ldr r3, [%1, #16] \n\t"                                 \
2680          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2681          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2682          "add sp, sp, #4 \n\t"                                    \
2683          "mov %0, r0"                                             \
2684          : /*out*/   "=r" (_res)                                  \
2685          : /*in*/    "0" (&_argvec[0])                            \
2686          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2687       );                                                          \
2688       lval = (__typeof__(lval)) _res;                             \
2689    } while (0)
2690
2691 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
2692    do {                                                           \
2693       volatile OrigFn        _orig = (orig);                      \
2694       volatile unsigned long _argvec[7];                          \
2695       volatile unsigned long _res;                                \
2696       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2697       _argvec[1] = (unsigned long)(arg1);                         \
2698       _argvec[2] = (unsigned long)(arg2);                         \
2699       _argvec[3] = (unsigned long)(arg3);                         \
2700       _argvec[4] = (unsigned long)(arg4);                         \
2701       _argvec[5] = (unsigned long)(arg5);                         \
2702       _argvec[6] = (unsigned long)(arg6);                         \
2703       __asm__ volatile(                                           \
2704          "ldr r0, [%1, #20] \n\t"                                 \
2705          "ldr r1, [%1, #24] \n\t"                                 \
2706          "push {r0, r1} \n\t"                                     \
2707          "ldr r0, [%1, #4] \n\t"                                  \
2708          "ldr r1, [%1, #8] \n\t"                                  \
2709          "ldr r2, [%1, #12] \n\t"                                 \
2710          "ldr r3, [%1, #16] \n\t"                                 \
2711          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2712          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2713          "add sp, sp, #8 \n\t"                                    \
2714          "mov %0, r0"                                             \
2715          : /*out*/   "=r" (_res)                                  \
2716          : /*in*/    "0" (&_argvec[0])                            \
2717          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2718       );                                                          \
2719       lval = (__typeof__(lval)) _res;                             \
2720    } while (0)
2721
2722 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2723                                  arg7)                            \
2724    do {                                                           \
2725       volatile OrigFn        _orig = (orig);                      \
2726       volatile unsigned long _argvec[8];                          \
2727       volatile unsigned long _res;                                \
2728       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2729       _argvec[1] = (unsigned long)(arg1);                         \
2730       _argvec[2] = (unsigned long)(arg2);                         \
2731       _argvec[3] = (unsigned long)(arg3);                         \
2732       _argvec[4] = (unsigned long)(arg4);                         \
2733       _argvec[5] = (unsigned long)(arg5);                         \
2734       _argvec[6] = (unsigned long)(arg6);                         \
2735       _argvec[7] = (unsigned long)(arg7);                         \
2736       __asm__ volatile(                                           \
2737          "ldr r0, [%1, #20] \n\t"                                 \
2738          "ldr r1, [%1, #24] \n\t"                                 \
2739          "ldr r2, [%1, #28] \n\t"                                 \
2740          "push {r0, r1, r2} \n\t"                                 \
2741          "ldr r0, [%1, #4] \n\t"                                  \
2742          "ldr r1, [%1, #8] \n\t"                                  \
2743          "ldr r2, [%1, #12] \n\t"                                 \
2744          "ldr r3, [%1, #16] \n\t"                                 \
2745          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2746          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2747          "add sp, sp, #12 \n\t"                                   \
2748          "mov %0, r0"                                             \
2749          : /*out*/   "=r" (_res)                                  \
2750          : /*in*/    "0" (&_argvec[0])                            \
2751          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2752       );                                                          \
2753       lval = (__typeof__(lval)) _res;                             \
2754    } while (0)
2755
2756 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2757                                  arg7,arg8)                       \
2758    do {                                                           \
2759       volatile OrigFn        _orig = (orig);                      \
2760       volatile unsigned long _argvec[9];                          \
2761       volatile unsigned long _res;                                \
2762       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2763       _argvec[1] = (unsigned long)(arg1);                         \
2764       _argvec[2] = (unsigned long)(arg2);                         \
2765       _argvec[3] = (unsigned long)(arg3);                         \
2766       _argvec[4] = (unsigned long)(arg4);                         \
2767       _argvec[5] = (unsigned long)(arg5);                         \
2768       _argvec[6] = (unsigned long)(arg6);                         \
2769       _argvec[7] = (unsigned long)(arg7);                         \
2770       _argvec[8] = (unsigned long)(arg8);                         \
2771       __asm__ volatile(                                           \
2772          "ldr r0, [%1, #20] \n\t"                                 \
2773          "ldr r1, [%1, #24] \n\t"                                 \
2774          "ldr r2, [%1, #28] \n\t"                                 \
2775          "ldr r3, [%1, #32] \n\t"                                 \
2776          "push {r0, r1, r2, r3} \n\t"                             \
2777          "ldr r0, [%1, #4] \n\t"                                  \
2778          "ldr r1, [%1, #8] \n\t"                                  \
2779          "ldr r2, [%1, #12] \n\t"                                 \
2780          "ldr r3, [%1, #16] \n\t"                                 \
2781          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2782          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2783          "add sp, sp, #16 \n\t"                                   \
2784          "mov %0, r0"                                             \
2785          : /*out*/   "=r" (_res)                                  \
2786          : /*in*/    "0" (&_argvec[0])                            \
2787          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2788       );                                                          \
2789       lval = (__typeof__(lval)) _res;                             \
2790    } while (0)
2791
2792 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2793                                  arg7,arg8,arg9)                  \
2794    do {                                                           \
2795       volatile OrigFn        _orig = (orig);                      \
2796       volatile unsigned long _argvec[10];                         \
2797       volatile unsigned long _res;                                \
2798       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2799       _argvec[1] = (unsigned long)(arg1);                         \
2800       _argvec[2] = (unsigned long)(arg2);                         \
2801       _argvec[3] = (unsigned long)(arg3);                         \
2802       _argvec[4] = (unsigned long)(arg4);                         \
2803       _argvec[5] = (unsigned long)(arg5);                         \
2804       _argvec[6] = (unsigned long)(arg6);                         \
2805       _argvec[7] = (unsigned long)(arg7);                         \
2806       _argvec[8] = (unsigned long)(arg8);                         \
2807       _argvec[9] = (unsigned long)(arg9);                         \
2808       __asm__ volatile(                                           \
2809          "ldr r0, [%1, #20] \n\t"                                 \
2810          "ldr r1, [%1, #24] \n\t"                                 \
2811          "ldr r2, [%1, #28] \n\t"                                 \
2812          "ldr r3, [%1, #32] \n\t"                                 \
2813          "ldr r4, [%1, #36] \n\t"                                 \
2814          "push {r0, r1, r2, r3, r4} \n\t"                         \
2815          "ldr r0, [%1, #4] \n\t"                                  \
2816          "ldr r1, [%1, #8] \n\t"                                  \
2817          "ldr r2, [%1, #12] \n\t"                                 \
2818          "ldr r3, [%1, #16] \n\t"                                 \
2819          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2820          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2821          "add sp, sp, #20 \n\t"                                   \
2822          "mov %0, r0"                                             \
2823          : /*out*/   "=r" (_res)                                  \
2824          : /*in*/    "0" (&_argvec[0])                            \
2825          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2826       );                                                          \
2827       lval = (__typeof__(lval)) _res;                             \
2828    } while (0)
2829
2830 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2831                                   arg7,arg8,arg9,arg10)           \
2832    do {                                                           \
2833       volatile OrigFn        _orig = (orig);                      \
2834       volatile unsigned long _argvec[11];                         \
2835       volatile unsigned long _res;                                \
2836       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2837       _argvec[1] = (unsigned long)(arg1);                         \
2838       _argvec[2] = (unsigned long)(arg2);                         \
2839       _argvec[3] = (unsigned long)(arg3);                         \
2840       _argvec[4] = (unsigned long)(arg4);                         \
2841       _argvec[5] = (unsigned long)(arg5);                         \
2842       _argvec[6] = (unsigned long)(arg6);                         \
2843       _argvec[7] = (unsigned long)(arg7);                         \
2844       _argvec[8] = (unsigned long)(arg8);                         \
2845       _argvec[9] = (unsigned long)(arg9);                         \
2846       _argvec[10] = (unsigned long)(arg10);                       \
2847       __asm__ volatile(                                           \
2848          "ldr r0, [%1, #40] \n\t"                                 \
2849          "push {r0} \n\t"                                         \
2850          "ldr r0, [%1, #20] \n\t"                                 \
2851          "ldr r1, [%1, #24] \n\t"                                 \
2852          "ldr r2, [%1, #28] \n\t"                                 \
2853          "ldr r3, [%1, #32] \n\t"                                 \
2854          "ldr r4, [%1, #36] \n\t"                                 \
2855          "push {r0, r1, r2, r3, r4} \n\t"                         \
2856          "ldr r0, [%1, #4] \n\t"                                  \
2857          "ldr r1, [%1, #8] \n\t"                                  \
2858          "ldr r2, [%1, #12] \n\t"                                 \
2859          "ldr r3, [%1, #16] \n\t"                                 \
2860          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2861          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2862          "add sp, sp, #24 \n\t"                                   \
2863          "mov %0, r0"                                             \
2864          : /*out*/   "=r" (_res)                                  \
2865          : /*in*/    "0" (&_argvec[0])                            \
2866          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2867       );                                                          \
2868       lval = (__typeof__(lval)) _res;                             \
2869    } while (0)
2870
2871 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
2872                                   arg6,arg7,arg8,arg9,arg10,      \
2873                                   arg11)                          \
2874    do {                                                           \
2875       volatile OrigFn        _orig = (orig);                      \
2876       volatile unsigned long _argvec[12];                         \
2877       volatile unsigned long _res;                                \
2878       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2879       _argvec[1] = (unsigned long)(arg1);                         \
2880       _argvec[2] = (unsigned long)(arg2);                         \
2881       _argvec[3] = (unsigned long)(arg3);                         \
2882       _argvec[4] = (unsigned long)(arg4);                         \
2883       _argvec[5] = (unsigned long)(arg5);                         \
2884       _argvec[6] = (unsigned long)(arg6);                         \
2885       _argvec[7] = (unsigned long)(arg7);                         \
2886       _argvec[8] = (unsigned long)(arg8);                         \
2887       _argvec[9] = (unsigned long)(arg9);                         \
2888       _argvec[10] = (unsigned long)(arg10);                       \
2889       _argvec[11] = (unsigned long)(arg11);                       \
2890       __asm__ volatile(                                           \
2891          "ldr r0, [%1, #40] \n\t"                                 \
2892          "ldr r1, [%1, #44] \n\t"                                 \
2893          "push {r0, r1} \n\t"                                     \
2894          "ldr r0, [%1, #20] \n\t"                                 \
2895          "ldr r1, [%1, #24] \n\t"                                 \
2896          "ldr r2, [%1, #28] \n\t"                                 \
2897          "ldr r3, [%1, #32] \n\t"                                 \
2898          "ldr r4, [%1, #36] \n\t"                                 \
2899          "push {r0, r1, r2, r3, r4} \n\t"                         \
2900          "ldr r0, [%1, #4] \n\t"                                  \
2901          "ldr r1, [%1, #8] \n\t"                                  \
2902          "ldr r2, [%1, #12] \n\t"                                 \
2903          "ldr r3, [%1, #16] \n\t"                                 \
2904          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2905          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2906          "add sp, sp, #28 \n\t"                                   \
2907          "mov %0, r0"                                             \
2908          : /*out*/   "=r" (_res)                                  \
2909          : /*in*/    "0" (&_argvec[0])                            \
2910          : /*trash*/ "cc", "memory",__CALLER_SAVED_REGS           \
2911       );                                                          \
2912       lval = (__typeof__(lval)) _res;                             \
2913    } while (0)
2914
2915 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
2916                                   arg6,arg7,arg8,arg9,arg10,      \
2917                                   arg11,arg12)                    \
2918    do {                                                           \
2919       volatile OrigFn        _orig = (orig);                      \
2920       volatile unsigned long _argvec[13];                         \
2921       volatile unsigned long _res;                                \
2922       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2923       _argvec[1] = (unsigned long)(arg1);                         \
2924       _argvec[2] = (unsigned long)(arg2);                         \
2925       _argvec[3] = (unsigned long)(arg3);                         \
2926       _argvec[4] = (unsigned long)(arg4);                         \
2927       _argvec[5] = (unsigned long)(arg5);                         \
2928       _argvec[6] = (unsigned long)(arg6);                         \
2929       _argvec[7] = (unsigned long)(arg7);                         \
2930       _argvec[8] = (unsigned long)(arg8);                         \
2931       _argvec[9] = (unsigned long)(arg9);                         \
2932       _argvec[10] = (unsigned long)(arg10);                       \
2933       _argvec[11] = (unsigned long)(arg11);                       \
2934       _argvec[12] = (unsigned long)(arg12);                       \
2935       __asm__ volatile(                                           \
2936          "ldr r0, [%1, #40] \n\t"                                 \
2937          "ldr r1, [%1, #44] \n\t"                                 \
2938          "ldr r2, [%1, #48] \n\t"                                 \
2939          "push {r0, r1, r2} \n\t"                                 \
2940          "ldr r0, [%1, #20] \n\t"                                 \
2941          "ldr r1, [%1, #24] \n\t"                                 \
2942          "ldr r2, [%1, #28] \n\t"                                 \
2943          "ldr r3, [%1, #32] \n\t"                                 \
2944          "ldr r4, [%1, #36] \n\t"                                 \
2945          "push {r0, r1, r2, r3, r4} \n\t"                         \
2946          "ldr r0, [%1, #4] \n\t"                                  \
2947          "ldr r1, [%1, #8] \n\t"                                  \
2948          "ldr r2, [%1, #12] \n\t"                                 \
2949          "ldr r3, [%1, #16] \n\t"                                 \
2950          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2951          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2952          "add sp, sp, #32 \n\t"                                   \
2953          "mov %0, r0"                                             \
2954          : /*out*/   "=r" (_res)                                  \
2955          : /*in*/    "0" (&_argvec[0])                            \
2956          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2957       );                                                          \
2958       lval = (__typeof__(lval)) _res;                             \
2959    } while (0)
2960
2961 #endif /* PLAT_arm_linux */
2962
2963 /* ------------------------ ppc32-aix5 ------------------------- */
2964
2965 #if defined(PLAT_ppc32_aix5)
2966
2967 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
2968
2969 /* These regs are trashed by the hidden call. */
2970 #define __CALLER_SAVED_REGS                                       \
2971    "lr", "ctr", "xer",                                            \
2972    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
2973    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
2974    "r11", "r12", "r13"
2975
2976 /* Expand the stack frame, copying enough info that unwinding
2977    still works.  Trashes r3. */
2978
2979 #define VG_EXPAND_FRAME_BY_trashes_r3(_n_fr)                      \
2980          "addi 1,1,-" #_n_fr "\n\t"                               \
2981          "lwz  3," #_n_fr "(1)\n\t"                               \
2982          "stw  3,0(1)\n\t"
2983
2984 #define VG_CONTRACT_FRAME_BY(_n_fr)                               \
2985          "addi 1,1," #_n_fr "\n\t"
2986
2987 /* These CALL_FN_ macros assume that on ppc32-aix5, sizeof(unsigned
2988    long) == 4. */
2989
2990 #define CALL_FN_W_v(lval, orig)                                   \
2991    do {                                                           \
2992       volatile OrigFn        _orig = (orig);                      \
2993       volatile unsigned long _argvec[3+0];                        \
2994       volatile unsigned long _res;                                \
2995       /* _argvec[0] holds current r2 across the call */           \
2996       _argvec[1] = (unsigned long)_orig.r2;                       \
2997       _argvec[2] = (unsigned long)_orig.nraddr;                   \
2998       __asm__ volatile(                                           \
2999          "mr 11,%1\n\t"                                           \
3000          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3001          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3002          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3003          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3004          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3005          "mr 11,%1\n\t"                                           \
3006          "mr %0,3\n\t"                                            \
3007          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3008          VG_CONTRACT_FRAME_BY(512)                                \
3009          : /*out*/   "=r" (_res)                                  \
3010          : /*in*/    "r" (&_argvec[2])                            \
3011          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3012       );                                                          \
3013       lval = (__typeof__(lval)) _res;                             \
3014    } while (0)
3015
3016 #define CALL_FN_W_W(lval, orig, arg1)                             \
3017    do {                                                           \
3018       volatile OrigFn        _orig = (orig);                      \
3019       volatile unsigned long _argvec[3+1];                        \
3020       volatile unsigned long _res;                                \
3021       /* _argvec[0] holds current r2 across the call */           \
3022       _argvec[1]   = (unsigned long)_orig.r2;                     \
3023       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3024       _argvec[2+1] = (unsigned long)arg1;                         \
3025       __asm__ volatile(                                           \
3026          "mr 11,%1\n\t"                                           \
3027          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3028          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3029          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3030          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3031          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3032          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3033          "mr 11,%1\n\t"                                           \
3034          "mr %0,3\n\t"                                            \
3035          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3036          VG_CONTRACT_FRAME_BY(512)                                \
3037          : /*out*/   "=r" (_res)                                  \
3038          : /*in*/    "r" (&_argvec[2])                            \
3039          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3040       );                                                          \
3041       lval = (__typeof__(lval)) _res;                             \
3042    } while (0)
3043
3044 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
3045    do {                                                           \
3046       volatile OrigFn        _orig = (orig);                      \
3047       volatile unsigned long _argvec[3+2];                        \
3048       volatile unsigned long _res;                                \
3049       /* _argvec[0] holds current r2 across the call */           \
3050       _argvec[1]   = (unsigned long)_orig.r2;                     \
3051       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3052       _argvec[2+1] = (unsigned long)arg1;                         \
3053       _argvec[2+2] = (unsigned long)arg2;                         \
3054       __asm__ volatile(                                           \
3055          "mr 11,%1\n\t"                                           \
3056          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3057          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3058          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3059          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3060          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3061          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3062          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3063          "mr 11,%1\n\t"                                           \
3064          "mr %0,3\n\t"                                            \
3065          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3066          VG_CONTRACT_FRAME_BY(512)                                \
3067          : /*out*/   "=r" (_res)                                  \
3068          : /*in*/    "r" (&_argvec[2])                            \
3069          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3070       );                                                          \
3071       lval = (__typeof__(lval)) _res;                             \
3072    } while (0)
3073
3074 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
3075    do {                                                           \
3076       volatile OrigFn        _orig = (orig);                      \
3077       volatile unsigned long _argvec[3+3];                        \
3078       volatile unsigned long _res;                                \
3079       /* _argvec[0] holds current r2 across the call */           \
3080       _argvec[1]   = (unsigned long)_orig.r2;                     \
3081       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3082       _argvec[2+1] = (unsigned long)arg1;                         \
3083       _argvec[2+2] = (unsigned long)arg2;                         \
3084       _argvec[2+3] = (unsigned long)arg3;                         \
3085       __asm__ volatile(                                           \
3086          "mr 11,%1\n\t"                                           \
3087          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3088          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3089          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3090          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3091          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3092          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3093          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3094          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3095          "mr 11,%1\n\t"                                           \
3096          "mr %0,3\n\t"                                            \
3097          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3098          VG_CONTRACT_FRAME_BY(512)                                \
3099          : /*out*/   "=r" (_res)                                  \
3100          : /*in*/    "r" (&_argvec[2])                            \
3101          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3102       );                                                          \
3103       lval = (__typeof__(lval)) _res;                             \
3104    } while (0)
3105
3106 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
3107    do {                                                           \
3108       volatile OrigFn        _orig = (orig);                      \
3109       volatile unsigned long _argvec[3+4];                        \
3110       volatile unsigned long _res;                                \
3111       /* _argvec[0] holds current r2 across the call */           \
3112       _argvec[1]   = (unsigned long)_orig.r2;                     \
3113       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3114       _argvec[2+1] = (unsigned long)arg1;                         \
3115       _argvec[2+2] = (unsigned long)arg2;                         \
3116       _argvec[2+3] = (unsigned long)arg3;                         \
3117       _argvec[2+4] = (unsigned long)arg4;                         \
3118       __asm__ volatile(                                           \
3119          "mr 11,%1\n\t"                                           \
3120          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3121          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3122          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3123          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3124          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3125          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3126          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3127          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3128          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3129          "mr 11,%1\n\t"                                           \
3130          "mr %0,3\n\t"                                            \
3131          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3132          VG_CONTRACT_FRAME_BY(512)                                \
3133          : /*out*/   "=r" (_res)                                  \
3134          : /*in*/    "r" (&_argvec[2])                            \
3135          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3136       );                                                          \
3137       lval = (__typeof__(lval)) _res;                             \
3138    } while (0)
3139
3140 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
3141    do {                                                           \
3142       volatile OrigFn        _orig = (orig);                      \
3143       volatile unsigned long _argvec[3+5];                        \
3144       volatile unsigned long _res;                                \
3145       /* _argvec[0] holds current r2 across the call */           \
3146       _argvec[1]   = (unsigned long)_orig.r2;                     \
3147       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3148       _argvec[2+1] = (unsigned long)arg1;                         \
3149       _argvec[2+2] = (unsigned long)arg2;                         \
3150       _argvec[2+3] = (unsigned long)arg3;                         \
3151       _argvec[2+4] = (unsigned long)arg4;                         \
3152       _argvec[2+5] = (unsigned long)arg5;                         \
3153       __asm__ volatile(                                           \
3154          "mr 11,%1\n\t"                                           \
3155          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3156          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3157          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3158          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3159          "lwz  4, 8(11)\n\t" /* arg2->r4 */                       \
3160          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3161          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3162          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3163          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3164          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3165          "mr 11,%1\n\t"                                           \
3166          "mr %0,3\n\t"                                            \
3167          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3168          VG_CONTRACT_FRAME_BY(512)                                \
3169          : /*out*/   "=r" (_res)                                  \
3170          : /*in*/    "r" (&_argvec[2])                            \
3171          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3172       );                                                          \
3173       lval = (__typeof__(lval)) _res;                             \
3174    } while (0)
3175
3176 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
3177    do {                                                           \
3178       volatile OrigFn        _orig = (orig);                      \
3179       volatile unsigned long _argvec[3+6];                        \
3180       volatile unsigned long _res;                                \
3181       /* _argvec[0] holds current r2 across the call */           \
3182       _argvec[1]   = (unsigned long)_orig.r2;                     \
3183       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3184       _argvec[2+1] = (unsigned long)arg1;                         \
3185       _argvec[2+2] = (unsigned long)arg2;                         \
3186       _argvec[2+3] = (unsigned long)arg3;                         \
3187       _argvec[2+4] = (unsigned long)arg4;                         \
3188       _argvec[2+5] = (unsigned long)arg5;                         \
3189       _argvec[2+6] = (unsigned long)arg6;                         \
3190       __asm__ volatile(                                           \
3191          "mr 11,%1\n\t"                                           \
3192          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3193          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3194          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3195          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3196          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3197          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3198          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3199          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3200          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3201          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3202          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3203          "mr 11,%1\n\t"                                           \
3204          "mr %0,3\n\t"                                            \
3205          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3206          VG_CONTRACT_FRAME_BY(512)                                \
3207          : /*out*/   "=r" (_res)                                  \
3208          : /*in*/    "r" (&_argvec[2])                            \
3209          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3210       );                                                          \
3211       lval = (__typeof__(lval)) _res;                             \
3212    } while (0)
3213
3214 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3215                                  arg7)                            \
3216    do {                                                           \
3217       volatile OrigFn        _orig = (orig);                      \
3218       volatile unsigned long _argvec[3+7];                        \
3219       volatile unsigned long _res;                                \
3220       /* _argvec[0] holds current r2 across the call */           \
3221       _argvec[1]   = (unsigned long)_orig.r2;                     \
3222       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3223       _argvec[2+1] = (unsigned long)arg1;                         \
3224       _argvec[2+2] = (unsigned long)arg2;                         \
3225       _argvec[2+3] = (unsigned long)arg3;                         \
3226       _argvec[2+4] = (unsigned long)arg4;                         \
3227       _argvec[2+5] = (unsigned long)arg5;                         \
3228       _argvec[2+6] = (unsigned long)arg6;                         \
3229       _argvec[2+7] = (unsigned long)arg7;                         \
3230       __asm__ volatile(                                           \
3231          "mr 11,%1\n\t"                                           \
3232          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3233          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3234          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3235          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3236          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3237          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3238          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3239          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3240          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3241          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3242          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3243          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3244          "mr 11,%1\n\t"                                           \
3245          "mr %0,3\n\t"                                            \
3246          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3247          VG_CONTRACT_FRAME_BY(512)                                \
3248          : /*out*/   "=r" (_res)                                  \
3249          : /*in*/    "r" (&_argvec[2])                            \
3250          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3251       );                                                          \
3252       lval = (__typeof__(lval)) _res;                             \
3253    } while (0)
3254
3255 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3256                                  arg7,arg8)                       \
3257    do {                                                           \
3258       volatile OrigFn        _orig = (orig);                      \
3259       volatile unsigned long _argvec[3+8];                        \
3260       volatile unsigned long _res;                                \
3261       /* _argvec[0] holds current r2 across the call */           \
3262       _argvec[1]   = (unsigned long)_orig.r2;                     \
3263       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3264       _argvec[2+1] = (unsigned long)arg1;                         \
3265       _argvec[2+2] = (unsigned long)arg2;                         \
3266       _argvec[2+3] = (unsigned long)arg3;                         \
3267       _argvec[2+4] = (unsigned long)arg4;                         \
3268       _argvec[2+5] = (unsigned long)arg5;                         \
3269       _argvec[2+6] = (unsigned long)arg6;                         \
3270       _argvec[2+7] = (unsigned long)arg7;                         \
3271       _argvec[2+8] = (unsigned long)arg8;                         \
3272       __asm__ volatile(                                           \
3273          "mr 11,%1\n\t"                                           \
3274          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3275          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3276          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3277          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3278          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3279          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3280          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3281          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3282          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3283          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3284          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3285          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3286          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3287          "mr 11,%1\n\t"                                           \
3288          "mr %0,3\n\t"                                            \
3289          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3290          VG_CONTRACT_FRAME_BY(512)                                \
3291          : /*out*/   "=r" (_res)                                  \
3292          : /*in*/    "r" (&_argvec[2])                            \
3293          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3294       );                                                          \
3295       lval = (__typeof__(lval)) _res;                             \
3296    } while (0)
3297
3298 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3299                                  arg7,arg8,arg9)                  \
3300    do {                                                           \
3301       volatile OrigFn        _orig = (orig);                      \
3302       volatile unsigned long _argvec[3+9];                        \
3303       volatile unsigned long _res;                                \
3304       /* _argvec[0] holds current r2 across the call */           \
3305       _argvec[1]   = (unsigned long)_orig.r2;                     \
3306       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3307       _argvec[2+1] = (unsigned long)arg1;                         \
3308       _argvec[2+2] = (unsigned long)arg2;                         \
3309       _argvec[2+3] = (unsigned long)arg3;                         \
3310       _argvec[2+4] = (unsigned long)arg4;                         \
3311       _argvec[2+5] = (unsigned long)arg5;                         \
3312       _argvec[2+6] = (unsigned long)arg6;                         \
3313       _argvec[2+7] = (unsigned long)arg7;                         \
3314       _argvec[2+8] = (unsigned long)arg8;                         \
3315       _argvec[2+9] = (unsigned long)arg9;                         \
3316       __asm__ volatile(                                           \
3317          "mr 11,%1\n\t"                                           \
3318          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3319          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3320          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3321          VG_EXPAND_FRAME_BY_trashes_r3(64)                        \
3322          /* arg9 */                                               \
3323          "lwz 3,36(11)\n\t"                                       \
3324          "stw 3,56(1)\n\t"                                        \
3325          /* args1-8 */                                            \
3326          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3327          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3328          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3329          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3330          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3331          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3332          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3333          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3334          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3335          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3336          "mr 11,%1\n\t"                                           \
3337          "mr %0,3\n\t"                                            \
3338          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3339          VG_CONTRACT_FRAME_BY(64)                                 \
3340          VG_CONTRACT_FRAME_BY(512)                                \
3341          : /*out*/   "=r" (_res)                                  \
3342          : /*in*/    "r" (&_argvec[2])                            \
3343          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3344       );                                                          \
3345       lval = (__typeof__(lval)) _res;                             \
3346    } while (0)
3347
3348 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3349                                   arg7,arg8,arg9,arg10)           \
3350    do {                                                           \
3351       volatile OrigFn        _orig = (orig);                      \
3352       volatile unsigned long _argvec[3+10];                       \
3353       volatile unsigned long _res;                                \
3354       /* _argvec[0] holds current r2 across the call */           \
3355       _argvec[1]   = (unsigned long)_orig.r2;                     \
3356       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3357       _argvec[2+1] = (unsigned long)arg1;                         \
3358       _argvec[2+2] = (unsigned long)arg2;                         \
3359       _argvec[2+3] = (unsigned long)arg3;                         \
3360       _argvec[2+4] = (unsigned long)arg4;                         \
3361       _argvec[2+5] = (unsigned long)arg5;                         \
3362       _argvec[2+6] = (unsigned long)arg6;                         \
3363       _argvec[2+7] = (unsigned long)arg7;                         \
3364       _argvec[2+8] = (unsigned long)arg8;                         \
3365       _argvec[2+9] = (unsigned long)arg9;                         \
3366       _argvec[2+10] = (unsigned long)arg10;                       \
3367       __asm__ volatile(                                           \
3368          "mr 11,%1\n\t"                                           \
3369          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3370          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3371          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3372          VG_EXPAND_FRAME_BY_trashes_r3(64)                        \
3373          /* arg10 */                                              \
3374          "lwz 3,40(11)\n\t"                                       \
3375          "stw 3,60(1)\n\t"                                        \
3376          /* arg9 */                                               \
3377          "lwz 3,36(11)\n\t"                                       \
3378          "stw 3,56(1)\n\t"                                        \
3379          /* args1-8 */                                            \
3380          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3381          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3382          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3383          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3384          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3385          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3386          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3387          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3388          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3389          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3390          "mr 11,%1\n\t"                                           \
3391          "mr %0,3\n\t"                                            \
3392          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3393          VG_CONTRACT_FRAME_BY(64)                                 \
3394          VG_CONTRACT_FRAME_BY(512)                                \
3395          : /*out*/   "=r" (_res)                                  \
3396          : /*in*/    "r" (&_argvec[2])                            \
3397          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3398       );                                                          \
3399       lval = (__typeof__(lval)) _res;                             \
3400    } while (0)
3401
3402 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3403                                   arg7,arg8,arg9,arg10,arg11)     \
3404    do {                                                           \
3405       volatile OrigFn        _orig = (orig);                      \
3406       volatile unsigned long _argvec[3+11];                       \
3407       volatile unsigned long _res;                                \
3408       /* _argvec[0] holds current r2 across the call */           \
3409       _argvec[1]   = (unsigned long)_orig.r2;                     \
3410       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3411       _argvec[2+1] = (unsigned long)arg1;                         \
3412       _argvec[2+2] = (unsigned long)arg2;                         \
3413       _argvec[2+3] = (unsigned long)arg3;                         \
3414       _argvec[2+4] = (unsigned long)arg4;                         \
3415       _argvec[2+5] = (unsigned long)arg5;                         \
3416       _argvec[2+6] = (unsigned long)arg6;                         \
3417       _argvec[2+7] = (unsigned long)arg7;                         \
3418       _argvec[2+8] = (unsigned long)arg8;                         \
3419       _argvec[2+9] = (unsigned long)arg9;                         \
3420       _argvec[2+10] = (unsigned long)arg10;                       \
3421       _argvec[2+11] = (unsigned long)arg11;                       \
3422       __asm__ volatile(                                           \
3423          "mr 11,%1\n\t"                                           \
3424          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3425          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3426          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3427          VG_EXPAND_FRAME_BY_trashes_r3(72)                        \
3428          /* arg11 */                                              \
3429          "lwz 3,44(11)\n\t"                                       \
3430          "stw 3,64(1)\n\t"                                        \
3431          /* arg10 */                                              \
3432          "lwz 3,40(11)\n\t"                                       \
3433          "stw 3,60(1)\n\t"                                        \
3434          /* arg9 */                                               \
3435          "lwz 3,36(11)\n\t"                                       \
3436          "stw 3,56(1)\n\t"                                        \
3437          /* args1-8 */                                            \
3438          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3439          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3440          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3441          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3442          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3443          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3444          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3445          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3446          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3447          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3448          "mr 11,%1\n\t"                                           \
3449          "mr %0,3\n\t"                                            \
3450          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3451          VG_CONTRACT_FRAME_BY(72)                                 \
3452          VG_CONTRACT_FRAME_BY(512)                                \
3453          : /*out*/   "=r" (_res)                                  \
3454          : /*in*/    "r" (&_argvec[2])                            \
3455          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3456       );                                                          \
3457       lval = (__typeof__(lval)) _res;                             \
3458    } while (0)
3459
3460 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3461                                 arg7,arg8,arg9,arg10,arg11,arg12) \
3462    do {                                                           \
3463       volatile OrigFn        _orig = (orig);                      \
3464       volatile unsigned long _argvec[3+12];                       \
3465       volatile unsigned long _res;                                \
3466       /* _argvec[0] holds current r2 across the call */           \
3467       _argvec[1]   = (unsigned long)_orig.r2;                     \
3468       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3469       _argvec[2+1] = (unsigned long)arg1;                         \
3470       _argvec[2+2] = (unsigned long)arg2;                         \
3471       _argvec[2+3] = (unsigned long)arg3;                         \
3472       _argvec[2+4] = (unsigned long)arg4;                         \
3473       _argvec[2+5] = (unsigned long)arg5;                         \
3474       _argvec[2+6] = (unsigned long)arg6;                         \
3475       _argvec[2+7] = (unsigned long)arg7;                         \
3476       _argvec[2+8] = (unsigned long)arg8;                         \
3477       _argvec[2+9] = (unsigned long)arg9;                         \
3478       _argvec[2+10] = (unsigned long)arg10;                       \
3479       _argvec[2+11] = (unsigned long)arg11;                       \
3480       _argvec[2+12] = (unsigned long)arg12;                       \
3481       __asm__ volatile(                                           \
3482          "mr 11,%1\n\t"                                           \
3483          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3484          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3485          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3486          VG_EXPAND_FRAME_BY_trashes_r3(72)                        \
3487          /* arg12 */                                              \
3488          "lwz 3,48(11)\n\t"                                       \
3489          "stw 3,68(1)\n\t"                                        \
3490          /* arg11 */                                              \
3491          "lwz 3,44(11)\n\t"                                       \
3492          "stw 3,64(1)\n\t"                                        \
3493          /* arg10 */                                              \
3494          "lwz 3,40(11)\n\t"                                       \
3495          "stw 3,60(1)\n\t"                                        \
3496          /* arg9 */                                               \
3497          "lwz 3,36(11)\n\t"                                       \
3498          "stw 3,56(1)\n\t"                                        \
3499          /* args1-8 */                                            \
3500          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3501          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3502          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3503          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3504          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3505          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3506          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3507          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3508          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3509          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3510          "mr 11,%1\n\t"                                           \
3511          "mr %0,3\n\t"                                            \
3512          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3513          VG_CONTRACT_FRAME_BY(72)                                 \
3514          VG_CONTRACT_FRAME_BY(512)                                \
3515          : /*out*/   "=r" (_res)                                  \
3516          : /*in*/    "r" (&_argvec[2])                            \
3517          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3518       );                                                          \
3519       lval = (__typeof__(lval)) _res;                             \
3520    } while (0)
3521
3522 #endif /* PLAT_ppc32_aix5 */
3523
3524 /* ------------------------ ppc64-aix5 ------------------------- */
3525
3526 #if defined(PLAT_ppc64_aix5)
3527
3528 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
3529
3530 /* These regs are trashed by the hidden call. */
3531 #define __CALLER_SAVED_REGS                                       \
3532    "lr", "ctr", "xer",                                            \
3533    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
3534    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
3535    "r11", "r12", "r13"
3536
3537 /* Expand the stack frame, copying enough info that unwinding
3538    still works.  Trashes r3. */
3539
3540 #define VG_EXPAND_FRAME_BY_trashes_r3(_n_fr)                      \
3541          "addi 1,1,-" #_n_fr "\n\t"                               \
3542          "ld   3," #_n_fr "(1)\n\t"                               \
3543          "std  3,0(1)\n\t"
3544
3545 #define VG_CONTRACT_FRAME_BY(_n_fr)                               \
3546          "addi 1,1," #_n_fr "\n\t"
3547
3548 /* These CALL_FN_ macros assume that on ppc64-aix5, sizeof(unsigned
3549    long) == 8. */
3550
3551 #define CALL_FN_W_v(lval, orig)                                   \
3552    do {                                                           \
3553       volatile OrigFn        _orig = (orig);                      \
3554       volatile unsigned long _argvec[3+0];                        \
3555       volatile unsigned long _res;                                \
3556       /* _argvec[0] holds current r2 across the call */           \
3557       _argvec[1] = (unsigned long)_orig.r2;                       \
3558       _argvec[2] = (unsigned long)_orig.nraddr;                   \
3559       __asm__ volatile(                                           \
3560          "mr 11,%1\n\t"                                           \
3561          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3562          "std  2,-16(11)\n\t" /* save tocptr */                   \
3563          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3564          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3565          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3566          "mr 11,%1\n\t"                                           \
3567          "mr %0,3\n\t"                                            \
3568          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3569          VG_CONTRACT_FRAME_BY(512)                                \
3570          : /*out*/   "=r" (_res)                                  \
3571          : /*in*/    "r" (&_argvec[2])                            \
3572          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3573       );                                                          \
3574       lval = (__typeof__(lval)) _res;                             \
3575    } while (0)
3576
3577 #define CALL_FN_W_W(lval, orig, arg1)                             \
3578    do {                                                           \
3579       volatile OrigFn        _orig = (orig);                      \
3580       volatile unsigned long _argvec[3+1];                        \
3581       volatile unsigned long _res;                                \
3582       /* _argvec[0] holds current r2 across the call */           \
3583       _argvec[1]   = (unsigned long)_orig.r2;                     \
3584       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3585       _argvec[2+1] = (unsigned long)arg1;                         \
3586       __asm__ volatile(                                           \
3587          "mr 11,%1\n\t"                                           \
3588          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3589          "std  2,-16(11)\n\t" /* save tocptr */                   \
3590          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3591          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3592          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3593          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3594          "mr 11,%1\n\t"                                           \
3595          "mr %0,3\n\t"                                            \
3596          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3597          VG_CONTRACT_FRAME_BY(512)                                \
3598          : /*out*/   "=r" (_res)                                  \
3599          : /*in*/    "r" (&_argvec[2])                            \
3600          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3601       );                                                          \
3602       lval = (__typeof__(lval)) _res;                             \
3603    } while (0)
3604
3605 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
3606    do {                                                           \
3607       volatile OrigFn        _orig = (orig);                      \
3608       volatile unsigned long _argvec[3+2];                        \
3609       volatile unsigned long _res;                                \
3610       /* _argvec[0] holds current r2 across the call */           \
3611       _argvec[1]   = (unsigned long)_orig.r2;                     \
3612       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3613       _argvec[2+1] = (unsigned long)arg1;                         \
3614       _argvec[2+2] = (unsigned long)arg2;                         \
3615       __asm__ volatile(                                           \
3616          "mr 11,%1\n\t"                                           \
3617          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3618          "std  2,-16(11)\n\t" /* save tocptr */                   \
3619          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3620          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3621          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3622          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3623          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3624          "mr 11,%1\n\t"                                           \
3625          "mr %0,3\n\t"                                            \
3626          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3627          VG_CONTRACT_FRAME_BY(512)                                \
3628          : /*out*/   "=r" (_res)                                  \
3629          : /*in*/    "r" (&_argvec[2])                            \
3630          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3631       );                                                          \
3632       lval = (__typeof__(lval)) _res;                             \
3633    } while (0)
3634
3635 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
3636    do {                                                           \
3637       volatile OrigFn        _orig = (orig);                      \
3638       volatile unsigned long _argvec[3+3];                        \
3639       volatile unsigned long _res;                                \
3640       /* _argvec[0] holds current r2 across the call */           \
3641       _argvec[1]   = (unsigned long)_orig.r2;                     \
3642       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3643       _argvec[2+1] = (unsigned long)arg1;                         \
3644       _argvec[2+2] = (unsigned long)arg2;                         \
3645       _argvec[2+3] = (unsigned long)arg3;                         \
3646       __asm__ volatile(                                           \
3647          "mr 11,%1\n\t"                                           \
3648          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3649          "std  2,-16(11)\n\t" /* save tocptr */                   \
3650          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3651          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3652          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3653          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3654          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3655          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3656          "mr 11,%1\n\t"                                           \
3657          "mr %0,3\n\t"                                            \
3658          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3659          VG_CONTRACT_FRAME_BY(512)                                \
3660          : /*out*/   "=r" (_res)                                  \
3661          : /*in*/    "r" (&_argvec[2])                            \
3662          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3663       );                                                          \
3664       lval = (__typeof__(lval)) _res;                             \
3665    } while (0)
3666
3667 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
3668    do {                                                           \
3669       volatile OrigFn        _orig = (orig);                      \
3670       volatile unsigned long _argvec[3+4];                        \
3671       volatile unsigned long _res;                                \
3672       /* _argvec[0] holds current r2 across the call */           \
3673       _argvec[1]   = (unsigned long)_orig.r2;                     \
3674       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3675       _argvec[2+1] = (unsigned long)arg1;                         \
3676       _argvec[2+2] = (unsigned long)arg2;                         \
3677       _argvec[2+3] = (unsigned long)arg3;                         \
3678       _argvec[2+4] = (unsigned long)arg4;                         \
3679       __asm__ volatile(                                           \
3680          "mr 11,%1\n\t"                                           \
3681          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3682          "std  2,-16(11)\n\t" /* save tocptr */                   \
3683          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3684          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3685          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3686          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3687          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3688          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3689          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3690          "mr 11,%1\n\t"                                           \
3691          "mr %0,3\n\t"                                            \
3692          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3693          VG_CONTRACT_FRAME_BY(512)                                \
3694          : /*out*/   "=r" (_res)                                  \
3695          : /*in*/    "r" (&_argvec[2])                            \
3696          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3697       );                                                          \
3698       lval = (__typeof__(lval)) _res;                             \
3699    } while (0)
3700
3701 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
3702    do {                                                           \
3703       volatile OrigFn        _orig = (orig);                      \
3704       volatile unsigned long _argvec[3+5];                        \
3705       volatile unsigned long _res;                                \
3706       /* _argvec[0] holds current r2 across the call */           \
3707       _argvec[1]   = (unsigned long)_orig.r2;                     \
3708       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3709       _argvec[2+1] = (unsigned long)arg1;                         \
3710       _argvec[2+2] = (unsigned long)arg2;                         \
3711       _argvec[2+3] = (unsigned long)arg3;                         \
3712       _argvec[2+4] = (unsigned long)arg4;                         \
3713       _argvec[2+5] = (unsigned long)arg5;                         \
3714       __asm__ volatile(                                           \
3715          "mr 11,%1\n\t"                                           \
3716          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3717          "std  2,-16(11)\n\t" /* save tocptr */                   \
3718          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3719          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3720          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3721          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3722          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3723          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3724          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3725          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3726          "mr 11,%1\n\t"                                           \
3727          "mr %0,3\n\t"                                            \
3728          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3729          VG_CONTRACT_FRAME_BY(512)                                \
3730          : /*out*/   "=r" (_res)                                  \
3731          : /*in*/    "r" (&_argvec[2])                            \
3732          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3733       );                                                          \
3734       lval = (__typeof__(lval)) _res;                             \
3735    } while (0)
3736
3737 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
3738    do {                                                           \
3739       volatile OrigFn        _orig = (orig);                      \
3740       volatile unsigned long _argvec[3+6];                        \
3741       volatile unsigned long _res;                                \
3742       /* _argvec[0] holds current r2 across the call */           \
3743       _argvec[1]   = (unsigned long)_orig.r2;                     \
3744       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3745       _argvec[2+1] = (unsigned long)arg1;                         \
3746       _argvec[2+2] = (unsigned long)arg2;                         \
3747       _argvec[2+3] = (unsigned long)arg3;                         \
3748       _argvec[2+4] = (unsigned long)arg4;                         \
3749       _argvec[2+5] = (unsigned long)arg5;                         \
3750       _argvec[2+6] = (unsigned long)arg6;                         \
3751       __asm__ volatile(                                           \
3752          "mr 11,%1\n\t"                                           \
3753          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3754          "std  2,-16(11)\n\t" /* save tocptr */                   \
3755          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3756          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3757          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3758          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3759          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3760          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3761          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3762          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3763          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3764          "mr 11,%1\n\t"                                           \
3765          "mr %0,3\n\t"                                            \
3766          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3767          VG_CONTRACT_FRAME_BY(512)                                \
3768          : /*out*/   "=r" (_res)                                  \
3769          : /*in*/    "r" (&_argvec[2])                            \
3770          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3771       );                                                          \
3772       lval = (__typeof__(lval)) _res;                             \
3773    } while (0)
3774
3775 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3776                                  arg7)                            \
3777    do {                                                           \
3778       volatile OrigFn        _orig = (orig);                      \
3779       volatile unsigned long _argvec[3+7];                        \
3780       volatile unsigned long _res;                                \
3781       /* _argvec[0] holds current r2 across the call */           \
3782       _argvec[1]   = (unsigned long)_orig.r2;                     \
3783       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3784       _argvec[2+1] = (unsigned long)arg1;                         \
3785       _argvec[2+2] = (unsigned long)arg2;                         \
3786       _argvec[2+3] = (unsigned long)arg3;                         \
3787       _argvec[2+4] = (unsigned long)arg4;                         \
3788       _argvec[2+5] = (unsigned long)arg5;                         \
3789       _argvec[2+6] = (unsigned long)arg6;                         \
3790       _argvec[2+7] = (unsigned long)arg7;                         \
3791       __asm__ volatile(                                           \
3792          "mr 11,%1\n\t"                                           \
3793          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3794          "std  2,-16(11)\n\t" /* save tocptr */                   \
3795          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3796          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3797          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3798          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3799          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3800          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3801          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3802          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3803          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3804          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3805          "mr 11,%1\n\t"                                           \
3806          "mr %0,3\n\t"                                            \
3807          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3808          VG_CONTRACT_FRAME_BY(512)                                \
3809          : /*out*/   "=r" (_res)                                  \
3810          : /*in*/    "r" (&_argvec[2])                            \
3811          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3812       );                                                          \
3813       lval = (__typeof__(lval)) _res;                             \
3814    } while (0)
3815
3816 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3817                                  arg7,arg8)                       \
3818    do {                                                           \
3819       volatile OrigFn        _orig = (orig);                      \
3820       volatile unsigned long _argvec[3+8];                        \
3821       volatile unsigned long _res;                                \
3822       /* _argvec[0] holds current r2 across the call */           \
3823       _argvec[1]   = (unsigned long)_orig.r2;                     \
3824       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3825       _argvec[2+1] = (unsigned long)arg1;                         \
3826       _argvec[2+2] = (unsigned long)arg2;                         \
3827       _argvec[2+3] = (unsigned long)arg3;                         \
3828       _argvec[2+4] = (unsigned long)arg4;                         \
3829       _argvec[2+5] = (unsigned long)arg5;                         \
3830       _argvec[2+6] = (unsigned long)arg6;                         \
3831       _argvec[2+7] = (unsigned long)arg7;                         \
3832       _argvec[2+8] = (unsigned long)arg8;                         \
3833       __asm__ volatile(                                           \
3834          "mr 11,%1\n\t"                                           \
3835          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3836          "std  2,-16(11)\n\t" /* save tocptr */                   \
3837          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3838          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3839          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3840          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3841          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3842          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3843          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3844          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3845          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3846          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3847          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3848          "mr 11,%1\n\t"                                           \
3849          "mr %0,3\n\t"                                            \
3850          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3851          VG_CONTRACT_FRAME_BY(512)                                \
3852          : /*out*/   "=r" (_res)                                  \
3853          : /*in*/    "r" (&_argvec[2])                            \
3854          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3855       );                                                          \
3856       lval = (__typeof__(lval)) _res;                             \
3857    } while (0)
3858
3859 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3860                                  arg7,arg8,arg9)                  \
3861    do {                                                           \
3862       volatile OrigFn        _orig = (orig);                      \
3863       volatile unsigned long _argvec[3+9];                        \
3864       volatile unsigned long _res;                                \
3865       /* _argvec[0] holds current r2 across the call */           \
3866       _argvec[1]   = (unsigned long)_orig.r2;                     \
3867       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3868       _argvec[2+1] = (unsigned long)arg1;                         \
3869       _argvec[2+2] = (unsigned long)arg2;                         \
3870       _argvec[2+3] = (unsigned long)arg3;                         \
3871       _argvec[2+4] = (unsigned long)arg4;                         \
3872       _argvec[2+5] = (unsigned long)arg5;                         \
3873       _argvec[2+6] = (unsigned long)arg6;                         \
3874       _argvec[2+7] = (unsigned long)arg7;                         \
3875       _argvec[2+8] = (unsigned long)arg8;                         \
3876       _argvec[2+9] = (unsigned long)arg9;                         \
3877       __asm__ volatile(                                           \
3878          "mr 11,%1\n\t"                                           \
3879          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3880          "std  2,-16(11)\n\t" /* save tocptr */                   \
3881          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3882          VG_EXPAND_FRAME_BY_trashes_r3(128)                       \
3883          /* arg9 */                                               \
3884          "ld  3,72(11)\n\t"                                       \
3885          "std 3,112(1)\n\t"                                       \
3886          /* args1-8 */                                            \
3887          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3888          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3889          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3890          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3891          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3892          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3893          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3894          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3895          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3896          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3897          "mr 11,%1\n\t"                                           \
3898          "mr %0,3\n\t"                                            \
3899          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3900          VG_CONTRACT_FRAME_BY(128)                                \
3901          VG_CONTRACT_FRAME_BY(512)                                \
3902          : /*out*/   "=r" (_res)                                  \
3903          : /*in*/    "r" (&_argvec[2])                            \
3904          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3905       );                                                          \
3906       lval = (__typeof__(lval)) _res;                             \
3907    } while (0)
3908
3909 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3910                                   arg7,arg8,arg9,arg10)           \
3911    do {                                                           \
3912       volatile OrigFn        _orig = (orig);                      \
3913       volatile unsigned long _argvec[3+10];                       \
3914       volatile unsigned long _res;                                \
3915       /* _argvec[0] holds current r2 across the call */           \
3916       _argvec[1]   = (unsigned long)_orig.r2;                     \
3917       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3918       _argvec[2+1] = (unsigned long)arg1;                         \
3919       _argvec[2+2] = (unsigned long)arg2;                         \
3920       _argvec[2+3] = (unsigned long)arg3;                         \
3921       _argvec[2+4] = (unsigned long)arg4;                         \
3922       _argvec[2+5] = (unsigned long)arg5;                         \
3923       _argvec[2+6] = (unsigned long)arg6;                         \
3924       _argvec[2+7] = (unsigned long)arg7;                         \
3925       _argvec[2+8] = (unsigned long)arg8;                         \
3926       _argvec[2+9] = (unsigned long)arg9;                         \
3927       _argvec[2+10] = (unsigned long)arg10;                       \
3928       __asm__ volatile(                                           \
3929          "mr 11,%1\n\t"                                           \
3930          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3931          "std  2,-16(11)\n\t" /* save tocptr */                   \
3932          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3933          VG_EXPAND_FRAME_BY_trashes_r3(128)                       \
3934          /* arg10 */                                              \
3935          "ld  3,80(11)\n\t"                                       \
3936          "std 3,120(1)\n\t"                                       \
3937          /* arg9 */                                               \
3938          "ld  3,72(11)\n\t"                                       \
3939          "std 3,112(1)\n\t"                                       \
3940          /* args1-8 */                                            \
3941          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3942          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3943          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3944          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3945          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3946          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3947          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3948          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3949          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3950          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3951          "mr 11,%1\n\t"                                           \
3952          "mr %0,3\n\t"                                            \
3953          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3954          VG_CONTRACT_FRAME_BY(128)                                \
3955          VG_CONTRACT_FRAME_BY(512)                                \
3956          : /*out*/   "=r" (_res)                                  \
3957          : /*in*/    "r" (&_argvec[2])                            \
3958          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3959       );                                                          \
3960       lval = (__typeof__(lval)) _res;                             \
3961    } while (0)
3962
3963 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3964                                   arg7,arg8,arg9,arg10,arg11)     \
3965    do {                                                           \
3966       volatile OrigFn        _orig = (orig);                      \
3967       volatile unsigned long _argvec[3+11];                       \
3968       volatile unsigned long _res;                                \
3969       /* _argvec[0] holds current r2 across the call */           \
3970       _argvec[1]   = (unsigned long)_orig.r2;                     \
3971       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3972       _argvec[2+1] = (unsigned long)arg1;                         \
3973       _argvec[2+2] = (unsigned long)arg2;                         \
3974       _argvec[2+3] = (unsigned long)arg3;                         \
3975       _argvec[2+4] = (unsigned long)arg4;                         \
3976       _argvec[2+5] = (unsigned long)arg5;                         \
3977       _argvec[2+6] = (unsigned long)arg6;                         \
3978       _argvec[2+7] = (unsigned long)arg7;                         \
3979       _argvec[2+8] = (unsigned long)arg8;                         \
3980       _argvec[2+9] = (unsigned long)arg9;                         \
3981       _argvec[2+10] = (unsigned long)arg10;                       \
3982       _argvec[2+11] = (unsigned long)arg11;                       \
3983       __asm__ volatile(                                           \
3984          "mr 11,%1\n\t"                                           \
3985          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3986          "std  2,-16(11)\n\t" /* save tocptr */                   \
3987          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3988          VG_EXPAND_FRAME_BY_trashes_r3(144)                       \
3989          /* arg11 */                                              \
3990          "ld  3,88(11)\n\t"                                       \
3991          "std 3,128(1)\n\t"                                       \
3992          /* arg10 */                                              \
3993          "ld  3,80(11)\n\t"                                       \
3994          "std 3,120(1)\n\t"                                       \
3995          /* arg9 */                                               \
3996          "ld  3,72(11)\n\t"                                       \
3997          "std 3,112(1)\n\t"                                       \
3998          /* args1-8 */                                            \
3999          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
4000          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
4001          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
4002          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
4003          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4004          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4005          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
4006          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
4007          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4008          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4009          "mr 11,%1\n\t"                                           \
4010          "mr %0,3\n\t"                                            \
4011          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4012          VG_CONTRACT_FRAME_BY(144)                                \
4013          VG_CONTRACT_FRAME_BY(512)                                \
4014          : /*out*/   "=r" (_res)                                  \
4015          : /*in*/    "r" (&_argvec[2])                            \
4016          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4017       );                                                          \
4018       lval = (__typeof__(lval)) _res;                             \
4019    } while (0)
4020
4021 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
4022                                 arg7,arg8,arg9,arg10,arg11,arg12) \
4023    do {                                                           \
4024       volatile OrigFn        _orig = (orig);                      \
4025       volatile unsigned long _argvec[3+12];                       \
4026       volatile unsigned long _res;                                \
4027       /* _argvec[0] holds current r2 across the call */           \
4028       _argvec[1]   = (unsigned long)_orig.r2;                     \
4029       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
4030       _argvec[2+1] = (unsigned long)arg1;                         \
4031       _argvec[2+2] = (unsigned long)arg2;                         \
4032       _argvec[2+3] = (unsigned long)arg3;                         \
4033       _argvec[2+4] = (unsigned long)arg4;                         \
4034       _argvec[2+5] = (unsigned long)arg5;                         \
4035       _argvec[2+6] = (unsigned long)arg6;                         \
4036       _argvec[2+7] = (unsigned long)arg7;                         \
4037       _argvec[2+8] = (unsigned long)arg8;                         \
4038       _argvec[2+9] = (unsigned long)arg9;                         \
4039       _argvec[2+10] = (unsigned long)arg10;                       \
4040       _argvec[2+11] = (unsigned long)arg11;                       \
4041       _argvec[2+12] = (unsigned long)arg12;                       \
4042       __asm__ volatile(                                           \
4043          "mr 11,%1\n\t"                                           \
4044          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
4045          "std  2,-16(11)\n\t" /* save tocptr */                   \
4046          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
4047          VG_EXPAND_FRAME_BY_trashes_r3(144)                       \
4048          /* arg12 */                                              \
4049          "ld  3,96(11)\n\t"                                       \
4050          "std 3,136(1)\n\t"                                       \
4051          /* arg11 */                                              \
4052          "ld  3,88(11)\n\t"                                       \
4053          "std 3,128(1)\n\t"                                       \
4054          /* arg10 */                                              \
4055          "ld  3,80(11)\n\t"                                       \
4056          "std 3,120(1)\n\t"                                       \
4057          /* arg9 */                                               \
4058          "ld  3,72(11)\n\t"                                       \
4059          "std 3,112(1)\n\t"                                       \
4060          /* args1-8 */                                            \
4061          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
4062          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
4063          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
4064          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
4065          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4066          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4067          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
4068          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
4069          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4070          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4071          "mr 11,%1\n\t"                                           \
4072          "mr %0,3\n\t"                                            \
4073          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4074          VG_CONTRACT_FRAME_BY(144)                                \
4075          VG_CONTRACT_FRAME_BY(512)                                \
4076          : /*out*/   "=r" (_res)                                  \
4077          : /*in*/    "r" (&_argvec[2])                            \
4078          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4079       );                                                          \
4080       lval = (__typeof__(lval)) _res;                             \
4081    } while (0)
4082
4083 #endif /* PLAT_ppc64_aix5 */
4084
4085
4086 /* ------------------------------------------------------------------ */
4087 /* ARCHITECTURE INDEPENDENT MACROS for CLIENT REQUESTS.               */
4088 /*                                                                    */
4089 /* ------------------------------------------------------------------ */
4090
4091 /* Some request codes.  There are many more of these, but most are not
4092    exposed to end-user view.  These are the public ones, all of the
4093    form 0x1000 + small_number.
4094
4095    Core ones are in the range 0x00000000--0x0000ffff.  The non-public
4096    ones start at 0x2000.
4097 */
4098
4099 /* These macros are used by tools -- they must be public, but don't
4100    embed them into other programs. */
4101 #define VG_USERREQ_TOOL_BASE(a,b) \
4102    ((unsigned int)(((a)&0xff) << 24 | ((b)&0xff) << 16))
4103 #define VG_IS_TOOL_USERREQ(a, b, v) \
4104    (VG_USERREQ_TOOL_BASE(a,b) == ((v) & 0xffff0000))
4105
4106 /* !! ABIWARNING !! ABIWARNING !! ABIWARNING !! ABIWARNING !!
4107    This enum comprises an ABI exported by Valgrind to programs
4108    which use client requests.  DO NOT CHANGE THE ORDER OF THESE
4109    ENTRIES, NOR DELETE ANY -- add new ones at the end. */
4110 typedef
4111    enum { VG_USERREQ__RUNNING_ON_VALGRIND  = 0x1001,
4112           VG_USERREQ__DISCARD_TRANSLATIONS = 0x1002,
4113
4114           /* These allow any function to be called from the simulated
4115              CPU but run on the real CPU.  Nb: the first arg passed to
4116              the function is always the ThreadId of the running
4117              thread!  So CLIENT_CALL0 actually requires a 1 arg
4118              function, etc. */
4119           VG_USERREQ__CLIENT_CALL0 = 0x1101,
4120           VG_USERREQ__CLIENT_CALL1 = 0x1102,
4121           VG_USERREQ__CLIENT_CALL2 = 0x1103,
4122           VG_USERREQ__CLIENT_CALL3 = 0x1104,
4123
4124           /* Can be useful in regression testing suites -- eg. can
4125              send Valgrind's output to /dev/null and still count
4126              errors. */
4127           VG_USERREQ__COUNT_ERRORS = 0x1201,
4128
4129           /* These are useful and can be interpreted by any tool that
4130              tracks malloc() et al, by using vg_replace_malloc.c. */
4131           VG_USERREQ__MALLOCLIKE_BLOCK = 0x1301,
4132           VG_USERREQ__FREELIKE_BLOCK   = 0x1302,
4133           /* Memory pool support. */
4134           VG_USERREQ__CREATE_MEMPOOL   = 0x1303,
4135           VG_USERREQ__DESTROY_MEMPOOL  = 0x1304,
4136           VG_USERREQ__MEMPOOL_ALLOC    = 0x1305,
4137           VG_USERREQ__MEMPOOL_FREE     = 0x1306,
4138           VG_USERREQ__MEMPOOL_TRIM     = 0x1307,
4139           VG_USERREQ__MOVE_MEMPOOL     = 0x1308,
4140           VG_USERREQ__MEMPOOL_CHANGE   = 0x1309,
4141           VG_USERREQ__MEMPOOL_EXISTS   = 0x130a,
4142
4143           /* Allow printfs to valgrind log. */
4144           /* The first two pass the va_list argument by value, which
4145              assumes it is the same size as or smaller than a UWord,
4146              which generally isn't the case.  Hence are deprecated.
4147              The second two pass the vargs by reference and so are
4148              immune to this problem. */
4149           /* both :: char* fmt, va_list vargs (DEPRECATED) */
4150           VG_USERREQ__PRINTF           = 0x1401,
4151           VG_USERREQ__PRINTF_BACKTRACE = 0x1402,
4152           /* both :: char* fmt, va_list* vargs */
4153           VG_USERREQ__PRINTF_VALIST_BY_REF = 0x1403,
4154           VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF = 0x1404,
4155
4156           /* Stack support. */
4157           VG_USERREQ__STACK_REGISTER   = 0x1501,
4158           VG_USERREQ__STACK_DEREGISTER = 0x1502,
4159           VG_USERREQ__STACK_CHANGE     = 0x1503,
4160
4161           /* Wine support */
4162           VG_USERREQ__LOAD_PDB_DEBUGINFO = 0x1601
4163    } Vg_ClientRequest;
4164
4165 #if !defined(__GNUC__)
4166 #  define __extension__ /* */
4167 #endif
4168
4169 /* Returns the number of Valgrinds this code is running under.  That
4170    is, 0 if running natively, 1 if running under Valgrind, 2 if
4171    running under Valgrind which is running under another Valgrind,
4172    etc. */
4173 #define RUNNING_ON_VALGRIND  __extension__                        \
4174    ({unsigned int _qzz_res;                                       \
4175     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0 /* if not */,          \
4176                                VG_USERREQ__RUNNING_ON_VALGRIND,   \
4177                                0, 0, 0, 0, 0);                    \
4178     _qzz_res;                                                     \
4179    })
4180
4181
4182 /* Discard translation of code in the range [_qzz_addr .. _qzz_addr +
4183    _qzz_len - 1].  Useful if you are debugging a JITter or some such,
4184    since it provides a way to make sure valgrind will retranslate the
4185    invalidated area.  Returns no value. */
4186 #define VALGRIND_DISCARD_TRANSLATIONS(_qzz_addr,_qzz_len)         \
4187    {unsigned int _qzz_res;                                        \
4188     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4189                                VG_USERREQ__DISCARD_TRANSLATIONS,  \
4190                                _qzz_addr, _qzz_len, 0, 0, 0);     \
4191    }
4192
4193
4194 /* These requests are for getting Valgrind itself to print something.
4195    Possibly with a backtrace.  This is a really ugly hack.  The return value
4196    is the number of characters printed, excluding the "**<pid>** " part at the
4197    start and the backtrace (if present). */
4198
4199 #if defined(NVALGRIND)
4200
4201 #  define VALGRIND_PRINTF(...)
4202 #  define VALGRIND_PRINTF_BACKTRACE(...)
4203
4204 #else /* NVALGRIND */
4205
4206 /* Modern GCC will optimize the static routine out if unused,
4207    and unused attribute will shut down warnings about it.  */
4208 static int VALGRIND_PRINTF(const char *format, ...)
4209    __attribute__((format(__printf__, 1, 2), __unused__));
4210 static int
4211 VALGRIND_PRINTF(const char *format, ...)
4212 {
4213    unsigned long _qzz_res;
4214    va_list vargs;
4215    va_start(vargs, format);
4216    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,
4217                               VG_USERREQ__PRINTF_VALIST_BY_REF,
4218                               (unsigned long)format,
4219                               (unsigned long)&vargs,
4220                               0, 0, 0);
4221    va_end(vargs);
4222    return (int)_qzz_res;
4223 }
4224
4225 static int VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
4226    __attribute__((format(__printf__, 1, 2), __unused__));
4227 static int
4228 VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
4229 {
4230    unsigned long _qzz_res;
4231    va_list vargs;
4232    va_start(vargs, format);
4233    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,
4234                               VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF,
4235                               (unsigned long)format,
4236                               (unsigned long)&vargs,
4237                               0, 0, 0);
4238    va_end(vargs);
4239    return (int)_qzz_res;
4240 }
4241
4242 #endif /* NVALGRIND */
4243
4244
4245 /* These requests allow control to move from the simulated CPU to the
4246    real CPU, calling an arbitary function.
4247
4248    Note that the current ThreadId is inserted as the first argument.
4249    So this call:
4250
4251      VALGRIND_NON_SIMD_CALL2(f, arg1, arg2)
4252
4253    requires f to have this signature:
4254
4255      Word f(Word tid, Word arg1, Word arg2)
4256
4257    where "Word" is a word-sized type.
4258
4259    Note that these client requests are not entirely reliable.  For example,
4260    if you call a function with them that subsequently calls printf(),
4261    there's a high chance Valgrind will crash.  Generally, your prospects of
4262    these working are made higher if the called function does not refer to
4263    any global variables, and does not refer to any libc or other functions
4264    (printf et al).  Any kind of entanglement with libc or dynamic linking is
4265    likely to have a bad outcome, for tricky reasons which we've grappled
4266    with a lot in the past.
4267 */
4268 #define VALGRIND_NON_SIMD_CALL0(_qyy_fn)                          \
4269    __extension__                                                  \
4270    ({unsigned long _qyy_res;                                      \
4271     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
4272                                VG_USERREQ__CLIENT_CALL0,          \
4273                                _qyy_fn,                           \
4274                                0, 0, 0, 0);                       \
4275     _qyy_res;                                                     \
4276    })
4277
4278 #define VALGRIND_NON_SIMD_CALL1(_qyy_fn, _qyy_arg1)               \
4279    __extension__                                                  \
4280    ({unsigned long _qyy_res;                                      \
4281     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
4282                                VG_USERREQ__CLIENT_CALL1,          \
4283                                _qyy_fn,                           \
4284                                _qyy_arg1, 0, 0, 0);               \
4285     _qyy_res;                                                     \
4286    })
4287
4288 #define VALGRIND_NON_SIMD_CALL2(_qyy_fn, _qyy_arg1, _qyy_arg2)    \
4289    __extension__                                                  \
4290    ({unsigned long _qyy_res;                                      \
4291     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
4292                                VG_USERREQ__CLIENT_CALL2,          \
4293                                _qyy_fn,                           \
4294                                _qyy_arg1, _qyy_arg2, 0, 0);       \
4295     _qyy_res;                                                     \
4296    })
4297
4298 #define VALGRIND_NON_SIMD_CALL3(_qyy_fn, _qyy_arg1, _qyy_arg2, _qyy_arg3) \
4299    __extension__                                                  \
4300    ({unsigned long _qyy_res;                                      \
4301     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
4302                                VG_USERREQ__CLIENT_CALL3,          \
4303                                _qyy_fn,                           \
4304                                _qyy_arg1, _qyy_arg2,              \
4305                                _qyy_arg3, 0);                     \
4306     _qyy_res;                                                     \
4307    })
4308
4309
4310 /* Counts the number of errors that have been recorded by a tool.  Nb:
4311    the tool must record the errors with VG_(maybe_record_error)() or
4312    VG_(unique_error)() for them to be counted. */
4313 #define VALGRIND_COUNT_ERRORS                                     \
4314    __extension__                                                  \
4315    ({unsigned int _qyy_res;                                       \
4316     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
4317                                VG_USERREQ__COUNT_ERRORS,          \
4318                                0, 0, 0, 0, 0);                    \
4319     _qyy_res;                                                     \
4320    })
4321
4322 /* Several Valgrind tools (Memcheck, Massif, Helgrind, DRD) rely on knowing
4323    when heap blocks are allocated in order to give accurate results.  This
4324    happens automatically for the standard allocator functions such as
4325    malloc(), calloc(), realloc(), memalign(), new, new[], free(), delete,
4326    delete[], etc.
4327
4328    But if your program uses a custom allocator, this doesn't automatically
4329    happen, and Valgrind will not do as well.  For example, if you allocate
4330    superblocks with mmap() and then allocates chunks of the superblocks, all
4331    Valgrind's observations will be at the mmap() level and it won't know that
4332    the chunks should be considered separate entities.  In Memcheck's case,
4333    that means you probably won't get heap block overrun detection (because
4334    there won't be redzones marked as unaddressable) and you definitely won't
4335    get any leak detection.
4336
4337    The following client requests allow a custom allocator to be annotated so
4338    that it can be handled accurately by Valgrind.
4339
4340    VALGRIND_MALLOCLIKE_BLOCK marks a region of memory as having been allocated
4341    by a malloc()-like function.  For Memcheck (an illustrative case), this
4342    does two things:
4343
4344    - It records that the block has been allocated.  This means any addresses
4345      within the block mentioned in error messages will be
4346      identified as belonging to the block.  It also means that if the block
4347      isn't freed it will be detected by the leak checker.
4348
4349    - It marks the block as being addressable and undefined (if 'is_zeroed' is
4350      not set), or addressable and defined (if 'is_zeroed' is set).  This
4351      controls how accesses to the block by the program are handled.
4352
4353    'addr' is the start of the usable block (ie. after any
4354    redzone), 'sizeB' is its size.  'rzB' is the redzone size if the allocator
4355    can apply redzones -- these are blocks of padding at the start and end of
4356    each block.  Adding redzones is recommended as it makes it much more likely
4357    Valgrind will spot block overruns.  `is_zeroed' indicates if the memory is
4358    zeroed (or filled with another predictable value), as is the case for
4359    calloc().
4360
4361    VALGRIND_MALLOCLIKE_BLOCK should be put immediately after the point where a
4362    heap block -- that will be used by the client program -- is allocated.
4363    It's best to put it at the outermost level of the allocator if possible;
4364    for example, if you have a function my_alloc() which calls
4365    internal_alloc(), and the client request is put inside internal_alloc(),
4366    stack traces relating to the heap block will contain entries for both
4367    my_alloc() and internal_alloc(), which is probably not what you want.
4368
4369    For Memcheck users: if you use VALGRIND_MALLOCLIKE_BLOCK to carve out
4370    custom blocks from within a heap block, B, that has been allocated with
4371    malloc/calloc/new/etc, then block B will be *ignored* during leak-checking
4372    -- the custom blocks will take precedence.
4373
4374    VALGRIND_FREELIKE_BLOCK is the partner to VALGRIND_MALLOCLIKE_BLOCK.  For
4375    Memcheck, it does two things:
4376
4377    - It records that the block has been deallocated.  This assumes that the
4378      block was annotated as having been allocated via
4379      VALGRIND_MALLOCLIKE_BLOCK.  Otherwise, an error will be issued.
4380
4381    - It marks the block as being unaddressable.
4382
4383    VALGRIND_FREELIKE_BLOCK should be put immediately after the point where a
4384    heap block is deallocated.
4385
4386    In many cases, these two client requests will not be enough to get your
4387    allocator working well with Memcheck.  More specifically, if your allocator
4388    writes to freed blocks in any way then a VALGRIND_MAKE_MEM_UNDEFINED call
4389    will be necessary to mark the memory as addressable just before the zeroing
4390    occurs, otherwise you'll get a lot of invalid write errors.  For example,
4391    you'll need to do this if your allocator recycles freed blocks, but it
4392    zeroes them before handing them back out (via VALGRIND_MALLOCLIKE_BLOCK).
4393    Alternatively, if your allocator reuses freed blocks for allocator-internal
4394    data structures, VALGRIND_MAKE_MEM_UNDEFINED calls will also be necessary.
4395
4396    Really, what's happening is a blurring of the lines between the client
4397    program and the allocator... after VALGRIND_FREELIKE_BLOCK is called, the
4398    memory should be considered unaddressable to the client program, but the
4399    allocator knows more than the rest of the client program and so may be able
4400    to safely access it.  Extra client requests are necessary for Valgrind to
4401    understand the distinction between the allocator and the rest of the
4402    program.
4403
4404    Note: there is currently no VALGRIND_REALLOCLIKE_BLOCK client request;  it
4405    has to be emulated with MALLOCLIKE/FREELIKE and memory copying.
4406
4407    Ignored if addr == 0.
4408 */
4409 #define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)    \
4410    {unsigned int _qzz_res;                                        \
4411     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4412                                VG_USERREQ__MALLOCLIKE_BLOCK,      \
4413                                addr, sizeB, rzB, is_zeroed, 0);   \
4414    }
4415
4416 /* See the comment for VALGRIND_MALLOCLIKE_BLOCK for details.
4417    Ignored if addr == 0.
4418 */
4419 #define VALGRIND_FREELIKE_BLOCK(addr, rzB)                        \
4420    {unsigned int _qzz_res;                                        \
4421     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4422                                VG_USERREQ__FREELIKE_BLOCK,        \
4423                                addr, rzB, 0, 0, 0);               \
4424    }
4425
4426 /* Create a memory pool. */
4427 #define VALGRIND_CREATE_MEMPOOL(pool, rzB, is_zeroed)             \
4428    {unsigned int _qzz_res;                                        \
4429     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4430                                VG_USERREQ__CREATE_MEMPOOL,        \
4431                                pool, rzB, is_zeroed, 0, 0);       \
4432    }
4433
4434 /* Destroy a memory pool. */
4435 #define VALGRIND_DESTROY_MEMPOOL(pool)                            \
4436    {unsigned int _qzz_res;                                        \
4437     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4438                                VG_USERREQ__DESTROY_MEMPOOL,       \
4439                                pool, 0, 0, 0, 0);                 \
4440    }
4441
4442 /* Associate a piece of memory with a memory pool. */
4443 #define VALGRIND_MEMPOOL_ALLOC(pool, addr, size)                  \
4444    {unsigned int _qzz_res;                                        \
4445     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4446                                VG_USERREQ__MEMPOOL_ALLOC,         \
4447                                pool, addr, size, 0, 0);           \
4448    }
4449
4450 /* Disassociate a piece of memory from a memory pool. */
4451 #define VALGRIND_MEMPOOL_FREE(pool, addr)                         \
4452    {unsigned int _qzz_res;                                        \
4453     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4454                                VG_USERREQ__MEMPOOL_FREE,          \
4455                                pool, addr, 0, 0, 0);              \
4456    }
4457
4458 /* Disassociate any pieces outside a particular range. */
4459 #define VALGRIND_MEMPOOL_TRIM(pool, addr, size)                   \
4460    {unsigned int _qzz_res;                                        \
4461     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4462                                VG_USERREQ__MEMPOOL_TRIM,          \
4463                                pool, addr, size, 0, 0);           \
4464    }
4465
4466 /* Resize and/or move a piece associated with a memory pool. */
4467 #define VALGRIND_MOVE_MEMPOOL(poolA, poolB)                       \
4468    {unsigned int _qzz_res;                                        \
4469     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4470                                VG_USERREQ__MOVE_MEMPOOL,          \
4471                                poolA, poolB, 0, 0, 0);            \
4472    }
4473
4474 /* Resize and/or move a piece associated with a memory pool. */
4475 #define VALGRIND_MEMPOOL_CHANGE(pool, addrA, addrB, size)         \
4476    {unsigned int _qzz_res;                                        \
4477     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4478                                VG_USERREQ__MEMPOOL_CHANGE,        \
4479                                pool, addrA, addrB, size, 0);      \
4480    }
4481
4482 /* Return 1 if a mempool exists, else 0. */
4483 #define VALGRIND_MEMPOOL_EXISTS(pool)                             \
4484    __extension__                                                  \
4485    ({unsigned int _qzz_res;                                       \
4486     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4487                                VG_USERREQ__MEMPOOL_EXISTS,        \
4488                                pool, 0, 0, 0, 0);                 \
4489     _qzz_res;                                                     \
4490    })
4491
4492 /* Mark a piece of memory as being a stack. Returns a stack id. */
4493 #define VALGRIND_STACK_REGISTER(start, end)                       \
4494    __extension__                                                  \
4495    ({unsigned int _qzz_res;                                       \
4496     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4497                                VG_USERREQ__STACK_REGISTER,        \
4498                                start, end, 0, 0, 0);              \
4499     _qzz_res;                                                     \
4500    })
4501
4502 /* Unmark the piece of memory associated with a stack id as being a
4503    stack. */
4504 #define VALGRIND_STACK_DEREGISTER(id)                             \
4505    {unsigned int _qzz_res;                                        \
4506     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4507                                VG_USERREQ__STACK_DEREGISTER,      \
4508                                id, 0, 0, 0, 0);                   \
4509    }
4510
4511 /* Change the start and end address of the stack id. */
4512 #define VALGRIND_STACK_CHANGE(id, start, end)                     \
4513    {unsigned int _qzz_res;                                        \
4514     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4515                                VG_USERREQ__STACK_CHANGE,          \
4516                                id, start, end, 0, 0);             \
4517    }
4518
4519 /* Load PDB debug info for Wine PE image_map. */
4520 #define VALGRIND_LOAD_PDB_DEBUGINFO(fd, ptr, total_size, delta)   \
4521    {unsigned int _qzz_res;                                        \
4522     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4523                                VG_USERREQ__LOAD_PDB_DEBUGINFO,    \
4524                                fd, ptr, total_size, delta, 0);    \
4525    }
4526
4527
4528 #undef PLAT_x86_linux
4529 #undef PLAT_amd64_linux
4530 #undef PLAT_ppc32_linux
4531 #undef PLAT_ppc64_linux
4532 #undef PLAT_arm_linux
4533 #undef PLAT_ppc32_aix5
4534 #undef PLAT_ppc64_aix5
4535
4536 #endif   /* __VALGRIND_H */