l4/pkg/valgrind/src/valgrind-3.6.0-svn/include/valgrind.h

   1 /* -*- c -*-
   2    ----------------------------------------------------------------
   3
   4    Notice that the following BSD-style license applies to this one
   5    file (valgrind.h) only.  The rest of Valgrind is licensed under the
   6    terms of the GNU General Public License, version 2, unless
   7    otherwise indicated.  See the COPYING file in the source
   8    distribution for details.
   9
  10    ----------------------------------------------------------------
  11
  12    This file is part of Valgrind, a dynamic binary instrumentation
  13    framework.
  14
  15    Copyright (C) 2000-2010 Julian Seward.  All rights reserved.
  16
  17    Redistribution and use in source and binary forms, with or without
  18    modification, are permitted provided that the following conditions
  19    are met:
  20
  21    1. Redistributions of source code must retain the above copyright
  22       notice, this list of conditions and the following disclaimer.
  23
  24    2. The origin of this software must not be misrepresented; you must
  25       not claim that you wrote the original software.  If you use this
  26       software in a product, an acknowledgment in the product
  27       documentation would be appreciated but is not required.
  28
  29    3. Altered source versions must be plainly marked as such, and must
  30       not be misrepresented as being the original software.
  31
  32    4. The name of the author may not be used to endorse or promote
  33       products derived from this software without specific prior written
  34       permission.
  35
  36    THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
  37    OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  38    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  39    ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
  40    DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  41    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
  42    GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  43    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  44    WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  45    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  46    SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  47
  48    ----------------------------------------------------------------
  49
  50    Notice that the above BSD-style license applies to this one file
  51    (valgrind.h) only.  The entire rest of Valgrind is licensed under
  52    the terms of the GNU General Public License, version 2.  See the
  53    COPYING file in the source distribution for details.
  54
  55    ----------------------------------------------------------------
  56 */
  57
  58
  59 /* This file is for inclusion into client (your!) code.
  60
  61    You can use these macros to manipulate and query Valgrind's
  62    execution inside your own programs.
  63
  64    The resulting executables will still run without Valgrind, just a
  65    little bit more slowly than they otherwise would, but otherwise
  66    unchanged.  When not running on valgrind, each client request
  67    consumes very few (eg. 7) instructions, so the resulting performance
  68    loss is negligible unless you plan to execute client requests
  69    millions of times per second.  Nevertheless, if that is still a
  70    problem, you can compile with the NVALGRIND symbol defined (gcc
  71    -DNVALGRIND) so that client requests are not even compiled in.  */
  72
  73 #ifndef __VALGRIND_H
  74 #define __VALGRIND_H
  75
  76
  77 /* ------------------------------------------------------------------ */
  78 /* VERSION NUMBER OF VALGRIND                                         */
  79 /* ------------------------------------------------------------------ */
  80
  81 /* Specify Valgrind's version number, so that user code can
  82    conditionally compile based on our version number.  Note that these
  83    were introduced at version 3.6 and so do not exist in version 3.5
  84    or earlier.  The recommended way to use them to check for "version
  85    X.Y or later" is (eg)
  86
  87 #if defined(__VALGRIND_MAJOR__) && defined(__VALGRIND_MINOR__)   \
  88     && (__VALGRIND_MAJOR__ > 3                                   \
  89         || (__VALGRIND_MAJOR__ == 3 && __VALGRIND_MINOR__ >= 6))
  90 */
  91 #define __VALGRIND_MAJOR__    3
  92 #define __VALGRIND_MINOR__    6
  93
  94
  95 #include <stdarg.h>
  96
  97 /* Nb: this file might be included in a file compiled with -ansi.  So
  98    we can't use C++ style "//" comments nor the "asm" keyword (instead
  99    use "__asm__"). */
 100
 101 /* Derive some tags indicating what the target platform is.  Note
 102    that in this file we're using the compiler's CPP symbols for
 103    identifying architectures, which are different to the ones we use
 104    within the rest of Valgrind.  Note, __powerpc__ is active for both
 105    32 and 64-bit PPC, whereas __powerpc64__ is only active for the
 106    latter (on Linux, that is).
 107
 108    Misc note: how to find out what's predefined in gcc by default:
 109    gcc -Wp,-dM somefile.c
 110 */
 111 #undef PLAT_ppc64_aix5
 112 #undef PLAT_ppc32_aix5
 113 #undef PLAT_x86_darwin
 114 #undef PLAT_amd64_darwin
 115 #undef PLAT_x86_win32
 116 #undef PLAT_x86_linux
 117 #undef PLAT_amd64_linux
 118 #undef PLAT_ppc32_linux
 119 #undef PLAT_ppc64_linux
 120 #undef PLAT_arm_linux
 121 #undef PLAT_s390x_linux
 122
 123
 124 #if defined(_AIX) && defined(__64BIT__)
 125 #  define PLAT_ppc64_aix5 1
 126 #elif defined(_AIX) && !defined(__64BIT__)
 127 #  define PLAT_ppc32_aix5 1
 128 #elif defined(__APPLE__) && defined(__i386__)
 129 #  define PLAT_x86_darwin 1
 130 #elif defined(__APPLE__) && defined(__x86_64__)
 131 #  define PLAT_amd64_darwin 1
 132 #elif defined(__MINGW32__) || defined(__CYGWIN32__) || defined(_WIN32) && defined(_M_IX86)
 133 #  define PLAT_x86_win32 1
 134 #elif defined(__linux__) && defined(__i386__)
 135 #  define PLAT_x86_linux 1
 136 #elif defined(__linux__) && defined(__x86_64__)
 137 #  define PLAT_amd64_linux 1
 138 #elif defined(__linux__) && defined(__powerpc__) && !defined(__powerpc64__)
 139 #  define PLAT_ppc32_linux 1
 140 #elif defined(__linux__) && defined(__powerpc__) && defined(__powerpc64__)
 141 #  define PLAT_ppc64_linux 1
 142 #elif defined(__linux__) && defined(__arm__)
 143 #  define PLAT_arm_linux 1
 144 #elif defined(__linux__) && defined(__s390__) && defined(__s390x__)
 145 #  define PLAT_s390x_linux 1
 146 #else
 147 /* If we're not compiling for our target platform, don't generate
 148    any inline asms.  */
 149 #  if !defined(NVALGRIND)
 150 #    define NVALGRIND 1
 151 #  endif
 152 #endif
 153
 154
 155 /* ------------------------------------------------------------------ */
 156 /* ARCHITECTURE SPECIFICS for SPECIAL INSTRUCTIONS.  There is nothing */
 157 /* in here of use to end-users -- skip to the next section.           */
 158 /* ------------------------------------------------------------------ */
 159
 160 /*
 161  * VALGRIND_DO_CLIENT_REQUEST(): a statement that invokes a Valgrind client
 162  * request. Accepts both pointers and integers as arguments.
 163  *
 164  * VALGRIND_DO_CLIENT_REQUEST_EXPR(): a C expression that invokes a Valgrind
 165  * client request and whose value equals the client request result. Accepts
 166  * both pointers and integers as arguments.
 167  */
 168
 169 #define VALGRIND_DO_CLIENT_REQUEST(_zzq_rlval, _zzq_default,            \
 170                                    _zzq_request, _zzq_arg1, _zzq_arg2,  \
 171                                    _zzq_arg3, _zzq_arg4, _zzq_arg5)     \
 172   { (_zzq_rlval) = VALGRIND_DO_CLIENT_REQUEST_EXPR((_zzq_default),      \
 173                         (_zzq_request), (_zzq_arg1), (_zzq_arg2),       \
 174                         (_zzq_arg3), (_zzq_arg4), (_zzq_arg5)); }
 175
 176 #if defined(NVALGRIND)
 177
 178 /* Define NVALGRIND to completely remove the Valgrind magic sequence
 179    from the compiled code (analogous to NDEBUG's effects on
 180    assert()) */
 181 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
 182         _zzq_default, _zzq_request,                               \
 183         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 184       (_zzq_default)
 185
 186 #else  /* ! NVALGRIND */
 187
 188 /* The following defines the magic code sequences which the JITter
 189    spots and handles magically.  Don't look too closely at them as
 190    they will rot your brain.
 191
 192    The assembly code sequences for all architectures is in this one
 193    file.  This is because this file must be stand-alone, and we don't
 194    want to have multiple files.
 195
 196    For VALGRIND_DO_CLIENT_REQUEST, we must ensure that the default
 197    value gets put in the return slot, so that everything works when
 198    this is executed not under Valgrind.  Args are passed in a memory
 199    block, and so there's no intrinsic limit to the number that could
 200    be passed, but it's currently five.
 201
 202    The macro args are:
 203       _zzq_rlval    result lvalue
 204       _zzq_default  default value (result returned when running on real CPU)
 205       _zzq_request  request code
 206       _zzq_arg1..5  request params
 207
 208    The other two macros are used to support function wrapping, and are
 209    a lot simpler.  VALGRIND_GET_NR_CONTEXT returns the value of the
 210    guest's NRADDR pseudo-register and whatever other information is
 211    needed to safely run the call original from the wrapper: on
 212    ppc64-linux, the R2 value at the divert point is also needed.  This
 213    information is abstracted into a user-visible type, OrigFn.
 214
 215    VALGRIND_CALL_NOREDIR_* behaves the same as the following on the
 216    guest, but guarantees that the branch instruction will not be
 217    redirected: x86: call *%eax, amd64: call *%rax, ppc32/ppc64:
 218    branch-and-link-to-r11.  VALGRIND_CALL_NOREDIR is just text, not a
 219    complete inline asm, since it needs to be combined with more magic
 220    inline asm stuff to be useful.
 221 */
 222
 223 /* ------------------------- x86-{linux,darwin} ---------------- */
 224
 225 #if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin)  \
 226     ||  (defined(PLAT_x86_win32) && defined(__GNUC__))
 227
 228 typedef
 229    struct {
 230       unsigned int nraddr; /* where's the code? */
 231    }
 232    OrigFn;
 233
 234 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 235                      "roll $3,  %%edi ; roll $13, %%edi\n\t"      \
 236                      "roll $29, %%edi ; roll $19, %%edi\n\t"
 237
 238 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
 239         _zzq_default, _zzq_request,                               \
 240         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 241   __extension__                                                   \
 242   ({volatile unsigned int _zzq_args[6];                           \
 243     volatile unsigned int _zzq_result;                            \
 244     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 245     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 246     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 247     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 248     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 249     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 250     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 251                      /* %EDX = client_request ( %EAX ) */         \
 252                      "xchgl %%ebx,%%ebx"                          \
 253                      : "=d" (_zzq_result)                         \
 254                      : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
 255                      : "cc", "memory"                             \
 256                     );                                            \
 257     _zzq_result;                                                  \
 258   })
 259
 260 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 261   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 262     volatile unsigned int __addr;                                 \
 263     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 264                      /* %EAX = guest_NRADDR */                    \
 265                      "xchgl %%ecx,%%ecx"                          \
 266                      : "=a" (__addr)                              \
 267                      :                                            \
 268                      : "cc", "memory"                             \
 269                     );                                            \
 270     _zzq_orig->nraddr = __addr;                                   \
 271   }
 272
 273 #define VALGRIND_CALL_NOREDIR_EAX                                 \
 274                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 275                      /* call-noredir *%EAX */                     \
 276                      "xchgl %%edx,%%edx\n\t"
 277 #endif /* PLAT_x86_linux || PLAT_x86_darwin || (PLAT_x86_win32 && __GNUC__) */
 278
 279 /* ------------------------- x86-Win32 ------------------------- */
 280
 281 #if defined(PLAT_x86_win32) && !defined(__GNUC__)
 282
 283 typedef
 284    struct {
 285       unsigned int nraddr; /* where's the code? */
 286    }
 287    OrigFn;
 288
 289 #if defined(_MSC_VER)
 290
 291 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 292                      __asm rol edi, 3  __asm rol edi, 13          \
 293                      __asm rol edi, 29 __asm rol edi, 19
 294
 295 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
 296         _zzq_default, _zzq_request,                               \
 297         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 298     valgrind_do_client_request_expr((uintptr_t)(_zzq_default),    \
 299         (uintptr_t)(_zzq_request), (uintptr_t)(_zzq_arg1),        \
 300         (uintptr_t)(_zzq_arg2), (uintptr_t)(_zzq_arg3),           \
 301         (uintptr_t)(_zzq_arg4), (uintptr_t)(_zzq_arg5))
 302
 303 static __inline uintptr_t
 304 valgrind_do_client_request_expr(uintptr_t _zzq_default, uintptr_t _zzq_request,
 305                                 uintptr_t _zzq_arg1, uintptr_t _zzq_arg2,
 306                                 uintptr_t _zzq_arg3, uintptr_t _zzq_arg4,
 307                                 uintptr_t _zzq_arg5)
 308 {
 309     volatile uintptr_t _zzq_args[6];
 310     volatile unsigned int _zzq_result;
 311     _zzq_args[0] = (uintptr_t)(_zzq_request);
 312     _zzq_args[1] = (uintptr_t)(_zzq_arg1);
 313     _zzq_args[2] = (uintptr_t)(_zzq_arg2);
 314     _zzq_args[3] = (uintptr_t)(_zzq_arg3);
 315     _zzq_args[4] = (uintptr_t)(_zzq_arg4);
 316     _zzq_args[5] = (uintptr_t)(_zzq_arg5);
 317     __asm { __asm lea eax, _zzq_args __asm mov edx, _zzq_default
 318             __SPECIAL_INSTRUCTION_PREAMBLE
 319             /* %EDX = client_request ( %EAX ) */
 320             __asm xchg ebx,ebx
 321             __asm mov _zzq_result, edx
 322     }
 323     return _zzq_result;
 324 }
 325
 326 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 327   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 328     volatile unsigned int __addr;                                 \
 329     __asm { __SPECIAL_INSTRUCTION_PREAMBLE                        \
 330             /* %EAX = guest_NRADDR */                             \
 331             __asm xchg ecx,ecx                                    \
 332             __asm mov __addr, eax                                 \
 333     }                                                             \
 334     _zzq_orig->nraddr = __addr;                                   \
 335   }
 336
 337 #define VALGRIND_CALL_NOREDIR_EAX ERROR
 338
 339 #else
 340 #error Unsupported compiler.
 341 #endif
 342
 343 #endif /* PLAT_x86_win32 */
 344
 345 /* ------------------------ amd64-{linux,darwin} --------------- */
 346
 347 #if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin)
 348
 349 typedef
 350    struct {
 351       unsigned long long int nraddr; /* where's the code? */
 352    }
 353    OrigFn;
 354
 355 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 356                      "rolq $3,  %%rdi ; rolq $13, %%rdi\n\t"      \
 357                      "rolq $61, %%rdi ; rolq $51, %%rdi\n\t"
 358
 359 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
 360         _zzq_default, _zzq_request,                               \
 361         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 362     __extension__                                                 \
 363     ({ volatile unsigned long long int _zzq_args[6];              \
 364     volatile unsigned long long int _zzq_result;                  \
 365     _zzq_args[0] = (unsigned long long int)(_zzq_request);        \
 366     _zzq_args[1] = (unsigned long long int)(_zzq_arg1);           \
 367     _zzq_args[2] = (unsigned long long int)(_zzq_arg2);           \
 368     _zzq_args[3] = (unsigned long long int)(_zzq_arg3);           \
 369     _zzq_args[4] = (unsigned long long int)(_zzq_arg4);           \
 370     _zzq_args[5] = (unsigned long long int)(_zzq_arg5);           \
 371     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 372                      /* %RDX = client_request ( %RAX ) */         \
 373                      "xchgq %%rbx,%%rbx"                          \
 374                      : "=d" (_zzq_result)                         \
 375                      : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
 376                      : "cc", "memory"                             \
 377                     );                                            \
 378     _zzq_result;                                                  \
 379     })
 380
 381 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 382   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 383     volatile unsigned long long int __addr;                       \
 384     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 385                      /* %RAX = guest_NRADDR */                    \
 386                      "xchgq %%rcx,%%rcx"                          \
 387                      : "=a" (__addr)                              \
 388                      :                                            \
 389                      : "cc", "memory"                             \
 390                     );                                            \
 391     _zzq_orig->nraddr = __addr;                                   \
 392   }
 393
 394 #define VALGRIND_CALL_NOREDIR_RAX                                 \
 395                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 396                      /* call-noredir *%RAX */                     \
 397                      "xchgq %%rdx,%%rdx\n\t"
 398 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin */
 399
 400 /* ------------------------ ppc32-linux ------------------------ */
 401
 402 #if defined(PLAT_ppc32_linux)
 403
 404 typedef
 405    struct {
 406       unsigned int nraddr; /* where's the code? */
 407    }
 408    OrigFn;
 409
 410 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 411                      "rlwinm 0,0,3,0,0  ; rlwinm 0,0,13,0,0\n\t"  \
 412                      "rlwinm 0,0,29,0,0 ; rlwinm 0,0,19,0,0\n\t"
 413
 414 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
 415         _zzq_default, _zzq_request,                               \
 416         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 417                                                                   \
 418     __extension__                                                 \
 419   ({         unsigned int  _zzq_args[6];                          \
 420              unsigned int  _zzq_result;                           \
 421              unsigned int* _zzq_ptr;                              \
 422     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 423     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 424     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 425     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 426     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 427     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 428     _zzq_ptr = _zzq_args;                                         \
 429     __asm__ volatile("mr 3,%1\n\t" /*default*/                    \
 430                      "mr 4,%2\n\t" /*ptr*/                        \
 431                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 432                      /* %R3 = client_request ( %R4 ) */           \
 433                      "or 1,1,1\n\t"                               \
 434                      "mr %0,3"     /*result*/                     \
 435                      : "=b" (_zzq_result)                         \
 436                      : "b" (_zzq_default), "b" (_zzq_ptr)         \
 437                      : "cc", "memory", "r3", "r4");               \
 438     _zzq_result;                                                  \
 439     })
 440
 441 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 442   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 443     unsigned int __addr;                                          \
 444     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 445                      /* %R3 = guest_NRADDR */                     \
 446                      "or 2,2,2\n\t"                               \
 447                      "mr %0,3"                                    \
 448                      : "=b" (__addr)                              \
 449                      :                                            \
 450                      : "cc", "memory", "r3"                       \
 451                     );                                            \
 452     _zzq_orig->nraddr = __addr;                                   \
 453   }
 454
 455 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 456                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 457                      /* branch-and-link-to-noredir *%R11 */       \
 458                      "or 3,3,3\n\t"
 459 #endif /* PLAT_ppc32_linux */
 460
 461 /* ------------------------ ppc64-linux ------------------------ */
 462
 463 #if defined(PLAT_ppc64_linux)
 464
 465 typedef
 466    struct {
 467       unsigned long long int nraddr; /* where's the code? */
 468       unsigned long long int r2;  /* what tocptr do we need? */
 469    }
 470    OrigFn;
 471
 472 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 473                      "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
 474                      "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
 475
 476 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
 477         _zzq_default, _zzq_request,                               \
 478         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 479                                                                   \
 480   __extension__                                                   \
 481   ({         unsigned long long int  _zzq_args[6];                \
 482     register unsigned long long int  _zzq_result __asm__("r3");   \
 483     register unsigned long long int* _zzq_ptr __asm__("r4");      \
 484     _zzq_args[0] = (unsigned long long int)(_zzq_request);        \
 485     _zzq_args[1] = (unsigned long long int)(_zzq_arg1);           \
 486     _zzq_args[2] = (unsigned long long int)(_zzq_arg2);           \
 487     _zzq_args[3] = (unsigned long long int)(_zzq_arg3);           \
 488     _zzq_args[4] = (unsigned long long int)(_zzq_arg4);           \
 489     _zzq_args[5] = (unsigned long long int)(_zzq_arg5);           \
 490     _zzq_ptr = _zzq_args;                                         \
 491     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 492                      /* %R3 = client_request ( %R4 ) */           \
 493                      "or 1,1,1"                                   \
 494                      : "=r" (_zzq_result)                         \
 495                      : "0" (_zzq_default), "r" (_zzq_ptr)         \
 496                      : "cc", "memory");                           \
 497     _zzq_result;                                                  \
 498   })
 499
 500 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 501   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 502     register unsigned long long int __addr __asm__("r3");         \
 503     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 504                      /* %R3 = guest_NRADDR */                     \
 505                      "or 2,2,2"                                   \
 506                      : "=r" (__addr)                              \
 507                      :                                            \
 508                      : "cc", "memory"                             \
 509                     );                                            \
 510     _zzq_orig->nraddr = __addr;                                   \
 511     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 512                      /* %R3 = guest_NRADDR_GPR2 */                \
 513                      "or 4,4,4"                                   \
 514                      : "=r" (__addr)                              \
 515                      :                                            \
 516                      : "cc", "memory"                             \
 517                     );                                            \
 518     _zzq_orig->r2 = __addr;                                       \
 519   }
 520
 521 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 522                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 523                      /* branch-and-link-to-noredir *%R11 */       \
 524                      "or 3,3,3\n\t"
 525
 526 #endif /* PLAT_ppc64_linux */
 527
 528 /* ------------------------- arm-linux ------------------------- */
 529
 530 #if defined(PLAT_arm_linux)
 531
 532 typedef
 533    struct {
 534       unsigned int nraddr; /* where's the code? */
 535    }
 536    OrigFn;
 537
 538 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 539             "mov r12, r12, ror #3  ; mov r12, r12, ror #13 \n\t"  \
 540             "mov r12, r12, ror #29 ; mov r12, r12, ror #19 \n\t"
 541
 542 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
 543         _zzq_default, _zzq_request,                               \
 544         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 545                                                                   \
 546   __extension__                                                   \
 547   ({volatile unsigned int  _zzq_args[6];                          \
 548     volatile unsigned int  _zzq_result;                           \
 549     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 550     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 551     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 552     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 553     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 554     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 555     __asm__ volatile("mov r3, %1\n\t" /*default*/                 \
 556                      "mov r4, %2\n\t" /*ptr*/                     \
 557                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 558                      /* R3 = client_request ( R4 ) */             \
 559                      "orr r10, r10, r10\n\t"                      \
 560                      "mov %0, r3"     /*result*/                  \
 561                      : "=r" (_zzq_result)                         \
 562                      : "r" (_zzq_default), "r" (&_zzq_args[0])    \
 563                      : "cc","memory", "r3", "r4");                \
 564     _zzq_result;                                                  \
 565   })
 566
 567 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 568   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 569     unsigned int __addr;                                          \
 570     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 571                      /* R3 = guest_NRADDR */                      \
 572                      "orr r11, r11, r11\n\t"                      \
 573                      "mov %0, r3"                                 \
 574                      : "=r" (__addr)                              \
 575                      :                                            \
 576                      : "cc", "memory", "r3"                       \
 577                     );                                            \
 578     _zzq_orig->nraddr = __addr;                                   \
 579   }
 580
 581 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                    \
 582                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 583                      /* branch-and-link-to-noredir *%R4 */        \
 584                      "orr r12, r12, r12\n\t"
 585
 586 #endif /* PLAT_arm_linux */
 587
 588 /* ------------------------ ppc32-aix5 ------------------------- */
 589
 590 #if defined(PLAT_ppc32_aix5)
 591
 592 typedef
 593    struct {
 594       unsigned int nraddr; /* where's the code? */
 595       unsigned int r2;  /* what tocptr do we need? */
 596    }
 597    OrigFn;
 598
 599 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 600                      "rlwinm 0,0,3,0,0  ; rlwinm 0,0,13,0,0\n\t"  \
 601                      "rlwinm 0,0,29,0,0 ; rlwinm 0,0,19,0,0\n\t"
 602
 603 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
 604         _zzq_default, _zzq_request,                               \
 605         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 606                                                                   \
 607   __extension__                                                   \
 608   ({         unsigned int  _zzq_args[7];                          \
 609     register unsigned int  _zzq_result;                           \
 610     register unsigned int* _zzq_ptr;                              \
 611     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 612     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 613     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 614     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 615     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 616     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 617     _zzq_args[6] = (unsigned int)(_zzq_default);                  \
 618     _zzq_ptr = _zzq_args;                                         \
 619     __asm__ volatile("mr 4,%1\n\t"                                \
 620                      "lwz 3, 24(4)\n\t"                           \
 621                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 622                      /* %R3 = client_request ( %R4 ) */           \
 623                      "or 1,1,1\n\t"                               \
 624                      "mr %0,3"                                    \
 625                      : "=b" (_zzq_result)                         \
 626                      : "b" (_zzq_ptr)                             \
 627                      : "r3", "r4", "cc", "memory");               \
 628     _zzq_result;                                                  \
 629   })
 630
 631 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 632   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 633     register unsigned int __addr;                                 \
 634     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 635                      /* %R3 = guest_NRADDR */                     \
 636                      "or 2,2,2\n\t"                               \
 637                      "mr %0,3"                                    \
 638                      : "=b" (__addr)                              \
 639                      :                                            \
 640                      : "r3", "cc", "memory"                       \
 641                     );                                            \
 642     _zzq_orig->nraddr = __addr;                                   \
 643     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 644                      /* %R3 = guest_NRADDR_GPR2 */                \
 645                      "or 4,4,4\n\t"                               \
 646                      "mr %0,3"                                    \
 647                      : "=b" (__addr)                              \
 648                      :                                            \
 649                      : "r3", "cc", "memory"                       \
 650                     );                                            \
 651     _zzq_orig->r2 = __addr;                                       \
 652   }
 653
 654 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 655                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 656                      /* branch-and-link-to-noredir *%R11 */       \
 657                      "or 3,3,3\n\t"
 658
 659 #endif /* PLAT_ppc32_aix5 */
 660
 661 /* ------------------------ ppc64-aix5 ------------------------- */
 662
 663 #if defined(PLAT_ppc64_aix5)
 664
 665 typedef
 666    struct {
 667       unsigned long long int nraddr; /* where's the code? */
 668       unsigned long long int r2;  /* what tocptr do we need? */
 669    }
 670    OrigFn;
 671
 672 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 673                      "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
 674                      "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
 675
 676 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
 677         _zzq_default, _zzq_request,                               \
 678         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 679                                                                   \
 680   __extension__                                                   \
 681   ({         unsigned long long int  _zzq_args[7];                \
 682     register unsigned long long int  _zzq_result;                 \
 683     register unsigned long long int* _zzq_ptr;                    \
 684     _zzq_args[0] = (unsigned int long long)(_zzq_request);        \
 685     _zzq_args[1] = (unsigned int long long)(_zzq_arg1);           \
 686     _zzq_args[2] = (unsigned int long long)(_zzq_arg2);           \
 687     _zzq_args[3] = (unsigned int long long)(_zzq_arg3);           \
 688     _zzq_args[4] = (unsigned int long long)(_zzq_arg4);           \
 689     _zzq_args[5] = (unsigned int long long)(_zzq_arg5);           \
 690     _zzq_args[6] = (unsigned int long long)(_zzq_default);        \
 691     _zzq_ptr = _zzq_args;                                         \
 692     __asm__ volatile("mr 4,%1\n\t"                                \
 693                      "ld 3, 48(4)\n\t"                            \
 694                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 695                      /* %R3 = client_request ( %R4 ) */           \
 696                      "or 1,1,1\n\t"                               \
 697                      "mr %0,3"                                    \
 698                      : "=b" (_zzq_result)                         \
 699                      : "b" (_zzq_ptr)                             \
 700                      : "r3", "r4", "cc", "memory");               \
 701     _zzq_result;                                                  \
 702   })
 703
 704 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 705   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 706     register unsigned long long int __addr;                       \
 707     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 708                      /* %R3 = guest_NRADDR */                     \
 709                      "or 2,2,2\n\t"                               \
 710                      "mr %0,3"                                    \
 711                      : "=b" (__addr)                              \
 712                      :                                            \
 713                      : "r3", "cc", "memory"                       \
 714                     );                                            \
 715     _zzq_orig->nraddr = __addr;                                   \
 716     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 717                      /* %R3 = guest_NRADDR_GPR2 */                \
 718                      "or 4,4,4\n\t"                               \
 719                      "mr %0,3"                                    \
 720                      : "=b" (__addr)                              \
 721                      :                                            \
 722                      : "r3", "cc", "memory"                       \
 723                     );                                            \
 724     _zzq_orig->r2 = __addr;                                       \
 725   }
 726
 727 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 728                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 729                      /* branch-and-link-to-noredir *%R11 */       \
 730                      "or 3,3,3\n\t"
 731
 732 #endif /* PLAT_ppc64_aix5 */
 733
 734 /* ------------------------ s390x-linux ------------------------ */
 735
 736 #if defined(PLAT_s390x_linux)
 737
 738 typedef
 739   struct {
 740      unsigned long long int nraddr; /* where's the code? */
 741   }
 742   OrigFn;
 743
 744 /* __SPECIAL_INSTRUCTION_PREAMBLE will be used to identify Valgrind specific
 745  * code. This detection is implemented in platform specific toIR.c
 746  * (e.g. VEX/priv/guest_s390_decoder.c).
 747  */
 748 #define __SPECIAL_INSTRUCTION_PREAMBLE                           \
 749                      "lr 15,15\n\t"                              \
 750                      "lr 1,1\n\t"                                \
 751                      "lr 2,2\n\t"                                \
 752                      "lr 3,3\n\t"
 753
 754 #define __CLIENT_REQUEST_CODE "lr 2,2\n\t"
 755 #define __GET_NR_CONTEXT_CODE "lr 3,3\n\t"
 756 #define __CALL_NO_REDIR_CODE  "lr 4,4\n\t"
 757
 758 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                         \
 759        _zzq_default, _zzq_request,                               \
 760        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 761   __extension__                                                  \
 762  ({volatile unsigned long long int _zzq_args[6];                 \
 763    volatile unsigned long long int _zzq_result;                  \
 764    _zzq_args[0] = (unsigned long long int)(_zzq_request);        \
 765    _zzq_args[1] = (unsigned long long int)(_zzq_arg1);           \
 766    _zzq_args[2] = (unsigned long long int)(_zzq_arg2);           \
 767    _zzq_args[3] = (unsigned long long int)(_zzq_arg3);           \
 768    _zzq_args[4] = (unsigned long long int)(_zzq_arg4);           \
 769    _zzq_args[5] = (unsigned long long int)(_zzq_arg5);           \
 770    __asm__ volatile(/* r2 = args */                              \
 771                     "lgr 2,%1\n\t"                               \
 772                     /* r3 = default */                           \
 773                     "lgr 3,%2\n\t"                               \
 774                     __SPECIAL_INSTRUCTION_PREAMBLE               \
 775                     __CLIENT_REQUEST_CODE                        \
 776                     /* results = r3 */                           \
 777                     "lgr %0, 3\n\t"                              \
 778                     : "=d" (_zzq_result)                         \
 779                     : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
 780                     : "cc", "2", "3", "memory"                   \
 781                    );                                            \
 782    _zzq_result;                                                  \
 783  })
 784
 785 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                      \
 786  { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 787    volatile unsigned long long int __addr;                       \
 788    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 789                     __GET_NR_CONTEXT_CODE                        \
 790                     "lgr %0, 3\n\t"                              \
 791                     : "=a" (__addr)                              \
 792                     :                                            \
 793                     : "cc", "3", "memory"                        \
 794                    );                                            \
 795    _zzq_orig->nraddr = __addr;                                   \
 796  }
 797
 798 #define VALGRIND_CALL_NOREDIR_R1                                 \
 799                     __SPECIAL_INSTRUCTION_PREAMBLE               \
 800                     __CALL_NO_REDIR_CODE
 801
 802 #endif /* PLAT_s390x_linux */
 803
 804 /* Insert assembly code for other platforms here... */
 805
 806 #endif /* NVALGRIND */
 807
 808
 809 /* ------------------------------------------------------------------ */
 810 /* PLATFORM SPECIFICS for FUNCTION WRAPPING.  This is all very        */
 811 /* ugly.  It's the least-worst tradeoff I can think of.               */
 812 /* ------------------------------------------------------------------ */
 813
 814 /* This section defines magic (a.k.a appalling-hack) macros for doing
 815    guaranteed-no-redirection macros, so as to get from function
 816    wrappers to the functions they are wrapping.  The whole point is to
 817    construct standard call sequences, but to do the call itself with a
 818    special no-redirect call pseudo-instruction that the JIT
 819    understands and handles specially.  This section is long and
 820    repetitious, and I can't see a way to make it shorter.
 821
 822    The naming scheme is as follows:
 823
 824       CALL_FN_{W,v}_{v,W,WW,WWW,WWWW,5W,6W,7W,etc}
 825
 826    'W' stands for "word" and 'v' for "void".  Hence there are
 827    different macros for calling arity 0, 1, 2, 3, 4, etc, functions,
 828    and for each, the possibility of returning a word-typed result, or
 829    no result.
 830 */
 831
 832 /* Use these to write the name of your wrapper.  NOTE: duplicates
 833    VG_WRAP_FUNCTION_Z{U,Z} in pub_tool_redir.h. */
 834
 835 /* Use an extra level of macroisation so as to ensure the soname/fnname
 836    args are fully macro-expanded before pasting them together. */
 837 #define VG_CONCAT4(_aa,_bb,_cc,_dd) _aa##_bb##_cc##_dd
 838
 839 #define I_WRAP_SONAME_FNNAME_ZU(soname,fnname)                    \
 840    VG_CONCAT4(_vgwZU_,soname,_,fnname)
 841
 842 #define I_WRAP_SONAME_FNNAME_ZZ(soname,fnname)                    \
 843    VG_CONCAT4(_vgwZZ_,soname,_,fnname)
 844
 845 /* Use this macro from within a wrapper function to collect the
 846    context (address and possibly other info) of the original function.
 847    Once you have that you can then use it in one of the CALL_FN_
 848    macros.  The type of the argument _lval is OrigFn. */
 849 #define VALGRIND_GET_ORIG_FN(_lval)  VALGRIND_GET_NR_CONTEXT(_lval)
 850
 851 /* Derivatives of the main macros below, for calling functions
 852    returning void. */
 853
 854 #define CALL_FN_v_v(fnptr)                                        \
 855    do { volatile unsigned long _junk;                             \
 856         CALL_FN_W_v(_junk,fnptr); } while (0)
 857
 858 #define CALL_FN_v_W(fnptr, arg1)                                  \
 859    do { volatile unsigned long _junk;                             \
 860         CALL_FN_W_W(_junk,fnptr,arg1); } while (0)
 861
 862 #define CALL_FN_v_WW(fnptr, arg1,arg2)                            \
 863    do { volatile unsigned long _junk;                             \
 864         CALL_FN_W_WW(_junk,fnptr,arg1,arg2); } while (0)
 865
 866 #define CALL_FN_v_WWW(fnptr, arg1,arg2,arg3)                      \
 867    do { volatile unsigned long _junk;                             \
 868         CALL_FN_W_WWW(_junk,fnptr,arg1,arg2,arg3); } while (0)
 869
 870 #define CALL_FN_v_WWWW(fnptr, arg1,arg2,arg3,arg4)                \
 871    do { volatile unsigned long _junk;                             \
 872         CALL_FN_W_WWWW(_junk,fnptr,arg1,arg2,arg3,arg4); } while (0)
 873
 874 #define CALL_FN_v_5W(fnptr, arg1,arg2,arg3,arg4,arg5)             \
 875    do { volatile unsigned long _junk;                             \
 876         CALL_FN_W_5W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5); } while (0)
 877
 878 #define CALL_FN_v_6W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6)        \
 879    do { volatile unsigned long _junk;                             \
 880         CALL_FN_W_6W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6); } while (0)
 881
 882 #define CALL_FN_v_7W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6,arg7)   \
 883    do { volatile unsigned long _junk;                             \
 884         CALL_FN_W_7W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6,arg7); } while (0)
 885
 886 /* ------------------------- x86-{linux,darwin} ---------------- */
 887
 888 #if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin)
 889
 890 /* These regs are trashed by the hidden call.  No need to mention eax
 891    as gcc can already see that, plus causes gcc to bomb. */
 892 #define __CALLER_SAVED_REGS /*"eax"*/ "ecx", "edx"
 893
 894 /* These CALL_FN_ macros assume that on x86-linux, sizeof(unsigned
 895    long) == 4. */
 896
 897 #define CALL_FN_W_v(lval, orig)                                   \
 898    do {                                                           \
 899       volatile OrigFn        _orig = (orig);                      \
 900       volatile unsigned long _argvec[1];                          \
 901       volatile unsigned long _res;                                \
 902       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 903       __asm__ volatile(                                           \
 904          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 905          VALGRIND_CALL_NOREDIR_EAX                                \
 906          : /*out*/   "=a" (_res)                                  \
 907          : /*in*/    "a" (&_argvec[0])                            \
 908          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 909       );                                                          \
 910       lval = (__typeof__(lval)) _res;                             \
 911    } while (0)
 912
 913 #define CALL_FN_W_W(lval, orig, arg1)                             \
 914    do {                                                           \
 915       volatile OrigFn        _orig = (orig);                      \
 916       volatile unsigned long _argvec[2];                          \
 917       volatile unsigned long _res;                                \
 918       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 919       _argvec[1] = (unsigned long)(arg1);                         \
 920       __asm__ volatile(                                           \
 921          "subl $12, %%esp\n\t"                                    \
 922          "pushl 4(%%eax)\n\t"                                     \
 923          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 924          VALGRIND_CALL_NOREDIR_EAX                                \
 925          "addl $16, %%esp\n"                                      \
 926          : /*out*/   "=a" (_res)                                  \
 927          : /*in*/    "a" (&_argvec[0])                            \
 928          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 929       );                                                          \
 930       lval = (__typeof__(lval)) _res;                             \
 931    } while (0)
 932
 933 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
 934    do {                                                           \
 935       volatile OrigFn        _orig = (orig);                      \
 936       volatile unsigned long _argvec[3];                          \
 937       volatile unsigned long _res;                                \
 938       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 939       _argvec[1] = (unsigned long)(arg1);                         \
 940       _argvec[2] = (unsigned long)(arg2);                         \
 941       __asm__ volatile(                                           \
 942          "subl $8, %%esp\n\t"                                     \
 943          "pushl 8(%%eax)\n\t"                                     \
 944          "pushl 4(%%eax)\n\t"                                     \
 945          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 946          VALGRIND_CALL_NOREDIR_EAX                                \
 947          "addl $16, %%esp\n"                                      \
 948          : /*out*/   "=a" (_res)                                  \
 949          : /*in*/    "a" (&_argvec[0])                            \
 950          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 951       );                                                          \
 952       lval = (__typeof__(lval)) _res;                             \
 953    } while (0)
 954
 955 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
 956    do {                                                           \
 957       volatile OrigFn        _orig = (orig);                      \
 958       volatile unsigned long _argvec[4];                          \
 959       volatile unsigned long _res;                                \
 960       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 961       _argvec[1] = (unsigned long)(arg1);                         \
 962       _argvec[2] = (unsigned long)(arg2);                         \
 963       _argvec[3] = (unsigned long)(arg3);                         \
 964       __asm__ volatile(                                           \
 965          "subl $4, %%esp\n\t"                                     \
 966          "pushl 12(%%eax)\n\t"                                    \
 967          "pushl 8(%%eax)\n\t"                                     \
 968          "pushl 4(%%eax)\n\t"                                     \
 969          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 970          VALGRIND_CALL_NOREDIR_EAX                                \
 971          "addl $16, %%esp\n"                                      \
 972          : /*out*/   "=a" (_res)                                  \
 973          : /*in*/    "a" (&_argvec[0])                            \
 974          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 975       );                                                          \
 976       lval = (__typeof__(lval)) _res;                             \
 977    } while (0)
 978
 979 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
 980    do {                                                           \
 981       volatile OrigFn        _orig = (orig);                      \
 982       volatile unsigned long _argvec[5];                          \
 983       volatile unsigned long _res;                                \
 984       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 985       _argvec[1] = (unsigned long)(arg1);                         \
 986       _argvec[2] = (unsigned long)(arg2);                         \
 987       _argvec[3] = (unsigned long)(arg3);                         \
 988       _argvec[4] = (unsigned long)(arg4);                         \
 989       __asm__ volatile(                                           \
 990          "pushl 16(%%eax)\n\t"                                    \
 991          "pushl 12(%%eax)\n\t"                                    \
 992          "pushl 8(%%eax)\n\t"                                     \
 993          "pushl 4(%%eax)\n\t"                                     \
 994          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 995          VALGRIND_CALL_NOREDIR_EAX                                \
 996          "addl $16, %%esp\n"                                      \
 997          : /*out*/   "=a" (_res)                                  \
 998          : /*in*/    "a" (&_argvec[0])                            \
 999          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1000       );                                                          \
1001       lval = (__typeof__(lval)) _res;                             \
1002    } while (0)
1003
1004 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
1005    do {                                                           \
1006       volatile OrigFn        _orig = (orig);                      \
1007       volatile unsigned long _argvec[6];                          \
1008       volatile unsigned long _res;                                \
1009       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1010       _argvec[1] = (unsigned long)(arg1);                         \
1011       _argvec[2] = (unsigned long)(arg2);                         \
1012       _argvec[3] = (unsigned long)(arg3);                         \
1013       _argvec[4] = (unsigned long)(arg4);                         \
1014       _argvec[5] = (unsigned long)(arg5);                         \
1015       __asm__ volatile(                                           \
1016          "subl $12, %%esp\n\t"                                    \
1017          "pushl 20(%%eax)\n\t"                                    \
1018          "pushl 16(%%eax)\n\t"                                    \
1019          "pushl 12(%%eax)\n\t"                                    \
1020          "pushl 8(%%eax)\n\t"                                     \
1021          "pushl 4(%%eax)\n\t"                                     \
1022          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1023          VALGRIND_CALL_NOREDIR_EAX                                \
1024          "addl $32, %%esp\n"                                      \
1025          : /*out*/   "=a" (_res)                                  \
1026          : /*in*/    "a" (&_argvec[0])                            \
1027          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1028       );                                                          \
1029       lval = (__typeof__(lval)) _res;                             \
1030    } while (0)
1031
1032 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
1033    do {                                                           \
1034       volatile OrigFn        _orig = (orig);                      \
1035       volatile unsigned long _argvec[7];                          \
1036       volatile unsigned long _res;                                \
1037       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1038       _argvec[1] = (unsigned long)(arg1);                         \
1039       _argvec[2] = (unsigned long)(arg2);                         \
1040       _argvec[3] = (unsigned long)(arg3);                         \
1041       _argvec[4] = (unsigned long)(arg4);                         \
1042       _argvec[5] = (unsigned long)(arg5);                         \
1043       _argvec[6] = (unsigned long)(arg6);                         \
1044       __asm__ volatile(                                           \
1045          "subl $8, %%esp\n\t"                                     \
1046          "pushl 24(%%eax)\n\t"                                    \
1047          "pushl 20(%%eax)\n\t"                                    \
1048          "pushl 16(%%eax)\n\t"                                    \
1049          "pushl 12(%%eax)\n\t"                                    \
1050          "pushl 8(%%eax)\n\t"                                     \
1051          "pushl 4(%%eax)\n\t"                                     \
1052          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1053          VALGRIND_CALL_NOREDIR_EAX                                \
1054          "addl $32, %%esp\n"                                      \
1055          : /*out*/   "=a" (_res)                                  \
1056          : /*in*/    "a" (&_argvec[0])                            \
1057          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1058       );                                                          \
1059       lval = (__typeof__(lval)) _res;                             \
1060    } while (0)
1061
1062 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1063                                  arg7)                            \
1064    do {                                                           \
1065       volatile OrigFn        _orig = (orig);                      \
1066       volatile unsigned long _argvec[8];                          \
1067       volatile unsigned long _res;                                \
1068       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1069       _argvec[1] = (unsigned long)(arg1);                         \
1070       _argvec[2] = (unsigned long)(arg2);                         \
1071       _argvec[3] = (unsigned long)(arg3);                         \
1072       _argvec[4] = (unsigned long)(arg4);                         \
1073       _argvec[5] = (unsigned long)(arg5);                         \
1074       _argvec[6] = (unsigned long)(arg6);                         \
1075       _argvec[7] = (unsigned long)(arg7);                         \
1076       __asm__ volatile(                                           \
1077          "subl $4, %%esp\n\t"                                     \
1078          "pushl 28(%%eax)\n\t"                                    \
1079          "pushl 24(%%eax)\n\t"                                    \
1080          "pushl 20(%%eax)\n\t"                                    \
1081          "pushl 16(%%eax)\n\t"                                    \
1082          "pushl 12(%%eax)\n\t"                                    \
1083          "pushl 8(%%eax)\n\t"                                     \
1084          "pushl 4(%%eax)\n\t"                                     \
1085          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1086          VALGRIND_CALL_NOREDIR_EAX                                \
1087          "addl $32, %%esp\n"                                      \
1088          : /*out*/   "=a" (_res)                                  \
1089          : /*in*/    "a" (&_argvec[0])                            \
1090          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1091       );                                                          \
1092       lval = (__typeof__(lval)) _res;                             \
1093    } while (0)
1094
1095 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1096                                  arg7,arg8)                       \
1097    do {                                                           \
1098       volatile OrigFn        _orig = (orig);                      \
1099       volatile unsigned long _argvec[9];                          \
1100       volatile unsigned long _res;                                \
1101       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1102       _argvec[1] = (unsigned long)(arg1);                         \
1103       _argvec[2] = (unsigned long)(arg2);                         \
1104       _argvec[3] = (unsigned long)(arg3);                         \
1105       _argvec[4] = (unsigned long)(arg4);                         \
1106       _argvec[5] = (unsigned long)(arg5);                         \
1107       _argvec[6] = (unsigned long)(arg6);                         \
1108       _argvec[7] = (unsigned long)(arg7);                         \
1109       _argvec[8] = (unsigned long)(arg8);                         \
1110       __asm__ volatile(                                           \
1111          "pushl 32(%%eax)\n\t"                                    \
1112          "pushl 28(%%eax)\n\t"                                    \
1113          "pushl 24(%%eax)\n\t"                                    \
1114          "pushl 20(%%eax)\n\t"                                    \
1115          "pushl 16(%%eax)\n\t"                                    \
1116          "pushl 12(%%eax)\n\t"                                    \
1117          "pushl 8(%%eax)\n\t"                                     \
1118          "pushl 4(%%eax)\n\t"                                     \
1119          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1120          VALGRIND_CALL_NOREDIR_EAX                                \
1121          "addl $32, %%esp\n"                                      \
1122          : /*out*/   "=a" (_res)                                  \
1123          : /*in*/    "a" (&_argvec[0])                            \
1124          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1125       );                                                          \
1126       lval = (__typeof__(lval)) _res;                             \
1127    } while (0)
1128
1129 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1130                                  arg7,arg8,arg9)                  \
1131    do {                                                           \
1132       volatile OrigFn        _orig = (orig);                      \
1133       volatile unsigned long _argvec[10];                         \
1134       volatile unsigned long _res;                                \
1135       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1136       _argvec[1] = (unsigned long)(arg1);                         \
1137       _argvec[2] = (unsigned long)(arg2);                         \
1138       _argvec[3] = (unsigned long)(arg3);                         \
1139       _argvec[4] = (unsigned long)(arg4);                         \
1140       _argvec[5] = (unsigned long)(arg5);                         \
1141       _argvec[6] = (unsigned long)(arg6);                         \
1142       _argvec[7] = (unsigned long)(arg7);                         \
1143       _argvec[8] = (unsigned long)(arg8);                         \
1144       _argvec[9] = (unsigned long)(arg9);                         \
1145       __asm__ volatile(                                           \
1146          "subl $12, %%esp\n\t"                                    \
1147          "pushl 36(%%eax)\n\t"                                    \
1148          "pushl 32(%%eax)\n\t"                                    \
1149          "pushl 28(%%eax)\n\t"                                    \
1150          "pushl 24(%%eax)\n\t"                                    \
1151          "pushl 20(%%eax)\n\t"                                    \
1152          "pushl 16(%%eax)\n\t"                                    \
1153          "pushl 12(%%eax)\n\t"                                    \
1154          "pushl 8(%%eax)\n\t"                                     \
1155          "pushl 4(%%eax)\n\t"                                     \
1156          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1157          VALGRIND_CALL_NOREDIR_EAX                                \
1158          "addl $48, %%esp\n"                                      \
1159          : /*out*/   "=a" (_res)                                  \
1160          : /*in*/    "a" (&_argvec[0])                            \
1161          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1162       );                                                          \
1163       lval = (__typeof__(lval)) _res;                             \
1164    } while (0)
1165
1166 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1167                                   arg7,arg8,arg9,arg10)           \
1168    do {                                                           \
1169       volatile OrigFn        _orig = (orig);                      \
1170       volatile unsigned long _argvec[11];                         \
1171       volatile unsigned long _res;                                \
1172       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1173       _argvec[1] = (unsigned long)(arg1);                         \
1174       _argvec[2] = (unsigned long)(arg2);                         \
1175       _argvec[3] = (unsigned long)(arg3);                         \
1176       _argvec[4] = (unsigned long)(arg4);                         \
1177       _argvec[5] = (unsigned long)(arg5);                         \
1178       _argvec[6] = (unsigned long)(arg6);                         \
1179       _argvec[7] = (unsigned long)(arg7);                         \
1180       _argvec[8] = (unsigned long)(arg8);                         \
1181       _argvec[9] = (unsigned long)(arg9);                         \
1182       _argvec[10] = (unsigned long)(arg10);                       \
1183       __asm__ volatile(                                           \
1184          "subl $8, %%esp\n\t"                                     \
1185          "pushl 40(%%eax)\n\t"                                    \
1186          "pushl 36(%%eax)\n\t"                                    \
1187          "pushl 32(%%eax)\n\t"                                    \
1188          "pushl 28(%%eax)\n\t"                                    \
1189          "pushl 24(%%eax)\n\t"                                    \
1190          "pushl 20(%%eax)\n\t"                                    \
1191          "pushl 16(%%eax)\n\t"                                    \
1192          "pushl 12(%%eax)\n\t"                                    \
1193          "pushl 8(%%eax)\n\t"                                     \
1194          "pushl 4(%%eax)\n\t"                                     \
1195          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1196          VALGRIND_CALL_NOREDIR_EAX                                \
1197          "addl $48, %%esp\n"                                      \
1198          : /*out*/   "=a" (_res)                                  \
1199          : /*in*/    "a" (&_argvec[0])                            \
1200          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1201       );                                                          \
1202       lval = (__typeof__(lval)) _res;                             \
1203    } while (0)
1204
1205 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
1206                                   arg6,arg7,arg8,arg9,arg10,      \
1207                                   arg11)                          \
1208    do {                                                           \
1209       volatile OrigFn        _orig = (orig);                      \
1210       volatile unsigned long _argvec[12];                         \
1211       volatile unsigned long _res;                                \
1212       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1213       _argvec[1] = (unsigned long)(arg1);                         \
1214       _argvec[2] = (unsigned long)(arg2);                         \
1215       _argvec[3] = (unsigned long)(arg3);                         \
1216       _argvec[4] = (unsigned long)(arg4);                         \
1217       _argvec[5] = (unsigned long)(arg5);                         \
1218       _argvec[6] = (unsigned long)(arg6);                         \
1219       _argvec[7] = (unsigned long)(arg7);                         \
1220       _argvec[8] = (unsigned long)(arg8);                         \
1221       _argvec[9] = (unsigned long)(arg9);                         \
1222       _argvec[10] = (unsigned long)(arg10);                       \
1223       _argvec[11] = (unsigned long)(arg11);                       \
1224       __asm__ volatile(                                           \
1225          "subl $4, %%esp\n\t"                                     \
1226          "pushl 44(%%eax)\n\t"                                    \
1227          "pushl 40(%%eax)\n\t"                                    \
1228          "pushl 36(%%eax)\n\t"                                    \
1229          "pushl 32(%%eax)\n\t"                                    \
1230          "pushl 28(%%eax)\n\t"                                    \
1231          "pushl 24(%%eax)\n\t"                                    \
1232          "pushl 20(%%eax)\n\t"                                    \
1233          "pushl 16(%%eax)\n\t"                                    \
1234          "pushl 12(%%eax)\n\t"                                    \
1235          "pushl 8(%%eax)\n\t"                                     \
1236          "pushl 4(%%eax)\n\t"                                     \
1237          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1238          VALGRIND_CALL_NOREDIR_EAX                                \
1239          "addl $48, %%esp\n"                                      \
1240          : /*out*/   "=a" (_res)                                  \
1241          : /*in*/    "a" (&_argvec[0])                            \
1242          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1243       );                                                          \
1244       lval = (__typeof__(lval)) _res;                             \
1245    } while (0)
1246
1247 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
1248                                   arg6,arg7,arg8,arg9,arg10,      \
1249                                   arg11,arg12)                    \
1250    do {                                                           \
1251       volatile OrigFn        _orig = (orig);                      \
1252       volatile unsigned long _argvec[13];                         \
1253       volatile unsigned long _res;                                \
1254       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1255       _argvec[1] = (unsigned long)(arg1);                         \
1256       _argvec[2] = (unsigned long)(arg2);                         \
1257       _argvec[3] = (unsigned long)(arg3);                         \
1258       _argvec[4] = (unsigned long)(arg4);                         \
1259       _argvec[5] = (unsigned long)(arg5);                         \
1260       _argvec[6] = (unsigned long)(arg6);                         \
1261       _argvec[7] = (unsigned long)(arg7);                         \
1262       _argvec[8] = (unsigned long)(arg8);                         \
1263       _argvec[9] = (unsigned long)(arg9);                         \
1264       _argvec[10] = (unsigned long)(arg10);                       \
1265       _argvec[11] = (unsigned long)(arg11);                       \
1266       _argvec[12] = (unsigned long)(arg12);                       \
1267       __asm__ volatile(                                           \
1268          "pushl 48(%%eax)\n\t"                                    \
1269          "pushl 44(%%eax)\n\t"                                    \
1270          "pushl 40(%%eax)\n\t"                                    \
1271          "pushl 36(%%eax)\n\t"                                    \
1272          "pushl 32(%%eax)\n\t"                                    \
1273          "pushl 28(%%eax)\n\t"                                    \
1274          "pushl 24(%%eax)\n\t"                                    \
1275          "pushl 20(%%eax)\n\t"                                    \
1276          "pushl 16(%%eax)\n\t"                                    \
1277          "pushl 12(%%eax)\n\t"                                    \
1278          "pushl 8(%%eax)\n\t"                                     \
1279          "pushl 4(%%eax)\n\t"                                     \
1280          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1281          VALGRIND_CALL_NOREDIR_EAX                                \
1282          "addl $48, %%esp\n"                                      \
1283          : /*out*/   "=a" (_res)                                  \
1284          : /*in*/    "a" (&_argvec[0])                            \
1285          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1286       );                                                          \
1287       lval = (__typeof__(lval)) _res;                             \
1288    } while (0)
1289
1290 #endif /* PLAT_x86_linux || PLAT_x86_darwin */
1291
1292 /* ------------------------ amd64-{linux,darwin} --------------- */
1293
1294 #if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin)
1295
1296 /* ARGREGS: rdi rsi rdx rcx r8 r9 (the rest on stack in R-to-L order) */
1297
1298 /* These regs are trashed by the hidden call. */
1299 #define __CALLER_SAVED_REGS /*"rax",*/ "rcx", "rdx", "rsi",       \
1300                             "rdi", "r8", "r9", "r10", "r11"
1301
1302 /* This is all pretty complex.  It's so as to make stack unwinding
1303    work reliably.  See bug 243270.  The basic problem is the sub and
1304    add of 128 of %rsp in all of the following macros.  If gcc believes
1305    the CFA is in %rsp, then unwinding may fail, because what's at the
1306    CFA is not what gcc "expected" when it constructs the CFIs for the
1307    places where the macros are instantiated.
1308
1309    But we can't just add a CFI annotation to increase the CFA offset
1310    by 128, to match the sub of 128 from %rsp, because we don't know
1311    whether gcc has chosen %rsp as the CFA at that point, or whether it
1312    has chosen some other register (eg, %rbp).  In the latter case,
1313    adding a CFI annotation to change the CFA offset is simply wrong.
1314
1315    So the solution is to get hold of the CFA using
1316    __builtin_dwarf_cfa(), put it in a known register, and add a
1317    CFI annotation to say what the register is.  We choose %rbp for
1318    this (perhaps perversely), because:
1319
1320    (1) %rbp is already subject to unwinding.  If a new register was
1321        chosen then the unwinder would have to unwind it in all stack
1322        traces, which is expensive, and
1323
1324    (2) %rbp is already subject to precise exception updates in the
1325        JIT.  If a new register was chosen, we'd have to have precise
1326        exceptions for it too, which reduces performance of the
1327        generated code.
1328
1329    However .. one extra complication.  We can't just whack the result
1330    of __builtin_dwarf_cfa() into %rbp and then add %rbp to the
1331    list of trashed registers at the end of the inline assembly
1332    fragments; gcc won't allow %rbp to appear in that list.  Hence
1333    instead we need to stash %rbp in %r15 for the duration of the asm,
1334    and say that %r15 is trashed instead.  gcc seems happy to go with
1335    that.
1336
1337    Oh .. and this all needs to be conditionalised so that it is
1338    unchanged from before this commit, when compiled with older gccs
1339    that don't support __builtin_dwarf_cfa.  Furthermore, since
1340    this header file is freestanding, it has to be independent of
1341    config.h, and so the following conditionalisation cannot depend on
1342    configure time checks.
1343
1344    Although it's not clear from
1345    'defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)',
1346    this expression excludes Darwin.
1347    .cfi directives in Darwin assembly appear to be completely
1348    different and I haven't investigated how they work.
1349
1350    For even more entertainment value, note we have to use the
1351    completely undocumented __builtin_dwarf_cfa(), which appears to
1352    really compute the CFA, whereas __builtin_frame_address(0) claims
1353    to but actually doesn't.  See
1354    https://bugs.kde.org/show_bug.cgi?id=243270#c47
1355 */
1356 #if defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)
1357 #  define __FRAME_POINTER                                         \
1358       ,"r"(__builtin_dwarf_cfa())
1359 #  define VALGRIND_CFI_PROLOGUE                                   \
1360       "movq %%rbp, %%r15\n\t"                                     \
1361       "movq %2, %%rbp\n\t"                                        \
1362       ".cfi_remember_state\n\t"                                   \
1363       ".cfi_def_cfa rbp, 0\n\t"
1364 #  define VALGRIND_CFI_EPILOGUE                                   \
1365       "movq %%r15, %%rbp\n\t"                                     \
1366       ".cfi_restore_state\n\t"
1367 #else
1368 #  define __FRAME_POINTER
1369 #  define VALGRIND_CFI_PROLOGUE
1370 #  define VALGRIND_CFI_EPILOGUE
1371 #endif
1372
1373
1374 /* These CALL_FN_ macros assume that on amd64-linux, sizeof(unsigned
1375    long) == 8. */
1376
1377 /* NB 9 Sept 07.  There is a nasty kludge here in all these CALL_FN_
1378    macros.  In order not to trash the stack redzone, we need to drop
1379    %rsp by 128 before the hidden call, and restore afterwards.  The
1380    nastyness is that it is only by luck that the stack still appears
1381    to be unwindable during the hidden call - since then the behaviour
1382    of any routine using this macro does not match what the CFI data
1383    says.  Sigh.
1384
1385    Why is this important?  Imagine that a wrapper has a stack
1386    allocated local, and passes to the hidden call, a pointer to it.
1387    Because gcc does not know about the hidden call, it may allocate
1388    that local in the redzone.  Unfortunately the hidden call may then
1389    trash it before it comes to use it.  So we must step clear of the
1390    redzone, for the duration of the hidden call, to make it safe.
1391
1392    Probably the same problem afflicts the other redzone-style ABIs too
1393    (ppc64-linux, ppc32-aix5, ppc64-aix5); but for those, the stack is
1394    self describing (none of this CFI nonsense) so at least messing
1395    with the stack pointer doesn't give a danger of non-unwindable
1396    stack. */
1397
1398 #define CALL_FN_W_v(lval, orig)                                   \
1399    do {                                                           \
1400       volatile OrigFn        _orig = (orig);                      \
1401       volatile unsigned long _argvec[1];                          \
1402       volatile unsigned long _res;                                \
1403       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1404       __asm__ volatile(                                           \
1405          VALGRIND_CFI_PROLOGUE                                    \
1406          "subq $128,%%rsp\n\t"                                    \
1407          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1408          VALGRIND_CALL_NOREDIR_RAX                                \
1409          "addq $128,%%rsp\n\t"                                    \
1410          VALGRIND_CFI_EPILOGUE                                    \
1411          : /*out*/   "=a" (_res)                                  \
1412          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1413          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1414       );                                                          \
1415       lval = (__typeof__(lval)) _res;                             \
1416    } while (0)
1417
1418 #define CALL_FN_W_W(lval, orig, arg1)                             \
1419    do {                                                           \
1420       volatile OrigFn        _orig = (orig);                      \
1421       volatile unsigned long _argvec[2];                          \
1422       volatile unsigned long _res;                                \
1423       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1424       _argvec[1] = (unsigned long)(arg1);                         \
1425       __asm__ volatile(                                           \
1426          VALGRIND_CFI_PROLOGUE                                    \
1427          "subq $128,%%rsp\n\t"                                    \
1428          "movq 8(%%rax), %%rdi\n\t"                               \
1429          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1430          VALGRIND_CALL_NOREDIR_RAX                                \
1431          "addq $128,%%rsp\n\t"                                    \
1432          VALGRIND_CFI_EPILOGUE                                    \
1433          : /*out*/   "=a" (_res)                                  \
1434          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1435          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1436       );                                                          \
1437       lval = (__typeof__(lval)) _res;                             \
1438    } while (0)
1439
1440 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
1441    do {                                                           \
1442       volatile OrigFn        _orig = (orig);                      \
1443       volatile unsigned long _argvec[3];                          \
1444       volatile unsigned long _res;                                \
1445       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1446       _argvec[1] = (unsigned long)(arg1);                         \
1447       _argvec[2] = (unsigned long)(arg2);                         \
1448       __asm__ volatile(                                           \
1449          VALGRIND_CFI_PROLOGUE                                    \
1450          "subq $128,%%rsp\n\t"                                    \
1451          "movq 16(%%rax), %%rsi\n\t"                              \
1452          "movq 8(%%rax), %%rdi\n\t"                               \
1453          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1454          VALGRIND_CALL_NOREDIR_RAX                                \
1455          "addq $128,%%rsp\n\t"                                    \
1456          VALGRIND_CFI_EPILOGUE                                    \
1457          : /*out*/   "=a" (_res)                                  \
1458          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1459          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1460       );                                                          \
1461       lval = (__typeof__(lval)) _res;                             \
1462    } while (0)
1463
1464 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
1465    do {                                                           \
1466       volatile OrigFn        _orig = (orig);                      \
1467       volatile unsigned long _argvec[4];                          \
1468       volatile unsigned long _res;                                \
1469       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1470       _argvec[1] = (unsigned long)(arg1);                         \
1471       _argvec[2] = (unsigned long)(arg2);                         \
1472       _argvec[3] = (unsigned long)(arg3);                         \
1473       __asm__ volatile(                                           \
1474          VALGRIND_CFI_PROLOGUE                                    \
1475          "subq $128,%%rsp\n\t"                                    \
1476          "movq 24(%%rax), %%rdx\n\t"                              \
1477          "movq 16(%%rax), %%rsi\n\t"                              \
1478          "movq 8(%%rax), %%rdi\n\t"                               \
1479          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1480          VALGRIND_CALL_NOREDIR_RAX                                \
1481          "addq $128,%%rsp\n\t"                                    \
1482          VALGRIND_CFI_EPILOGUE                                    \
1483          : /*out*/   "=a" (_res)                                  \
1484          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1485          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1486       );                                                          \
1487       lval = (__typeof__(lval)) _res;                             \
1488    } while (0)
1489
1490 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
1491    do {                                                           \
1492       volatile OrigFn        _orig = (orig);                      \
1493       volatile unsigned long _argvec[5];                          \
1494       volatile unsigned long _res;                                \
1495       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1496       _argvec[1] = (unsigned long)(arg1);                         \
1497       _argvec[2] = (unsigned long)(arg2);                         \
1498       _argvec[3] = (unsigned long)(arg3);                         \
1499       _argvec[4] = (unsigned long)(arg4);                         \
1500       __asm__ volatile(                                           \
1501          VALGRIND_CFI_PROLOGUE                                    \
1502          "subq $128,%%rsp\n\t"                                    \
1503          "movq 32(%%rax), %%rcx\n\t"                              \
1504          "movq 24(%%rax), %%rdx\n\t"                              \
1505          "movq 16(%%rax), %%rsi\n\t"                              \
1506          "movq 8(%%rax), %%rdi\n\t"                               \
1507          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1508          VALGRIND_CALL_NOREDIR_RAX                                \
1509          "addq $128,%%rsp\n\t"                                    \
1510          VALGRIND_CFI_EPILOGUE                                    \
1511          : /*out*/   "=a" (_res)                                  \
1512          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1513          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1514       );                                                          \
1515       lval = (__typeof__(lval)) _res;                             \
1516    } while (0)
1517
1518 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
1519    do {                                                           \
1520       volatile OrigFn        _orig = (orig);                      \
1521       volatile unsigned long _argvec[6];                          \
1522       volatile unsigned long _res;                                \
1523       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1524       _argvec[1] = (unsigned long)(arg1);                         \
1525       _argvec[2] = (unsigned long)(arg2);                         \
1526       _argvec[3] = (unsigned long)(arg3);                         \
1527       _argvec[4] = (unsigned long)(arg4);                         \
1528       _argvec[5] = (unsigned long)(arg5);                         \
1529       __asm__ volatile(                                           \
1530          VALGRIND_CFI_PROLOGUE                                    \
1531          "subq $128,%%rsp\n\t"                                    \
1532          "movq 40(%%rax), %%r8\n\t"                               \
1533          "movq 32(%%rax), %%rcx\n\t"                              \
1534          "movq 24(%%rax), %%rdx\n\t"                              \
1535          "movq 16(%%rax), %%rsi\n\t"                              \
1536          "movq 8(%%rax), %%rdi\n\t"                               \
1537          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1538          VALGRIND_CALL_NOREDIR_RAX                                \
1539          "addq $128,%%rsp\n\t"                                    \
1540          VALGRIND_CFI_EPILOGUE                                    \
1541          : /*out*/   "=a" (_res)                                  \
1542          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1543          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1544       );                                                          \
1545       lval = (__typeof__(lval)) _res;                             \
1546    } while (0)
1547
1548 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
1549    do {                                                           \
1550       volatile OrigFn        _orig = (orig);                      \
1551       volatile unsigned long _argvec[7];                          \
1552       volatile unsigned long _res;                                \
1553       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1554       _argvec[1] = (unsigned long)(arg1);                         \
1555       _argvec[2] = (unsigned long)(arg2);                         \
1556       _argvec[3] = (unsigned long)(arg3);                         \
1557       _argvec[4] = (unsigned long)(arg4);                         \
1558       _argvec[5] = (unsigned long)(arg5);                         \
1559       _argvec[6] = (unsigned long)(arg6);                         \
1560       __asm__ volatile(                                           \
1561          VALGRIND_CFI_PROLOGUE                                    \
1562          "subq $128,%%rsp\n\t"                                    \
1563          "movq 48(%%rax), %%r9\n\t"                               \
1564          "movq 40(%%rax), %%r8\n\t"                               \
1565          "movq 32(%%rax), %%rcx\n\t"                              \
1566          "movq 24(%%rax), %%rdx\n\t"                              \
1567          "movq 16(%%rax), %%rsi\n\t"                              \
1568          "movq 8(%%rax), %%rdi\n\t"                               \
1569          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1570          VALGRIND_CALL_NOREDIR_RAX                                \
1571          "addq $128,%%rsp\n\t"                                    \
1572          VALGRIND_CFI_EPILOGUE                                    \
1573          : /*out*/   "=a" (_res)                                  \
1574          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1575          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1576       );                                                          \
1577       lval = (__typeof__(lval)) _res;                             \
1578    } while (0)
1579
1580 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1581                                  arg7)                            \
1582    do {                                                           \
1583       volatile OrigFn        _orig = (orig);                      \
1584       volatile unsigned long _argvec[8];                          \
1585       volatile unsigned long _res;                                \
1586       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1587       _argvec[1] = (unsigned long)(arg1);                         \
1588       _argvec[2] = (unsigned long)(arg2);                         \
1589       _argvec[3] = (unsigned long)(arg3);                         \
1590       _argvec[4] = (unsigned long)(arg4);                         \
1591       _argvec[5] = (unsigned long)(arg5);                         \
1592       _argvec[6] = (unsigned long)(arg6);                         \
1593       _argvec[7] = (unsigned long)(arg7);                         \
1594       __asm__ volatile(                                           \
1595          VALGRIND_CFI_PROLOGUE                                    \
1596          "subq $136,%%rsp\n\t"                                    \
1597          "pushq 56(%%rax)\n\t"                                    \
1598          "movq 48(%%rax), %%r9\n\t"                               \
1599          "movq 40(%%rax), %%r8\n\t"                               \
1600          "movq 32(%%rax), %%rcx\n\t"                              \
1601          "movq 24(%%rax), %%rdx\n\t"                              \
1602          "movq 16(%%rax), %%rsi\n\t"                              \
1603          "movq 8(%%rax), %%rdi\n\t"                               \
1604          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1605          VALGRIND_CALL_NOREDIR_RAX                                \
1606          "addq $8, %%rsp\n"                                       \
1607          "addq $136,%%rsp\n\t"                                    \
1608          VALGRIND_CFI_EPILOGUE                                    \
1609          : /*out*/   "=a" (_res)                                  \
1610          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1611          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1612       );                                                          \
1613       lval = (__typeof__(lval)) _res;                             \
1614    } while (0)
1615
1616 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1617                                  arg7,arg8)                       \
1618    do {                                                           \
1619       volatile OrigFn        _orig = (orig);                      \
1620       volatile unsigned long _argvec[9];                          \
1621       volatile unsigned long _res;                                \
1622       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1623       _argvec[1] = (unsigned long)(arg1);                         \
1624       _argvec[2] = (unsigned long)(arg2);                         \
1625       _argvec[3] = (unsigned long)(arg3);                         \
1626       _argvec[4] = (unsigned long)(arg4);                         \
1627       _argvec[5] = (unsigned long)(arg5);                         \
1628       _argvec[6] = (unsigned long)(arg6);                         \
1629       _argvec[7] = (unsigned long)(arg7);                         \
1630       _argvec[8] = (unsigned long)(arg8);                         \
1631       __asm__ volatile(                                           \
1632          VALGRIND_CFI_PROLOGUE                                    \
1633          "subq $128,%%rsp\n\t"                                    \
1634          "pushq 64(%%rax)\n\t"                                    \
1635          "pushq 56(%%rax)\n\t"                                    \
1636          "movq 48(%%rax), %%r9\n\t"                               \
1637          "movq 40(%%rax), %%r8\n\t"                               \
1638          "movq 32(%%rax), %%rcx\n\t"                              \
1639          "movq 24(%%rax), %%rdx\n\t"                              \
1640          "movq 16(%%rax), %%rsi\n\t"                              \
1641          "movq 8(%%rax), %%rdi\n\t"                               \
1642          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1643          VALGRIND_CALL_NOREDIR_RAX                                \
1644          "addq $16, %%rsp\n"                                      \
1645          "addq $128,%%rsp\n\t"                                    \
1646          VALGRIND_CFI_EPILOGUE                                    \
1647          : /*out*/   "=a" (_res)                                  \
1648          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1649          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1650       );                                                          \
1651       lval = (__typeof__(lval)) _res;                             \
1652    } while (0)
1653
1654 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1655                                  arg7,arg8,arg9)                  \
1656    do {                                                           \
1657       volatile OrigFn        _orig = (orig);                      \
1658       volatile unsigned long _argvec[10];                         \
1659       volatile unsigned long _res;                                \
1660       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1661       _argvec[1] = (unsigned long)(arg1);                         \
1662       _argvec[2] = (unsigned long)(arg2);                         \
1663       _argvec[3] = (unsigned long)(arg3);                         \
1664       _argvec[4] = (unsigned long)(arg4);                         \
1665       _argvec[5] = (unsigned long)(arg5);                         \
1666       _argvec[6] = (unsigned long)(arg6);                         \
1667       _argvec[7] = (unsigned long)(arg7);                         \
1668       _argvec[8] = (unsigned long)(arg8);                         \
1669       _argvec[9] = (unsigned long)(arg9);                         \
1670       __asm__ volatile(                                           \
1671          VALGRIND_CFI_PROLOGUE                                    \
1672          "subq $136,%%rsp\n\t"                                    \
1673          "pushq 72(%%rax)\n\t"                                    \
1674          "pushq 64(%%rax)\n\t"                                    \
1675          "pushq 56(%%rax)\n\t"                                    \
1676          "movq 48(%%rax), %%r9\n\t"                               \
1677          "movq 40(%%rax), %%r8\n\t"                               \
1678          "movq 32(%%rax), %%rcx\n\t"                              \
1679          "movq 24(%%rax), %%rdx\n\t"                              \
1680          "movq 16(%%rax), %%rsi\n\t"                              \
1681          "movq 8(%%rax), %%rdi\n\t"                               \
1682          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1683          VALGRIND_CALL_NOREDIR_RAX                                \
1684          "addq $24, %%rsp\n"                                      \
1685          "addq $136,%%rsp\n\t"                                    \
1686          VALGRIND_CFI_EPILOGUE                                    \
1687          : /*out*/   "=a" (_res)                                  \
1688          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1689          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1690       );                                                          \
1691       lval = (__typeof__(lval)) _res;                             \
1692    } while (0)
1693
1694 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1695                                   arg7,arg8,arg9,arg10)           \
1696    do {                                                           \
1697       volatile OrigFn        _orig = (orig);                      \
1698       volatile unsigned long _argvec[11];                         \
1699       volatile unsigned long _res;                                \
1700       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1701       _argvec[1] = (unsigned long)(arg1);                         \
1702       _argvec[2] = (unsigned long)(arg2);                         \
1703       _argvec[3] = (unsigned long)(arg3);                         \
1704       _argvec[4] = (unsigned long)(arg4);                         \
1705       _argvec[5] = (unsigned long)(arg5);                         \
1706       _argvec[6] = (unsigned long)(arg6);                         \
1707       _argvec[7] = (unsigned long)(arg7);                         \
1708       _argvec[8] = (unsigned long)(arg8);                         \
1709       _argvec[9] = (unsigned long)(arg9);                         \
1710       _argvec[10] = (unsigned long)(arg10);                       \
1711       __asm__ volatile(                                           \
1712          VALGRIND_CFI_PROLOGUE                                    \
1713          "subq $128,%%rsp\n\t"                                    \
1714          "pushq 80(%%rax)\n\t"                                    \
1715          "pushq 72(%%rax)\n\t"                                    \
1716          "pushq 64(%%rax)\n\t"                                    \
1717          "pushq 56(%%rax)\n\t"                                    \
1718          "movq 48(%%rax), %%r9\n\t"                               \
1719          "movq 40(%%rax), %%r8\n\t"                               \
1720          "movq 32(%%rax), %%rcx\n\t"                              \
1721          "movq 24(%%rax), %%rdx\n\t"                              \
1722          "movq 16(%%rax), %%rsi\n\t"                              \
1723          "movq 8(%%rax), %%rdi\n\t"                               \
1724          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1725          VALGRIND_CALL_NOREDIR_RAX                                \
1726          "addq $32, %%rsp\n"                                      \
1727          "addq $128,%%rsp\n\t"                                    \
1728          VALGRIND_CFI_EPILOGUE                                    \
1729          : /*out*/   "=a" (_res)                                  \
1730          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1731          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1732       );                                                          \
1733       lval = (__typeof__(lval)) _res;                             \
1734    } while (0)
1735
1736 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1737                                   arg7,arg8,arg9,arg10,arg11)     \
1738    do {                                                           \
1739       volatile OrigFn        _orig = (orig);                      \
1740       volatile unsigned long _argvec[12];                         \
1741       volatile unsigned long _res;                                \
1742       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1743       _argvec[1] = (unsigned long)(arg1);                         \
1744       _argvec[2] = (unsigned long)(arg2);                         \
1745       _argvec[3] = (unsigned long)(arg3);                         \
1746       _argvec[4] = (unsigned long)(arg4);                         \
1747       _argvec[5] = (unsigned long)(arg5);                         \
1748       _argvec[6] = (unsigned long)(arg6);                         \
1749       _argvec[7] = (unsigned long)(arg7);                         \
1750       _argvec[8] = (unsigned long)(arg8);                         \
1751       _argvec[9] = (unsigned long)(arg9);                         \
1752       _argvec[10] = (unsigned long)(arg10);                       \
1753       _argvec[11] = (unsigned long)(arg11);                       \
1754       __asm__ volatile(                                           \
1755          VALGRIND_CFI_PROLOGUE                                    \
1756          "subq $136,%%rsp\n\t"                                    \
1757          "pushq 88(%%rax)\n\t"                                    \
1758          "pushq 80(%%rax)\n\t"                                    \
1759          "pushq 72(%%rax)\n\t"                                    \
1760          "pushq 64(%%rax)\n\t"                                    \
1761          "pushq 56(%%rax)\n\t"                                    \
1762          "movq 48(%%rax), %%r9\n\t"                               \
1763          "movq 40(%%rax), %%r8\n\t"                               \
1764          "movq 32(%%rax), %%rcx\n\t"                              \
1765          "movq 24(%%rax), %%rdx\n\t"                              \
1766          "movq 16(%%rax), %%rsi\n\t"                              \
1767          "movq 8(%%rax), %%rdi\n\t"                               \
1768          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1769          VALGRIND_CALL_NOREDIR_RAX                                \
1770          "addq $40, %%rsp\n"                                      \
1771          "addq $136,%%rsp\n\t"                                    \
1772          VALGRIND_CFI_EPILOGUE                                    \
1773          : /*out*/   "=a" (_res)                                  \
1774          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1775          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1776       );                                                          \
1777       lval = (__typeof__(lval)) _res;                             \
1778    } while (0)
1779
1780 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1781                                 arg7,arg8,arg9,arg10,arg11,arg12) \
1782    do {                                                           \
1783       volatile OrigFn        _orig = (orig);                      \
1784       volatile unsigned long _argvec[13];                         \
1785       volatile unsigned long _res;                                \
1786       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1787       _argvec[1] = (unsigned long)(arg1);                         \
1788       _argvec[2] = (unsigned long)(arg2);                         \
1789       _argvec[3] = (unsigned long)(arg3);                         \
1790       _argvec[4] = (unsigned long)(arg4);                         \
1791       _argvec[5] = (unsigned long)(arg5);                         \
1792       _argvec[6] = (unsigned long)(arg6);                         \
1793       _argvec[7] = (unsigned long)(arg7);                         \
1794       _argvec[8] = (unsigned long)(arg8);                         \
1795       _argvec[9] = (unsigned long)(arg9);                         \
1796       _argvec[10] = (unsigned long)(arg10);                       \
1797       _argvec[11] = (unsigned long)(arg11);                       \
1798       _argvec[12] = (unsigned long)(arg12);                       \
1799       __asm__ volatile(                                           \
1800          VALGRIND_CFI_PROLOGUE                                    \
1801          "subq $128,%%rsp\n\t"                                    \
1802          "pushq 96(%%rax)\n\t"                                    \
1803          "pushq 88(%%rax)\n\t"                                    \
1804          "pushq 80(%%rax)\n\t"                                    \
1805          "pushq 72(%%rax)\n\t"                                    \
1806          "pushq 64(%%rax)\n\t"                                    \
1807          "pushq 56(%%rax)\n\t"                                    \
1808          "movq 48(%%rax), %%r9\n\t"                               \
1809          "movq 40(%%rax), %%r8\n\t"                               \
1810          "movq 32(%%rax), %%rcx\n\t"                              \
1811          "movq 24(%%rax), %%rdx\n\t"                              \
1812          "movq 16(%%rax), %%rsi\n\t"                              \
1813          "movq 8(%%rax), %%rdi\n\t"                               \
1814          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1815          VALGRIND_CALL_NOREDIR_RAX                                \
1816          "addq $48, %%rsp\n"                                      \
1817          "addq $128,%%rsp\n\t"                                    \
1818          VALGRIND_CFI_EPILOGUE                                    \
1819          : /*out*/   "=a" (_res)                                  \
1820          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER            \
1821          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r15"   \
1822       );                                                          \
1823       lval = (__typeof__(lval)) _res;                             \
1824    } while (0)
1825
1826 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin */
1827
1828 /* ------------------------ ppc32-linux ------------------------ */
1829
1830 #if defined(PLAT_ppc32_linux)
1831
1832 /* This is useful for finding out about the on-stack stuff:
1833
1834    extern int f9  ( int,int,int,int,int,int,int,int,int );
1835    extern int f10 ( int,int,int,int,int,int,int,int,int,int );
1836    extern int f11 ( int,int,int,int,int,int,int,int,int,int,int );
1837    extern int f12 ( int,int,int,int,int,int,int,int,int,int,int,int );
1838
1839    int g9 ( void ) {
1840       return f9(11,22,33,44,55,66,77,88,99);
1841    }
1842    int g10 ( void ) {
1843       return f10(11,22,33,44,55,66,77,88,99,110);
1844    }
1845    int g11 ( void ) {
1846       return f11(11,22,33,44,55,66,77,88,99,110,121);
1847    }
1848    int g12 ( void ) {
1849       return f12(11,22,33,44,55,66,77,88,99,110,121,132);
1850    }
1851 */
1852
1853 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
1854
1855 /* These regs are trashed by the hidden call. */
1856 #define __CALLER_SAVED_REGS                                       \
1857    "lr", "ctr", "xer",                                            \
1858    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
1859    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
1860    "r11", "r12", "r13"
1861
1862 /* These CALL_FN_ macros assume that on ppc32-linux,
1863    sizeof(unsigned long) == 4. */
1864
1865 #define CALL_FN_W_v(lval, orig)                                   \
1866    do {                                                           \
1867       volatile OrigFn        _orig = (orig);                      \
1868       volatile unsigned long _argvec[1];                          \
1869       volatile unsigned long _res;                                \
1870       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1871       __asm__ volatile(                                           \
1872          "mr 11,%1\n\t"                                           \
1873          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1874          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1875          "mr %0,3"                                                \
1876          : /*out*/   "=r" (_res)                                  \
1877          : /*in*/    "r" (&_argvec[0])                            \
1878          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1879       );                                                          \
1880       lval = (__typeof__(lval)) _res;                             \
1881    } while (0)
1882
1883 #define CALL_FN_W_W(lval, orig, arg1)                             \
1884    do {                                                           \
1885       volatile OrigFn        _orig = (orig);                      \
1886       volatile unsigned long _argvec[2];                          \
1887       volatile unsigned long _res;                                \
1888       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1889       _argvec[1] = (unsigned long)arg1;                           \
1890       __asm__ volatile(                                           \
1891          "mr 11,%1\n\t"                                           \
1892          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1893          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1894          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1895          "mr %0,3"                                                \
1896          : /*out*/   "=r" (_res)                                  \
1897          : /*in*/    "r" (&_argvec[0])                            \
1898          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1899       );                                                          \
1900       lval = (__typeof__(lval)) _res;                             \
1901    } while (0)
1902
1903 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
1904    do {                                                           \
1905       volatile OrigFn        _orig = (orig);                      \
1906       volatile unsigned long _argvec[3];                          \
1907       volatile unsigned long _res;                                \
1908       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1909       _argvec[1] = (unsigned long)arg1;                           \
1910       _argvec[2] = (unsigned long)arg2;                           \
1911       __asm__ volatile(                                           \
1912          "mr 11,%1\n\t"                                           \
1913          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1914          "lwz 4,8(11)\n\t"                                        \
1915          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1916          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1917          "mr %0,3"                                                \
1918          : /*out*/   "=r" (_res)                                  \
1919          : /*in*/    "r" (&_argvec[0])                            \
1920          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1921       );                                                          \
1922       lval = (__typeof__(lval)) _res;                             \
1923    } while (0)
1924
1925 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
1926    do {                                                           \
1927       volatile OrigFn        _orig = (orig);                      \
1928       volatile unsigned long _argvec[4];                          \
1929       volatile unsigned long _res;                                \
1930       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1931       _argvec[1] = (unsigned long)arg1;                           \
1932       _argvec[2] = (unsigned long)arg2;                           \
1933       _argvec[3] = (unsigned long)arg3;                           \
1934       __asm__ volatile(                                           \
1935          "mr 11,%1\n\t"                                           \
1936          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1937          "lwz 4,8(11)\n\t"                                        \
1938          "lwz 5,12(11)\n\t"                                       \
1939          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1940          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1941          "mr %0,3"                                                \
1942          : /*out*/   "=r" (_res)                                  \
1943          : /*in*/    "r" (&_argvec[0])                            \
1944          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1945       );                                                          \
1946       lval = (__typeof__(lval)) _res;                             \
1947    } while (0)
1948
1949 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
1950    do {                                                           \
1951       volatile OrigFn        _orig = (orig);                      \
1952       volatile unsigned long _argvec[5];                          \
1953       volatile unsigned long _res;                                \
1954       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1955       _argvec[1] = (unsigned long)arg1;                           \
1956       _argvec[2] = (unsigned long)arg2;                           \
1957       _argvec[3] = (unsigned long)arg3;                           \
1958       _argvec[4] = (unsigned long)arg4;                           \
1959       __asm__ volatile(                                           \
1960          "mr 11,%1\n\t"                                           \
1961          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1962          "lwz 4,8(11)\n\t"                                        \
1963          "lwz 5,12(11)\n\t"                                       \
1964          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1965          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1966          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1967          "mr %0,3"                                                \
1968          : /*out*/   "=r" (_res)                                  \
1969          : /*in*/    "r" (&_argvec[0])                            \
1970          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1971       );                                                          \
1972       lval = (__typeof__(lval)) _res;                             \
1973    } while (0)
1974
1975 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
1976    do {                                                           \
1977       volatile OrigFn        _orig = (orig);                      \
1978       volatile unsigned long _argvec[6];                          \
1979       volatile unsigned long _res;                                \
1980       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1981       _argvec[1] = (unsigned long)arg1;                           \
1982       _argvec[2] = (unsigned long)arg2;                           \
1983       _argvec[3] = (unsigned long)arg3;                           \
1984       _argvec[4] = (unsigned long)arg4;                           \
1985       _argvec[5] = (unsigned long)arg5;                           \
1986       __asm__ volatile(                                           \
1987          "mr 11,%1\n\t"                                           \
1988          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1989          "lwz 4,8(11)\n\t"                                        \
1990          "lwz 5,12(11)\n\t"                                       \
1991          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1992          "lwz 7,20(11)\n\t"                                       \
1993          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1994          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1995          "mr %0,3"                                                \
1996          : /*out*/   "=r" (_res)                                  \
1997          : /*in*/    "r" (&_argvec[0])                            \
1998          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1999       );                                                          \
2000       lval = (__typeof__(lval)) _res;                             \
2001    } while (0)
2002
2003 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
2004    do {                                                           \
2005       volatile OrigFn        _orig = (orig);                      \
2006       volatile unsigned long _argvec[7];                          \
2007       volatile unsigned long _res;                                \
2008       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2009       _argvec[1] = (unsigned long)arg1;                           \
2010       _argvec[2] = (unsigned long)arg2;                           \
2011       _argvec[3] = (unsigned long)arg3;                           \
2012       _argvec[4] = (unsigned long)arg4;                           \
2013       _argvec[5] = (unsigned long)arg5;                           \
2014       _argvec[6] = (unsigned long)arg6;                           \
2015       __asm__ volatile(                                           \
2016          "mr 11,%1\n\t"                                           \
2017          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2018          "lwz 4,8(11)\n\t"                                        \
2019          "lwz 5,12(11)\n\t"                                       \
2020          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2021          "lwz 7,20(11)\n\t"                                       \
2022          "lwz 8,24(11)\n\t"                                       \
2023          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2024          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2025          "mr %0,3"                                                \
2026          : /*out*/   "=r" (_res)                                  \
2027          : /*in*/    "r" (&_argvec[0])                            \
2028          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2029       );                                                          \
2030       lval = (__typeof__(lval)) _res;                             \
2031    } while (0)
2032
2033 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2034                                  arg7)                            \
2035    do {                                                           \
2036       volatile OrigFn        _orig = (orig);                      \
2037       volatile unsigned long _argvec[8];                          \
2038       volatile unsigned long _res;                                \
2039       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2040       _argvec[1] = (unsigned long)arg1;                           \
2041       _argvec[2] = (unsigned long)arg2;                           \
2042       _argvec[3] = (unsigned long)arg3;                           \
2043       _argvec[4] = (unsigned long)arg4;                           \
2044       _argvec[5] = (unsigned long)arg5;                           \
2045       _argvec[6] = (unsigned long)arg6;                           \
2046       _argvec[7] = (unsigned long)arg7;                           \
2047       __asm__ volatile(                                           \
2048          "mr 11,%1\n\t"                                           \
2049          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2050          "lwz 4,8(11)\n\t"                                        \
2051          "lwz 5,12(11)\n\t"                                       \
2052          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2053          "lwz 7,20(11)\n\t"                                       \
2054          "lwz 8,24(11)\n\t"                                       \
2055          "lwz 9,28(11)\n\t"                                       \
2056          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2057          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2058          "mr %0,3"                                                \
2059          : /*out*/   "=r" (_res)                                  \
2060          : /*in*/    "r" (&_argvec[0])                            \
2061          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2062       );                                                          \
2063       lval = (__typeof__(lval)) _res;                             \
2064    } while (0)
2065
2066 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2067                                  arg7,arg8)                       \
2068    do {                                                           \
2069       volatile OrigFn        _orig = (orig);                      \
2070       volatile unsigned long _argvec[9];                          \
2071       volatile unsigned long _res;                                \
2072       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2073       _argvec[1] = (unsigned long)arg1;                           \
2074       _argvec[2] = (unsigned long)arg2;                           \
2075       _argvec[3] = (unsigned long)arg3;                           \
2076       _argvec[4] = (unsigned long)arg4;                           \
2077       _argvec[5] = (unsigned long)arg5;                           \
2078       _argvec[6] = (unsigned long)arg6;                           \
2079       _argvec[7] = (unsigned long)arg7;                           \
2080       _argvec[8] = (unsigned long)arg8;                           \
2081       __asm__ volatile(                                           \
2082          "mr 11,%1\n\t"                                           \
2083          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2084          "lwz 4,8(11)\n\t"                                        \
2085          "lwz 5,12(11)\n\t"                                       \
2086          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2087          "lwz 7,20(11)\n\t"                                       \
2088          "lwz 8,24(11)\n\t"                                       \
2089          "lwz 9,28(11)\n\t"                                       \
2090          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2091          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2092          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2093          "mr %0,3"                                                \
2094          : /*out*/   "=r" (_res)                                  \
2095          : /*in*/    "r" (&_argvec[0])                            \
2096          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2097       );                                                          \
2098       lval = (__typeof__(lval)) _res;                             \
2099    } while (0)
2100
2101 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2102                                  arg7,arg8,arg9)                  \
2103    do {                                                           \
2104       volatile OrigFn        _orig = (orig);                      \
2105       volatile unsigned long _argvec[10];                         \
2106       volatile unsigned long _res;                                \
2107       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2108       _argvec[1] = (unsigned long)arg1;                           \
2109       _argvec[2] = (unsigned long)arg2;                           \
2110       _argvec[3] = (unsigned long)arg3;                           \
2111       _argvec[4] = (unsigned long)arg4;                           \
2112       _argvec[5] = (unsigned long)arg5;                           \
2113       _argvec[6] = (unsigned long)arg6;                           \
2114       _argvec[7] = (unsigned long)arg7;                           \
2115       _argvec[8] = (unsigned long)arg8;                           \
2116       _argvec[9] = (unsigned long)arg9;                           \
2117       __asm__ volatile(                                           \
2118          "mr 11,%1\n\t"                                           \
2119          "addi 1,1,-16\n\t"                                       \
2120          /* arg9 */                                               \
2121          "lwz 3,36(11)\n\t"                                       \
2122          "stw 3,8(1)\n\t"                                         \
2123          /* args1-8 */                                            \
2124          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2125          "lwz 4,8(11)\n\t"                                        \
2126          "lwz 5,12(11)\n\t"                                       \
2127          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2128          "lwz 7,20(11)\n\t"                                       \
2129          "lwz 8,24(11)\n\t"                                       \
2130          "lwz 9,28(11)\n\t"                                       \
2131          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2132          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2133          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2134          "addi 1,1,16\n\t"                                        \
2135          "mr %0,3"                                                \
2136          : /*out*/   "=r" (_res)                                  \
2137          : /*in*/    "r" (&_argvec[0])                            \
2138          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2139       );                                                          \
2140       lval = (__typeof__(lval)) _res;                             \
2141    } while (0)
2142
2143 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2144                                   arg7,arg8,arg9,arg10)           \
2145    do {                                                           \
2146       volatile OrigFn        _orig = (orig);                      \
2147       volatile unsigned long _argvec[11];                         \
2148       volatile unsigned long _res;                                \
2149       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2150       _argvec[1] = (unsigned long)arg1;                           \
2151       _argvec[2] = (unsigned long)arg2;                           \
2152       _argvec[3] = (unsigned long)arg3;                           \
2153       _argvec[4] = (unsigned long)arg4;                           \
2154       _argvec[5] = (unsigned long)arg5;                           \
2155       _argvec[6] = (unsigned long)arg6;                           \
2156       _argvec[7] = (unsigned long)arg7;                           \
2157       _argvec[8] = (unsigned long)arg8;                           \
2158       _argvec[9] = (unsigned long)arg9;                           \
2159       _argvec[10] = (unsigned long)arg10;                         \
2160       __asm__ volatile(                                           \
2161          "mr 11,%1\n\t"                                           \
2162          "addi 1,1,-16\n\t"                                       \
2163          /* arg10 */                                              \
2164          "lwz 3,40(11)\n\t"                                       \
2165          "stw 3,12(1)\n\t"                                        \
2166          /* arg9 */                                               \
2167          "lwz 3,36(11)\n\t"                                       \
2168          "stw 3,8(1)\n\t"                                         \
2169          /* args1-8 */                                            \
2170          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2171          "lwz 4,8(11)\n\t"                                        \
2172          "lwz 5,12(11)\n\t"                                       \
2173          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2174          "lwz 7,20(11)\n\t"                                       \
2175          "lwz 8,24(11)\n\t"                                       \
2176          "lwz 9,28(11)\n\t"                                       \
2177          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2178          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2179          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2180          "addi 1,1,16\n\t"                                        \
2181          "mr %0,3"                                                \
2182          : /*out*/   "=r" (_res)                                  \
2183          : /*in*/    "r" (&_argvec[0])                            \
2184          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2185       );                                                          \
2186       lval = (__typeof__(lval)) _res;                             \
2187    } while (0)
2188
2189 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2190                                   arg7,arg8,arg9,arg10,arg11)     \
2191    do {                                                           \
2192       volatile OrigFn        _orig = (orig);                      \
2193       volatile unsigned long _argvec[12];                         \
2194       volatile unsigned long _res;                                \
2195       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2196       _argvec[1] = (unsigned long)arg1;                           \
2197       _argvec[2] = (unsigned long)arg2;                           \
2198       _argvec[3] = (unsigned long)arg3;                           \
2199       _argvec[4] = (unsigned long)arg4;                           \
2200       _argvec[5] = (unsigned long)arg5;                           \
2201       _argvec[6] = (unsigned long)arg6;                           \
2202       _argvec[7] = (unsigned long)arg7;                           \
2203       _argvec[8] = (unsigned long)arg8;                           \
2204       _argvec[9] = (unsigned long)arg9;                           \
2205       _argvec[10] = (unsigned long)arg10;                         \
2206       _argvec[11] = (unsigned long)arg11;                         \
2207       __asm__ volatile(                                           \
2208          "mr 11,%1\n\t"                                           \
2209          "addi 1,1,-32\n\t"                                       \
2210          /* arg11 */                                              \
2211          "lwz 3,44(11)\n\t"                                       \
2212          "stw 3,16(1)\n\t"                                        \
2213          /* arg10 */                                              \
2214          "lwz 3,40(11)\n\t"                                       \
2215          "stw 3,12(1)\n\t"                                        \
2216          /* arg9 */                                               \
2217          "lwz 3,36(11)\n\t"                                       \
2218          "stw 3,8(1)\n\t"                                         \
2219          /* args1-8 */                                            \
2220          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2221          "lwz 4,8(11)\n\t"                                        \
2222          "lwz 5,12(11)\n\t"                                       \
2223          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2224          "lwz 7,20(11)\n\t"                                       \
2225          "lwz 8,24(11)\n\t"                                       \
2226          "lwz 9,28(11)\n\t"                                       \
2227          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2228          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2229          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2230          "addi 1,1,32\n\t"                                        \
2231          "mr %0,3"                                                \
2232          : /*out*/   "=r" (_res)                                  \
2233          : /*in*/    "r" (&_argvec[0])                            \
2234          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2235       );                                                          \
2236       lval = (__typeof__(lval)) _res;                             \
2237    } while (0)
2238
2239 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2240                                 arg7,arg8,arg9,arg10,arg11,arg12) \
2241    do {                                                           \
2242       volatile OrigFn        _orig = (orig);                      \
2243       volatile unsigned long _argvec[13];                         \
2244       volatile unsigned long _res;                                \
2245       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2246       _argvec[1] = (unsigned long)arg1;                           \
2247       _argvec[2] = (unsigned long)arg2;                           \
2248       _argvec[3] = (unsigned long)arg3;                           \
2249       _argvec[4] = (unsigned long)arg4;                           \
2250       _argvec[5] = (unsigned long)arg5;                           \
2251       _argvec[6] = (unsigned long)arg6;                           \
2252       _argvec[7] = (unsigned long)arg7;                           \
2253       _argvec[8] = (unsigned long)arg8;                           \
2254       _argvec[9] = (unsigned long)arg9;                           \
2255       _argvec[10] = (unsigned long)arg10;                         \
2256       _argvec[11] = (unsigned long)arg11;                         \
2257       _argvec[12] = (unsigned long)arg12;                         \
2258       __asm__ volatile(                                           \
2259          "mr 11,%1\n\t"                                           \
2260          "addi 1,1,-32\n\t"                                       \
2261          /* arg12 */                                              \
2262          "lwz 3,48(11)\n\t"                                       \
2263          "stw 3,20(1)\n\t"                                        \
2264          /* arg11 */                                              \
2265          "lwz 3,44(11)\n\t"                                       \
2266          "stw 3,16(1)\n\t"                                        \
2267          /* arg10 */                                              \
2268          "lwz 3,40(11)\n\t"                                       \
2269          "stw 3,12(1)\n\t"                                        \
2270          /* arg9 */                                               \
2271          "lwz 3,36(11)\n\t"                                       \
2272          "stw 3,8(1)\n\t"                                         \
2273          /* args1-8 */                                            \
2274          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2275          "lwz 4,8(11)\n\t"                                        \
2276          "lwz 5,12(11)\n\t"                                       \
2277          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2278          "lwz 7,20(11)\n\t"                                       \
2279          "lwz 8,24(11)\n\t"                                       \
2280          "lwz 9,28(11)\n\t"                                       \
2281          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2282          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2283          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2284          "addi 1,1,32\n\t"                                        \
2285          "mr %0,3"                                                \
2286          : /*out*/   "=r" (_res)                                  \
2287          : /*in*/    "r" (&_argvec[0])                            \
2288          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2289       );                                                          \
2290       lval = (__typeof__(lval)) _res;                             \
2291    } while (0)
2292
2293 #endif /* PLAT_ppc32_linux */
2294
2295 /* ------------------------ ppc64-linux ------------------------ */
2296
2297 #if defined(PLAT_ppc64_linux)
2298
2299 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
2300
2301 /* These regs are trashed by the hidden call. */
2302 #define __CALLER_SAVED_REGS                                       \
2303    "lr", "ctr", "xer",                                            \
2304    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
2305    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
2306    "r11", "r12", "r13"
2307
2308 /* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned
2309    long) == 8. */
2310
2311 #define CALL_FN_W_v(lval, orig)                                   \
2312    do {                                                           \
2313       volatile OrigFn        _orig = (orig);                      \
2314       volatile unsigned long _argvec[3+0];                        \
2315       volatile unsigned long _res;                                \
2316       /* _argvec[0] holds current r2 across the call */           \
2317       _argvec[1] = (unsigned long)_orig.r2;                       \
2318       _argvec[2] = (unsigned long)_orig.nraddr;                   \
2319       __asm__ volatile(                                           \
2320          "mr 11,%1\n\t"                                           \
2321          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2322          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2323          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2324          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2325          "mr 11,%1\n\t"                                           \
2326          "mr %0,3\n\t"                                            \
2327          "ld 2,-16(11)" /* restore tocptr */                      \
2328          : /*out*/   "=r" (_res)                                  \
2329          : /*in*/    "r" (&_argvec[2])                            \
2330          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2331       );                                                          \
2332       lval = (__typeof__(lval)) _res;                             \
2333    } while (0)
2334
2335 #define CALL_FN_W_W(lval, orig, arg1)                             \
2336    do {                                                           \
2337       volatile OrigFn        _orig = (orig);                      \
2338       volatile unsigned long _argvec[3+1];                        \
2339       volatile unsigned long _res;                                \
2340       /* _argvec[0] holds current r2 across the call */           \
2341       _argvec[1]   = (unsigned long)_orig.r2;                     \
2342       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2343       _argvec[2+1] = (unsigned long)arg1;                         \
2344       __asm__ volatile(                                           \
2345          "mr 11,%1\n\t"                                           \
2346          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2347          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2348          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2349          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2350          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2351          "mr 11,%1\n\t"                                           \
2352          "mr %0,3\n\t"                                            \
2353          "ld 2,-16(11)" /* restore tocptr */                      \
2354          : /*out*/   "=r" (_res)                                  \
2355          : /*in*/    "r" (&_argvec[2])                            \
2356          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2357       );                                                          \
2358       lval = (__typeof__(lval)) _res;                             \
2359    } while (0)
2360
2361 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
2362    do {                                                           \
2363       volatile OrigFn        _orig = (orig);                      \
2364       volatile unsigned long _argvec[3+2];                        \
2365       volatile unsigned long _res;                                \
2366       /* _argvec[0] holds current r2 across the call */           \
2367       _argvec[1]   = (unsigned long)_orig.r2;                     \
2368       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2369       _argvec[2+1] = (unsigned long)arg1;                         \
2370       _argvec[2+2] = (unsigned long)arg2;                         \
2371       __asm__ volatile(                                           \
2372          "mr 11,%1\n\t"                                           \
2373          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2374          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2375          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2376          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2377          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2378          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2379          "mr 11,%1\n\t"                                           \
2380          "mr %0,3\n\t"                                            \
2381          "ld 2,-16(11)" /* restore tocptr */                      \
2382          : /*out*/   "=r" (_res)                                  \
2383          : /*in*/    "r" (&_argvec[2])                            \
2384          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2385       );                                                          \
2386       lval = (__typeof__(lval)) _res;                             \
2387    } while (0)
2388
2389 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
2390    do {                                                           \
2391       volatile OrigFn        _orig = (orig);                      \
2392       volatile unsigned long _argvec[3+3];                        \
2393       volatile unsigned long _res;                                \
2394       /* _argvec[0] holds current r2 across the call */           \
2395       _argvec[1]   = (unsigned long)_orig.r2;                     \
2396       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2397       _argvec[2+1] = (unsigned long)arg1;                         \
2398       _argvec[2+2] = (unsigned long)arg2;                         \
2399       _argvec[2+3] = (unsigned long)arg3;                         \
2400       __asm__ volatile(                                           \
2401          "mr 11,%1\n\t"                                           \
2402          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2403          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2404          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2405          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2406          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2407          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2408          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2409          "mr 11,%1\n\t"                                           \
2410          "mr %0,3\n\t"                                            \
2411          "ld 2,-16(11)" /* restore tocptr */                      \
2412          : /*out*/   "=r" (_res)                                  \
2413          : /*in*/    "r" (&_argvec[2])                            \
2414          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2415       );                                                          \
2416       lval = (__typeof__(lval)) _res;                             \
2417    } while (0)
2418
2419 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
2420    do {                                                           \
2421       volatile OrigFn        _orig = (orig);                      \
2422       volatile unsigned long _argvec[3+4];                        \
2423       volatile unsigned long _res;                                \
2424       /* _argvec[0] holds current r2 across the call */           \
2425       _argvec[1]   = (unsigned long)_orig.r2;                     \
2426       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2427       _argvec[2+1] = (unsigned long)arg1;                         \
2428       _argvec[2+2] = (unsigned long)arg2;                         \
2429       _argvec[2+3] = (unsigned long)arg3;                         \
2430       _argvec[2+4] = (unsigned long)arg4;                         \
2431       __asm__ volatile(                                           \
2432          "mr 11,%1\n\t"                                           \
2433          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2434          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2435          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2436          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2437          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2438          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2439          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2440          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2441          "mr 11,%1\n\t"                                           \
2442          "mr %0,3\n\t"                                            \
2443          "ld 2,-16(11)" /* restore tocptr */                      \
2444          : /*out*/   "=r" (_res)                                  \
2445          : /*in*/    "r" (&_argvec[2])                            \
2446          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2447       );                                                          \
2448       lval = (__typeof__(lval)) _res;                             \
2449    } while (0)
2450
2451 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
2452    do {                                                           \
2453       volatile OrigFn        _orig = (orig);                      \
2454       volatile unsigned long _argvec[3+5];                        \
2455       volatile unsigned long _res;                                \
2456       /* _argvec[0] holds current r2 across the call */           \
2457       _argvec[1]   = (unsigned long)_orig.r2;                     \
2458       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2459       _argvec[2+1] = (unsigned long)arg1;                         \
2460       _argvec[2+2] = (unsigned long)arg2;                         \
2461       _argvec[2+3] = (unsigned long)arg3;                         \
2462       _argvec[2+4] = (unsigned long)arg4;                         \
2463       _argvec[2+5] = (unsigned long)arg5;                         \
2464       __asm__ volatile(                                           \
2465          "mr 11,%1\n\t"                                           \
2466          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2467          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2468          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2469          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2470          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2471          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2472          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2473          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2474          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2475          "mr 11,%1\n\t"                                           \
2476          "mr %0,3\n\t"                                            \
2477          "ld 2,-16(11)" /* restore tocptr */                      \
2478          : /*out*/   "=r" (_res)                                  \
2479          : /*in*/    "r" (&_argvec[2])                            \
2480          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2481       );                                                          \
2482       lval = (__typeof__(lval)) _res;                             \
2483    } while (0)
2484
2485 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
2486    do {                                                           \
2487       volatile OrigFn        _orig = (orig);                      \
2488       volatile unsigned long _argvec[3+6];                        \
2489       volatile unsigned long _res;                                \
2490       /* _argvec[0] holds current r2 across the call */           \
2491       _argvec[1]   = (unsigned long)_orig.r2;                     \
2492       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2493       _argvec[2+1] = (unsigned long)arg1;                         \
2494       _argvec[2+2] = (unsigned long)arg2;                         \
2495       _argvec[2+3] = (unsigned long)arg3;                         \
2496       _argvec[2+4] = (unsigned long)arg4;                         \
2497       _argvec[2+5] = (unsigned long)arg5;                         \
2498       _argvec[2+6] = (unsigned long)arg6;                         \
2499       __asm__ volatile(                                           \
2500          "mr 11,%1\n\t"                                           \
2501          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2502          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2503          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2504          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2505          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2506          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2507          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2508          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2509          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2510          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2511          "mr 11,%1\n\t"                                           \
2512          "mr %0,3\n\t"                                            \
2513          "ld 2,-16(11)" /* restore tocptr */                      \
2514          : /*out*/   "=r" (_res)                                  \
2515          : /*in*/    "r" (&_argvec[2])                            \
2516          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2517       );                                                          \
2518       lval = (__typeof__(lval)) _res;                             \
2519    } while (0)
2520
2521 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2522                                  arg7)                            \
2523    do {                                                           \
2524       volatile OrigFn        _orig = (orig);                      \
2525       volatile unsigned long _argvec[3+7];                        \
2526       volatile unsigned long _res;                                \
2527       /* _argvec[0] holds current r2 across the call */           \
2528       _argvec[1]   = (unsigned long)_orig.r2;                     \
2529       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2530       _argvec[2+1] = (unsigned long)arg1;                         \
2531       _argvec[2+2] = (unsigned long)arg2;                         \
2532       _argvec[2+3] = (unsigned long)arg3;                         \
2533       _argvec[2+4] = (unsigned long)arg4;                         \
2534       _argvec[2+5] = (unsigned long)arg5;                         \
2535       _argvec[2+6] = (unsigned long)arg6;                         \
2536       _argvec[2+7] = (unsigned long)arg7;                         \
2537       __asm__ volatile(                                           \
2538          "mr 11,%1\n\t"                                           \
2539          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2540          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2541          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2542          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2543          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2544          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2545          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2546          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2547          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2548          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2549          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2550          "mr 11,%1\n\t"                                           \
2551          "mr %0,3\n\t"                                            \
2552          "ld 2,-16(11)" /* restore tocptr */                      \
2553          : /*out*/   "=r" (_res)                                  \
2554          : /*in*/    "r" (&_argvec[2])                            \
2555          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2556       );                                                          \
2557       lval = (__typeof__(lval)) _res;                             \
2558    } while (0)
2559
2560 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2561                                  arg7,arg8)                       \
2562    do {                                                           \
2563       volatile OrigFn        _orig = (orig);                      \
2564       volatile unsigned long _argvec[3+8];                        \
2565       volatile unsigned long _res;                                \
2566       /* _argvec[0] holds current r2 across the call */           \
2567       _argvec[1]   = (unsigned long)_orig.r2;                     \
2568       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2569       _argvec[2+1] = (unsigned long)arg1;                         \
2570       _argvec[2+2] = (unsigned long)arg2;                         \
2571       _argvec[2+3] = (unsigned long)arg3;                         \
2572       _argvec[2+4] = (unsigned long)arg4;                         \
2573       _argvec[2+5] = (unsigned long)arg5;                         \
2574       _argvec[2+6] = (unsigned long)arg6;                         \
2575       _argvec[2+7] = (unsigned long)arg7;                         \
2576       _argvec[2+8] = (unsigned long)arg8;                         \
2577       __asm__ volatile(                                           \
2578          "mr 11,%1\n\t"                                           \
2579          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2580          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2581          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2582          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2583          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2584          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2585          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2586          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2587          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2588          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2589          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2590          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2591          "mr 11,%1\n\t"                                           \
2592          "mr %0,3\n\t"                                            \
2593          "ld 2,-16(11)" /* restore tocptr */                      \
2594          : /*out*/   "=r" (_res)                                  \
2595          : /*in*/    "r" (&_argvec[2])                            \
2596          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2597       );                                                          \
2598       lval = (__typeof__(lval)) _res;                             \
2599    } while (0)
2600
2601 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2602                                  arg7,arg8,arg9)                  \
2603    do {                                                           \
2604       volatile OrigFn        _orig = (orig);                      \
2605       volatile unsigned long _argvec[3+9];                        \
2606       volatile unsigned long _res;                                \
2607       /* _argvec[0] holds current r2 across the call */           \
2608       _argvec[1]   = (unsigned long)_orig.r2;                     \
2609       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2610       _argvec[2+1] = (unsigned long)arg1;                         \
2611       _argvec[2+2] = (unsigned long)arg2;                         \
2612       _argvec[2+3] = (unsigned long)arg3;                         \
2613       _argvec[2+4] = (unsigned long)arg4;                         \
2614       _argvec[2+5] = (unsigned long)arg5;                         \
2615       _argvec[2+6] = (unsigned long)arg6;                         \
2616       _argvec[2+7] = (unsigned long)arg7;                         \
2617       _argvec[2+8] = (unsigned long)arg8;                         \
2618       _argvec[2+9] = (unsigned long)arg9;                         \
2619       __asm__ volatile(                                           \
2620          "mr 11,%1\n\t"                                           \
2621          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2622          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2623          "addi 1,1,-128\n\t"  /* expand stack frame */            \
2624          /* arg9 */                                               \
2625          "ld  3,72(11)\n\t"                                       \
2626          "std 3,112(1)\n\t"                                       \
2627          /* args1-8 */                                            \
2628          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2629          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2630          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2631          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2632          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2633          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2634          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2635          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2636          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2637          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2638          "mr 11,%1\n\t"                                           \
2639          "mr %0,3\n\t"                                            \
2640          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2641          "addi 1,1,128"     /* restore frame */                   \
2642          : /*out*/   "=r" (_res)                                  \
2643          : /*in*/    "r" (&_argvec[2])                            \
2644          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2645       );                                                          \
2646       lval = (__typeof__(lval)) _res;                             \
2647    } while (0)
2648
2649 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2650                                   arg7,arg8,arg9,arg10)           \
2651    do {                                                           \
2652       volatile OrigFn        _orig = (orig);                      \
2653       volatile unsigned long _argvec[3+10];                       \
2654       volatile unsigned long _res;                                \
2655       /* _argvec[0] holds current r2 across the call */           \
2656       _argvec[1]   = (unsigned long)_orig.r2;                     \
2657       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2658       _argvec[2+1] = (unsigned long)arg1;                         \
2659       _argvec[2+2] = (unsigned long)arg2;                         \
2660       _argvec[2+3] = (unsigned long)arg3;                         \
2661       _argvec[2+4] = (unsigned long)arg4;                         \
2662       _argvec[2+5] = (unsigned long)arg5;                         \
2663       _argvec[2+6] = (unsigned long)arg6;                         \
2664       _argvec[2+7] = (unsigned long)arg7;                         \
2665       _argvec[2+8] = (unsigned long)arg8;                         \
2666       _argvec[2+9] = (unsigned long)arg9;                         \
2667       _argvec[2+10] = (unsigned long)arg10;                       \
2668       __asm__ volatile(                                           \
2669          "mr 11,%1\n\t"                                           \
2670          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2671          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2672          "addi 1,1,-128\n\t"  /* expand stack frame */            \
2673          /* arg10 */                                              \
2674          "ld  3,80(11)\n\t"                                       \
2675          "std 3,120(1)\n\t"                                       \
2676          /* arg9 */                                               \
2677          "ld  3,72(11)\n\t"                                       \
2678          "std 3,112(1)\n\t"                                       \
2679          /* args1-8 */                                            \
2680          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2681          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2682          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2683          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2684          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2685          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2686          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2687          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2688          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2689          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2690          "mr 11,%1\n\t"                                           \
2691          "mr %0,3\n\t"                                            \
2692          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2693          "addi 1,1,128"     /* restore frame */                   \
2694          : /*out*/   "=r" (_res)                                  \
2695          : /*in*/    "r" (&_argvec[2])                            \
2696          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2697       );                                                          \
2698       lval = (__typeof__(lval)) _res;                             \
2699    } while (0)
2700
2701 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2702                                   arg7,arg8,arg9,arg10,arg11)     \
2703    do {                                                           \
2704       volatile OrigFn        _orig = (orig);                      \
2705       volatile unsigned long _argvec[3+11];                       \
2706       volatile unsigned long _res;                                \
2707       /* _argvec[0] holds current r2 across the call */           \
2708       _argvec[1]   = (unsigned long)_orig.r2;                     \
2709       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2710       _argvec[2+1] = (unsigned long)arg1;                         \
2711       _argvec[2+2] = (unsigned long)arg2;                         \
2712       _argvec[2+3] = (unsigned long)arg3;                         \
2713       _argvec[2+4] = (unsigned long)arg4;                         \
2714       _argvec[2+5] = (unsigned long)arg5;                         \
2715       _argvec[2+6] = (unsigned long)arg6;                         \
2716       _argvec[2+7] = (unsigned long)arg7;                         \
2717       _argvec[2+8] = (unsigned long)arg8;                         \
2718       _argvec[2+9] = (unsigned long)arg9;                         \
2719       _argvec[2+10] = (unsigned long)arg10;                       \
2720       _argvec[2+11] = (unsigned long)arg11;                       \
2721       __asm__ volatile(                                           \
2722          "mr 11,%1\n\t"                                           \
2723          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2724          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2725          "addi 1,1,-144\n\t"  /* expand stack frame */            \
2726          /* arg11 */                                              \
2727          "ld  3,88(11)\n\t"                                       \
2728          "std 3,128(1)\n\t"                                       \
2729          /* arg10 */                                              \
2730          "ld  3,80(11)\n\t"                                       \
2731          "std 3,120(1)\n\t"                                       \
2732          /* arg9 */                                               \
2733          "ld  3,72(11)\n\t"                                       \
2734          "std 3,112(1)\n\t"                                       \
2735          /* args1-8 */                                            \
2736          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2737          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2738          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2739          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2740          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2741          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2742          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2743          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2744          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2745          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2746          "mr 11,%1\n\t"                                           \
2747          "mr %0,3\n\t"                                            \
2748          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2749          "addi 1,1,144"     /* restore frame */                   \
2750          : /*out*/   "=r" (_res)                                  \
2751          : /*in*/    "r" (&_argvec[2])                            \
2752          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2753       );                                                          \
2754       lval = (__typeof__(lval)) _res;                             \
2755    } while (0)
2756
2757 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2758                                 arg7,arg8,arg9,arg10,arg11,arg12) \
2759    do {                                                           \
2760       volatile OrigFn        _orig = (orig);                      \
2761       volatile unsigned long _argvec[3+12];                       \
2762       volatile unsigned long _res;                                \
2763       /* _argvec[0] holds current r2 across the call */           \
2764       _argvec[1]   = (unsigned long)_orig.r2;                     \
2765       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2766       _argvec[2+1] = (unsigned long)arg1;                         \
2767       _argvec[2+2] = (unsigned long)arg2;                         \
2768       _argvec[2+3] = (unsigned long)arg3;                         \
2769       _argvec[2+4] = (unsigned long)arg4;                         \
2770       _argvec[2+5] = (unsigned long)arg5;                         \
2771       _argvec[2+6] = (unsigned long)arg6;                         \
2772       _argvec[2+7] = (unsigned long)arg7;                         \
2773       _argvec[2+8] = (unsigned long)arg8;                         \
2774       _argvec[2+9] = (unsigned long)arg9;                         \
2775       _argvec[2+10] = (unsigned long)arg10;                       \
2776       _argvec[2+11] = (unsigned long)arg11;                       \
2777       _argvec[2+12] = (unsigned long)arg12;                       \
2778       __asm__ volatile(                                           \
2779          "mr 11,%1\n\t"                                           \
2780          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2781          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2782          "addi 1,1,-144\n\t"  /* expand stack frame */            \
2783          /* arg12 */                                              \
2784          "ld  3,96(11)\n\t"                                       \
2785          "std 3,136(1)\n\t"                                       \
2786          /* arg11 */                                              \
2787          "ld  3,88(11)\n\t"                                       \
2788          "std 3,128(1)\n\t"                                       \
2789          /* arg10 */                                              \
2790          "ld  3,80(11)\n\t"                                       \
2791          "std 3,120(1)\n\t"                                       \
2792          /* arg9 */                                               \
2793          "ld  3,72(11)\n\t"                                       \
2794          "std 3,112(1)\n\t"                                       \
2795          /* args1-8 */                                            \
2796          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2797          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2798          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2799          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2800          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2801          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2802          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2803          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2804          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2805          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2806          "mr 11,%1\n\t"                                           \
2807          "mr %0,3\n\t"                                            \
2808          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2809          "addi 1,1,144"     /* restore frame */                   \
2810          : /*out*/   "=r" (_res)                                  \
2811          : /*in*/    "r" (&_argvec[2])                            \
2812          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2813       );                                                          \
2814       lval = (__typeof__(lval)) _res;                             \
2815    } while (0)
2816
2817 #endif /* PLAT_ppc64_linux */
2818
2819 /* ------------------------- arm-linux ------------------------- */
2820
2821 #if defined(PLAT_arm_linux)
2822
2823 /* These regs are trashed by the hidden call. */
2824 #define __CALLER_SAVED_REGS "r0", "r1", "r2", "r3","r4","r14"
2825
2826 /* These CALL_FN_ macros assume that on arm-linux, sizeof(unsigned
2827    long) == 4. */
2828
2829 #define CALL_FN_W_v(lval, orig)                                   \
2830    do {                                                           \
2831       volatile OrigFn        _orig = (orig);                      \
2832       volatile unsigned long _argvec[1];                          \
2833       volatile unsigned long _res;                                \
2834       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2835       __asm__ volatile(                                           \
2836          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2837          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2838          "mov %0, r0\n"                                           \
2839          : /*out*/   "=r" (_res)                                  \
2840          : /*in*/    "0" (&_argvec[0])                            \
2841          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2842       );                                                          \
2843       lval = (__typeof__(lval)) _res;                             \
2844    } while (0)
2845
2846 #define CALL_FN_W_W(lval, orig, arg1)                             \
2847    do {                                                           \
2848       volatile OrigFn        _orig = (orig);                      \
2849       volatile unsigned long _argvec[2];                          \
2850       volatile unsigned long _res;                                \
2851       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2852       _argvec[1] = (unsigned long)(arg1);                         \
2853       __asm__ volatile(                                           \
2854          "ldr r0, [%1, #4] \n\t"                                  \
2855          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2856          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2857          "mov %0, r0\n"                                           \
2858          : /*out*/   "=r" (_res)                                  \
2859          : /*in*/    "0" (&_argvec[0])                            \
2860          : /*trash*/ "cc", "memory",  __CALLER_SAVED_REGS         \
2861       );                                                          \
2862       lval = (__typeof__(lval)) _res;                             \
2863    } while (0)
2864
2865 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
2866    do {                                                           \
2867       volatile OrigFn        _orig = (orig);                      \
2868       volatile unsigned long _argvec[3];                          \
2869       volatile unsigned long _res;                                \
2870       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2871       _argvec[1] = (unsigned long)(arg1);                         \
2872       _argvec[2] = (unsigned long)(arg2);                         \
2873       __asm__ volatile(                                           \
2874          "ldr r0, [%1, #4] \n\t"                                  \
2875          "ldr r1, [%1, #8] \n\t"                                  \
2876          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2877          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2878          "mov %0, r0\n"                                           \
2879          : /*out*/   "=r" (_res)                                  \
2880          : /*in*/    "0" (&_argvec[0])                            \
2881          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2882       );                                                          \
2883       lval = (__typeof__(lval)) _res;                             \
2884    } while (0)
2885
2886 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
2887    do {                                                           \
2888       volatile OrigFn        _orig = (orig);                      \
2889       volatile unsigned long _argvec[4];                          \
2890       volatile unsigned long _res;                                \
2891       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2892       _argvec[1] = (unsigned long)(arg1);                         \
2893       _argvec[2] = (unsigned long)(arg2);                         \
2894       _argvec[3] = (unsigned long)(arg3);                         \
2895       __asm__ volatile(                                           \
2896          "ldr r0, [%1, #4] \n\t"                                  \
2897          "ldr r1, [%1, #8] \n\t"                                  \
2898          "ldr r2, [%1, #12] \n\t"                                 \
2899          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2900          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2901          "mov %0, r0\n"                                           \
2902          : /*out*/   "=r" (_res)                                  \
2903          : /*in*/    "0" (&_argvec[0])                            \
2904          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2905       );                                                          \
2906       lval = (__typeof__(lval)) _res;                             \
2907    } while (0)
2908
2909 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
2910    do {                                                           \
2911       volatile OrigFn        _orig = (orig);                      \
2912       volatile unsigned long _argvec[5];                          \
2913       volatile unsigned long _res;                                \
2914       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2915       _argvec[1] = (unsigned long)(arg1);                         \
2916       _argvec[2] = (unsigned long)(arg2);                         \
2917       _argvec[3] = (unsigned long)(arg3);                         \
2918       _argvec[4] = (unsigned long)(arg4);                         \
2919       __asm__ volatile(                                           \
2920          "ldr r0, [%1, #4] \n\t"                                  \
2921          "ldr r1, [%1, #8] \n\t"                                  \
2922          "ldr r2, [%1, #12] \n\t"                                 \
2923          "ldr r3, [%1, #16] \n\t"                                 \
2924          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2925          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2926          "mov %0, r0"                                             \
2927          : /*out*/   "=r" (_res)                                  \
2928          : /*in*/    "0" (&_argvec[0])                            \
2929          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2930       );                                                          \
2931       lval = (__typeof__(lval)) _res;                             \
2932    } while (0)
2933
2934 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
2935    do {                                                           \
2936       volatile OrigFn        _orig = (orig);                      \
2937       volatile unsigned long _argvec[6];                          \
2938       volatile unsigned long _res;                                \
2939       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2940       _argvec[1] = (unsigned long)(arg1);                         \
2941       _argvec[2] = (unsigned long)(arg2);                         \
2942       _argvec[3] = (unsigned long)(arg3);                         \
2943       _argvec[4] = (unsigned long)(arg4);                         \
2944       _argvec[5] = (unsigned long)(arg5);                         \
2945       __asm__ volatile(                                           \
2946          "ldr r0, [%1, #20] \n\t"                                 \
2947          "push {r0} \n\t"                                         \
2948          "ldr r0, [%1, #4] \n\t"                                  \
2949          "ldr r1, [%1, #8] \n\t"                                  \
2950          "ldr r2, [%1, #12] \n\t"                                 \
2951          "ldr r3, [%1, #16] \n\t"                                 \
2952          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2953          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2954          "add sp, sp, #4 \n\t"                                    \
2955          "mov %0, r0"                                             \
2956          : /*out*/   "=r" (_res)                                  \
2957          : /*in*/    "0" (&_argvec[0])                            \
2958          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2959       );                                                          \
2960       lval = (__typeof__(lval)) _res;                             \
2961    } while (0)
2962
2963 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
2964    do {                                                           \
2965       volatile OrigFn        _orig = (orig);                      \
2966       volatile unsigned long _argvec[7];                          \
2967       volatile unsigned long _res;                                \
2968       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2969       _argvec[1] = (unsigned long)(arg1);                         \
2970       _argvec[2] = (unsigned long)(arg2);                         \
2971       _argvec[3] = (unsigned long)(arg3);                         \
2972       _argvec[4] = (unsigned long)(arg4);                         \
2973       _argvec[5] = (unsigned long)(arg5);                         \
2974       _argvec[6] = (unsigned long)(arg6);                         \
2975       __asm__ volatile(                                           \
2976          "ldr r0, [%1, #20] \n\t"                                 \
2977          "ldr r1, [%1, #24] \n\t"                                 \
2978          "push {r0, r1} \n\t"                                     \
2979          "ldr r0, [%1, #4] \n\t"                                  \
2980          "ldr r1, [%1, #8] \n\t"                                  \
2981          "ldr r2, [%1, #12] \n\t"                                 \
2982          "ldr r3, [%1, #16] \n\t"                                 \
2983          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2984          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2985          "add sp, sp, #8 \n\t"                                    \
2986          "mov %0, r0"                                             \
2987          : /*out*/   "=r" (_res)                                  \
2988          : /*in*/    "0" (&_argvec[0])                            \
2989          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2990       );                                                          \
2991       lval = (__typeof__(lval)) _res;                             \
2992    } while (0)
2993
2994 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2995                                  arg7)                            \
2996    do {                                                           \
2997       volatile OrigFn        _orig = (orig);                      \
2998       volatile unsigned long _argvec[8];                          \
2999       volatile unsigned long _res;                                \
3000       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3001       _argvec[1] = (unsigned long)(arg1);                         \
3002       _argvec[2] = (unsigned long)(arg2);                         \
3003       _argvec[3] = (unsigned long)(arg3);                         \
3004       _argvec[4] = (unsigned long)(arg4);                         \
3005       _argvec[5] = (unsigned long)(arg5);                         \
3006       _argvec[6] = (unsigned long)(arg6);                         \
3007       _argvec[7] = (unsigned long)(arg7);                         \
3008       __asm__ volatile(                                           \
3009          "ldr r0, [%1, #20] \n\t"                                 \
3010          "ldr r1, [%1, #24] \n\t"                                 \
3011          "ldr r2, [%1, #28] \n\t"                                 \
3012          "push {r0, r1, r2} \n\t"                                 \
3013          "ldr r0, [%1, #4] \n\t"                                  \
3014          "ldr r1, [%1, #8] \n\t"                                  \
3015          "ldr r2, [%1, #12] \n\t"                                 \
3016          "ldr r3, [%1, #16] \n\t"                                 \
3017          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3018          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3019          "add sp, sp, #12 \n\t"                                   \
3020          "mov %0, r0"                                             \
3021          : /*out*/   "=r" (_res)                                  \
3022          : /*in*/    "0" (&_argvec[0])                            \
3023          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3024       );                                                          \
3025       lval = (__typeof__(lval)) _res;                             \
3026    } while (0)
3027
3028 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3029                                  arg7,arg8)                       \
3030    do {                                                           \
3031       volatile OrigFn        _orig = (orig);                      \
3032       volatile unsigned long _argvec[9];                          \
3033       volatile unsigned long _res;                                \
3034       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3035       _argvec[1] = (unsigned long)(arg1);                         \
3036       _argvec[2] = (unsigned long)(arg2);                         \
3037       _argvec[3] = (unsigned long)(arg3);                         \
3038       _argvec[4] = (unsigned long)(arg4);                         \
3039       _argvec[5] = (unsigned long)(arg5);                         \
3040       _argvec[6] = (unsigned long)(arg6);                         \
3041       _argvec[7] = (unsigned long)(arg7);                         \
3042       _argvec[8] = (unsigned long)(arg8);                         \
3043       __asm__ volatile(                                           \
3044          "ldr r0, [%1, #20] \n\t"                                 \
3045          "ldr r1, [%1, #24] \n\t"                                 \
3046          "ldr r2, [%1, #28] \n\t"                                 \
3047          "ldr r3, [%1, #32] \n\t"                                 \
3048          "push {r0, r1, r2, r3} \n\t"                             \
3049          "ldr r0, [%1, #4] \n\t"                                  \
3050          "ldr r1, [%1, #8] \n\t"                                  \
3051          "ldr r2, [%1, #12] \n\t"                                 \
3052          "ldr r3, [%1, #16] \n\t"                                 \
3053          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3054          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3055          "add sp, sp, #16 \n\t"                                   \
3056          "mov %0, r0"                                             \
3057          : /*out*/   "=r" (_res)                                  \
3058          : /*in*/    "0" (&_argvec[0])                            \
3059          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3060       );                                                          \
3061       lval = (__typeof__(lval)) _res;                             \
3062    } while (0)
3063
3064 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3065                                  arg7,arg8,arg9)                  \
3066    do {                                                           \
3067       volatile OrigFn        _orig = (orig);                      \
3068       volatile unsigned long _argvec[10];                         \
3069       volatile unsigned long _res;                                \
3070       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3071       _argvec[1] = (unsigned long)(arg1);                         \
3072       _argvec[2] = (unsigned long)(arg2);                         \
3073       _argvec[3] = (unsigned long)(arg3);                         \
3074       _argvec[4] = (unsigned long)(arg4);                         \
3075       _argvec[5] = (unsigned long)(arg5);                         \
3076       _argvec[6] = (unsigned long)(arg6);                         \
3077       _argvec[7] = (unsigned long)(arg7);                         \
3078       _argvec[8] = (unsigned long)(arg8);                         \
3079       _argvec[9] = (unsigned long)(arg9);                         \
3080       __asm__ volatile(                                           \
3081          "ldr r0, [%1, #20] \n\t"                                 \
3082          "ldr r1, [%1, #24] \n\t"                                 \
3083          "ldr r2, [%1, #28] \n\t"                                 \
3084          "ldr r3, [%1, #32] \n\t"                                 \
3085          "ldr r4, [%1, #36] \n\t"                                 \
3086          "push {r0, r1, r2, r3, r4} \n\t"                         \
3087          "ldr r0, [%1, #4] \n\t"                                  \
3088          "ldr r1, [%1, #8] \n\t"                                  \
3089          "ldr r2, [%1, #12] \n\t"                                 \
3090          "ldr r3, [%1, #16] \n\t"                                 \
3091          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3092          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3093          "add sp, sp, #20 \n\t"                                   \
3094          "mov %0, r0"                                             \
3095          : /*out*/   "=r" (_res)                                  \
3096          : /*in*/    "0" (&_argvec[0])                            \
3097          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3098       );                                                          \
3099       lval = (__typeof__(lval)) _res;                             \
3100    } while (0)
3101
3102 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3103                                   arg7,arg8,arg9,arg10)           \
3104    do {                                                           \
3105       volatile OrigFn        _orig = (orig);                      \
3106       volatile unsigned long _argvec[11];                         \
3107       volatile unsigned long _res;                                \
3108       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3109       _argvec[1] = (unsigned long)(arg1);                         \
3110       _argvec[2] = (unsigned long)(arg2);                         \
3111       _argvec[3] = (unsigned long)(arg3);                         \
3112       _argvec[4] = (unsigned long)(arg4);                         \
3113       _argvec[5] = (unsigned long)(arg5);                         \
3114       _argvec[6] = (unsigned long)(arg6);                         \
3115       _argvec[7] = (unsigned long)(arg7);                         \
3116       _argvec[8] = (unsigned long)(arg8);                         \
3117       _argvec[9] = (unsigned long)(arg9);                         \
3118       _argvec[10] = (unsigned long)(arg10);                       \
3119       __asm__ volatile(                                           \
3120          "ldr r0, [%1, #40] \n\t"                                 \
3121          "push {r0} \n\t"                                         \
3122          "ldr r0, [%1, #20] \n\t"                                 \
3123          "ldr r1, [%1, #24] \n\t"                                 \
3124          "ldr r2, [%1, #28] \n\t"                                 \
3125          "ldr r3, [%1, #32] \n\t"                                 \
3126          "ldr r4, [%1, #36] \n\t"                                 \
3127          "push {r0, r1, r2, r3, r4} \n\t"                         \
3128          "ldr r0, [%1, #4] \n\t"                                  \
3129          "ldr r1, [%1, #8] \n\t"                                  \
3130          "ldr r2, [%1, #12] \n\t"                                 \
3131          "ldr r3, [%1, #16] \n\t"                                 \
3132          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3133          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3134          "add sp, sp, #24 \n\t"                                   \
3135          "mov %0, r0"                                             \
3136          : /*out*/   "=r" (_res)                                  \
3137          : /*in*/    "0" (&_argvec[0])                            \
3138          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3139       );                                                          \
3140       lval = (__typeof__(lval)) _res;                             \
3141    } while (0)
3142
3143 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
3144                                   arg6,arg7,arg8,arg9,arg10,      \
3145                                   arg11)                          \
3146    do {                                                           \
3147       volatile OrigFn        _orig = (orig);                      \
3148       volatile unsigned long _argvec[12];                         \
3149       volatile unsigned long _res;                                \
3150       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3151       _argvec[1] = (unsigned long)(arg1);                         \
3152       _argvec[2] = (unsigned long)(arg2);                         \
3153       _argvec[3] = (unsigned long)(arg3);                         \
3154       _argvec[4] = (unsigned long)(arg4);                         \
3155       _argvec[5] = (unsigned long)(arg5);                         \
3156       _argvec[6] = (unsigned long)(arg6);                         \
3157       _argvec[7] = (unsigned long)(arg7);                         \
3158       _argvec[8] = (unsigned long)(arg8);                         \
3159       _argvec[9] = (unsigned long)(arg9);                         \
3160       _argvec[10] = (unsigned long)(arg10);                       \
3161       _argvec[11] = (unsigned long)(arg11);                       \
3162       __asm__ volatile(                                           \
3163          "ldr r0, [%1, #40] \n\t"                                 \
3164          "ldr r1, [%1, #44] \n\t"                                 \
3165          "push {r0, r1} \n\t"                                     \
3166          "ldr r0, [%1, #20] \n\t"                                 \
3167          "ldr r1, [%1, #24] \n\t"                                 \
3168          "ldr r2, [%1, #28] \n\t"                                 \
3169          "ldr r3, [%1, #32] \n\t"                                 \
3170          "ldr r4, [%1, #36] \n\t"                                 \
3171          "push {r0, r1, r2, r3, r4} \n\t"                         \
3172          "ldr r0, [%1, #4] \n\t"                                  \
3173          "ldr r1, [%1, #8] \n\t"                                  \
3174          "ldr r2, [%1, #12] \n\t"                                 \
3175          "ldr r3, [%1, #16] \n\t"                                 \
3176          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3177          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3178          "add sp, sp, #28 \n\t"                                   \
3179          "mov %0, r0"                                             \
3180          : /*out*/   "=r" (_res)                                  \
3181          : /*in*/    "0" (&_argvec[0])                            \
3182          : /*trash*/ "cc", "memory",__CALLER_SAVED_REGS           \
3183       );                                                          \
3184       lval = (__typeof__(lval)) _res;                             \
3185    } while (0)
3186
3187 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
3188                                   arg6,arg7,arg8,arg9,arg10,      \
3189                                   arg11,arg12)                    \
3190    do {                                                           \
3191       volatile OrigFn        _orig = (orig);                      \
3192       volatile unsigned long _argvec[13];                         \
3193       volatile unsigned long _res;                                \
3194       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3195       _argvec[1] = (unsigned long)(arg1);                         \
3196       _argvec[2] = (unsigned long)(arg2);                         \
3197       _argvec[3] = (unsigned long)(arg3);                         \
3198       _argvec[4] = (unsigned long)(arg4);                         \
3199       _argvec[5] = (unsigned long)(arg5);                         \
3200       _argvec[6] = (unsigned long)(arg6);                         \
3201       _argvec[7] = (unsigned long)(arg7);                         \
3202       _argvec[8] = (unsigned long)(arg8);                         \
3203       _argvec[9] = (unsigned long)(arg9);                         \
3204       _argvec[10] = (unsigned long)(arg10);                       \
3205       _argvec[11] = (unsigned long)(arg11);                       \
3206       _argvec[12] = (unsigned long)(arg12);                       \
3207       __asm__ volatile(                                           \
3208          "ldr r0, [%1, #40] \n\t"                                 \
3209          "ldr r1, [%1, #44] \n\t"                                 \
3210          "ldr r2, [%1, #48] \n\t"                                 \
3211          "push {r0, r1, r2} \n\t"                                 \
3212          "ldr r0, [%1, #20] \n\t"                                 \
3213          "ldr r1, [%1, #24] \n\t"                                 \
3214          "ldr r2, [%1, #28] \n\t"                                 \
3215          "ldr r3, [%1, #32] \n\t"                                 \
3216          "ldr r4, [%1, #36] \n\t"                                 \
3217          "push {r0, r1, r2, r3, r4} \n\t"                         \
3218          "ldr r0, [%1, #4] \n\t"                                  \
3219          "ldr r1, [%1, #8] \n\t"                                  \
3220          "ldr r2, [%1, #12] \n\t"                                 \
3221          "ldr r3, [%1, #16] \n\t"                                 \
3222          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3223          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3224          "add sp, sp, #32 \n\t"                                   \
3225          "mov %0, r0"                                             \
3226          : /*out*/   "=r" (_res)                                  \
3227          : /*in*/    "0" (&_argvec[0])                            \
3228          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3229       );                                                          \
3230       lval = (__typeof__(lval)) _res;                             \
3231    } while (0)
3232
3233 #endif /* PLAT_arm_linux */
3234
3235 /* ------------------------ ppc32-aix5 ------------------------- */
3236
3237 #if defined(PLAT_ppc32_aix5)
3238
3239 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
3240
3241 /* These regs are trashed by the hidden call. */
3242 #define __CALLER_SAVED_REGS                                       \
3243    "lr", "ctr", "xer",                                            \
3244    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
3245    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
3246    "r11", "r12", "r13"
3247
3248 /* Expand the stack frame, copying enough info that unwinding
3249    still works.  Trashes r3. */
3250
3251 #define VG_EXPAND_FRAME_BY_trashes_r3(_n_fr)                      \
3252          "addi 1,1,-" #_n_fr "\n\t"                               \
3253          "lwz  3," #_n_fr "(1)\n\t"                               \
3254          "stw  3,0(1)\n\t"
3255
3256 #define VG_CONTRACT_FRAME_BY(_n_fr)                               \
3257          "addi 1,1," #_n_fr "\n\t"
3258
3259 /* These CALL_FN_ macros assume that on ppc32-aix5, sizeof(unsigned
3260    long) == 4. */
3261
3262 #define CALL_FN_W_v(lval, orig)                                   \
3263    do {                                                           \
3264       volatile OrigFn        _orig = (orig);                      \
3265       volatile unsigned long _argvec[3+0];                        \
3266       volatile unsigned long _res;                                \
3267       /* _argvec[0] holds current r2 across the call */           \
3268       _argvec[1] = (unsigned long)_orig.r2;                       \
3269       _argvec[2] = (unsigned long)_orig.nraddr;                   \
3270       __asm__ volatile(                                           \
3271          "mr 11,%1\n\t"                                           \
3272          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3273          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3274          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3275          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3276          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3277          "mr 11,%1\n\t"                                           \
3278          "mr %0,3\n\t"                                            \
3279          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3280          VG_CONTRACT_FRAME_BY(512)                                \
3281          : /*out*/   "=r" (_res)                                  \
3282          : /*in*/    "r" (&_argvec[2])                            \
3283          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3284       );                                                          \
3285       lval = (__typeof__(lval)) _res;                             \
3286    } while (0)
3287
3288 #define CALL_FN_W_W(lval, orig, arg1)                             \
3289    do {                                                           \
3290       volatile OrigFn        _orig = (orig);                      \
3291       volatile unsigned long _argvec[3+1];                        \
3292       volatile unsigned long _res;                                \
3293       /* _argvec[0] holds current r2 across the call */           \
3294       _argvec[1]   = (unsigned long)_orig.r2;                     \
3295       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3296       _argvec[2+1] = (unsigned long)arg1;                         \
3297       __asm__ volatile(                                           \
3298          "mr 11,%1\n\t"                                           \
3299          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3300          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3301          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3302          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3303          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3304          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3305          "mr 11,%1\n\t"                                           \
3306          "mr %0,3\n\t"                                            \
3307          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3308          VG_CONTRACT_FRAME_BY(512)                                \
3309          : /*out*/   "=r" (_res)                                  \
3310          : /*in*/    "r" (&_argvec[2])                            \
3311          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3312       );                                                          \
3313       lval = (__typeof__(lval)) _res;                             \
3314    } while (0)
3315
3316 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
3317    do {                                                           \
3318       volatile OrigFn        _orig = (orig);                      \
3319       volatile unsigned long _argvec[3+2];                        \
3320       volatile unsigned long _res;                                \
3321       /* _argvec[0] holds current r2 across the call */           \
3322       _argvec[1]   = (unsigned long)_orig.r2;                     \
3323       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3324       _argvec[2+1] = (unsigned long)arg1;                         \
3325       _argvec[2+2] = (unsigned long)arg2;                         \
3326       __asm__ volatile(                                           \
3327          "mr 11,%1\n\t"                                           \
3328          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3329          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3330          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3331          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3332          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3333          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3334          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3335          "mr 11,%1\n\t"                                           \
3336          "mr %0,3\n\t"                                            \
3337          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3338          VG_CONTRACT_FRAME_BY(512)                                \
3339          : /*out*/   "=r" (_res)                                  \
3340          : /*in*/    "r" (&_argvec[2])                            \
3341          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3342       );                                                          \
3343       lval = (__typeof__(lval)) _res;                             \
3344    } while (0)
3345
3346 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
3347    do {                                                           \
3348       volatile OrigFn        _orig = (orig);                      \
3349       volatile unsigned long _argvec[3+3];                        \
3350       volatile unsigned long _res;                                \
3351       /* _argvec[0] holds current r2 across the call */           \
3352       _argvec[1]   = (unsigned long)_orig.r2;                     \
3353       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3354       _argvec[2+1] = (unsigned long)arg1;                         \
3355       _argvec[2+2] = (unsigned long)arg2;                         \
3356       _argvec[2+3] = (unsigned long)arg3;                         \
3357       __asm__ volatile(                                           \
3358          "mr 11,%1\n\t"                                           \
3359          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3360          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3361          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3362          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3363          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3364          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3365          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3366          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3367          "mr 11,%1\n\t"                                           \
3368          "mr %0,3\n\t"                                            \
3369          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3370          VG_CONTRACT_FRAME_BY(512)                                \
3371          : /*out*/   "=r" (_res)                                  \
3372          : /*in*/    "r" (&_argvec[2])                            \
3373          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3374       );                                                          \
3375       lval = (__typeof__(lval)) _res;                             \
3376    } while (0)
3377
3378 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
3379    do {                                                           \
3380       volatile OrigFn        _orig = (orig);                      \
3381       volatile unsigned long _argvec[3+4];                        \
3382       volatile unsigned long _res;                                \
3383       /* _argvec[0] holds current r2 across the call */           \
3384       _argvec[1]   = (unsigned long)_orig.r2;                     \
3385       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3386       _argvec[2+1] = (unsigned long)arg1;                         \
3387       _argvec[2+2] = (unsigned long)arg2;                         \
3388       _argvec[2+3] = (unsigned long)arg3;                         \
3389       _argvec[2+4] = (unsigned long)arg4;                         \
3390       __asm__ volatile(                                           \
3391          "mr 11,%1\n\t"                                           \
3392          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3393          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3394          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3395          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3396          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3397          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3398          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3399          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3400          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3401          "mr 11,%1\n\t"                                           \
3402          "mr %0,3\n\t"                                            \
3403          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3404          VG_CONTRACT_FRAME_BY(512)                                \
3405          : /*out*/   "=r" (_res)                                  \
3406          : /*in*/    "r" (&_argvec[2])                            \
3407          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3408       );                                                          \
3409       lval = (__typeof__(lval)) _res;                             \
3410    } while (0)
3411
3412 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
3413    do {                                                           \
3414       volatile OrigFn        _orig = (orig);                      \
3415       volatile unsigned long _argvec[3+5];                        \
3416       volatile unsigned long _res;                                \
3417       /* _argvec[0] holds current r2 across the call */           \
3418       _argvec[1]   = (unsigned long)_orig.r2;                     \
3419       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3420       _argvec[2+1] = (unsigned long)arg1;                         \
3421       _argvec[2+2] = (unsigned long)arg2;                         \
3422       _argvec[2+3] = (unsigned long)arg3;                         \
3423       _argvec[2+4] = (unsigned long)arg4;                         \
3424       _argvec[2+5] = (unsigned long)arg5;                         \
3425       __asm__ volatile(                                           \
3426          "mr 11,%1\n\t"                                           \
3427          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3428          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3429          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3430          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3431          "lwz  4, 8(11)\n\t" /* arg2->r4 */                       \
3432          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3433          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3434          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3435          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3436          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3437          "mr 11,%1\n\t"                                           \
3438          "mr %0,3\n\t"                                            \
3439          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3440          VG_CONTRACT_FRAME_BY(512)                                \
3441          : /*out*/   "=r" (_res)                                  \
3442          : /*in*/    "r" (&_argvec[2])                            \
3443          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3444       );                                                          \
3445       lval = (__typeof__(lval)) _res;                             \
3446    } while (0)
3447
3448 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
3449    do {                                                           \
3450       volatile OrigFn        _orig = (orig);                      \
3451       volatile unsigned long _argvec[3+6];                        \
3452       volatile unsigned long _res;                                \
3453       /* _argvec[0] holds current r2 across the call */           \
3454       _argvec[1]   = (unsigned long)_orig.r2;                     \
3455       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3456       _argvec[2+1] = (unsigned long)arg1;                         \
3457       _argvec[2+2] = (unsigned long)arg2;                         \
3458       _argvec[2+3] = (unsigned long)arg3;                         \
3459       _argvec[2+4] = (unsigned long)arg4;                         \
3460       _argvec[2+5] = (unsigned long)arg5;                         \
3461       _argvec[2+6] = (unsigned long)arg6;                         \
3462       __asm__ volatile(                                           \
3463          "mr 11,%1\n\t"                                           \
3464          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3465          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3466          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3467          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3468          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3469          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3470          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3471          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3472          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3473          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3474          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3475          "mr 11,%1\n\t"                                           \
3476          "mr %0,3\n\t"                                            \
3477          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3478          VG_CONTRACT_FRAME_BY(512)                                \
3479          : /*out*/   "=r" (_res)                                  \
3480          : /*in*/    "r" (&_argvec[2])                            \
3481          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3482       );                                                          \
3483       lval = (__typeof__(lval)) _res;                             \
3484    } while (0)
3485
3486 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3487                                  arg7)                            \
3488    do {                                                           \
3489       volatile OrigFn        _orig = (orig);                      \
3490       volatile unsigned long _argvec[3+7];                        \
3491       volatile unsigned long _res;                                \
3492       /* _argvec[0] holds current r2 across the call */           \
3493       _argvec[1]   = (unsigned long)_orig.r2;                     \
3494       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3495       _argvec[2+1] = (unsigned long)arg1;                         \
3496       _argvec[2+2] = (unsigned long)arg2;                         \
3497       _argvec[2+3] = (unsigned long)arg3;                         \
3498       _argvec[2+4] = (unsigned long)arg4;                         \
3499       _argvec[2+5] = (unsigned long)arg5;                         \
3500       _argvec[2+6] = (unsigned long)arg6;                         \
3501       _argvec[2+7] = (unsigned long)arg7;                         \
3502       __asm__ volatile(                                           \
3503          "mr 11,%1\n\t"                                           \
3504          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3505          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3506          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3507          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3508          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3509          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3510          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3511          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3512          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3513          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3514          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3515          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3516          "mr 11,%1\n\t"                                           \
3517          "mr %0,3\n\t"                                            \
3518          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3519          VG_CONTRACT_FRAME_BY(512)                                \
3520          : /*out*/   "=r" (_res)                                  \
3521          : /*in*/    "r" (&_argvec[2])                            \
3522          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3523       );                                                          \
3524       lval = (__typeof__(lval)) _res;                             \
3525    } while (0)
3526
3527 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3528                                  arg7,arg8)                       \
3529    do {                                                           \
3530       volatile OrigFn        _orig = (orig);                      \
3531       volatile unsigned long _argvec[3+8];                        \
3532       volatile unsigned long _res;                                \
3533       /* _argvec[0] holds current r2 across the call */           \
3534       _argvec[1]   = (unsigned long)_orig.r2;                     \
3535       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3536       _argvec[2+1] = (unsigned long)arg1;                         \
3537       _argvec[2+2] = (unsigned long)arg2;                         \
3538       _argvec[2+3] = (unsigned long)arg3;                         \
3539       _argvec[2+4] = (unsigned long)arg4;                         \
3540       _argvec[2+5] = (unsigned long)arg5;                         \
3541       _argvec[2+6] = (unsigned long)arg6;                         \
3542       _argvec[2+7] = (unsigned long)arg7;                         \
3543       _argvec[2+8] = (unsigned long)arg8;                         \
3544       __asm__ volatile(                                           \
3545          "mr 11,%1\n\t"                                           \
3546          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3547          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3548          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3549          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3550          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3551          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3552          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3553          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3554          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3555          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3556          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3557          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3558          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3559          "mr 11,%1\n\t"                                           \
3560          "mr %0,3\n\t"                                            \
3561          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3562          VG_CONTRACT_FRAME_BY(512)                                \
3563          : /*out*/   "=r" (_res)                                  \
3564          : /*in*/    "r" (&_argvec[2])                            \
3565          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3566       );                                                          \
3567       lval = (__typeof__(lval)) _res;                             \
3568    } while (0)
3569
3570 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3571                                  arg7,arg8,arg9)                  \
3572    do {                                                           \
3573       volatile OrigFn        _orig = (orig);                      \
3574       volatile unsigned long _argvec[3+9];                        \
3575       volatile unsigned long _res;                                \
3576       /* _argvec[0] holds current r2 across the call */           \
3577       _argvec[1]   = (unsigned long)_orig.r2;                     \
3578       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3579       _argvec[2+1] = (unsigned long)arg1;                         \
3580       _argvec[2+2] = (unsigned long)arg2;                         \
3581       _argvec[2+3] = (unsigned long)arg3;                         \
3582       _argvec[2+4] = (unsigned long)arg4;                         \
3583       _argvec[2+5] = (unsigned long)arg5;                         \
3584       _argvec[2+6] = (unsigned long)arg6;                         \
3585       _argvec[2+7] = (unsigned long)arg7;                         \
3586       _argvec[2+8] = (unsigned long)arg8;                         \
3587       _argvec[2+9] = (unsigned long)arg9;                         \
3588       __asm__ volatile(                                           \
3589          "mr 11,%1\n\t"                                           \
3590          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3591          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3592          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3593          VG_EXPAND_FRAME_BY_trashes_r3(64)                        \
3594          /* arg9 */                                               \
3595          "lwz 3,36(11)\n\t"                                       \
3596          "stw 3,56(1)\n\t"                                        \
3597          /* args1-8 */                                            \
3598          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3599          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3600          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3601          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3602          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3603          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3604          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3605          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3606          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3607          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3608          "mr 11,%1\n\t"                                           \
3609          "mr %0,3\n\t"                                            \
3610          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3611          VG_CONTRACT_FRAME_BY(64)                                 \
3612          VG_CONTRACT_FRAME_BY(512)                                \
3613          : /*out*/   "=r" (_res)                                  \
3614          : /*in*/    "r" (&_argvec[2])                            \
3615          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3616       );                                                          \
3617       lval = (__typeof__(lval)) _res;                             \
3618    } while (0)
3619
3620 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3621                                   arg7,arg8,arg9,arg10)           \
3622    do {                                                           \
3623       volatile OrigFn        _orig = (orig);                      \
3624       volatile unsigned long _argvec[3+10];                       \
3625       volatile unsigned long _res;                                \
3626       /* _argvec[0] holds current r2 across the call */           \
3627       _argvec[1]   = (unsigned long)_orig.r2;                     \
3628       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3629       _argvec[2+1] = (unsigned long)arg1;                         \
3630       _argvec[2+2] = (unsigned long)arg2;                         \
3631       _argvec[2+3] = (unsigned long)arg3;                         \
3632       _argvec[2+4] = (unsigned long)arg4;                         \
3633       _argvec[2+5] = (unsigned long)arg5;                         \
3634       _argvec[2+6] = (unsigned long)arg6;                         \
3635       _argvec[2+7] = (unsigned long)arg7;                         \
3636       _argvec[2+8] = (unsigned long)arg8;                         \
3637       _argvec[2+9] = (unsigned long)arg9;                         \
3638       _argvec[2+10] = (unsigned long)arg10;                       \
3639       __asm__ volatile(                                           \
3640          "mr 11,%1\n\t"                                           \
3641          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3642          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3643          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3644          VG_EXPAND_FRAME_BY_trashes_r3(64)                        \
3645          /* arg10 */                                              \
3646          "lwz 3,40(11)\n\t"                                       \
3647          "stw 3,60(1)\n\t"                                        \
3648          /* arg9 */                                               \
3649          "lwz 3,36(11)\n\t"                                       \
3650          "stw 3,56(1)\n\t"                                        \
3651          /* args1-8 */                                            \
3652          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3653          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3654          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3655          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3656          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3657          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3658          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3659          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3660          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3661          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3662          "mr 11,%1\n\t"                                           \
3663          "mr %0,3\n\t"                                            \
3664          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3665          VG_CONTRACT_FRAME_BY(64)                                 \
3666          VG_CONTRACT_FRAME_BY(512)                                \
3667          : /*out*/   "=r" (_res)                                  \
3668          : /*in*/    "r" (&_argvec[2])                            \
3669          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3670       );                                                          \
3671       lval = (__typeof__(lval)) _res;                             \
3672    } while (0)
3673
3674 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3675                                   arg7,arg8,arg9,arg10,arg11)     \
3676    do {                                                           \
3677       volatile OrigFn        _orig = (orig);                      \
3678       volatile unsigned long _argvec[3+11];                       \
3679       volatile unsigned long _res;                                \
3680       /* _argvec[0] holds current r2 across the call */           \
3681       _argvec[1]   = (unsigned long)_orig.r2;                     \
3682       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3683       _argvec[2+1] = (unsigned long)arg1;                         \
3684       _argvec[2+2] = (unsigned long)arg2;                         \
3685       _argvec[2+3] = (unsigned long)arg3;                         \
3686       _argvec[2+4] = (unsigned long)arg4;                         \
3687       _argvec[2+5] = (unsigned long)arg5;                         \
3688       _argvec[2+6] = (unsigned long)arg6;                         \
3689       _argvec[2+7] = (unsigned long)arg7;                         \
3690       _argvec[2+8] = (unsigned long)arg8;                         \
3691       _argvec[2+9] = (unsigned long)arg9;                         \
3692       _argvec[2+10] = (unsigned long)arg10;                       \
3693       _argvec[2+11] = (unsigned long)arg11;                       \
3694       __asm__ volatile(                                           \
3695          "mr 11,%1\n\t"                                           \
3696          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3697          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3698          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3699          VG_EXPAND_FRAME_BY_trashes_r3(72)                        \
3700          /* arg11 */                                              \
3701          "lwz 3,44(11)\n\t"                                       \
3702          "stw 3,64(1)\n\t"                                        \
3703          /* arg10 */                                              \
3704          "lwz 3,40(11)\n\t"                                       \
3705          "stw 3,60(1)\n\t"                                        \
3706          /* arg9 */                                               \
3707          "lwz 3,36(11)\n\t"                                       \
3708          "stw 3,56(1)\n\t"                                        \
3709          /* args1-8 */                                            \
3710          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3711          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3712          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3713          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3714          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3715          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3716          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3717          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3718          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3719          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3720          "mr 11,%1\n\t"                                           \
3721          "mr %0,3\n\t"                                            \
3722          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3723          VG_CONTRACT_FRAME_BY(72)                                 \
3724          VG_CONTRACT_FRAME_BY(512)                                \
3725          : /*out*/   "=r" (_res)                                  \
3726          : /*in*/    "r" (&_argvec[2])                            \
3727          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3728       );                                                          \
3729       lval = (__typeof__(lval)) _res;                             \
3730    } while (0)
3731
3732 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3733                                 arg7,arg8,arg9,arg10,arg11,arg12) \
3734    do {                                                           \
3735       volatile OrigFn        _orig = (orig);                      \
3736       volatile unsigned long _argvec[3+12];                       \
3737       volatile unsigned long _res;                                \
3738       /* _argvec[0] holds current r2 across the call */           \
3739       _argvec[1]   = (unsigned long)_orig.r2;                     \
3740       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3741       _argvec[2+1] = (unsigned long)arg1;                         \
3742       _argvec[2+2] = (unsigned long)arg2;                         \
3743       _argvec[2+3] = (unsigned long)arg3;                         \
3744       _argvec[2+4] = (unsigned long)arg4;                         \
3745       _argvec[2+5] = (unsigned long)arg5;                         \
3746       _argvec[2+6] = (unsigned long)arg6;                         \
3747       _argvec[2+7] = (unsigned long)arg7;                         \
3748       _argvec[2+8] = (unsigned long)arg8;                         \
3749       _argvec[2+9] = (unsigned long)arg9;                         \
3750       _argvec[2+10] = (unsigned long)arg10;                       \
3751       _argvec[2+11] = (unsigned long)arg11;                       \
3752       _argvec[2+12] = (unsigned long)arg12;                       \
3753       __asm__ volatile(                                           \
3754          "mr 11,%1\n\t"                                           \
3755          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3756          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3757          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3758          VG_EXPAND_FRAME_BY_trashes_r3(72)                        \
3759          /* arg12 */                                              \
3760          "lwz 3,48(11)\n\t"                                       \
3761          "stw 3,68(1)\n\t"                                        \
3762          /* arg11 */                                              \
3763          "lwz 3,44(11)\n\t"                                       \
3764          "stw 3,64(1)\n\t"                                        \
3765          /* arg10 */                                              \
3766          "lwz 3,40(11)\n\t"                                       \
3767          "stw 3,60(1)\n\t"                                        \
3768          /* arg9 */                                               \
3769          "lwz 3,36(11)\n\t"                                       \
3770          "stw 3,56(1)\n\t"                                        \
3771          /* args1-8 */                                            \
3772          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3773          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3774          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3775          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3776          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3777          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3778          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3779          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3780          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3781          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3782          "mr 11,%1\n\t"                                           \
3783          "mr %0,3\n\t"                                            \
3784          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3785          VG_CONTRACT_FRAME_BY(72)                                 \
3786          VG_CONTRACT_FRAME_BY(512)                                \
3787          : /*out*/   "=r" (_res)                                  \
3788          : /*in*/    "r" (&_argvec[2])                            \
3789          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3790       );                                                          \
3791       lval = (__typeof__(lval)) _res;                             \
3792    } while (0)
3793
3794 #endif /* PLAT_ppc32_aix5 */
3795
3796 /* ------------------------ ppc64-aix5 ------------------------- */
3797
3798 #if defined(PLAT_ppc64_aix5)
3799
3800 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
3801
3802 /* These regs are trashed by the hidden call. */
3803 #define __CALLER_SAVED_REGS                                       \
3804    "lr", "ctr", "xer",                                            \
3805    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
3806    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
3807    "r11", "r12", "r13"
3808
3809 /* Expand the stack frame, copying enough info that unwinding
3810    still works.  Trashes r3. */
3811
3812 #define VG_EXPAND_FRAME_BY_trashes_r3(_n_fr)                      \
3813          "addi 1,1,-" #_n_fr "\n\t"                               \
3814          "ld   3," #_n_fr "(1)\n\t"                               \
3815          "std  3,0(1)\n\t"
3816
3817 #define VG_CONTRACT_FRAME_BY(_n_fr)                               \
3818          "addi 1,1," #_n_fr "\n\t"
3819
3820 /* These CALL_FN_ macros assume that on ppc64-aix5, sizeof(unsigned
3821    long) == 8. */
3822
3823 #define CALL_FN_W_v(lval, orig)                                   \
3824    do {                                                           \
3825       volatile OrigFn        _orig = (orig);                      \
3826       volatile unsigned long _argvec[3+0];                        \
3827       volatile unsigned long _res;                                \
3828       /* _argvec[0] holds current r2 across the call */           \
3829       _argvec[1] = (unsigned long)_orig.r2;                       \
3830       _argvec[2] = (unsigned long)_orig.nraddr;                   \
3831       __asm__ volatile(                                           \
3832          "mr 11,%1\n\t"                                           \
3833          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3834          "std  2,-16(11)\n\t" /* save tocptr */                   \
3835          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3836          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3837          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3838          "mr 11,%1\n\t"                                           \
3839          "mr %0,3\n\t"                                            \
3840          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3841          VG_CONTRACT_FRAME_BY(512)                                \
3842          : /*out*/   "=r" (_res)                                  \
3843          : /*in*/    "r" (&_argvec[2])                            \
3844          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3845       );                                                          \
3846       lval = (__typeof__(lval)) _res;                             \
3847    } while (0)
3848
3849 #define CALL_FN_W_W(lval, orig, arg1)                             \
3850    do {                                                           \
3851       volatile OrigFn        _orig = (orig);                      \
3852       volatile unsigned long _argvec[3+1];                        \
3853       volatile unsigned long _res;                                \
3854       /* _argvec[0] holds current r2 across the call */           \
3855       _argvec[1]   = (unsigned long)_orig.r2;                     \
3856       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3857       _argvec[2+1] = (unsigned long)arg1;                         \
3858       __asm__ volatile(                                           \
3859          "mr 11,%1\n\t"                                           \
3860          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3861          "std  2,-16(11)\n\t" /* save tocptr */                   \
3862          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3863          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3864          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3865          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3866          "mr 11,%1\n\t"                                           \
3867          "mr %0,3\n\t"                                            \
3868          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3869          VG_CONTRACT_FRAME_BY(512)                                \
3870          : /*out*/   "=r" (_res)                                  \
3871          : /*in*/    "r" (&_argvec[2])                            \
3872          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3873       );                                                          \
3874       lval = (__typeof__(lval)) _res;                             \
3875    } while (0)
3876
3877 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
3878    do {                                                           \
3879       volatile OrigFn        _orig = (orig);                      \
3880       volatile unsigned long _argvec[3+2];                        \
3881       volatile unsigned long _res;                                \
3882       /* _argvec[0] holds current r2 across the call */           \
3883       _argvec[1]   = (unsigned long)_orig.r2;                     \
3884       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3885       _argvec[2+1] = (unsigned long)arg1;                         \
3886       _argvec[2+2] = (unsigned long)arg2;                         \
3887       __asm__ volatile(                                           \
3888          "mr 11,%1\n\t"                                           \
3889          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3890          "std  2,-16(11)\n\t" /* save tocptr */                   \
3891          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3892          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3893          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3894          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3895          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3896          "mr 11,%1\n\t"                                           \
3897          "mr %0,3\n\t"                                            \
3898          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3899          VG_CONTRACT_FRAME_BY(512)                                \
3900          : /*out*/   "=r" (_res)                                  \
3901          : /*in*/    "r" (&_argvec[2])                            \
3902          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3903       );                                                          \
3904       lval = (__typeof__(lval)) _res;                             \
3905    } while (0)
3906
3907 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
3908    do {                                                           \
3909       volatile OrigFn        _orig = (orig);                      \
3910       volatile unsigned long _argvec[3+3];                        \
3911       volatile unsigned long _res;                                \
3912       /* _argvec[0] holds current r2 across the call */           \
3913       _argvec[1]   = (unsigned long)_orig.r2;                     \
3914       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3915       _argvec[2+1] = (unsigned long)arg1;                         \
3916       _argvec[2+2] = (unsigned long)arg2;                         \
3917       _argvec[2+3] = (unsigned long)arg3;                         \
3918       __asm__ volatile(                                           \
3919          "mr 11,%1\n\t"                                           \
3920          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3921          "std  2,-16(11)\n\t" /* save tocptr */                   \
3922          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3923          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3924          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3925          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3926          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3927          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3928          "mr 11,%1\n\t"                                           \
3929          "mr %0,3\n\t"                                            \
3930          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3931          VG_CONTRACT_FRAME_BY(512)                                \
3932          : /*out*/   "=r" (_res)                                  \
3933          : /*in*/    "r" (&_argvec[2])                            \
3934          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3935       );                                                          \
3936       lval = (__typeof__(lval)) _res;                             \
3937    } while (0)
3938
3939 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
3940    do {                                                           \
3941       volatile OrigFn        _orig = (orig);                      \
3942       volatile unsigned long _argvec[3+4];                        \
3943       volatile unsigned long _res;                                \
3944       /* _argvec[0] holds current r2 across the call */           \
3945       _argvec[1]   = (unsigned long)_orig.r2;                     \
3946       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3947       _argvec[2+1] = (unsigned long)arg1;                         \
3948       _argvec[2+2] = (unsigned long)arg2;                         \
3949       _argvec[2+3] = (unsigned long)arg3;                         \
3950       _argvec[2+4] = (unsigned long)arg4;                         \
3951       __asm__ volatile(                                           \
3952          "mr 11,%1\n\t"                                           \
3953          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3954          "std  2,-16(11)\n\t" /* save tocptr */                   \
3955          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3956          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3957          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3958          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3959          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3960          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3961          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3962          "mr 11,%1\n\t"                                           \
3963          "mr %0,3\n\t"                                            \
3964          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3965          VG_CONTRACT_FRAME_BY(512)                                \
3966          : /*out*/   "=r" (_res)                                  \
3967          : /*in*/    "r" (&_argvec[2])                            \
3968          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3969       );                                                          \
3970       lval = (__typeof__(lval)) _res;                             \
3971    } while (0)
3972
3973 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
3974    do {                                                           \
3975       volatile OrigFn        _orig = (orig);                      \
3976       volatile unsigned long _argvec[3+5];                        \
3977       volatile unsigned long _res;                                \
3978       /* _argvec[0] holds current r2 across the call */           \
3979       _argvec[1]   = (unsigned long)_orig.r2;                     \
3980       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3981       _argvec[2+1] = (unsigned long)arg1;                         \
3982       _argvec[2+2] = (unsigned long)arg2;                         \
3983       _argvec[2+3] = (unsigned long)arg3;                         \
3984       _argvec[2+4] = (unsigned long)arg4;                         \
3985       _argvec[2+5] = (unsigned long)arg5;                         \
3986       __asm__ volatile(                                           \
3987          "mr 11,%1\n\t"                                           \
3988          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3989          "std  2,-16(11)\n\t" /* save tocptr */                   \
3990          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3991          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3992          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3993          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3994          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3995          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3996          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3997          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3998          "mr 11,%1\n\t"                                           \
3999          "mr %0,3\n\t"                                            \
4000          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4001          VG_CONTRACT_FRAME_BY(512)                                \
4002          : /*out*/   "=r" (_res)                                  \
4003          : /*in*/    "r" (&_argvec[2])                            \
4004          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4005       );                                                          \
4006       lval = (__typeof__(lval)) _res;                             \
4007    } while (0)
4008
4009 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
4010    do {                                                           \
4011       volatile OrigFn        _orig = (orig);                      \
4012       volatile unsigned long _argvec[3+6];                        \
4013       volatile unsigned long _res;                                \
4014       /* _argvec[0] holds current r2 across the call */           \
4015       _argvec[1]   = (unsigned long)_orig.r2;                     \
4016       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
4017       _argvec[2+1] = (unsigned long)arg1;                         \
4018       _argvec[2+2] = (unsigned long)arg2;                         \
4019       _argvec[2+3] = (unsigned long)arg3;                         \
4020       _argvec[2+4] = (unsigned long)arg4;                         \
4021       _argvec[2+5] = (unsigned long)arg5;                         \
4022       _argvec[2+6] = (unsigned long)arg6;                         \
4023       __asm__ volatile(                                           \
4024          "mr 11,%1\n\t"                                           \
4025          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
4026          "std  2,-16(11)\n\t" /* save tocptr */                   \
4027          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
4028          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
4029          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
4030          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
4031          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
4032          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4033          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4034          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4035          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4036          "mr 11,%1\n\t"                                           \
4037          "mr %0,3\n\t"                                            \
4038          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4039          VG_CONTRACT_FRAME_BY(512)                                \
4040          : /*out*/   "=r" (_res)                                  \
4041          : /*in*/    "r" (&_argvec[2])                            \
4042          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4043       );                                                          \
4044       lval = (__typeof__(lval)) _res;                             \
4045    } while (0)
4046
4047 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
4048                                  arg7)                            \
4049    do {                                                           \
4050       volatile OrigFn        _orig = (orig);                      \
4051       volatile unsigned long _argvec[3+7];                        \
4052       volatile unsigned long _res;                                \
4053       /* _argvec[0] holds current r2 across the call */           \
4054       _argvec[1]   = (unsigned long)_orig.r2;                     \
4055       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
4056       _argvec[2+1] = (unsigned long)arg1;                         \
4057       _argvec[2+2] = (unsigned long)arg2;                         \
4058       _argvec[2+3] = (unsigned long)arg3;                         \
4059       _argvec[2+4] = (unsigned long)arg4;                         \
4060       _argvec[2+5] = (unsigned long)arg5;                         \
4061       _argvec[2+6] = (unsigned long)arg6;                         \
4062       _argvec[2+7] = (unsigned long)arg7;                         \
4063       __asm__ volatile(                                           \
4064          "mr 11,%1\n\t"                                           \
4065          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
4066          "std  2,-16(11)\n\t" /* save tocptr */                   \
4067          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
4068          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
4069          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
4070          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
4071          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
4072          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4073          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4074          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
4075          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4076          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4077          "mr 11,%1\n\t"                                           \
4078          "mr %0,3\n\t"                                            \
4079          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4080          VG_CONTRACT_FRAME_BY(512)                                \
4081          : /*out*/   "=r" (_res)                                  \
4082          : /*in*/    "r" (&_argvec[2])                            \
4083          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4084       );                                                          \
4085       lval = (__typeof__(lval)) _res;                             \
4086    } while (0)
4087
4088 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
4089                                  arg7,arg8)                       \
4090    do {                                                           \
4091       volatile OrigFn        _orig = (orig);                      \
4092       volatile unsigned long _argvec[3+8];                        \
4093       volatile unsigned long _res;                                \
4094       /* _argvec[0] holds current r2 across the call */           \
4095       _argvec[1]   = (unsigned long)_orig.r2;                     \
4096       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
4097       _argvec[2+1] = (unsigned long)arg1;                         \
4098       _argvec[2+2] = (unsigned long)arg2;                         \
4099       _argvec[2+3] = (unsigned long)arg3;                         \
4100       _argvec[2+4] = (unsigned long)arg4;                         \
4101       _argvec[2+5] = (unsigned long)arg5;                         \
4102       _argvec[2+6] = (unsigned long)arg6;                         \
4103       _argvec[2+7] = (unsigned long)arg7;                         \
4104       _argvec[2+8] = (unsigned long)arg8;                         \
4105       __asm__ volatile(                                           \
4106          "mr 11,%1\n\t"                                           \
4107          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
4108          "std  2,-16(11)\n\t" /* save tocptr */                   \
4109          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
4110          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
4111          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
4112          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
4113          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
4114          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4115          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4116          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
4117          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
4118          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4119          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4120          "mr 11,%1\n\t"                                           \
4121          "mr %0,3\n\t"                                            \
4122          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4123          VG_CONTRACT_FRAME_BY(512)                                \
4124          : /*out*/   "=r" (_res)                                  \
4125          : /*in*/    "r" (&_argvec[2])                            \
4126          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4127       );                                                          \
4128       lval = (__typeof__(lval)) _res;                             \
4129    } while (0)
4130
4131 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
4132                                  arg7,arg8,arg9)                  \
4133    do {                                                           \
4134       volatile OrigFn        _orig = (orig);                      \
4135       volatile unsigned long _argvec[3+9];                        \
4136       volatile unsigned long _res;                                \
4137       /* _argvec[0] holds current r2 across the call */           \
4138       _argvec[1]   = (unsigned long)_orig.r2;                     \
4139       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
4140       _argvec[2+1] = (unsigned long)arg1;                         \
4141       _argvec[2+2] = (unsigned long)arg2;                         \
4142       _argvec[2+3] = (unsigned long)arg3;                         \
4143       _argvec[2+4] = (unsigned long)arg4;                         \
4144       _argvec[2+5] = (unsigned long)arg5;                         \
4145       _argvec[2+6] = (unsigned long)arg6;                         \
4146       _argvec[2+7] = (unsigned long)arg7;                         \
4147       _argvec[2+8] = (unsigned long)arg8;                         \
4148       _argvec[2+9] = (unsigned long)arg9;                         \
4149       __asm__ volatile(                                           \
4150          "mr 11,%1\n\t"                                           \
4151          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
4152          "std  2,-16(11)\n\t" /* save tocptr */                   \
4153          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
4154          VG_EXPAND_FRAME_BY_trashes_r3(128)                       \
4155          /* arg9 */                                               \
4156          "ld  3,72(11)\n\t"                                       \
4157          "std 3,112(1)\n\t"                                       \
4158          /* args1-8 */                                            \
4159          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
4160          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
4161          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
4162          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
4163          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4164          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4165          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
4166          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
4167          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4168          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4169          "mr 11,%1\n\t"                                           \
4170          "mr %0,3\n\t"                                            \
4171          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4172          VG_CONTRACT_FRAME_BY(128)                                \
4173          VG_CONTRACT_FRAME_BY(512)                                \
4174          : /*out*/   "=r" (_res)                                  \
4175          : /*in*/    "r" (&_argvec[2])                            \
4176          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4177       );                                                          \
4178       lval = (__typeof__(lval)) _res;                             \
4179    } while (0)
4180
4181 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
4182                                   arg7,arg8,arg9,arg10)           \
4183    do {                                                           \
4184       volatile OrigFn        _orig = (orig);                      \
4185       volatile unsigned long _argvec[3+10];                       \
4186       volatile unsigned long _res;                                \
4187       /* _argvec[0] holds current r2 across the call */           \
4188       _argvec[1]   = (unsigned long)_orig.r2;                     \
4189       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
4190       _argvec[2+1] = (unsigned long)arg1;                         \
4191       _argvec[2+2] = (unsigned long)arg2;                         \
4192       _argvec[2+3] = (unsigned long)arg3;                         \
4193       _argvec[2+4] = (unsigned long)arg4;                         \
4194       _argvec[2+5] = (unsigned long)arg5;                         \
4195       _argvec[2+6] = (unsigned long)arg6;                         \
4196       _argvec[2+7] = (unsigned long)arg7;                         \
4197       _argvec[2+8] = (unsigned long)arg8;                         \
4198       _argvec[2+9] = (unsigned long)arg9;                         \
4199       _argvec[2+10] = (unsigned long)arg10;                       \
4200       __asm__ volatile(                                           \
4201          "mr 11,%1\n\t"                                           \
4202          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
4203          "std  2,-16(11)\n\t" /* save tocptr */                   \
4204          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
4205          VG_EXPAND_FRAME_BY_trashes_r3(128)                       \
4206          /* arg10 */                                              \
4207          "ld  3,80(11)\n\t"                                       \
4208          "std 3,120(1)\n\t"                                       \
4209          /* arg9 */                                               \
4210          "ld  3,72(11)\n\t"                                       \
4211          "std 3,112(1)\n\t"                                       \
4212          /* args1-8 */                                            \
4213          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
4214          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
4215          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
4216          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
4217          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4218          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4219          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
4220          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
4221          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4222          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4223          "mr 11,%1\n\t"                                           \
4224          "mr %0,3\n\t"                                            \
4225          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4226          VG_CONTRACT_FRAME_BY(128)                                \
4227          VG_CONTRACT_FRAME_BY(512)                                \
4228          : /*out*/   "=r" (_res)                                  \
4229          : /*in*/    "r" (&_argvec[2])                            \
4230          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4231       );                                                          \
4232       lval = (__typeof__(lval)) _res;                             \
4233    } while (0)
4234
4235 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
4236                                   arg7,arg8,arg9,arg10,arg11)     \
4237    do {                                                           \
4238       volatile OrigFn        _orig = (orig);                      \
4239       volatile unsigned long _argvec[3+11];                       \
4240       volatile unsigned long _res;                                \
4241       /* _argvec[0] holds current r2 across the call */           \
4242       _argvec[1]   = (unsigned long)_orig.r2;                     \
4243       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
4244       _argvec[2+1] = (unsigned long)arg1;                         \
4245       _argvec[2+2] = (unsigned long)arg2;                         \
4246       _argvec[2+3] = (unsigned long)arg3;                         \
4247       _argvec[2+4] = (unsigned long)arg4;                         \
4248       _argvec[2+5] = (unsigned long)arg5;                         \
4249       _argvec[2+6] = (unsigned long)arg6;                         \
4250       _argvec[2+7] = (unsigned long)arg7;                         \
4251       _argvec[2+8] = (unsigned long)arg8;                         \
4252       _argvec[2+9] = (unsigned long)arg9;                         \
4253       _argvec[2+10] = (unsigned long)arg10;                       \
4254       _argvec[2+11] = (unsigned long)arg11;                       \
4255       __asm__ volatile(                                           \
4256          "mr 11,%1\n\t"                                           \
4257          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
4258          "std  2,-16(11)\n\t" /* save tocptr */                   \
4259          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
4260          VG_EXPAND_FRAME_BY_trashes_r3(144)                       \
4261          /* arg11 */                                              \
4262          "ld  3,88(11)\n\t"                                       \
4263          "std 3,128(1)\n\t"                                       \
4264          /* arg10 */                                              \
4265          "ld  3,80(11)\n\t"                                       \
4266          "std 3,120(1)\n\t"                                       \
4267          /* arg9 */                                               \
4268          "ld  3,72(11)\n\t"                                       \
4269          "std 3,112(1)\n\t"                                       \
4270          /* args1-8 */                                            \
4271          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
4272          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
4273          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
4274          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
4275          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4276          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4277          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
4278          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
4279          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4280          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4281          "mr 11,%1\n\t"                                           \
4282          "mr %0,3\n\t"                                            \
4283          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4284          VG_CONTRACT_FRAME_BY(144)                                \
4285          VG_CONTRACT_FRAME_BY(512)                                \
4286          : /*out*/   "=r" (_res)                                  \
4287          : /*in*/    "r" (&_argvec[2])                            \
4288          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4289       );                                                          \
4290       lval = (__typeof__(lval)) _res;                             \
4291    } while (0)
4292
4293 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
4294                                 arg7,arg8,arg9,arg10,arg11,arg12) \
4295    do {                                                           \
4296       volatile OrigFn        _orig = (orig);                      \
4297       volatile unsigned long _argvec[3+12];                       \
4298       volatile unsigned long _res;                                \
4299       /* _argvec[0] holds current r2 across the call */           \
4300       _argvec[1]   = (unsigned long)_orig.r2;                     \
4301       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
4302       _argvec[2+1] = (unsigned long)arg1;                         \
4303       _argvec[2+2] = (unsigned long)arg2;                         \
4304       _argvec[2+3] = (unsigned long)arg3;                         \
4305       _argvec[2+4] = (unsigned long)arg4;                         \
4306       _argvec[2+5] = (unsigned long)arg5;                         \
4307       _argvec[2+6] = (unsigned long)arg6;                         \
4308       _argvec[2+7] = (unsigned long)arg7;                         \
4309       _argvec[2+8] = (unsigned long)arg8;                         \
4310       _argvec[2+9] = (unsigned long)arg9;                         \
4311       _argvec[2+10] = (unsigned long)arg10;                       \
4312       _argvec[2+11] = (unsigned long)arg11;                       \
4313       _argvec[2+12] = (unsigned long)arg12;                       \
4314       __asm__ volatile(                                           \
4315          "mr 11,%1\n\t"                                           \
4316          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
4317          "std  2,-16(11)\n\t" /* save tocptr */                   \
4318          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
4319          VG_EXPAND_FRAME_BY_trashes_r3(144)                       \
4320          /* arg12 */                                              \
4321          "ld  3,96(11)\n\t"                                       \
4322          "std 3,136(1)\n\t"                                       \
4323          /* arg11 */                                              \
4324          "ld  3,88(11)\n\t"                                       \
4325          "std 3,128(1)\n\t"                                       \
4326          /* arg10 */                                              \
4327          "ld  3,80(11)\n\t"                                       \
4328          "std 3,120(1)\n\t"                                       \
4329          /* arg9 */                                               \
4330          "ld  3,72(11)\n\t"                                       \
4331          "std 3,112(1)\n\t"                                       \
4332          /* args1-8 */                                            \
4333          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
4334          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
4335          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
4336          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
4337          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4338          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4339          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
4340          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
4341          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4342          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4343          "mr 11,%1\n\t"                                           \
4344          "mr %0,3\n\t"                                            \
4345          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4346          VG_CONTRACT_FRAME_BY(144)                                \
4347          VG_CONTRACT_FRAME_BY(512)                                \
4348          : /*out*/   "=r" (_res)                                  \
4349          : /*in*/    "r" (&_argvec[2])                            \
4350          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4351       );                                                          \
4352       lval = (__typeof__(lval)) _res;                             \
4353    } while (0)
4354
4355 #endif /* PLAT_ppc64_aix5 */
4356
4357 /* ------------------------- s390x-linux ------------------------- */
4358
4359 #if defined(PLAT_s390x_linux)
4360
4361 /* Similar workaround as amd64 (see above), but we use r11 as frame
4362    pointer and save the old r11 in r7. r11 might be used for
4363    argvec, therefore we copy argvec in r1 since r1 is clobbered
4364    after the call anyway.  */
4365 #if defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)
4366 #  define __FRAME_POINTER                                         \
4367       ,"d"(__builtin_dwarf_cfa())
4368 #  define VALGRIND_CFI_PROLOGUE                                   \
4369       ".cfi_remember_state\n\t"                                   \
4370       "lgr 1,%1\n\t" /* copy the argvec pointer in r1 */          \
4371       "lgr 7,11\n\t"                                              \
4372       "lgr 11,%2\n\t"                                             \
4373       ".cfi_def_cfa r11, 0\n\t"
4374 #  define VALGRIND_CFI_EPILOGUE                                   \
4375       "lgr 11, 7\n\t"                                             \
4376       ".cfi_restore_state\n\t"
4377 #else
4378 #  define __FRAME_POINTER
4379 #  define VALGRIND_CFI_PROLOGUE                                   \
4380       "lgr 1,%1\n\t"
4381 #  define VALGRIND_CFI_EPILOGUE
4382 #endif
4383
4384
4385
4386
4387 /* These regs are trashed by the hidden call. Note that we overwrite
4388    r14 in s390_irgen_noredir (VEX/priv/guest_s390_irgen.c) to give the
4389    function a proper return address. All others are ABI defined call
4390    clobbers. */
4391 #define __CALLER_SAVED_REGS "0","1","2","3","4","5","14", \
4392                            "f0","f1","f2","f3","f4","f5","f6","f7"
4393
4394
4395 #define CALL_FN_W_v(lval, orig)                                  \
4396    do {                                                          \
4397       volatile OrigFn        _orig = (orig);                     \
4398       volatile unsigned long  _argvec[1];                        \
4399       volatile unsigned long _res;                               \
4400       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4401       __asm__ volatile(                                          \
4402          VALGRIND_CFI_PROLOGUE                                   \
4403          "aghi 15,-160\n\t"                                      \
4404          "lg 1, 0(1)\n\t"  /* target->r1 */                      \
4405          VALGRIND_CALL_NOREDIR_R1                                \
4406          "lgr %0, 2\n\t"                                         \
4407          "aghi 15,160\n\t"                                       \
4408          VALGRIND_CFI_EPILOGUE                                   \
4409          : /*out*/   "=d" (_res)                                 \
4410          : /*in*/    "d" (&_argvec[0]) __FRAME_POINTER           \
4411          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4412       );                                                         \
4413       lval = (__typeof__(lval)) _res;                            \
4414    } while (0)
4415
4416 /* The call abi has the arguments in r2-r6 and stack */
4417 #define CALL_FN_W_W(lval, orig, arg1)                            \
4418    do {                                                          \
4419       volatile OrigFn        _orig = (orig);                     \
4420       volatile unsigned long _argvec[2];                         \
4421       volatile unsigned long _res;                               \
4422       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4423       _argvec[1] = (unsigned long)arg1;                          \
4424       __asm__ volatile(                                          \
4425          VALGRIND_CFI_PROLOGUE                                   \
4426          "aghi 15,-160\n\t"                                      \
4427          "lg 2, 8(1)\n\t"                                        \
4428          "lg 1, 0(1)\n\t"                                        \
4429          VALGRIND_CALL_NOREDIR_R1                                \
4430          "lgr %0, 2\n\t"                                         \
4431          "aghi 15,160\n\t"                                       \
4432          VALGRIND_CFI_EPILOGUE                                   \
4433          : /*out*/   "=d" (_res)                                 \
4434          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4435          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4436       );                                                         \
4437       lval = (__typeof__(lval)) _res;                            \
4438    } while (0)
4439
4440 #define CALL_FN_W_WW(lval, orig, arg1, arg2)                     \
4441    do {                                                          \
4442       volatile OrigFn        _orig = (orig);                     \
4443       volatile unsigned long _argvec[3];                         \
4444       volatile unsigned long _res;                               \
4445       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4446       _argvec[1] = (unsigned long)arg1;                          \
4447       _argvec[2] = (unsigned long)arg2;                          \
4448       __asm__ volatile(                                          \
4449          VALGRIND_CFI_PROLOGUE                                   \
4450          "aghi 15,-160\n\t"                                      \
4451          "lg 2, 8(1)\n\t"                                        \
4452          "lg 3,16(1)\n\t"                                        \
4453          "lg 1, 0(1)\n\t"                                        \
4454          VALGRIND_CALL_NOREDIR_R1                                \
4455          "lgr %0, 2\n\t"                                         \
4456          "aghi 15,160\n\t"                                       \
4457          VALGRIND_CFI_EPILOGUE                                   \
4458          : /*out*/   "=d" (_res)                                 \
4459          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4460          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4461       );                                                         \
4462       lval = (__typeof__(lval)) _res;                            \
4463    } while (0)
4464
4465 #define CALL_FN_W_WWW(lval, orig, arg1, arg2, arg3)              \
4466    do {                                                          \
4467       volatile OrigFn        _orig = (orig);                     \
4468       volatile unsigned long _argvec[4];                         \
4469       volatile unsigned long _res;                               \
4470       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4471       _argvec[1] = (unsigned long)arg1;                          \
4472       _argvec[2] = (unsigned long)arg2;                          \
4473       _argvec[3] = (unsigned long)arg3;                          \
4474       __asm__ volatile(                                          \
4475          VALGRIND_CFI_PROLOGUE                                   \
4476          "aghi 15,-160\n\t"                                      \
4477          "lg 2, 8(1)\n\t"                                        \
4478          "lg 3,16(1)\n\t"                                        \
4479          "lg 4,24(1)\n\t"                                        \
4480          "lg 1, 0(1)\n\t"                                        \
4481          VALGRIND_CALL_NOREDIR_R1                                \
4482          "lgr %0, 2\n\t"                                         \
4483          "aghi 15,160\n\t"                                       \
4484          VALGRIND_CFI_EPILOGUE                                   \
4485          : /*out*/   "=d" (_res)                                 \
4486          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4487          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4488       );                                                         \
4489       lval = (__typeof__(lval)) _res;                            \
4490    } while (0)
4491
4492 #define CALL_FN_W_WWWW(lval, orig, arg1, arg2, arg3, arg4)       \
4493    do {                                                          \
4494       volatile OrigFn        _orig = (orig);                     \
4495       volatile unsigned long _argvec[5];                         \
4496       volatile unsigned long _res;                               \
4497       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4498       _argvec[1] = (unsigned long)arg1;                          \
4499       _argvec[2] = (unsigned long)arg2;                          \
4500       _argvec[3] = (unsigned long)arg3;                          \
4501       _argvec[4] = (unsigned long)arg4;                          \
4502       __asm__ volatile(                                          \
4503          VALGRIND_CFI_PROLOGUE                                   \
4504          "aghi 15,-160\n\t"                                      \
4505          "lg 2, 8(1)\n\t"                                        \
4506          "lg 3,16(1)\n\t"                                        \
4507          "lg 4,24(1)\n\t"                                        \
4508          "lg 5,32(1)\n\t"                                        \
4509          "lg 1, 0(1)\n\t"                                        \
4510          VALGRIND_CALL_NOREDIR_R1                                \
4511          "lgr %0, 2\n\t"                                         \
4512          "aghi 15,160\n\t"                                       \
4513          VALGRIND_CFI_EPILOGUE                                   \
4514          : /*out*/   "=d" (_res)                                 \
4515          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4516          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4517       );                                                         \
4518       lval = (__typeof__(lval)) _res;                            \
4519    } while (0)
4520
4521 #define CALL_FN_W_5W(lval, orig, arg1, arg2, arg3, arg4, arg5)   \
4522    do {                                                          \
4523       volatile OrigFn        _orig = (orig);                     \
4524       volatile unsigned long _argvec[6];                         \
4525       volatile unsigned long _res;                               \
4526       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4527       _argvec[1] = (unsigned long)arg1;                          \
4528       _argvec[2] = (unsigned long)arg2;                          \
4529       _argvec[3] = (unsigned long)arg3;                          \
4530       _argvec[4] = (unsigned long)arg4;                          \
4531       _argvec[5] = (unsigned long)arg5;                          \
4532       __asm__ volatile(                                          \
4533          VALGRIND_CFI_PROLOGUE                                   \
4534          "aghi 15,-160\n\t"                                      \
4535          "lg 2, 8(1)\n\t"                                        \
4536          "lg 3,16(1)\n\t"                                        \
4537          "lg 4,24(1)\n\t"                                        \
4538          "lg 5,32(1)\n\t"                                        \
4539          "lg 6,40(1)\n\t"                                        \
4540          "lg 1, 0(1)\n\t"                                        \
4541          VALGRIND_CALL_NOREDIR_R1                                \
4542          "lgr %0, 2\n\t"                                         \
4543          "aghi 15,160\n\t"                                       \
4544          VALGRIND_CFI_EPILOGUE                                   \
4545          : /*out*/   "=d" (_res)                                 \
4546          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4547          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4548       );                                                         \
4549       lval = (__typeof__(lval)) _res;                            \
4550    } while (0)
4551
4552 #define CALL_FN_W_6W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
4553                      arg6)                                       \
4554    do {                                                          \
4555       volatile OrigFn        _orig = (orig);                     \
4556       volatile unsigned long _argvec[7];                         \
4557       volatile unsigned long _res;                               \
4558       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4559       _argvec[1] = (unsigned long)arg1;                          \
4560       _argvec[2] = (unsigned long)arg2;                          \
4561       _argvec[3] = (unsigned long)arg3;                          \
4562       _argvec[4] = (unsigned long)arg4;                          \
4563       _argvec[5] = (unsigned long)arg5;                          \
4564       _argvec[6] = (unsigned long)arg6;                          \
4565       __asm__ volatile(                                          \
4566          VALGRIND_CFI_PROLOGUE                                   \
4567          "aghi 15,-168\n\t"                                      \
4568          "lg 2, 8(1)\n\t"                                        \
4569          "lg 3,16(1)\n\t"                                        \
4570          "lg 4,24(1)\n\t"                                        \
4571          "lg 5,32(1)\n\t"                                        \
4572          "lg 6,40(1)\n\t"                                        \
4573          "mvc 160(8,15), 48(1)\n\t"                              \
4574          "lg 1, 0(1)\n\t"                                        \
4575          VALGRIND_CALL_NOREDIR_R1                                \
4576          "lgr %0, 2\n\t"                                         \
4577          "aghi 15,168\n\t"                                       \
4578          VALGRIND_CFI_EPILOGUE                                   \
4579          : /*out*/   "=d" (_res)                                 \
4580          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4581          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4582       );                                                         \
4583       lval = (__typeof__(lval)) _res;                            \
4584    } while (0)
4585
4586 #define CALL_FN_W_7W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
4587                      arg6, arg7)                                 \
4588    do {                                                          \
4589       volatile OrigFn        _orig = (orig);                     \
4590       volatile unsigned long _argvec[8];                         \
4591       volatile unsigned long _res;                               \
4592       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4593       _argvec[1] = (unsigned long)arg1;                          \
4594       _argvec[2] = (unsigned long)arg2;                          \
4595       _argvec[3] = (unsigned long)arg3;                          \
4596       _argvec[4] = (unsigned long)arg4;                          \
4597       _argvec[5] = (unsigned long)arg5;                          \
4598       _argvec[6] = (unsigned long)arg6;                          \
4599       _argvec[7] = (unsigned long)arg7;                          \
4600       __asm__ volatile(                                          \
4601          VALGRIND_CFI_PROLOGUE                                   \
4602          "aghi 15,-176\n\t"                                      \
4603          "lg 2, 8(1)\n\t"                                        \
4604          "lg 3,16(1)\n\t"                                        \
4605          "lg 4,24(1)\n\t"                                        \
4606          "lg 5,32(1)\n\t"                                        \
4607          "lg 6,40(1)\n\t"                                        \
4608          "mvc 160(8,15), 48(1)\n\t"                              \
4609          "mvc 168(8,15), 56(1)\n\t"                              \
4610          "lg 1, 0(1)\n\t"                                        \
4611          VALGRIND_CALL_NOREDIR_R1                                \
4612          "lgr %0, 2\n\t"                                         \
4613          "aghi 15,176\n\t"                                       \
4614          VALGRIND_CFI_EPILOGUE                                   \
4615          : /*out*/   "=d" (_res)                                 \
4616          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4617          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4618       );                                                         \
4619       lval = (__typeof__(lval)) _res;                            \
4620    } while (0)
4621
4622 #define CALL_FN_W_8W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
4623                      arg6, arg7 ,arg8)                           \
4624    do {                                                          \
4625       volatile OrigFn        _orig = (orig);                     \
4626       volatile unsigned long _argvec[9];                         \
4627       volatile unsigned long _res;                               \
4628       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4629       _argvec[1] = (unsigned long)arg1;                          \
4630       _argvec[2] = (unsigned long)arg2;                          \
4631       _argvec[3] = (unsigned long)arg3;                          \
4632       _argvec[4] = (unsigned long)arg4;                          \
4633       _argvec[5] = (unsigned long)arg5;                          \
4634       _argvec[6] = (unsigned long)arg6;                          \
4635       _argvec[7] = (unsigned long)arg7;                          \
4636       _argvec[8] = (unsigned long)arg8;                          \
4637       __asm__ volatile(                                          \
4638          VALGRIND_CFI_PROLOGUE                                   \
4639          "aghi 15,-184\n\t"                                      \
4640          "lg 2, 8(1)\n\t"                                        \
4641          "lg 3,16(1)\n\t"                                        \
4642          "lg 4,24(1)\n\t"                                        \
4643          "lg 5,32(1)\n\t"                                        \
4644          "lg 6,40(1)\n\t"                                        \
4645          "mvc 160(8,15), 48(1)\n\t"                              \
4646          "mvc 168(8,15), 56(1)\n\t"                              \
4647          "mvc 176(8,15), 64(1)\n\t"                              \
4648          "lg 1, 0(1)\n\t"                                        \
4649          VALGRIND_CALL_NOREDIR_R1                                \
4650          "lgr %0, 2\n\t"                                         \
4651          "aghi 15,184\n\t"                                       \
4652          VALGRIND_CFI_EPILOGUE                                   \
4653          : /*out*/   "=d" (_res)                                 \
4654          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4655          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4656       );                                                         \
4657       lval = (__typeof__(lval)) _res;                            \
4658    } while (0)
4659
4660 #define CALL_FN_W_9W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
4661                      arg6, arg7 ,arg8, arg9)                     \
4662    do {                                                          \
4663       volatile OrigFn        _orig = (orig);                     \
4664       volatile unsigned long _argvec[10];                        \
4665       volatile unsigned long _res;                               \
4666       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4667       _argvec[1] = (unsigned long)arg1;                          \
4668       _argvec[2] = (unsigned long)arg2;                          \
4669       _argvec[3] = (unsigned long)arg3;                          \
4670       _argvec[4] = (unsigned long)arg4;                          \
4671       _argvec[5] = (unsigned long)arg5;                          \
4672       _argvec[6] = (unsigned long)arg6;                          \
4673       _argvec[7] = (unsigned long)arg7;                          \
4674       _argvec[8] = (unsigned long)arg8;                          \
4675       _argvec[9] = (unsigned long)arg9;                          \
4676       __asm__ volatile(                                          \
4677          VALGRIND_CFI_PROLOGUE                                   \
4678          "aghi 15,-192\n\t"                                      \
4679          "lg 2, 8(1)\n\t"                                        \
4680          "lg 3,16(1)\n\t"                                        \
4681          "lg 4,24(1)\n\t"                                        \
4682          "lg 5,32(1)\n\t"                                        \
4683          "lg 6,40(1)\n\t"                                        \
4684          "mvc 160(8,15), 48(1)\n\t"                              \
4685          "mvc 168(8,15), 56(1)\n\t"                              \
4686          "mvc 176(8,15), 64(1)\n\t"                              \
4687          "mvc 184(8,15), 72(1)\n\t"                              \
4688          "lg 1, 0(1)\n\t"                                        \
4689          VALGRIND_CALL_NOREDIR_R1                                \
4690          "lgr %0, 2\n\t"                                         \
4691          "aghi 15,192\n\t"                                       \
4692          VALGRIND_CFI_EPILOGUE                                   \
4693          : /*out*/   "=d" (_res)                                 \
4694          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4695          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4696       );                                                         \
4697       lval = (__typeof__(lval)) _res;                            \
4698    } while (0)
4699
4700 #define CALL_FN_W_10W(lval, orig, arg1, arg2, arg3, arg4, arg5,  \
4701                      arg6, arg7 ,arg8, arg9, arg10)              \
4702    do {                                                          \
4703       volatile OrigFn        _orig = (orig);                     \
4704       volatile unsigned long _argvec[11];                        \
4705       volatile unsigned long _res;                               \
4706       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4707       _argvec[1] = (unsigned long)arg1;                          \
4708       _argvec[2] = (unsigned long)arg2;                          \
4709       _argvec[3] = (unsigned long)arg3;                          \
4710       _argvec[4] = (unsigned long)arg4;                          \
4711       _argvec[5] = (unsigned long)arg5;                          \
4712       _argvec[6] = (unsigned long)arg6;                          \
4713       _argvec[7] = (unsigned long)arg7;                          \
4714       _argvec[8] = (unsigned long)arg8;                          \
4715       _argvec[9] = (unsigned long)arg9;                          \
4716       _argvec[10] = (unsigned long)arg10;                        \
4717       __asm__ volatile(                                          \
4718          VALGRIND_CFI_PROLOGUE                                   \
4719          "aghi 15,-200\n\t"                                      \
4720          "lg 2, 8(1)\n\t"                                        \
4721          "lg 3,16(1)\n\t"                                        \
4722          "lg 4,24(1)\n\t"                                        \
4723          "lg 5,32(1)\n\t"                                        \
4724          "lg 6,40(1)\n\t"                                        \
4725          "mvc 160(8,15), 48(1)\n\t"                              \
4726          "mvc 168(8,15), 56(1)\n\t"                              \
4727          "mvc 176(8,15), 64(1)\n\t"                              \
4728          "mvc 184(8,15), 72(1)\n\t"                              \
4729          "mvc 192(8,15), 80(1)\n\t"                              \
4730          "lg 1, 0(1)\n\t"                                        \
4731          VALGRIND_CALL_NOREDIR_R1                                \
4732          "lgr %0, 2\n\t"                                         \
4733          "aghi 15,200\n\t"                                       \
4734          VALGRIND_CFI_EPILOGUE                                   \
4735          : /*out*/   "=d" (_res)                                 \
4736          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4737          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4738       );                                                         \
4739       lval = (__typeof__(lval)) _res;                            \
4740    } while (0)
4741
4742 #define CALL_FN_W_11W(lval, orig, arg1, arg2, arg3, arg4, arg5,  \
4743                      arg6, arg7 ,arg8, arg9, arg10, arg11)       \
4744    do {                                                          \
4745       volatile OrigFn        _orig = (orig);                     \
4746       volatile unsigned long _argvec[12];                        \
4747       volatile unsigned long _res;                               \
4748       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4749       _argvec[1] = (unsigned long)arg1;                          \
4750       _argvec[2] = (unsigned long)arg2;                          \
4751       _argvec[3] = (unsigned long)arg3;                          \
4752       _argvec[4] = (unsigned long)arg4;                          \
4753       _argvec[5] = (unsigned long)arg5;                          \
4754       _argvec[6] = (unsigned long)arg6;                          \
4755       _argvec[7] = (unsigned long)arg7;                          \
4756       _argvec[8] = (unsigned long)arg8;                          \
4757       _argvec[9] = (unsigned long)arg9;                          \
4758       _argvec[10] = (unsigned long)arg10;                        \
4759       _argvec[11] = (unsigned long)arg11;                        \
4760       __asm__ volatile(                                          \
4761          VALGRIND_CFI_PROLOGUE                                   \
4762          "aghi 15,-208\n\t"                                      \
4763          "lg 2, 8(1)\n\t"                                        \
4764          "lg 3,16(1)\n\t"                                        \
4765          "lg 4,24(1)\n\t"                                        \
4766          "lg 5,32(1)\n\t"                                        \
4767          "lg 6,40(1)\n\t"                                        \
4768          "mvc 160(8,15), 48(1)\n\t"                              \
4769          "mvc 168(8,15), 56(1)\n\t"                              \
4770          "mvc 176(8,15), 64(1)\n\t"                              \
4771          "mvc 184(8,15), 72(1)\n\t"                              \
4772          "mvc 192(8,15), 80(1)\n\t"                              \
4773          "mvc 200(8,15), 88(1)\n\t"                              \
4774          "lg 1, 0(1)\n\t"                                        \
4775          VALGRIND_CALL_NOREDIR_R1                                \
4776          "lgr %0, 2\n\t"                                         \
4777          "aghi 15,208\n\t"                                       \
4778          VALGRIND_CFI_EPILOGUE                                   \
4779          : /*out*/   "=d" (_res)                                 \
4780          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4781          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4782       );                                                         \
4783       lval = (__typeof__(lval)) _res;                            \
4784    } while (0)
4785
4786 #define CALL_FN_W_12W(lval, orig, arg1, arg2, arg3, arg4, arg5,  \
4787                      arg6, arg7 ,arg8, arg9, arg10, arg11, arg12)\
4788    do {                                                          \
4789       volatile OrigFn        _orig = (orig);                     \
4790       volatile unsigned long _argvec[13];                        \
4791       volatile unsigned long _res;                               \
4792       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4793       _argvec[1] = (unsigned long)arg1;                          \
4794       _argvec[2] = (unsigned long)arg2;                          \
4795       _argvec[3] = (unsigned long)arg3;                          \
4796       _argvec[4] = (unsigned long)arg4;                          \
4797       _argvec[5] = (unsigned long)arg5;                          \
4798       _argvec[6] = (unsigned long)arg6;                          \
4799       _argvec[7] = (unsigned long)arg7;                          \
4800       _argvec[8] = (unsigned long)arg8;                          \
4801       _argvec[9] = (unsigned long)arg9;                          \
4802       _argvec[10] = (unsigned long)arg10;                        \
4803       _argvec[11] = (unsigned long)arg11;                        \
4804       _argvec[12] = (unsigned long)arg12;                        \
4805       __asm__ volatile(                                          \
4806          VALGRIND_CFI_PROLOGUE                                   \
4807          "aghi 15,-216\n\t"                                      \
4808          "lg 2, 8(1)\n\t"                                        \
4809          "lg 3,16(1)\n\t"                                        \
4810          "lg 4,24(1)\n\t"                                        \
4811          "lg 5,32(1)\n\t"                                        \
4812          "lg 6,40(1)\n\t"                                        \
4813          "mvc 160(8,15), 48(1)\n\t"                              \
4814          "mvc 168(8,15), 56(1)\n\t"                              \
4815          "mvc 176(8,15), 64(1)\n\t"                              \
4816          "mvc 184(8,15), 72(1)\n\t"                              \
4817          "mvc 192(8,15), 80(1)\n\t"                              \
4818          "mvc 200(8,15), 88(1)\n\t"                              \
4819          "mvc 208(8,15), 96(1)\n\t"                              \
4820          "lg 1, 0(1)\n\t"                                        \
4821          VALGRIND_CALL_NOREDIR_R1                                \
4822          "lgr %0, 2\n\t"                                         \
4823          "aghi 15,216\n\t"                                       \
4824          VALGRIND_CFI_EPILOGUE                                   \
4825          : /*out*/   "=d" (_res)                                 \
4826          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4827          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4828       );                                                         \
4829       lval = (__typeof__(lval)) _res;                            \
4830    } while (0)
4831
4832
4833 #endif /* PLAT_s390x_linux */
4834
4835
4836 /* ------------------------------------------------------------------ */
4837 /* ARCHITECTURE INDEPENDENT MACROS for CLIENT REQUESTS.               */
4838 /*                                                                    */
4839 /* ------------------------------------------------------------------ */
4840
4841 /* Some request codes.  There are many more of these, but most are not
4842    exposed to end-user view.  These are the public ones, all of the
4843    form 0x1000 + small_number.
4844
4845    Core ones are in the range 0x00000000--0x0000ffff.  The non-public
4846    ones start at 0x2000.
4847 */
4848
4849 /* These macros are used by tools -- they must be public, but don't
4850    embed them into other programs. */
4851 #define VG_USERREQ_TOOL_BASE(a,b) \
4852    ((unsigned int)(((a)&0xff) << 24 | ((b)&0xff) << 16))
4853 #define VG_IS_TOOL_USERREQ(a, b, v) \
4854    (VG_USERREQ_TOOL_BASE(a,b) == ((v) & 0xffff0000))
4855
4856 /* !! ABIWARNING !! ABIWARNING !! ABIWARNING !! ABIWARNING !!
4857    This enum comprises an ABI exported by Valgrind to programs
4858    which use client requests.  DO NOT CHANGE THE ORDER OF THESE
4859    ENTRIES, NOR DELETE ANY -- add new ones at the end. */
4860 typedef
4861    enum { VG_USERREQ__RUNNING_ON_VALGRIND  = 0x1001,
4862           VG_USERREQ__DISCARD_TRANSLATIONS = 0x1002,
4863
4864           /* These allow any function to be called from the simulated
4865              CPU but run on the real CPU.  Nb: the first arg passed to
4866              the function is always the ThreadId of the running
4867              thread!  So CLIENT_CALL0 actually requires a 1 arg
4868              function, etc. */
4869           VG_USERREQ__CLIENT_CALL0 = 0x1101,
4870           VG_USERREQ__CLIENT_CALL1 = 0x1102,
4871           VG_USERREQ__CLIENT_CALL2 = 0x1103,
4872           VG_USERREQ__CLIENT_CALL3 = 0x1104,
4873
4874           /* Can be useful in regression testing suites -- eg. can
4875              send Valgrind's output to /dev/null and still count
4876              errors. */
4877           VG_USERREQ__COUNT_ERRORS = 0x1201,
4878
4879           /* Allows a string (gdb monitor command) to be passed to the tool
4880              Used for interaction with vgdb/gdb */
4881           VG_USERREQ__GDB_MONITOR_COMMAND = 0x1202,
4882
4883           /* These are useful and can be interpreted by any tool that
4884              tracks malloc() et al, by using vg_replace_malloc.c. */
4885           VG_USERREQ__MALLOCLIKE_BLOCK = 0x1301,
4886           VG_USERREQ__RESIZEINPLACE_BLOCK = 0x130b,
4887           VG_USERREQ__FREELIKE_BLOCK   = 0x1302,
4888           /* Memory pool support. */
4889           VG_USERREQ__CREATE_MEMPOOL   = 0x1303,
4890           VG_USERREQ__DESTROY_MEMPOOL  = 0x1304,
4891           VG_USERREQ__MEMPOOL_ALLOC    = 0x1305,
4892           VG_USERREQ__MEMPOOL_FREE     = 0x1306,
4893           VG_USERREQ__MEMPOOL_TRIM     = 0x1307,
4894           VG_USERREQ__MOVE_MEMPOOL     = 0x1308,
4895           VG_USERREQ__MEMPOOL_CHANGE   = 0x1309,
4896           VG_USERREQ__MEMPOOL_EXISTS   = 0x130a,
4897
4898           /* Allow printfs to valgrind log. */
4899           /* The first two pass the va_list argument by value, which
4900              assumes it is the same size as or smaller than a UWord,
4901              which generally isn't the case.  Hence are deprecated.
4902              The second two pass the vargs by reference and so are
4903              immune to this problem. */
4904           /* both :: char* fmt, va_list vargs (DEPRECATED) */
4905           VG_USERREQ__PRINTF           = 0x1401,
4906           VG_USERREQ__PRINTF_BACKTRACE = 0x1402,
4907           /* both :: char* fmt, va_list* vargs */
4908           VG_USERREQ__PRINTF_VALIST_BY_REF = 0x1403,
4909           VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF = 0x1404,
4910
4911           /* Stack support. */
4912           VG_USERREQ__STACK_REGISTER   = 0x1501,
4913           VG_USERREQ__STACK_DEREGISTER = 0x1502,
4914           VG_USERREQ__STACK_CHANGE     = 0x1503,
4915
4916           /* Wine support */
4917           VG_USERREQ__LOAD_PDB_DEBUGINFO = 0x1601,
4918
4919           /* Querying of debug info. */
4920           VG_USERREQ__MAP_IP_TO_SRCLOC = 0x1701
4921    } Vg_ClientRequest;
4922
4923 #if !defined(__GNUC__)
4924 #  define __extension__ /* */
4925 #endif
4926
4927
4928 /* Returns the number of Valgrinds this code is running under.  That
4929    is, 0 if running natively, 1 if running under Valgrind, 2 if
4930    running under Valgrind which is running under another Valgrind,
4931    etc. */
4932 #define RUNNING_ON_VALGRIND                                           \
4933     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* if not */,         \
4934                                     VG_USERREQ__RUNNING_ON_VALGRIND,  \
4935                                     0, 0, 0, 0, 0)                    \
4936
4937
4938 /* Discard translation of code in the range [_qzz_addr .. _qzz_addr +
4939    _qzz_len - 1].  Useful if you are debugging a JITter or some such,
4940    since it provides a way to make sure valgrind will retranslate the
4941    invalidated area.  Returns no value. */
4942 #define VALGRIND_DISCARD_TRANSLATIONS(_qzz_addr,_qzz_len)         \
4943     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                  \
4944                                VG_USERREQ__DISCARD_TRANSLATIONS,  \
4945                                _qzz_addr, _qzz_len, 0, 0, 0)
4946
4947
4948 /* These requests are for getting Valgrind itself to print something.
4949    Possibly with a backtrace.  This is a really ugly hack.  The return value
4950    is the number of characters printed, excluding the "**<pid>** " part at the
4951    start and the backtrace (if present). */
4952
4953 #if defined(__GNUC__) || defined(__INTEL_COMPILER)
4954 /* Modern GCC will optimize the static routine out if unused,
4955    and unused attribute will shut down warnings about it.  */
4956 static int VALGRIND_PRINTF(const char *format, ...)
4957    __attribute__((format(__printf__, 1, 2), __unused__));
4958 #endif
4959 static int
4960 #if defined(_MSC_VER)
4961 __inline
4962 #endif
4963 VALGRIND_PRINTF(const char *format, ...)
4964 {
4965 #if defined(NVALGRIND)
4966    return 0;
4967 #else /* NVALGRIND */
4968 #if defined(_MSC_VER)
4969    uintptr_t _qzz_res;
4970 #else
4971    unsigned long _qzz_res;
4972 #endif
4973    va_list vargs;
4974    va_start(vargs, format);
4975 #if defined(_MSC_VER)
4976    _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
4977                               VG_USERREQ__PRINTF_VALIST_BY_REF,
4978                               (uintptr_t)format,
4979                               (uintptr_t)&vargs,
4980                               0, 0, 0);
4981 #else
4982    _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
4983                               VG_USERREQ__PRINTF_VALIST_BY_REF,
4984                               (unsigned long)format,
4985                               (unsigned long)&vargs,
4986                               0, 0, 0);
4987 #endif
4988    va_end(vargs);
4989    return (int)_qzz_res;
4990 #endif /* NVALGRIND */
4991 }
4992
4993 #if defined(__GNUC__) || defined(__INTEL_COMPILER)
4994 static int VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
4995    __attribute__((format(__printf__, 1, 2), __unused__));
4996 #endif
4997 static int
4998 #if defined(_MSC_VER)
4999 __inline
5000 #endif
5001 VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
5002 {
5003 #if defined(NVALGRIND)
5004    return 0;
5005 #else /* NVALGRIND */
5006 #if defined(_MSC_VER)
5007    uintptr_t _qzz_res;
5008 #else
5009    unsigned long _qzz_res;
5010 #endif
5011    va_list vargs;
5012    va_start(vargs, format);
5013 #if defined(_MSC_VER)
5014    _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
5015                               VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF,
5016                               (uintptr_t)format,
5017                               (uintptr_t)&vargs,
5018                               0, 0, 0);
5019 #else
5020    _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
5021                               VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF,
5022                               (unsigned long)format,
5023                               (unsigned long)&vargs,
5024                               0, 0, 0);
5025 #endif
5026    va_end(vargs);
5027    return (int)_qzz_res;
5028 #endif /* NVALGRIND */
5029 }
5030
5031
5032 /* These requests allow control to move from the simulated CPU to the
5033    real CPU, calling an arbitary function.
5034
5035    Note that the current ThreadId is inserted as the first argument.
5036    So this call:
5037
5038      VALGRIND_NON_SIMD_CALL2(f, arg1, arg2)
5039
5040    requires f to have this signature:
5041
5042      Word f(Word tid, Word arg1, Word arg2)
5043
5044    where "Word" is a word-sized type.
5045
5046    Note that these client requests are not entirely reliable.  For example,
5047    if you call a function with them that subsequently calls printf(),
5048    there's a high chance Valgrind will crash.  Generally, your prospects of
5049    these working are made higher if the called function does not refer to
5050    any global variables, and does not refer to any libc or other functions
5051    (printf et al).  Any kind of entanglement with libc or dynamic linking is
5052    likely to have a bad outcome, for tricky reasons which we've grappled
5053    with a lot in the past.
5054 */
5055 #define VALGRIND_NON_SIMD_CALL0(_qyy_fn)                          \
5056     VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */,       \
5057                                     VG_USERREQ__CLIENT_CALL0,     \
5058                                     _qyy_fn,                      \
5059                                     0, 0, 0, 0)
5060
5061 #define VALGRIND_NON_SIMD_CALL1(_qyy_fn, _qyy_arg1)                    \
5062     VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */,            \
5063                                     VG_USERREQ__CLIENT_CALL1,          \
5064                                     _qyy_fn,                           \
5065                                     _qyy_arg1, 0, 0, 0)
5066
5067 #define VALGRIND_NON_SIMD_CALL2(_qyy_fn, _qyy_arg1, _qyy_arg2)         \
5068     VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */,            \
5069                                     VG_USERREQ__CLIENT_CALL2,          \
5070                                     _qyy_fn,                           \
5071                                     _qyy_arg1, _qyy_arg2, 0, 0)
5072
5073 #define VALGRIND_NON_SIMD_CALL3(_qyy_fn, _qyy_arg1, _qyy_arg2, _qyy_arg3) \
5074     VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */,             \
5075                                     VG_USERREQ__CLIENT_CALL3,           \
5076                                     _qyy_fn,                            \
5077                                     _qyy_arg1, _qyy_arg2,               \
5078                                     _qyy_arg3, 0)
5079
5080
5081 /* Counts the number of errors that have been recorded by a tool.  Nb:
5082    the tool must record the errors with VG_(maybe_record_error)() or
5083    VG_(unique_error)() for them to be counted. */
5084 #define VALGRIND_COUNT_ERRORS                                     \
5085     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(                    \
5086                                0 /* default return */,            \
5087                                VG_USERREQ__COUNT_ERRORS,          \
5088                                0, 0, 0, 0, 0)
5089
5090 /* Several Valgrind tools (Memcheck, Massif, Helgrind, DRD) rely on knowing
5091    when heap blocks are allocated in order to give accurate results.  This
5092    happens automatically for the standard allocator functions such as
5093    malloc(), calloc(), realloc(), memalign(), new, new[], free(), delete,
5094    delete[], etc.
5095
5096    But if your program uses a custom allocator, this doesn't automatically
5097    happen, and Valgrind will not do as well.  For example, if you allocate
5098    superblocks with mmap() and then allocates chunks of the superblocks, all
5099    Valgrind's observations will be at the mmap() level and it won't know that
5100    the chunks should be considered separate entities.  In Memcheck's case,
5101    that means you probably won't get heap block overrun detection (because
5102    there won't be redzones marked as unaddressable) and you definitely won't
5103    get any leak detection.
5104
5105    The following client requests allow a custom allocator to be annotated so
5106    that it can be handled accurately by Valgrind.
5107
5108    VALGRIND_MALLOCLIKE_BLOCK marks a region of memory as having been allocated
5109    by a malloc()-like function.  For Memcheck (an illustrative case), this
5110    does two things:
5111
5112    - It records that the block has been allocated.  This means any addresses
5113      within the block mentioned in error messages will be
5114      identified as belonging to the block.  It also means that if the block
5115      isn't freed it will be detected by the leak checker.
5116
5117    - It marks the block as being addressable and undefined (if 'is_zeroed' is
5118      not set), or addressable and defined (if 'is_zeroed' is set).  This
5119      controls how accesses to the block by the program are handled.
5120
5121    'addr' is the start of the usable block (ie. after any
5122    redzone), 'sizeB' is its size.  'rzB' is the redzone size if the allocator
5123    can apply redzones -- these are blocks of padding at the start and end of
5124    each block.  Adding redzones is recommended as it makes it much more likely
5125    Valgrind will spot block overruns.  `is_zeroed' indicates if the memory is
5126    zeroed (or filled with another predictable value), as is the case for
5127    calloc().
5128
5129    VALGRIND_MALLOCLIKE_BLOCK should be put immediately after the point where a
5130    heap block -- that will be used by the client program -- is allocated.
5131    It's best to put it at the outermost level of the allocator if possible;
5132    for example, if you have a function my_alloc() which calls
5133    internal_alloc(), and the client request is put inside internal_alloc(),
5134    stack traces relating to the heap block will contain entries for both
5135    my_alloc() and internal_alloc(), which is probably not what you want.
5136
5137    For Memcheck users: if you use VALGRIND_MALLOCLIKE_BLOCK to carve out
5138    custom blocks from within a heap block, B, that has been allocated with
5139    malloc/calloc/new/etc, then block B will be *ignored* during leak-checking
5140    -- the custom blocks will take precedence.
5141
5142    VALGRIND_FREELIKE_BLOCK is the partner to VALGRIND_MALLOCLIKE_BLOCK.  For
5143    Memcheck, it does two things:
5144
5145    - It records that the block has been deallocated.  This assumes that the
5146      block was annotated as having been allocated via
5147      VALGRIND_MALLOCLIKE_BLOCK.  Otherwise, an error will be issued.
5148
5149    - It marks the block as being unaddressable.
5150
5151    VALGRIND_FREELIKE_BLOCK should be put immediately after the point where a
5152    heap block is deallocated.
5153
5154    VALGRIND_RESIZEINPLACE_BLOCK informs a tool about reallocation. For
5155    Memcheck, it does four things:
5156
5157    - It records that the size of a block has been changed.  This assumes that
5158      the block was annotated as having been allocated via
5159      VALGRIND_MALLOCLIKE_BLOCK.  Otherwise, an error will be issued.
5160
5161    - If the block shrunk, it marks the freed memory as being unaddressable.
5162
5163    - If the block grew, it marks the new area as undefined and defines a red
5164      zone past the end of the new block.
5165
5166    - The V-bits of the overlap between the old and the new block are preserved.
5167
5168    VALGRIND_RESIZEINPLACE_BLOCK should be put after allocation of the new block
5169    and before deallocation of the old block.
5170
5171    In many cases, these three client requests will not be enough to get your
5172    allocator working well with Memcheck.  More specifically, if your allocator
5173    writes to freed blocks in any way then a VALGRIND_MAKE_MEM_UNDEFINED call
5174    will be necessary to mark the memory as addressable just before the zeroing
5175    occurs, otherwise you'll get a lot of invalid write errors.  For example,
5176    you'll need to do this if your allocator recycles freed blocks, but it
5177    zeroes them before handing them back out (via VALGRIND_MALLOCLIKE_BLOCK).
5178    Alternatively, if your allocator reuses freed blocks for allocator-internal
5179    data structures, VALGRIND_MAKE_MEM_UNDEFINED calls will also be necessary.
5180
5181    Really, what's happening is a blurring of the lines between the client
5182    program and the allocator... after VALGRIND_FREELIKE_BLOCK is called, the
5183    memory should be considered unaddressable to the client program, but the
5184    allocator knows more than the rest of the client program and so may be able
5185    to safely access it.  Extra client requests are necessary for Valgrind to
5186    understand the distinction between the allocator and the rest of the
5187    program.
5188
5189    Ignored if addr == 0.
5190 */
5191 #define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)    \
5192     VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                            \
5193                                VG_USERREQ__MALLOCLIKE_BLOCK,      \
5194                                addr, sizeB, rzB, is_zeroed, 0)
5195
5196 /* See the comment for VALGRIND_MALLOCLIKE_BLOCK for details.
5197    Ignored if addr == 0.
5198 */
5199 #define VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB) \
5200     VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                            \
5201                                VG_USERREQ__RESIZEINPLACE_BLOCK,   \
5202                                addr, oldSizeB, newSizeB, rzB, 0)
5203
5204 /* See the comment for VALGRIND_MALLOCLIKE_BLOCK for details.
5205    Ignored if addr == 0.
5206 */
5207 #define VALGRIND_FREELIKE_BLOCK(addr, rzB)                        \
5208     VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                            \
5209                                VG_USERREQ__FREELIKE_BLOCK,        \
5210                                addr, rzB, 0, 0, 0)
5211
5212 /* Create a memory pool. */
5213 #define VALGRIND_CREATE_MEMPOOL(pool, rzB, is_zeroed)             \
5214     VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                            \
5215                                VG_USERREQ__CREATE_MEMPOOL,        \
5216                                pool, rzB, is_zeroed, 0, 0)
5217
5218 /* Destroy a memory pool. */
5219 #define VALGRIND_DESTROY_MEMPOOL(pool)                            \
5220     VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                            \
5221                                VG_USERREQ__DESTROY_MEMPOOL,       \
5222                                pool, 0, 0, 0, 0)
5223
5224 /* Associate a piece of memory with a memory pool. */
5225 #define VALGRIND_MEMPOOL_ALLOC(pool, addr, size)                  \
5226     VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                            \
5227                                VG_USERREQ__MEMPOOL_ALLOC,         \
5228                                pool, addr, size, 0, 0)
5229
5230 /* Disassociate a piece of memory from a memory pool. */
5231 #define VALGRIND_MEMPOOL_FREE(pool, addr)                         \
5232     VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                            \
5233                                VG_USERREQ__MEMPOOL_FREE,          \
5234                                pool, addr, 0, 0, 0)
5235
5236 /* Disassociate any pieces outside a particular range. */
5237 #define VALGRIND_MEMPOOL_TRIM(pool, addr, size)                   \
5238     VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                            \
5239                                VG_USERREQ__MEMPOOL_TRIM,          \
5240                                pool, addr, size, 0, 0)
5241
5242 /* Resize and/or move a piece associated with a memory pool. */
5243 #define VALGRIND_MOVE_MEMPOOL(poolA, poolB)                       \
5244     VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                            \
5245                                VG_USERREQ__MOVE_MEMPOOL,          \
5246                                poolA, poolB, 0, 0, 0)
5247
5248 /* Resize and/or move a piece associated with a memory pool. */
5249 #define VALGRIND_MEMPOOL_CHANGE(pool, addrA, addrB, size)         \
5250     VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                            \
5251                                VG_USERREQ__MEMPOOL_CHANGE,        \
5252                                pool, addrA, addrB, size, 0)
5253
5254 /* Return 1 if a mempool exists, else 0. */
5255 #define VALGRIND_MEMPOOL_EXISTS(pool)                             \
5256     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                  \
5257                                VG_USERREQ__MEMPOOL_EXISTS,        \
5258                                pool, 0, 0, 0, 0)
5259
5260 /* Mark a piece of memory as being a stack. Returns a stack id. */
5261 #define VALGRIND_STACK_REGISTER(start, end)                       \
5262     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                  \
5263                                VG_USERREQ__STACK_REGISTER,        \
5264                                start, end, 0, 0, 0)
5265
5266 /* Unmark the piece of memory associated with a stack id as being a
5267    stack. */
5268 #define VALGRIND_STACK_DEREGISTER(id)                             \
5269     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                  \
5270                                VG_USERREQ__STACK_DEREGISTER,      \
5271                                id, 0, 0, 0, 0)
5272
5273 /* Change the start and end address of the stack id. */
5274 #define VALGRIND_STACK_CHANGE(id, start, end)                     \
5275     VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                            \
5276                                VG_USERREQ__STACK_CHANGE,          \
5277                                id, start, end, 0, 0)
5278
5279 /* Load PDB debug info for Wine PE image_map. */
5280 #define VALGRIND_LOAD_PDB_DEBUGINFO(fd, ptr, total_size, delta)   \
5281     VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                            \
5282                                VG_USERREQ__LOAD_PDB_DEBUGINFO,    \
5283                                fd, ptr, total_size, delta, 0)
5284
5285 /* Map a code address to a source file name and line number.  buf64
5286    must point to a 64-byte buffer in the caller's address space.  The
5287    result will be dumped in there and is guaranteed to be zero
5288    terminated.  If no info is found, the first byte is set to zero. */
5289 #define VALGRIND_MAP_IP_TO_SRCLOC(addr, buf64)                    \
5290     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                  \
5291                                VG_USERREQ__MAP_IP_TO_SRCLOC,      \
5292                                addr, buf64, 0, 0, 0)
5293
5294
5295 #undef PLAT_x86_linux
5296 #undef PLAT_amd64_linux
5297 #undef PLAT_ppc32_linux
5298 #undef PLAT_ppc64_linux
5299 #undef PLAT_arm_linux
5300 #undef PLAT_s390x_linux
5301 #undef PLAT_ppc32_aix5
5302 #undef PLAT_ppc64_aix5
5303
5304 #endif   /* __VALGRIND_H */