2 /*--------------------------------------------------------------------*/
3 /*--- MemCheck: Maintain bitmaps of memory, tracking the ---*/
4 /*--- accessibility (A) and validity (V) status of each byte. ---*/
6 /*--------------------------------------------------------------------*/
9 This file is part of MemCheck, a heavyweight Valgrind tool for
10 detecting memory errors.
12 Copyright (C) 2000-2010 Julian Seward
15 This program is free software; you can redistribute it and/or
16 modify it under the terms of the GNU General Public License as
17 published by the Free Software Foundation; either version 2 of the
18 License, or (at your option) any later version.
20 This program is distributed in the hope that it will be useful, but
21 WITHOUT ANY WARRANTY; without even the implied warranty of
22 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
23 General Public License for more details.
25 You should have received a copy of the GNU General Public License
26 along with this program; if not, write to the Free Software
27 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
30 The GNU General Public License is contained in the file COPYING.
33 #include "pub_tool_basics.h"
34 #include "pub_tool_aspacemgr.h"
35 #include "pub_tool_gdbserver.h"
36 #include "pub_tool_hashtable.h" // For mc_include.h
37 #include "pub_tool_libcbase.h"
38 #include "pub_tool_libcassert.h"
39 #include "pub_tool_libcprint.h"
40 #include "pub_tool_machine.h"
41 #include "pub_tool_mallocfree.h"
42 #include "pub_tool_options.h"
43 #include "pub_tool_oset.h"
44 #include "pub_tool_replacemalloc.h"
45 #include "pub_tool_tooliface.h"
46 #include "pub_tool_threadstate.h"
48 #include "mc_include.h"
49 #include "memcheck.h" /* for client requests */
52 HChar *VG_(toolname)="memcheck";
55 /* Set to 1 to do a little more sanity checking */
56 #define VG_DEBUG_MEMORY 0
58 #define DEBUG(fmt, args...) //VG_(printf)(fmt, ## args)
60 static void ocache_sarp_Set_Origins ( Addr, UWord, UInt ); /* fwds */
61 static void ocache_sarp_Clear_Origins ( Addr, UWord ); /* fwds */
64 /*------------------------------------------------------------*/
65 /*--- Fast-case knobs ---*/
66 /*------------------------------------------------------------*/
68 // Comment these out to disable the fast cases (don't just set them to zero).
70 #define PERF_FAST_LOADV 1
71 #define PERF_FAST_STOREV 1
73 #define PERF_FAST_SARP 1
75 #define PERF_FAST_STACK 1
76 #define PERF_FAST_STACK2 1
78 /* Change this to 1 to enable assertions on origin tracking cache fast
80 #define OC_ENABLE_ASSERTIONS 0
83 /*------------------------------------------------------------*/
84 /*--- Comments on the origin tracking implementation ---*/
85 /*------------------------------------------------------------*/
87 /* See detailed comment entitled
88 AN OVERVIEW OF THE ORIGIN TRACKING IMPLEMENTATION
89 which is contained further on in this file. */
92 /*------------------------------------------------------------*/
93 /*--- V bits and A bits ---*/
94 /*------------------------------------------------------------*/
96 /* Conceptually, every byte value has 8 V bits, which track whether Memcheck
97 thinks the corresponding value bit is defined. And every memory byte
98 has an A bit, which tracks whether Memcheck thinks the program can access
99 it safely (ie. it's mapped, and has at least one of the RWX permission bits
100 set). So every N-bit register is shadowed with N V bits, and every memory
101 byte is shadowed with 8 V bits and one A bit.
103 In the implementation, we use two forms of compression (compressed V bits
104 and distinguished secondary maps) to avoid the 9-bit-per-byte overhead
107 Memcheck also tracks extra information about each heap block that is
108 allocated, for detecting memory leaks and other purposes.
111 /*------------------------------------------------------------*/
112 /*--- Basic A/V bitmap representation. ---*/
113 /*------------------------------------------------------------*/
115 /* All reads and writes are checked against a memory map (a.k.a. shadow
116 memory), which records the state of all memory in the process.
118 On 32-bit machines the memory map is organised as follows.
119 The top 16 bits of an address are used to index into a top-level
120 map table, containing 65536 entries. Each entry is a pointer to a
121 second-level map, which records the accesibililty and validity
122 permissions for the 65536 bytes indexed by the lower 16 bits of the
123 address. Each byte is represented by two bits (details are below). So
124 each second-level map contains 16384 bytes. This two-level arrangement
125 conveniently divides the 4G address space into 64k lumps, each size 64k
128 All entries in the primary (top-level) map must point to a valid
129 secondary (second-level) map. Since many of the 64kB chunks will
130 have the same status for every bit -- ie. noaccess (for unused
131 address space) or entirely addressable and defined (for code segments) --
132 there are three distinguished secondary maps, which indicate 'noaccess',
133 'undefined' and 'defined'. For these uniform 64kB chunks, the primary
134 map entry points to the relevant distinguished map. In practice,
135 typically more than half of the addressable memory is represented with
136 the 'undefined' or 'defined' distinguished secondary map, so it gives a
137 good saving. It also lets us set the V+A bits of large address regions
138 quickly in set_address_range_perms().
140 On 64-bit machines it's more complicated. If we followed the same basic
141 scheme we'd have a four-level table which would require too many memory
142 accesses. So instead the top-level map table has 2^19 entries (indexed
143 using bits 16..34 of the address); this covers the bottom 32GB. Any
144 accesses above 32GB are handled with a slow, sparse auxiliary table.
145 Valgrind's address space manager tries very hard to keep things below
146 this 32GB barrier so that performance doesn't suffer too much.
148 Note that this file has a lot of different functions for reading and
149 writing shadow memory. Only a couple are strictly necessary (eg.
150 get_vabits2 and set_vabits2), most are just specialised for specific
151 common cases to improve performance.
153 Aside: the V+A bits are less precise than they could be -- we have no way
154 of marking memory as read-only. It would be great if we could add an
155 extra state VA_BITSn_READONLY. But then we'd have 5 different states,
156 which requires 2.3 bits to hold, and there's no way to do that elegantly
157 -- we'd have to double up to 4 bits of metadata per byte, which doesn't
161 /* --------------- Basic configuration --------------- */
163 /* Only change this. N_PRIMARY_MAP *must* be a power of 2. */
167 /* cover the entire address space */
168 # define N_PRIMARY_BITS 16
172 /* Just handle the first 32G fast and the rest via auxiliary
173 primaries. If you change this, Memcheck will assert at startup.
174 See the definition of UNALIGNED_OR_HIGH for extensive comments. */
175 # define N_PRIMARY_BITS 19
180 /* Do not change this. */
181 #define N_PRIMARY_MAP ( ((UWord)1) << N_PRIMARY_BITS)
183 /* Do not change this. */
184 #define MAX_PRIMARY_ADDRESS (Addr)((((Addr)65536) * N_PRIMARY_MAP)-1)
187 /* --------------- Secondary maps --------------- */
189 // Each byte of memory conceptually has an A bit, which indicates its
190 // addressability, and 8 V bits, which indicates its definedness.
192 // But because very few bytes are partially defined, we can use a nice
193 // compression scheme to reduce the size of shadow memory. Each byte of
194 // memory has 2 bits which indicates its state (ie. V+A bits):
196 // 00: noaccess (unaddressable but treated as fully defined)
197 // 01: undefined (addressable and fully undefined)
198 // 10: defined (addressable and fully defined)
199 // 11: partdefined (addressable and partially defined)
201 // In the "partdefined" case, we use a secondary table to store the V bits.
202 // Each entry in the secondary-V-bits table maps a byte address to its 8 V
205 // We store the compressed V+A bits in 8-bit chunks, ie. the V+A bits for
206 // four bytes (32 bits) of memory are in each chunk. Hence the name
207 // "vabits8". This lets us get the V+A bits for four bytes at a time
208 // easily (without having to do any shifting and/or masking), and that is a
209 // very common operation. (Note that although each vabits8 chunk
210 // is 8 bits in size, it represents 32 bits of memory.)
212 // The representation is "inverse" little-endian... each 4 bytes of
213 // memory is represented by a 1 byte value, where:
215 // - the status of byte (a+0) is held in bits [1..0]
216 // - the status of byte (a+1) is held in bits [3..2]
217 // - the status of byte (a+2) is held in bits [5..4]
218 // - the status of byte (a+3) is held in bits [7..6]
220 // It's "inverse" because endianness normally describes a mapping from
221 // value bits to memory addresses; in this case the mapping is inverted.
222 // Ie. instead of particular value bits being held in certain addresses, in
223 // this case certain addresses are represented by particular value bits.
224 // See insert_vabits2_into_vabits8() for an example.
226 // But note that we don't compress the V bits stored in registers; they
227 // need to be explicit to made the shadow operations possible. Therefore
228 // when moving values between registers and memory we need to convert
229 // between the expanded in-register format and the compressed in-memory
230 // format. This isn't so difficult, it just requires careful attention in a
233 // These represent eight bits of memory.
234 #define VA_BITS2_NOACCESS 0x0 // 00b
235 #define VA_BITS2_UNDEFINED 0x1 // 01b
236 #define VA_BITS2_DEFINED 0x2 // 10b
237 #define VA_BITS2_PARTDEFINED 0x3 // 11b
239 // These represent 16 bits of memory.
240 #define VA_BITS4_NOACCESS 0x0 // 00_00b
241 #define VA_BITS4_UNDEFINED 0x5 // 01_01b
242 #define VA_BITS4_DEFINED 0xa // 10_10b
244 // These represent 32 bits of memory.
245 #define VA_BITS8_NOACCESS 0x00 // 00_00_00_00b
246 #define VA_BITS8_UNDEFINED 0x55 // 01_01_01_01b
247 #define VA_BITS8_DEFINED 0xaa // 10_10_10_10b
249 // These represent 64 bits of memory.
250 #define VA_BITS16_NOACCESS 0x0000 // 00_00_00_00b x 2
251 #define VA_BITS16_UNDEFINED 0x5555 // 01_01_01_01b x 2
252 #define VA_BITS16_DEFINED 0xaaaa // 10_10_10_10b x 2
255 #define SM_CHUNKS 16384
256 #define SM_OFF(aaa) (((aaa) & 0xffff) >> 2)
257 #define SM_OFF_16(aaa) (((aaa) & 0xffff) >> 3)
259 // Paranoia: it's critical for performance that the requested inlining
260 // occurs. So try extra hard.
261 #define INLINE inline __attribute__((always_inline))
263 static INLINE Addr start_of_this_sm ( Addr a ) {
264 return (a & (~SM_MASK));
266 static INLINE Bool is_start_of_sm ( Addr a ) {
267 return (start_of_this_sm(a) == a);
272 UChar vabits8[SM_CHUNKS];
276 // 3 distinguished secondary maps, one for no-access, one for
277 // accessible but undefined, and one for accessible and defined.
278 // Distinguished secondaries may never be modified.
279 #define SM_DIST_NOACCESS 0
280 #define SM_DIST_UNDEFINED 1
281 #define SM_DIST_DEFINED 2
283 static SecMap sm_distinguished[3];
285 static INLINE Bool is_distinguished_sm ( SecMap* sm ) {
286 return sm >= &sm_distinguished[0] && sm <= &sm_distinguished[2];
289 // Forward declaration
290 static void update_SM_counts(SecMap* oldSM, SecMap* newSM);
292 /* dist_sm points to one of our three distinguished secondaries. Make
293 a copy of it so that we can write to it.
295 static SecMap* copy_for_writing ( SecMap* dist_sm )
298 tl_assert(dist_sm == &sm_distinguished[0]
299 || dist_sm == &sm_distinguished[1]
300 || dist_sm == &sm_distinguished[2]);
302 new_sm = VG_(am_shadow_alloc)(sizeof(SecMap));
304 VG_(out_of_memory_NORETURN)( "memcheck:allocate new SecMap",
306 VG_(memcpy)(new_sm, dist_sm, sizeof(SecMap));
307 update_SM_counts(dist_sm, new_sm);
311 /* --------------- Stats --------------- */
313 static Int n_issued_SMs = 0;
314 static Int n_deissued_SMs = 0;
315 static Int n_noaccess_SMs = N_PRIMARY_MAP; // start with many noaccess DSMs
316 static Int n_undefined_SMs = 0;
317 static Int n_defined_SMs = 0;
318 static Int n_non_DSM_SMs = 0;
319 static Int max_noaccess_SMs = 0;
320 static Int max_undefined_SMs = 0;
321 static Int max_defined_SMs = 0;
322 static Int max_non_DSM_SMs = 0;
324 /* # searches initiated in auxmap_L1, and # base cmps required */
325 static ULong n_auxmap_L1_searches = 0;
326 static ULong n_auxmap_L1_cmps = 0;
327 /* # of searches that missed in auxmap_L1 and therefore had to
328 be handed to auxmap_L2. And the number of nodes inserted. */
329 static ULong n_auxmap_L2_searches = 0;
330 static ULong n_auxmap_L2_nodes = 0;
332 static Int n_sanity_cheap = 0;
333 static Int n_sanity_expensive = 0;
335 static Int n_secVBit_nodes = 0;
336 static Int max_secVBit_nodes = 0;
338 static void update_SM_counts(SecMap* oldSM, SecMap* newSM)
340 if (oldSM == &sm_distinguished[SM_DIST_NOACCESS ]) n_noaccess_SMs --;
341 else if (oldSM == &sm_distinguished[SM_DIST_UNDEFINED]) n_undefined_SMs--;
342 else if (oldSM == &sm_distinguished[SM_DIST_DEFINED ]) n_defined_SMs --;
343 else { n_non_DSM_SMs --;
346 if (newSM == &sm_distinguished[SM_DIST_NOACCESS ]) n_noaccess_SMs ++;
347 else if (newSM == &sm_distinguished[SM_DIST_UNDEFINED]) n_undefined_SMs++;
348 else if (newSM == &sm_distinguished[SM_DIST_DEFINED ]) n_defined_SMs ++;
349 else { n_non_DSM_SMs ++;
352 if (n_noaccess_SMs > max_noaccess_SMs ) max_noaccess_SMs = n_noaccess_SMs;
353 if (n_undefined_SMs > max_undefined_SMs) max_undefined_SMs = n_undefined_SMs;
354 if (n_defined_SMs > max_defined_SMs ) max_defined_SMs = n_defined_SMs;
355 if (n_non_DSM_SMs > max_non_DSM_SMs ) max_non_DSM_SMs = n_non_DSM_SMs;
358 /* --------------- Primary maps --------------- */
360 /* The main primary map. This covers some initial part of the address
361 space, addresses 0 .. (N_PRIMARY_MAP << 16)-1. The rest of it is
362 handled using the auxiliary primary map.
364 static SecMap* primary_map[N_PRIMARY_MAP];
367 /* An entry in the auxiliary primary map. base must be a 64k-aligned
368 value, and sm points at the relevant secondary map. As with the
369 main primary map, the secondary may be either a real secondary, or
370 one of the three distinguished secondaries. DO NOT CHANGE THIS
371 LAYOUT: the first word has to be the key for OSet fast lookups.
380 /* Tunable parameter: How big is the L1 queue? */
381 #define N_AUXMAP_L1 24
383 /* Tunable parameter: How far along the L1 queue to insert
384 entries resulting from L2 lookups? */
385 #define AUXMAP_L1_INSERT_IX 12
389 AuxMapEnt* ent; // pointer to the matching auxmap_L2 node
391 auxmap_L1[N_AUXMAP_L1];
393 static OSet* auxmap_L2 = NULL;
395 static void init_auxmap_L1_L2 ( void )
398 for (i = 0; i < N_AUXMAP_L1; i++) {
399 auxmap_L1[i].base = 0;
400 auxmap_L1[i].ent = NULL;
403 tl_assert(0 == offsetof(AuxMapEnt,base));
404 tl_assert(sizeof(Addr) == sizeof(void*));
405 auxmap_L2 = VG_(OSetGen_Create)( /*keyOff*/ offsetof(AuxMapEnt,base),
407 VG_(malloc), "mc.iaLL.1", VG_(free) );
410 /* Check representation invariants; if OK return NULL; else a
411 descriptive bit of text. Also return the number of
412 non-distinguished secondary maps referred to from the auxiliary
415 static HChar* check_auxmap_L1_L2_sanity ( Word* n_secmaps_found )
418 /* On a 32-bit platform, the L2 and L1 tables should
419 both remain empty forever.
421 On a 64-bit platform:
423 all .base & 0xFFFF == 0
424 all .base > MAX_PRIMARY_ADDRESS
426 all .base & 0xFFFF == 0
427 all (.base > MAX_PRIMARY_ADDRESS
429 and .ent points to an AuxMapEnt with the same .base)
431 (.base == 0 and .ent == NULL)
433 *n_secmaps_found = 0;
434 if (sizeof(void*) == 4) {
435 /* 32-bit platform */
436 if (VG_(OSetGen_Size)(auxmap_L2) != 0)
437 return "32-bit: auxmap_L2 is non-empty";
438 for (i = 0; i < N_AUXMAP_L1; i++)
439 if (auxmap_L1[i].base != 0 || auxmap_L1[i].ent != NULL)
440 return "32-bit: auxmap_L1 is non-empty";
442 /* 64-bit platform */
443 UWord elems_seen = 0;
444 AuxMapEnt *elem, *res;
447 VG_(OSetGen_ResetIter)(auxmap_L2);
448 while ( (elem = VG_(OSetGen_Next)(auxmap_L2)) ) {
450 if (0 != (elem->base & (Addr)0xFFFF))
451 return "64-bit: nonzero .base & 0xFFFF in auxmap_L2";
452 if (elem->base <= MAX_PRIMARY_ADDRESS)
453 return "64-bit: .base <= MAX_PRIMARY_ADDRESS in auxmap_L2";
454 if (elem->sm == NULL)
455 return "64-bit: .sm in _L2 is NULL";
456 if (!is_distinguished_sm(elem->sm))
457 (*n_secmaps_found)++;
459 if (elems_seen != n_auxmap_L2_nodes)
460 return "64-bit: disagreement on number of elems in _L2";
461 /* Check L1-L2 correspondence */
462 for (i = 0; i < N_AUXMAP_L1; i++) {
463 if (auxmap_L1[i].base == 0 && auxmap_L1[i].ent == NULL)
465 if (0 != (auxmap_L1[i].base & (Addr)0xFFFF))
466 return "64-bit: nonzero .base & 0xFFFF in auxmap_L1";
467 if (auxmap_L1[i].base <= MAX_PRIMARY_ADDRESS)
468 return "64-bit: .base <= MAX_PRIMARY_ADDRESS in auxmap_L1";
469 if (auxmap_L1[i].ent == NULL)
470 return "64-bit: .ent is NULL in auxmap_L1";
471 if (auxmap_L1[i].ent->base != auxmap_L1[i].base)
472 return "64-bit: _L1 and _L2 bases are inconsistent";
473 /* Look it up in auxmap_L2. */
474 key.base = auxmap_L1[i].base;
476 res = VG_(OSetGen_Lookup)(auxmap_L2, &key);
478 return "64-bit: _L1 .base not found in _L2";
479 if (res != auxmap_L1[i].ent)
480 return "64-bit: _L1 .ent disagrees with _L2 entry";
482 /* Check L1 contains no duplicates */
483 for (i = 0; i < N_AUXMAP_L1; i++) {
484 if (auxmap_L1[i].base == 0)
486 for (j = i+1; j < N_AUXMAP_L1; j++) {
487 if (auxmap_L1[j].base == 0)
489 if (auxmap_L1[j].base == auxmap_L1[i].base)
490 return "64-bit: duplicate _L1 .base entries";
494 return NULL; /* ok */
497 static void insert_into_auxmap_L1_at ( Word rank, AuxMapEnt* ent )
501 tl_assert(rank >= 0 && rank < N_AUXMAP_L1);
502 for (i = N_AUXMAP_L1-1; i > rank; i--)
503 auxmap_L1[i] = auxmap_L1[i-1];
504 auxmap_L1[rank].base = ent->base;
505 auxmap_L1[rank].ent = ent;
508 static INLINE AuxMapEnt* maybe_find_in_auxmap ( Addr a )
514 tl_assert(a > MAX_PRIMARY_ADDRESS);
517 /* First search the front-cache, which is a self-organising
518 list containing the most popular entries. */
520 if (LIKELY(auxmap_L1[0].base == a))
521 return auxmap_L1[0].ent;
522 if (LIKELY(auxmap_L1[1].base == a)) {
523 Addr t_base = auxmap_L1[0].base;
524 AuxMapEnt* t_ent = auxmap_L1[0].ent;
525 auxmap_L1[0].base = auxmap_L1[1].base;
526 auxmap_L1[0].ent = auxmap_L1[1].ent;
527 auxmap_L1[1].base = t_base;
528 auxmap_L1[1].ent = t_ent;
529 return auxmap_L1[0].ent;
532 n_auxmap_L1_searches++;
534 for (i = 0; i < N_AUXMAP_L1; i++) {
535 if (auxmap_L1[i].base == a) {
539 tl_assert(i >= 0 && i <= N_AUXMAP_L1);
541 n_auxmap_L1_cmps += (ULong)(i+1);
543 if (i < N_AUXMAP_L1) {
545 Addr t_base = auxmap_L1[i-1].base;
546 AuxMapEnt* t_ent = auxmap_L1[i-1].ent;
547 auxmap_L1[i-1].base = auxmap_L1[i-0].base;
548 auxmap_L1[i-1].ent = auxmap_L1[i-0].ent;
549 auxmap_L1[i-0].base = t_base;
550 auxmap_L1[i-0].ent = t_ent;
553 return auxmap_L1[i].ent;
556 n_auxmap_L2_searches++;
558 /* First see if we already have it. */
562 res = VG_(OSetGen_Lookup)(auxmap_L2, &key);
564 insert_into_auxmap_L1_at( AUXMAP_L1_INSERT_IX, res );
568 static AuxMapEnt* find_or_alloc_in_auxmap ( Addr a )
570 AuxMapEnt *nyu, *res;
572 /* First see if we already have it. */
573 res = maybe_find_in_auxmap( a );
577 /* Ok, there's no entry in the secondary map, so we'll have
581 nyu = (AuxMapEnt*) VG_(OSetGen_AllocNode)( auxmap_L2, sizeof(AuxMapEnt) );
584 nyu->sm = &sm_distinguished[SM_DIST_NOACCESS];
585 VG_(OSetGen_Insert)( auxmap_L2, nyu );
586 insert_into_auxmap_L1_at( AUXMAP_L1_INSERT_IX, nyu );
591 /* --------------- SecMap fundamentals --------------- */
593 // In all these, 'low' means it's definitely in the main primary map,
594 // 'high' means it's definitely in the auxiliary table.
596 static INLINE SecMap** get_secmap_low_ptr ( Addr a )
598 UWord pm_off = a >> 16;
599 # if VG_DEBUG_MEMORY >= 1
600 tl_assert(pm_off < N_PRIMARY_MAP);
602 return &primary_map[ pm_off ];
605 static INLINE SecMap** get_secmap_high_ptr ( Addr a )
607 AuxMapEnt* am = find_or_alloc_in_auxmap(a);
611 static SecMap** get_secmap_ptr ( Addr a )
613 return ( a <= MAX_PRIMARY_ADDRESS
614 ? get_secmap_low_ptr(a)
615 : get_secmap_high_ptr(a));
618 static INLINE SecMap* get_secmap_for_reading_low ( Addr a )
620 return *get_secmap_low_ptr(a);
623 static INLINE SecMap* get_secmap_for_reading_high ( Addr a )
625 return *get_secmap_high_ptr(a);
628 static INLINE SecMap* get_secmap_for_writing_low(Addr a)
630 SecMap** p = get_secmap_low_ptr(a);
631 if (UNLIKELY(is_distinguished_sm(*p)))
632 *p = copy_for_writing(*p);
636 static INLINE SecMap* get_secmap_for_writing_high ( Addr a )
638 SecMap** p = get_secmap_high_ptr(a);
639 if (UNLIKELY(is_distinguished_sm(*p)))
640 *p = copy_for_writing(*p);
644 /* Produce the secmap for 'a', either from the primary map or by
645 ensuring there is an entry for it in the aux primary map. The
646 secmap may be a distinguished one as the caller will only want to
649 static INLINE SecMap* get_secmap_for_reading ( Addr a )
651 return ( a <= MAX_PRIMARY_ADDRESS
652 ? get_secmap_for_reading_low (a)
653 : get_secmap_for_reading_high(a) );
656 /* Produce the secmap for 'a', either from the primary map or by
657 ensuring there is an entry for it in the aux primary map. The
658 secmap may not be a distinguished one, since the caller will want
659 to be able to write it. If it is a distinguished secondary, make a
660 writable copy of it, install it, and return the copy instead. (COW
663 static SecMap* get_secmap_for_writing ( Addr a )
665 return ( a <= MAX_PRIMARY_ADDRESS
666 ? get_secmap_for_writing_low (a)
667 : get_secmap_for_writing_high(a) );
670 /* If 'a' has a SecMap, produce it. Else produce NULL. But don't
671 allocate one if one doesn't already exist. This is used by the
674 static SecMap* maybe_get_secmap_for ( Addr a )
676 if (a <= MAX_PRIMARY_ADDRESS) {
677 return get_secmap_for_reading_low(a);
679 AuxMapEnt* am = maybe_find_in_auxmap(a);
680 return am ? am->sm : NULL;
684 /* --------------- Fundamental functions --------------- */
687 void insert_vabits2_into_vabits8 ( Addr a, UChar vabits2, UChar* vabits8 )
689 UInt shift = (a & 3) << 1; // shift by 0, 2, 4, or 6
690 *vabits8 &= ~(0x3 << shift); // mask out the two old bits
691 *vabits8 |= (vabits2 << shift); // mask in the two new bits
695 void insert_vabits4_into_vabits8 ( Addr a, UChar vabits4, UChar* vabits8 )
698 tl_assert(VG_IS_2_ALIGNED(a)); // Must be 2-aligned
699 shift = (a & 2) << 1; // shift by 0 or 4
700 *vabits8 &= ~(0xf << shift); // mask out the four old bits
701 *vabits8 |= (vabits4 << shift); // mask in the four new bits
705 UChar extract_vabits2_from_vabits8 ( Addr a, UChar vabits8 )
707 UInt shift = (a & 3) << 1; // shift by 0, 2, 4, or 6
708 vabits8 >>= shift; // shift the two bits to the bottom
709 return 0x3 & vabits8; // mask out the rest
713 UChar extract_vabits4_from_vabits8 ( Addr a, UChar vabits8 )
716 tl_assert(VG_IS_2_ALIGNED(a)); // Must be 2-aligned
717 shift = (a & 2) << 1; // shift by 0 or 4
718 vabits8 >>= shift; // shift the four bits to the bottom
719 return 0xf & vabits8; // mask out the rest
722 // Note that these four are only used in slow cases. The fast cases do
723 // clever things like combine the auxmap check (in
724 // get_secmap_{read,writ}able) with alignment checks.
727 // Any time this function is called, if it is possible that vabits2
728 // is equal to VA_BITS2_PARTDEFINED, then the corresponding entry in the
729 // sec-V-bits table must also be set!
731 void set_vabits2 ( Addr a, UChar vabits2 )
733 SecMap* sm = get_secmap_for_writing(a);
734 UWord sm_off = SM_OFF(a);
735 insert_vabits2_into_vabits8( a, vabits2, &(sm->vabits8[sm_off]) );
739 UChar get_vabits2 ( Addr a )
741 SecMap* sm = get_secmap_for_reading(a);
742 UWord sm_off = SM_OFF(a);
743 UChar vabits8 = sm->vabits8[sm_off];
744 return extract_vabits2_from_vabits8(a, vabits8);
748 // Any time this function is called, if it is possible that any of the
749 // 4 2-bit fields in vabits8 are equal to VA_BITS2_PARTDEFINED, then the
750 // corresponding entry(s) in the sec-V-bits table must also be set!
752 UChar get_vabits8_for_aligned_word32 ( Addr a )
754 SecMap* sm = get_secmap_for_reading(a);
755 UWord sm_off = SM_OFF(a);
756 UChar vabits8 = sm->vabits8[sm_off];
761 void set_vabits8_for_aligned_word32 ( Addr a, UChar vabits8 )
763 SecMap* sm = get_secmap_for_writing(a);
764 UWord sm_off = SM_OFF(a);
765 sm->vabits8[sm_off] = vabits8;
769 // Forward declarations
770 static UWord get_sec_vbits8(Addr a);
771 static void set_sec_vbits8(Addr a, UWord vbits8);
773 // Returns False if there was an addressability error.
775 Bool set_vbits8 ( Addr a, UChar vbits8 )
778 UChar vabits2 = get_vabits2(a);
779 if ( VA_BITS2_NOACCESS != vabits2 ) {
780 // Addressable. Convert in-register format to in-memory format.
781 // Also remove any existing sec V bit entry for the byte if no
783 if ( V_BITS8_DEFINED == vbits8 ) { vabits2 = VA_BITS2_DEFINED; }
784 else if ( V_BITS8_UNDEFINED == vbits8 ) { vabits2 = VA_BITS2_UNDEFINED; }
785 else { vabits2 = VA_BITS2_PARTDEFINED;
786 set_sec_vbits8(a, vbits8); }
787 set_vabits2(a, vabits2);
790 // Unaddressable! Do nothing -- when writing to unaddressable
791 // memory it acts as a black hole, and the V bits can never be seen
792 // again. So we don't have to write them at all.
798 // Returns False if there was an addressability error. In that case, we put
799 // all defined bits into vbits8.
801 Bool get_vbits8 ( Addr a, UChar* vbits8 )
804 UChar vabits2 = get_vabits2(a);
806 // Convert the in-memory format to in-register format.
807 if ( VA_BITS2_DEFINED == vabits2 ) { *vbits8 = V_BITS8_DEFINED; }
808 else if ( VA_BITS2_UNDEFINED == vabits2 ) { *vbits8 = V_BITS8_UNDEFINED; }
809 else if ( VA_BITS2_NOACCESS == vabits2 ) {
810 *vbits8 = V_BITS8_DEFINED; // Make V bits defined!
813 tl_assert( VA_BITS2_PARTDEFINED == vabits2 );
814 *vbits8 = get_sec_vbits8(a);
820 /* --------------- Secondary V bit table ------------ */
822 // This table holds the full V bit pattern for partially-defined bytes
823 // (PDBs) that are represented by VA_BITS2_PARTDEFINED in the main shadow
826 // Note: the nodes in this table can become stale. Eg. if you write a PDB,
827 // then overwrite the same address with a fully defined byte, the sec-V-bit
828 // node will not necessarily be removed. This is because checking for
829 // whether removal is necessary would slow down the fast paths.
831 // To avoid the stale nodes building up too much, we periodically (once the
832 // table reaches a certain size) garbage collect (GC) the table by
833 // traversing it and evicting any "sufficiently stale" nodes, ie. nodes that
834 // are stale and haven't been touched for a certain number of collections.
835 // If more than a certain proportion of nodes survived, we increase the
836 // table size so that GCs occur less often.
838 // (So this a bit different to a traditional GC, where you definitely want
839 // to remove any dead nodes. It's more like we have a resizable cache and
840 // we're trying to find the right balance how many elements to evict and how
841 // big to make the cache.)
843 // This policy is designed to avoid bad table bloat in the worst case where
844 // a program creates huge numbers of stale PDBs -- we would get this bloat
845 // if we had no GC -- while handling well the case where a node becomes
846 // stale but shortly afterwards is rewritten with a PDB and so becomes
847 // non-stale again (which happens quite often, eg. in perf/bz2). If we just
848 // remove all stale nodes as soon as possible, we just end up re-adding a
849 // lot of them in later again. The "sufficiently stale" approach avoids
850 // this. (If a program has many live PDBs, performance will just suck,
851 // there's no way around that.)
853 static OSet* secVBitTable;
856 static ULong sec_vbits_new_nodes = 0;
857 static ULong sec_vbits_updates = 0;
859 // This must be a power of two; this is checked in mc_pre_clo_init().
860 // The size chosen here is a trade-off: if the nodes are bigger (ie. cover
861 // a larger address range) they take more space but we can get multiple
862 // partially-defined bytes in one if they are close to each other, reducing
863 // the number of total nodes. In practice sometimes they are clustered (eg.
864 // perf/bz2 repeatedly writes then reads more than 20,000 in a contiguous
865 // row), but often not. So we choose something intermediate.
866 #define BYTES_PER_SEC_VBIT_NODE 16
868 // We make the table bigger if more than this many nodes survive a GC.
869 #define MAX_SURVIVOR_PROPORTION 0.5
871 // Each time we make the table bigger, we increase it by this much.
872 #define TABLE_GROWTH_FACTOR 2
874 // This defines "sufficiently stale" -- any node that hasn't been touched in
875 // this many GCs will be removed.
876 #define MAX_STALE_AGE 2
878 // We GC the table when it gets this many nodes in it, ie. it's effectively
879 // the table size. It can change.
880 static Int secVBitLimit = 1024;
882 // The number of GCs done, used to age sec-V-bit nodes for eviction.
883 // Because it's unsigned, wrapping doesn't matter -- the right answer will
885 static UInt GCs_done = 0;
890 UChar vbits8[BYTES_PER_SEC_VBIT_NODE];
895 static OSet* createSecVBitTable(void)
897 return VG_(OSetGen_Create)( offsetof(SecVBitNode, a),
898 NULL, // use fast comparisons
899 VG_(malloc), "mc.cSVT.1 (sec VBit table)",
903 static void gcSecVBitTable(void)
907 Int i, n_nodes = 0, n_survivors = 0;
911 // Create the new table.
912 secVBitTable2 = createSecVBitTable();
914 // Traverse the table, moving fresh nodes into the new table.
915 VG_(OSetGen_ResetIter)(secVBitTable);
916 while ( (n = VG_(OSetGen_Next)(secVBitTable)) ) {
918 if ( (GCs_done - n->last_touched) <= MAX_STALE_AGE ) {
919 // Keep node if it's been touched recently enough (regardless of
920 // freshness/staleness).
923 // Keep node if any of its bytes are non-stale. Using
924 // get_vabits2() for the lookup is not very efficient, but I don't
926 for (i = 0; i < BYTES_PER_SEC_VBIT_NODE; i++) {
927 if (VA_BITS2_PARTDEFINED == get_vabits2(n->a + i)) {
928 keep = True; // Found a non-stale byte, so keep
935 // Insert a copy of the node into the new table.
937 VG_(OSetGen_AllocNode)(secVBitTable2, sizeof(SecVBitNode));
939 VG_(OSetGen_Insert)(secVBitTable2, n2);
943 // Get the before and after sizes.
944 n_nodes = VG_(OSetGen_Size)(secVBitTable);
945 n_survivors = VG_(OSetGen_Size)(secVBitTable2);
947 // Destroy the old table, and put the new one in its place.
948 VG_(OSetGen_Destroy)(secVBitTable);
949 secVBitTable = secVBitTable2;
951 if (VG_(clo_verbosity) > 1) {
953 VG_(percentify)(n_survivors, n_nodes, 1, 6, percbuf);
954 VG_(message)(Vg_DebugMsg, "memcheck GC: %d nodes, %d survivors (%s)\n",
955 n_nodes, n_survivors, percbuf);
958 // Increase table size if necessary.
959 if (n_survivors > (secVBitLimit * MAX_SURVIVOR_PROPORTION)) {
960 secVBitLimit *= TABLE_GROWTH_FACTOR;
961 if (VG_(clo_verbosity) > 1)
962 VG_(message)(Vg_DebugMsg, "memcheck GC: increase table size to %d\n",
967 static UWord get_sec_vbits8(Addr a)
969 Addr aAligned = VG_ROUNDDN(a, BYTES_PER_SEC_VBIT_NODE);
970 Int amod = a % BYTES_PER_SEC_VBIT_NODE;
971 SecVBitNode* n = VG_(OSetGen_Lookup)(secVBitTable, &aAligned);
973 tl_assert2(n, "get_sec_vbits8: no node for address %p (%p)\n", aAligned, a);
974 // Shouldn't be fully defined or fully undefined -- those cases shouldn't
975 // make it to the secondary V bits table.
976 vbits8 = n->vbits8[amod];
977 tl_assert(V_BITS8_DEFINED != vbits8 && V_BITS8_UNDEFINED != vbits8);
981 static void set_sec_vbits8(Addr a, UWord vbits8)
983 Addr aAligned = VG_ROUNDDN(a, BYTES_PER_SEC_VBIT_NODE);
984 Int i, amod = a % BYTES_PER_SEC_VBIT_NODE;
985 SecVBitNode* n = VG_(OSetGen_Lookup)(secVBitTable, &aAligned);
986 // Shouldn't be fully defined or fully undefined -- those cases shouldn't
987 // make it to the secondary V bits table.
988 tl_assert(V_BITS8_DEFINED != vbits8 && V_BITS8_UNDEFINED != vbits8);
990 n->vbits8[amod] = vbits8; // update
991 n->last_touched = GCs_done;
994 // New node: assign the specific byte, make the rest invalid (they
995 // should never be read as-is, but be cautious).
996 n = VG_(OSetGen_AllocNode)(secVBitTable, sizeof(SecVBitNode));
998 for (i = 0; i < BYTES_PER_SEC_VBIT_NODE; i++) {
999 n->vbits8[i] = V_BITS8_UNDEFINED;
1001 n->vbits8[amod] = vbits8;
1002 n->last_touched = GCs_done;
1004 // Do a table GC if necessary. Nb: do this before inserting the new
1005 // node, to avoid erroneously GC'ing the new node.
1006 if (secVBitLimit == VG_(OSetGen_Size)(secVBitTable)) {
1010 // Insert the new node.
1011 VG_(OSetGen_Insert)(secVBitTable, n);
1012 sec_vbits_new_nodes++;
1014 n_secVBit_nodes = VG_(OSetGen_Size)(secVBitTable);
1015 if (n_secVBit_nodes > max_secVBit_nodes)
1016 max_secVBit_nodes = n_secVBit_nodes;
1020 /* --------------- Endianness helpers --------------- */
1022 /* Returns the offset in memory of the byteno-th most significant byte
1023 in a wordszB-sized word, given the specified endianness. */
1024 static INLINE UWord byte_offset_w ( UWord wordszB, Bool bigendian,
1026 return bigendian ? (wordszB-1-byteno) : byteno;
1030 /* --------------- Ignored address ranges --------------- */
1032 #define M_IGNORE_RANGES 4
1037 Addr start[M_IGNORE_RANGES];
1038 Addr end[M_IGNORE_RANGES];
1042 static IgnoreRanges ignoreRanges;
1044 INLINE Bool MC_(in_ignored_range) ( Addr a )
1047 if (LIKELY(ignoreRanges.used == 0))
1049 for (i = 0; i < ignoreRanges.used; i++) {
1050 if (a >= ignoreRanges.start[i] && a < ignoreRanges.end[i])
1056 /* Parse two Addr separated by a dash, or fail. */
1058 static Bool parse_range ( UChar** ppc, Addr* result1, Addr* result2 )
1060 Bool ok = VG_(parse_Addr) (ppc, result1);
1066 ok = VG_(parse_Addr) (ppc, result2);
1072 /* Parse a set of ranges separated by commas into 'ignoreRanges', or
1075 static Bool parse_ignore_ranges ( UChar* str0 )
1081 ignoreRanges.used = 0;
1083 ok = parse_range(ppc, &start, &end);
1086 if (ignoreRanges.used >= M_IGNORE_RANGES)
1088 ignoreRanges.start[ignoreRanges.used] = start;
1089 ignoreRanges.end[ignoreRanges.used] = end;
1090 ignoreRanges.used++;
1102 /* --------------- Load/store slow cases. --------------- */
1105 #ifndef PERF_FAST_LOADV
1108 ULong mc_LOADVn_slow ( Addr a, SizeT nBits, Bool bigendian )
1110 /* Make up a 64-bit result V word, which contains the loaded data for
1111 valid addresses and Defined for invalid addresses. Iterate over
1112 the bytes in the word, from the most significant down to the
1114 ULong vbits64 = V_BITS64_UNDEFINED;
1115 SizeT szB = nBits / 8;
1116 SSizeT i; // Must be signed.
1117 SizeT n_addrs_bad = 0;
1119 Bool partial_load_exemption_applies;
1123 PROF_EVENT(30, "mc_LOADVn_slow");
1125 /* ------------ BEGIN semi-fast cases ------------ */
1126 /* These deal quickly-ish with the common auxiliary primary map
1127 cases on 64-bit platforms. Are merely a speedup hack; can be
1128 omitted without loss of correctness/functionality. Note that in
1129 both cases the "sizeof(void*) == 8" causes these cases to be
1130 folded out by compilers on 32-bit platforms. These are derived
1131 from LOADV64 and LOADV32.
1133 if (LIKELY(sizeof(void*) == 8
1134 && nBits == 64 && VG_IS_8_ALIGNED(a))) {
1135 SecMap* sm = get_secmap_for_reading(a);
1136 UWord sm_off16 = SM_OFF_16(a);
1137 UWord vabits16 = ((UShort*)(sm->vabits8))[sm_off16];
1138 if (LIKELY(vabits16 == VA_BITS16_DEFINED))
1139 return V_BITS64_DEFINED;
1140 if (LIKELY(vabits16 == VA_BITS16_UNDEFINED))
1141 return V_BITS64_UNDEFINED;
1142 /* else fall into the slow case */
1144 if (LIKELY(sizeof(void*) == 8
1145 && nBits == 32 && VG_IS_4_ALIGNED(a))) {
1146 SecMap* sm = get_secmap_for_reading(a);
1147 UWord sm_off = SM_OFF(a);
1148 UWord vabits8 = sm->vabits8[sm_off];
1149 if (LIKELY(vabits8 == VA_BITS8_DEFINED))
1150 return ((UWord)0xFFFFFFFF00000000ULL | (UWord)V_BITS32_DEFINED);
1151 if (LIKELY(vabits8 == VA_BITS8_UNDEFINED))
1152 return ((UWord)0xFFFFFFFF00000000ULL | (UWord)V_BITS32_UNDEFINED);
1153 /* else fall into slow case */
1155 /* ------------ END semi-fast cases ------------ */
1157 tl_assert(nBits == 64 || nBits == 32 || nBits == 16 || nBits == 8);
1159 for (i = szB-1; i >= 0; i--) {
1160 PROF_EVENT(31, "mc_LOADVn_slow(loop)");
1161 ai = a + byte_offset_w(szB, bigendian, i);
1162 ok = get_vbits8(ai, &vbits8);
1163 if (!ok) n_addrs_bad++;
1168 /* This is a hack which avoids producing errors for code which
1169 insists in stepping along byte strings in aligned word-sized
1170 chunks, and there is a partially defined word at the end. (eg,
1171 optimised strlen). Such code is basically broken at least WRT
1172 semantics of ANSI C, but sometimes users don't have the option
1173 to fix it, and so this option is provided. Note it is now
1174 defaulted to not-engaged.
1176 A load from a partially-addressible place is allowed if:
1177 - the command-line flag is set
1178 - it's a word-sized, word-aligned load
1179 - at least one of the addresses in the word *is* valid
1181 partial_load_exemption_applies
1182 = MC_(clo_partial_loads_ok) && szB == VG_WORDSIZE
1183 && VG_IS_WORD_ALIGNED(a)
1184 && n_addrs_bad < VG_WORDSIZE;
1186 if (n_addrs_bad > 0 && !partial_load_exemption_applies)
1187 MC_(record_address_error)( VG_(get_running_tid)(), a, szB, False );
1194 #ifndef PERF_FAST_STOREV
1197 void mc_STOREVn_slow ( Addr a, SizeT nBits, ULong vbytes, Bool bigendian )
1199 SizeT szB = nBits / 8;
1200 SizeT i, n_addrs_bad = 0;
1205 PROF_EVENT(35, "mc_STOREVn_slow");
1207 /* ------------ BEGIN semi-fast cases ------------ */
1208 /* These deal quickly-ish with the common auxiliary primary map
1209 cases on 64-bit platforms. Are merely a speedup hack; can be
1210 omitted without loss of correctness/functionality. Note that in
1211 both cases the "sizeof(void*) == 8" causes these cases to be
1212 folded out by compilers on 32-bit platforms. These are derived
1213 from STOREV64 and STOREV32.
1215 if (LIKELY(sizeof(void*) == 8
1216 && nBits == 64 && VG_IS_8_ALIGNED(a))) {
1217 SecMap* sm = get_secmap_for_reading(a);
1218 UWord sm_off16 = SM_OFF_16(a);
1219 UWord vabits16 = ((UShort*)(sm->vabits8))[sm_off16];
1220 if (LIKELY( !is_distinguished_sm(sm) &&
1221 (VA_BITS16_DEFINED == vabits16 ||
1222 VA_BITS16_UNDEFINED == vabits16) )) {
1223 /* Handle common case quickly: a is suitably aligned, */
1224 /* is mapped, and is addressible. */
1225 // Convert full V-bits in register to compact 2-bit form.
1226 if (LIKELY(V_BITS64_DEFINED == vbytes)) {
1227 ((UShort*)(sm->vabits8))[sm_off16] = (UShort)VA_BITS16_DEFINED;
1229 } else if (V_BITS64_UNDEFINED == vbytes) {
1230 ((UShort*)(sm->vabits8))[sm_off16] = (UShort)VA_BITS16_UNDEFINED;
1233 /* else fall into the slow case */
1235 /* else fall into the slow case */
1237 if (LIKELY(sizeof(void*) == 8
1238 && nBits == 32 && VG_IS_4_ALIGNED(a))) {
1239 SecMap* sm = get_secmap_for_reading(a);
1240 UWord sm_off = SM_OFF(a);
1241 UWord vabits8 = sm->vabits8[sm_off];
1242 if (LIKELY( !is_distinguished_sm(sm) &&
1243 (VA_BITS8_DEFINED == vabits8 ||
1244 VA_BITS8_UNDEFINED == vabits8) )) {
1245 /* Handle common case quickly: a is suitably aligned, */
1246 /* is mapped, and is addressible. */
1247 // Convert full V-bits in register to compact 2-bit form.
1248 if (LIKELY(V_BITS32_DEFINED == (vbytes & 0xFFFFFFFF))) {
1249 sm->vabits8[sm_off] = VA_BITS8_DEFINED;
1251 } else if (V_BITS32_UNDEFINED == (vbytes & 0xFFFFFFFF)) {
1252 sm->vabits8[sm_off] = VA_BITS8_UNDEFINED;
1255 /* else fall into the slow case */
1257 /* else fall into the slow case */
1259 /* ------------ END semi-fast cases ------------ */
1261 tl_assert(nBits == 64 || nBits == 32 || nBits == 16 || nBits == 8);
1263 /* Dump vbytes in memory, iterating from least to most significant
1264 byte. At the same time establish addressibility of the location. */
1265 for (i = 0; i < szB; i++) {
1266 PROF_EVENT(36, "mc_STOREVn_slow(loop)");
1267 ai = a + byte_offset_w(szB, bigendian, i);
1268 vbits8 = vbytes & 0xff;
1269 ok = set_vbits8(ai, vbits8);
1270 if (!ok) n_addrs_bad++;
1274 /* If an address error has happened, report it. */
1275 if (n_addrs_bad > 0)
1276 MC_(record_address_error)( VG_(get_running_tid)(), a, szB, True );
1280 /*------------------------------------------------------------*/
1281 /*--- Setting permissions over address ranges. ---*/
1282 /*------------------------------------------------------------*/
1284 static void set_address_range_perms ( Addr a, SizeT lenT, UWord vabits16,
1287 UWord sm_off, sm_off16;
1288 UWord vabits2 = vabits16 & 0x3;
1289 SizeT lenA, lenB, len_to_next_secmap;
1293 SecMap* example_dsm;
1295 PROF_EVENT(150, "set_address_range_perms");
1297 /* Check the V+A bits make sense. */
1298 tl_assert(VA_BITS16_NOACCESS == vabits16 ||
1299 VA_BITS16_UNDEFINED == vabits16 ||
1300 VA_BITS16_DEFINED == vabits16);
1302 // This code should never write PDBs; ensure this. (See comment above
1304 tl_assert(VA_BITS2_PARTDEFINED != vabits2);
1309 if (lenT > 256 * 1024 * 1024) {
1310 if (VG_(clo_verbosity) > 0 && !VG_(clo_xml)) {
1311 Char* s = "unknown???";
1312 if (vabits16 == VA_BITS16_NOACCESS ) s = "noaccess";
1313 if (vabits16 == VA_BITS16_UNDEFINED) s = "undefined";
1314 if (vabits16 == VA_BITS16_DEFINED ) s = "defined";
1315 VG_(message)(Vg_UserMsg, "Warning: set address range perms: "
1316 "large range [0x%lx, 0x%lx) (%s)\n",
1321 #ifndef PERF_FAST_SARP
1322 /*------------------ debug-only case ------------------ */
1324 // Endianness doesn't matter here because all bytes are being set to
1326 // Nb: We don't have to worry about updating the sec-V-bits table
1327 // after these set_vabits2() calls because this code never writes
1328 // VA_BITS2_PARTDEFINED values.
1330 for (i = 0; i < lenT; i++) {
1331 set_vabits2(a + i, vabits2);
1337 /*------------------ standard handling ------------------ */
1339 /* Get the distinguished secondary that we might want
1340 to use (part of the space-compression scheme). */
1341 example_dsm = &sm_distinguished[dsm_num];
1343 // We have to handle ranges covering various combinations of partial and
1344 // whole sec-maps. Here is how parts 1, 2 and 3 are used in each case.
1345 // Cases marked with a '*' are common.
1349 // * one partial sec-map (p) 1
1350 // - one whole sec-map (P) 2
1352 // * two partial sec-maps (pp) 1,3
1353 // - one partial, one whole sec-map (pP) 1,2
1354 // - one whole, one partial sec-map (Pp) 2,3
1355 // - two whole sec-maps (PP) 2,2
1357 // * one partial, one whole, one partial (pPp) 1,2,3
1358 // - one partial, two whole (pPP) 1,2,2
1359 // - two whole, one partial (PPp) 2,2,3
1360 // - three whole (PPP) 2,2,2
1362 // * one partial, N-2 whole, one partial (pP...Pp) 1,2...2,3
1363 // - one partial, N-1 whole (pP...PP) 1,2...2,2
1364 // - N-1 whole, one partial (PP...Pp) 2,2...2,3
1365 // - N whole (PP...PP) 2,2...2,3
1367 // Break up total length (lenT) into two parts: length in the first
1368 // sec-map (lenA), and the rest (lenB); lenT == lenA + lenB.
1369 aNext = start_of_this_sm(a) + SM_SIZE;
1370 len_to_next_secmap = aNext - a;
1371 if ( lenT <= len_to_next_secmap ) {
1372 // Range entirely within one sec-map. Covers almost all cases.
1373 PROF_EVENT(151, "set_address_range_perms-single-secmap");
1376 } else if (is_start_of_sm(a)) {
1377 // Range spans at least one whole sec-map, and starts at the beginning
1378 // of a sec-map; skip to Part 2.
1379 PROF_EVENT(152, "set_address_range_perms-startof-secmap");
1384 // Range spans two or more sec-maps, first one is partial.
1385 PROF_EVENT(153, "set_address_range_perms-multiple-secmaps");
1386 lenA = len_to_next_secmap;
1390 //------------------------------------------------------------------------
1391 // Part 1: Deal with the first sec_map. Most of the time the range will be
1392 // entirely within a sec_map and this part alone will suffice. Also,
1393 // doing it this way lets us avoid repeatedly testing for the crossing of
1394 // a sec-map boundary within these loops.
1395 //------------------------------------------------------------------------
1397 // If it's distinguished, make it undistinguished if necessary.
1398 sm_ptr = get_secmap_ptr(a);
1399 if (is_distinguished_sm(*sm_ptr)) {
1400 if (*sm_ptr == example_dsm) {
1401 // Sec-map already has the V+A bits that we want, so skip.
1402 PROF_EVENT(154, "set_address_range_perms-dist-sm1-quick");
1406 PROF_EVENT(155, "set_address_range_perms-dist-sm1");
1407 *sm_ptr = copy_for_writing(*sm_ptr);
1414 if (VG_IS_8_ALIGNED(a)) break;
1415 if (lenA < 1) break;
1416 PROF_EVENT(156, "set_address_range_perms-loop1a");
1418 insert_vabits2_into_vabits8( a, vabits2, &(sm->vabits8[sm_off]) );
1422 // 8-aligned, 8 byte steps
1424 if (lenA < 8) break;
1425 PROF_EVENT(157, "set_address_range_perms-loop8a");
1426 sm_off16 = SM_OFF_16(a);
1427 ((UShort*)(sm->vabits8))[sm_off16] = vabits16;
1433 if (lenA < 1) break;
1434 PROF_EVENT(158, "set_address_range_perms-loop1b");
1436 insert_vabits2_into_vabits8( a, vabits2, &(sm->vabits8[sm_off]) );
1441 // We've finished the first sec-map. Is that it?
1445 //------------------------------------------------------------------------
1446 // Part 2: Fast-set entire sec-maps at a time.
1447 //------------------------------------------------------------------------
1449 // 64KB-aligned, 64KB steps.
1450 // Nb: we can reach here with lenB < SM_SIZE
1451 tl_assert(0 == lenA);
1453 if (lenB < SM_SIZE) break;
1454 tl_assert(is_start_of_sm(a));
1455 PROF_EVENT(159, "set_address_range_perms-loop64K");
1456 sm_ptr = get_secmap_ptr(a);
1457 if (!is_distinguished_sm(*sm_ptr)) {
1458 PROF_EVENT(160, "set_address_range_perms-loop64K-free-dist-sm");
1459 // Free the non-distinguished sec-map that we're replacing. This
1460 // case happens moderately often, enough to be worthwhile.
1461 VG_(am_munmap_valgrind)((Addr)*sm_ptr, sizeof(SecMap));
1463 update_SM_counts(*sm_ptr, example_dsm);
1464 // Make the sec-map entry point to the example DSM
1465 *sm_ptr = example_dsm;
1470 // We've finished the whole sec-maps. Is that it?
1474 //------------------------------------------------------------------------
1475 // Part 3: Finish off the final partial sec-map, if necessary.
1476 //------------------------------------------------------------------------
1478 tl_assert(is_start_of_sm(a) && lenB < SM_SIZE);
1480 // If it's distinguished, make it undistinguished if necessary.
1481 sm_ptr = get_secmap_ptr(a);
1482 if (is_distinguished_sm(*sm_ptr)) {
1483 if (*sm_ptr == example_dsm) {
1484 // Sec-map already has the V+A bits that we want, so stop.
1485 PROF_EVENT(161, "set_address_range_perms-dist-sm2-quick");
1488 PROF_EVENT(162, "set_address_range_perms-dist-sm2");
1489 *sm_ptr = copy_for_writing(*sm_ptr);
1494 // 8-aligned, 8 byte steps
1496 if (lenB < 8) break;
1497 PROF_EVENT(163, "set_address_range_perms-loop8b");
1498 sm_off16 = SM_OFF_16(a);
1499 ((UShort*)(sm->vabits8))[sm_off16] = vabits16;
1505 if (lenB < 1) return;
1506 PROF_EVENT(164, "set_address_range_perms-loop1c");
1508 insert_vabits2_into_vabits8( a, vabits2, &(sm->vabits8[sm_off]) );
1515 /* --- Set permissions for arbitrary address ranges --- */
1517 void MC_(make_mem_noaccess) ( Addr a, SizeT len )
1519 PROF_EVENT(40, "MC_(make_mem_noaccess)");
1520 DEBUG("MC_(make_mem_noaccess)(%p, %lu)\n", a, len);
1521 set_address_range_perms ( a, len, VA_BITS16_NOACCESS, SM_DIST_NOACCESS );
1522 if (UNLIKELY( MC_(clo_mc_level) == 3 ))
1523 ocache_sarp_Clear_Origins ( a, len );
1526 static void make_mem_undefined ( Addr a, SizeT len )
1528 PROF_EVENT(41, "make_mem_undefined");
1529 DEBUG("make_mem_undefined(%p, %lu)\n", a, len);
1530 set_address_range_perms ( a, len, VA_BITS16_UNDEFINED, SM_DIST_UNDEFINED );
1533 void MC_(make_mem_undefined_w_otag) ( Addr a, SizeT len, UInt otag )
1535 PROF_EVENT(41, "MC_(make_mem_undefined)");
1536 DEBUG("MC_(make_mem_undefined)(%p, %lu)\n", a, len);
1537 set_address_range_perms ( a, len, VA_BITS16_UNDEFINED, SM_DIST_UNDEFINED );
1538 if (UNLIKELY( MC_(clo_mc_level) == 3 ))
1539 ocache_sarp_Set_Origins ( a, len, otag );
1543 void make_mem_undefined_w_tid_and_okind ( Addr a, SizeT len,
1544 ThreadId tid, UInt okind )
1548 /* VG_(record_ExeContext) checks for validity of tid, and asserts
1549 if it is invalid. So no need to do it here. */
1550 tl_assert(okind <= 3);
1551 here = VG_(record_ExeContext)( tid, 0/*first_ip_delta*/ );
1553 ecu = VG_(get_ECU_from_ExeContext)(here);
1554 tl_assert(VG_(is_plausible_ECU)(ecu));
1555 MC_(make_mem_undefined_w_otag) ( a, len, ecu | okind );
1559 void make_mem_undefined_w_tid ( Addr a, SizeT len, ThreadId tid ) {
1560 make_mem_undefined_w_tid_and_okind ( a, len, tid, MC_OKIND_UNKNOWN );
1564 void MC_(make_mem_defined) ( Addr a, SizeT len )
1566 PROF_EVENT(42, "MC_(make_mem_defined)");
1567 DEBUG("MC_(make_mem_defined)(%p, %lu)\n", a, len);
1568 set_address_range_perms ( a, len, VA_BITS16_DEFINED, SM_DIST_DEFINED );
1569 if (UNLIKELY( MC_(clo_mc_level) == 3 ))
1570 ocache_sarp_Clear_Origins ( a, len );
1573 /* For each byte in [a,a+len), if the byte is addressable, make it be
1574 defined, but if it isn't addressible, leave it alone. In other
1575 words a version of MC_(make_mem_defined) that doesn't mess with
1576 addressibility. Low-performance implementation. */
1577 static void make_mem_defined_if_addressable ( Addr a, SizeT len )
1581 DEBUG("make_mem_defined_if_addressable(%p, %llu)\n", a, (ULong)len);
1582 for (i = 0; i < len; i++) {
1583 vabits2 = get_vabits2( a+i );
1584 if (LIKELY(VA_BITS2_NOACCESS != vabits2)) {
1585 set_vabits2(a+i, VA_BITS2_DEFINED);
1586 if (UNLIKELY(MC_(clo_mc_level) >= 3)) {
1587 MC_(helperc_b_store1)( a+i, 0 ); /* clear the origin tag */
1593 /* Similarly (needed for mprotect handling ..) */
1594 static void make_mem_defined_if_noaccess ( Addr a, SizeT len )
1598 DEBUG("make_mem_defined_if_noaccess(%p, %llu)\n", a, (ULong)len);
1599 for (i = 0; i < len; i++) {
1600 vabits2 = get_vabits2( a+i );
1601 if (LIKELY(VA_BITS2_NOACCESS == vabits2)) {
1602 set_vabits2(a+i, VA_BITS2_DEFINED);
1603 if (UNLIKELY(MC_(clo_mc_level) >= 3)) {
1604 MC_(helperc_b_store1)( a+i, 0 ); /* clear the origin tag */
1610 /* --- Block-copy permissions (needed for implementing realloc() and
1613 void MC_(copy_address_range_state) ( Addr src, Addr dst, SizeT len )
1616 UChar vabits2, vabits8;
1617 Bool aligned, nooverlap;
1619 DEBUG("MC_(copy_address_range_state)\n");
1620 PROF_EVENT(50, "MC_(copy_address_range_state)");
1622 if (len == 0 || src == dst)
1625 aligned = VG_IS_4_ALIGNED(src) && VG_IS_4_ALIGNED(dst);
1626 nooverlap = src+len <= dst || dst+len <= src;
1628 if (nooverlap && aligned) {
1630 /* Vectorised fast case, when no overlap and suitably aligned */
1634 vabits8 = get_vabits8_for_aligned_word32( src+i );
1635 set_vabits8_for_aligned_word32( dst+i, vabits8 );
1636 if (LIKELY(VA_BITS8_DEFINED == vabits8
1637 || VA_BITS8_UNDEFINED == vabits8
1638 || VA_BITS8_NOACCESS == vabits8)) {
1641 /* have to copy secondary map info */
1642 if (VA_BITS2_PARTDEFINED == get_vabits2( src+i+0 ))
1643 set_sec_vbits8( dst+i+0, get_sec_vbits8( src+i+0 ) );
1644 if (VA_BITS2_PARTDEFINED == get_vabits2( src+i+1 ))
1645 set_sec_vbits8( dst+i+1, get_sec_vbits8( src+i+1 ) );
1646 if (VA_BITS2_PARTDEFINED == get_vabits2( src+i+2 ))
1647 set_sec_vbits8( dst+i+2, get_sec_vbits8( src+i+2 ) );
1648 if (VA_BITS2_PARTDEFINED == get_vabits2( src+i+3 ))
1649 set_sec_vbits8( dst+i+3, get_sec_vbits8( src+i+3 ) );
1656 vabits2 = get_vabits2( src+i );
1657 set_vabits2( dst+i, vabits2 );
1658 if (VA_BITS2_PARTDEFINED == vabits2) {
1659 set_sec_vbits8( dst+i, get_sec_vbits8( src+i ) );
1667 /* We have to do things the slow way */
1669 for (i = 0, j = len-1; i < len; i++, j--) {
1670 PROF_EVENT(51, "MC_(copy_address_range_state)(loop)");
1671 vabits2 = get_vabits2( src+j );
1672 set_vabits2( dst+j, vabits2 );
1673 if (VA_BITS2_PARTDEFINED == vabits2) {
1674 set_sec_vbits8( dst+j, get_sec_vbits8( src+j ) );
1680 for (i = 0; i < len; i++) {
1681 PROF_EVENT(52, "MC_(copy_address_range_state)(loop)");
1682 vabits2 = get_vabits2( src+i );
1683 set_vabits2( dst+i, vabits2 );
1684 if (VA_BITS2_PARTDEFINED == vabits2) {
1685 set_sec_vbits8( dst+i, get_sec_vbits8( src+i ) );
1694 /*------------------------------------------------------------*/
1695 /*--- Origin tracking stuff - cache basics ---*/
1696 /*------------------------------------------------------------*/
1698 /* AN OVERVIEW OF THE ORIGIN TRACKING IMPLEMENTATION
1699 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1701 Note that this implementation draws inspiration from the "origin
1702 tracking by value piggybacking" scheme described in "Tracking Bad
1703 Apples: Reporting the Origin of Null and Undefined Value Errors"
1704 (Michael Bond, Nicholas Nethercote, Stephen Kent, Samuel Guyer,
1705 Kathryn McKinley, OOPSLA07, Montreal, Oct 2007) but in fact it is
1706 implemented completely differently.
1708 Origin tags and ECUs -- about the shadow values
1709 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1711 This implementation tracks the defining point of all uninitialised
1712 values using so called "origin tags", which are 32-bit integers,
1713 rather than using the values themselves to encode the origins. The
1714 latter, so-called value piggybacking", is what the OOPSLA07 paper
1717 Origin tags, as tracked by the machinery below, are 32-bit unsigned
1718 ints (UInts), regardless of the machine's word size. Each tag
1719 comprises an upper 30-bit ECU field and a lower 2-bit
1720 'kind' field. The ECU field is a number given out by m_execontext
1721 and has a 1-1 mapping with ExeContext*s. An ECU can be used
1722 directly as an origin tag (otag), but in fact we want to put
1723 additional information 'kind' field to indicate roughly where the
1724 tag came from. This helps print more understandable error messages
1725 for the user -- it has no other purpose. In summary:
1727 * Both ECUs and origin tags are represented as 32-bit words
1729 * m_execontext and the core-tool interface deal purely in ECUs.
1730 They have no knowledge of origin tags - that is a purely
1731 Memcheck-internal matter.
1733 * all valid ECUs have the lowest 2 bits zero and at least
1734 one of the upper 30 bits nonzero (see VG_(is_plausible_ECU))
1736 * to convert from an ECU to an otag, OR in one of the MC_OKIND_
1737 constants defined in mc_include.h.
1739 * to convert an otag back to an ECU, AND it with ~3
1741 One important fact is that no valid otag is zero. A zero otag is
1742 used by the implementation to indicate "no origin", which could
1743 mean that either the value is defined, or it is undefined but the
1744 implementation somehow managed to lose the origin.
1746 The ECU used for memory created by malloc etc is derived from the
1747 stack trace at the time the malloc etc happens. This means the
1748 mechanism can show the exact allocation point for heap-created
1749 uninitialised values.
1751 In contrast, it is simply too expensive to create a complete
1752 backtrace for each stack allocation. Therefore we merely use a
1753 depth-1 backtrace for stack allocations, which can be done once at
1754 translation time, rather than N times at run time. The result of
1755 this is that, for stack created uninitialised values, Memcheck can
1756 only show the allocating function, and not what called it.
1757 Furthermore, compilers tend to move the stack pointer just once at
1758 the start of the function, to allocate all locals, and so in fact
1759 the stack origin almost always simply points to the opening brace
1760 of the function. Net result is, for stack origins, the mechanism
1761 can tell you in which function the undefined value was created, but
1762 that's all. Users will need to carefully check all locals in the
1765 Shadowing registers and memory
1766 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1768 Memory is shadowed using a two level cache structure (ocacheL1 and
1769 ocacheL2). Memory references are first directed to ocacheL1. This
1770 is a traditional 2-way set associative cache with 32-byte lines and
1771 approximate LRU replacement within each set.
1773 A naive implementation would require storing one 32 bit otag for
1774 each byte of memory covered, a 4:1 space overhead. Instead, there
1775 is one otag for every 4 bytes of memory covered, plus a 4-bit mask
1776 that shows which of the 4 bytes have that shadow value and which
1777 have a shadow value of zero (indicating no origin). Hence a lot of
1778 space is saved, but the cost is that only one different origin per
1779 4 bytes of address space can be represented. This is a source of
1780 imprecision, but how much of a problem it really is remains to be
1783 A cache line that contains all zeroes ("no origins") contains no
1784 useful information, and can be ejected from the L1 cache "for
1785 free", in the sense that a read miss on the L1 causes a line of
1786 zeroes to be installed. However, ejecting a line containing
1787 nonzeroes risks losing origin information permanently. In order to
1788 prevent such lossage, ejected nonzero lines are placed in a
1789 secondary cache (ocacheL2), which is an OSet (AVL tree) of cache
1790 lines. This can grow arbitrarily large, and so should ensure that
1791 Memcheck runs out of memory in preference to losing useful origin
1792 info due to cache size limitations.
1794 Shadowing registers is a bit tricky, because the shadow values are
1795 32 bits, regardless of the size of the register. That gives a
1796 problem for registers smaller than 32 bits. The solution is to
1797 find spaces in the guest state that are unused, and use those to
1798 shadow guest state fragments smaller than 32 bits. For example, on
1799 ppc32/64, each vector register is 16 bytes long. If 4 bytes of the
1800 shadow are allocated for the register's otag, then there are still
1801 12 bytes left over which could be used to shadow 3 other values.
1803 This implies there is some non-obvious mapping from guest state
1804 (start,length) pairs to the relevant shadow offset (for the origin
1805 tags). And it is unfortunately guest-architecture specific. The
1806 mapping is contained in mc_machine.c, which is quite lengthy but
1809 Instrumenting the IR
1810 ~~~~~~~~~~~~~~~~~~~~
1812 Instrumentation is largely straightforward, and done by the
1813 functions schemeE and schemeS in mc_translate.c. These generate
1814 code for handling the origin tags of expressions (E) and statements
1815 (S) respectively. The rather strange names are a reference to the
1816 "compilation schemes" shown in Simon Peyton Jones' book "The
1817 Implementation of Functional Programming Languages" (Prentice Hall,
1819 http://research.microsoft.com/~simonpj/papers/slpj-book-1987/index.htm).
1821 schemeS merely arranges to move shadow values around the guest
1822 state to track the incoming IR. schemeE is largely trivial too.
1823 The only significant point is how to compute the otag corresponding
1824 to binary (or ternary, quaternary, etc) operator applications. The
1825 rule is simple: just take whichever value is larger (32-bit
1826 unsigned max). Constants get the special value zero. Hence this
1827 rule always propagates a nonzero (known) otag in preference to a
1828 zero (unknown, or more likely, value-is-defined) tag, as we want.
1829 If two different undefined values are inputs to a binary operator
1830 application, then which is propagated is arbitrary, but that
1831 doesn't matter, since the program is erroneous in using either of
1832 the values, and so there's no point in attempting to propagate
1835 Since constants are abstracted to (otag) zero, much of the
1836 instrumentation code can be folded out without difficulty by the
1837 generic post-instrumentation IR cleanup pass, using these rules:
1838 Max32U(0,x) -> x, Max32U(x,0) -> x, Max32(x,y) where x and y are
1839 constants is evaluated at JIT time. And the resulting dead code
1840 removal. In practice this causes surprisingly few Max32Us to
1841 survive through to backend code generation.
1843 Integration with the V-bits machinery
1844 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1846 This is again largely straightforward. Mostly the otag and V bits
1847 stuff are independent. The only point of interaction is when the V
1848 bits instrumenter creates a call to a helper function to report an
1849 uninitialised value error -- in that case it must first use schemeE
1850 to get hold of the origin tag expression for the value, and pass
1851 that to the helper too.
1853 There is the usual stuff to do with setting address range
1854 permissions. When memory is painted undefined, we must also know
1855 the origin tag to paint with, which involves some tedious plumbing,
1856 particularly to do with the fast case stack handlers. When memory
1857 is painted defined or noaccess then the origin tags must be forced
1860 One of the goals of the implementation was to ensure that the
1861 non-origin tracking mode isn't slowed down at all. To do this,
1862 various functions to do with memory permissions setting (again,
1863 mostly pertaining to the stack) are duplicated for the with- and
1866 Dealing with stack redzones, and the NIA cache
1867 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1869 This is one of the few non-obvious parts of the implementation.
1871 Some ABIs (amd64-ELF, ppc64-ELF, ppc32/64-XCOFF) define a small
1872 reserved area below the stack pointer, that can be used as scratch
1873 space by compiler generated code for functions. In the Memcheck
1874 sources this is referred to as the "stack redzone". The important
1875 thing here is that such redzones are considered volatile across
1876 function calls and returns. So Memcheck takes care to mark them as
1877 undefined for each call and return, on the afflicted platforms.
1878 Past experience shows this is essential in order to get reliable
1879 messages about uninitialised values that come from the stack.
1881 So the question is, when we paint a redzone undefined, what origin
1882 tag should we use for it? Consider a function f() calling g(). If
1883 we paint the redzone using an otag derived from the ExeContext of
1884 the CALL/BL instruction in f, then any errors in g causing it to
1885 use uninitialised values that happen to lie in the redzone, will be
1886 reported as having their origin in f. Which is highly confusing.
1888 The same applies for returns: if, on a return, we paint the redzone
1889 using a origin tag derived from the ExeContext of the RET/BLR
1890 instruction in g, then any later errors in f causing it to use
1891 uninitialised values in the redzone, will be reported as having
1892 their origin in g. Which is just as confusing.
1894 To do it right, in both cases we need to use an origin tag which
1895 pertains to the instruction which dynamically follows the CALL/BL
1896 or RET/BLR. In short, one derived from the NIA - the "next
1897 instruction address".
1899 To make this work, Memcheck's redzone-painting helper,
1900 MC_(helperc_MAKE_STACK_UNINIT), now takes a third argument, the
1901 NIA. It converts the NIA to a 1-element ExeContext, and uses that
1902 ExeContext's ECU as the basis for the otag used to paint the
1903 redzone. The expensive part of this is converting an NIA into an
1904 ECU, since this happens once for every call and every return. So
1905 we use a simple 511-line, 2-way set associative cache
1906 (nia_to_ecu_cache) to cache the mappings, and that knocks most of
1909 Further background comments
1910 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
1912 > Question: why is otag a UInt? Wouldn't a UWord be better? Isn't
1913 > it really just the address of the relevant ExeContext?
1915 Well, it's not the address, but a value which has a 1-1 mapping
1916 with ExeContexts, and is guaranteed not to be zero, since zero
1917 denotes (to memcheck) "unknown origin or defined value". So these
1918 UInts are just numbers starting at 4 and incrementing by 4; each
1919 ExeContext is given a number when it is created. (*** NOTE this
1920 confuses otags and ECUs; see comments above ***).
1922 Making these otags 32-bit regardless of the machine's word size
1923 makes the 64-bit implementation easier (next para). And it doesn't
1924 really limit us in any way, since for the tags to overflow would
1925 require that the program somehow caused 2^30-1 different
1926 ExeContexts to be created, in which case it is probably in deep
1927 trouble. Not to mention V will have soaked up many tens of
1928 gigabytes of memory merely to store them all.
1930 So having 64-bit origins doesn't really buy you anything, and has
1931 the following downsides:
1933 Suppose that instead, an otag is a UWord. This would mean that, on
1936 1. It becomes hard to shadow any element of guest state which is
1937 smaller than 8 bytes. To do so means you'd need to find some
1938 8-byte-sized hole in the guest state which you don't want to
1939 shadow, and use that instead to hold the otag. On ppc64, the
1940 condition code register(s) are split into 20 UChar sized pieces,
1941 all of which need to be tracked (guest_XER_SO .. guest_CR7_0)
1942 and so that would entail finding 160 bytes somewhere else in the
1945 Even on x86, I want to track origins for %AH .. %DH (bits 15:8
1946 of %EAX .. %EDX) that are separate from %AL .. %DL (bits 7:0 of
1947 same) and so I had to look for 4 untracked otag-sized areas in
1948 the guest state to make that possible.
1950 The same problem exists of course when origin tags are only 32
1951 bits, but it's less extreme.
1953 2. (More compelling) it doubles the size of the origin shadow
1954 memory. Given that the shadow memory is organised as a fixed
1955 size cache, and that accuracy of tracking is limited by origins
1956 falling out the cache due to space conflicts, this isn't good.
1958 > Another question: is the origin tracking perfect, or are there
1959 > cases where it fails to determine an origin?
1961 It is imperfect for at least for the following reasons, and
1964 * Insufficient capacity in the origin cache. When a line is
1965 evicted from the cache it is gone forever, and so subsequent
1966 queries for the line produce zero, indicating no origin
1967 information. Interestingly, a line containing all zeroes can be
1968 evicted "free" from the cache, since it contains no useful
1969 information, so there is scope perhaps for some cleverer cache
1970 management schemes. (*** NOTE, with the introduction of the
1971 second level origin tag cache, ocacheL2, this is no longer a
1974 * The origin cache only stores one otag per 32-bits of address
1975 space, plus 4 bits indicating which of the 4 bytes has that tag
1976 and which are considered defined. The result is that if two
1977 undefined bytes in the same word are stored in memory, the first
1978 stored byte's origin will be lost and replaced by the origin for
1981 * Nonzero origin tags for defined values. Consider a binary
1982 operator application op(x,y). Suppose y is undefined (and so has
1983 a valid nonzero origin tag), and x is defined, but erroneously
1984 has a nonzero origin tag (defined values should have tag zero).
1985 If the erroneous tag has a numeric value greater than y's tag,
1986 then the rule for propagating origin tags though binary
1987 operations, which is simply to take the unsigned max of the two
1988 tags, will erroneously propagate x's tag rather than y's.
1990 * Some obscure uses of x86/amd64 byte registers can cause lossage
1991 or confusion of origins. %AH .. %DH are treated as different
1992 from, and unrelated to, their parent registers, %EAX .. %EDX.
1993 So some wierd sequences like
1995 movb undefined-value, %AH
1996 movb defined-value, %AL
1997 .. use %AX or %EAX ..
1999 will cause the origin attributed to %AH to be ignored, since %AL,
2000 %AX, %EAX are treated as the same register, and %AH as a
2001 completely separate one.
2003 But having said all that, it actually seems to work fairly well in
2007 static UWord stats_ocacheL1_find = 0;
2008 static UWord stats_ocacheL1_found_at_1 = 0;
2009 static UWord stats_ocacheL1_found_at_N = 0;
2010 static UWord stats_ocacheL1_misses = 0;
2011 static UWord stats_ocacheL1_lossage = 0;
2012 static UWord stats_ocacheL1_movefwds = 0;
2014 static UWord stats__ocacheL2_refs = 0;
2015 static UWord stats__ocacheL2_misses = 0;
2016 static UWord stats__ocacheL2_n_nodes_max = 0;
2018 /* Cache of 32-bit values, one every 32 bits of address space */
2020 #define OC_BITS_PER_LINE 5
2021 #define OC_W32S_PER_LINE (1 << (OC_BITS_PER_LINE - 2))
2023 static INLINE UWord oc_line_offset ( Addr a ) {
2024 return (a >> 2) & (OC_W32S_PER_LINE - 1);
2026 static INLINE Bool is_valid_oc_tag ( Addr tag ) {
2027 return 0 == (tag & ((1 << OC_BITS_PER_LINE) - 1));
2030 #define OC_LINES_PER_SET 2
2032 #define OC_N_SET_BITS 20
2033 #define OC_N_SETS (1 << OC_N_SET_BITS)
2035 /* These settings give:
2036 64 bit host: ocache: 100,663,296 sizeB 67,108,864 useful
2037 32 bit host: ocache: 92,274,688 sizeB 67,108,864 useful
2040 #define OC_MOVE_FORWARDS_EVERY_BITS 7
2046 UInt w32[OC_W32S_PER_LINE];
2047 UChar descr[OC_W32S_PER_LINE];
2051 /* Classify and also sanity-check 'line'. Return 'e' (empty) if not
2052 in use, 'n' (nonzero) if it contains at least one valid origin tag,
2053 and 'z' if all the represented tags are zero. */
2054 static UChar classify_OCacheLine ( OCacheLine* line )
2057 if (line->tag == 1/*invalid*/)
2058 return 'e'; /* EMPTY */
2059 tl_assert(is_valid_oc_tag(line->tag));
2060 for (i = 0; i < OC_W32S_PER_LINE; i++) {
2061 tl_assert(0 == ((~0xF) & line->descr[i]));
2062 if (line->w32[i] > 0 && line->descr[i] > 0)
2063 return 'n'; /* NONZERO - contains useful info */
2065 return 'z'; /* ZERO - no useful info */
2070 OCacheLine line[OC_LINES_PER_SET];
2076 OCacheSet set[OC_N_SETS];
2080 static OCache* ocacheL1 = NULL;
2081 static UWord ocacheL1_event_ctr = 0;
2083 static void init_ocacheL2 ( void ); /* fwds */
2084 static void init_OCache ( void )
2087 tl_assert(MC_(clo_mc_level) >= 3);
2088 tl_assert(ocacheL1 == NULL);
2089 ocacheL1 = VG_(am_shadow_alloc)(sizeof(OCache));
2090 if (ocacheL1 == NULL) {
2091 VG_(out_of_memory_NORETURN)( "memcheck:allocating ocacheL1",
2094 tl_assert(ocacheL1 != NULL);
2095 for (set = 0; set < OC_N_SETS; set++) {
2096 for (line = 0; line < OC_LINES_PER_SET; line++) {
2097 ocacheL1->set[set].line[line].tag = 1/*invalid*/;
2103 static void moveLineForwards ( OCacheSet* set, UWord lineno )
2106 stats_ocacheL1_movefwds++;
2107 tl_assert(lineno > 0 && lineno < OC_LINES_PER_SET);
2108 tmp = set->line[lineno-1];
2109 set->line[lineno-1] = set->line[lineno];
2110 set->line[lineno] = tmp;
2113 static void zeroise_OCacheLine ( OCacheLine* line, Addr tag ) {
2115 for (i = 0; i < OC_W32S_PER_LINE; i++) {
2116 line->w32[i] = 0; /* NO ORIGIN */
2117 line->descr[i] = 0; /* REALLY REALLY NO ORIGIN! */
2122 //////////////////////////////////////////////////////////////
2123 //// OCache backing store
2125 static OSet* ocacheL2 = NULL;
2127 static void* ocacheL2_malloc ( HChar* cc, SizeT szB ) {
2128 return VG_(malloc)(cc, szB);
2130 static void ocacheL2_free ( void* v ) {
2134 /* Stats: # nodes currently in tree */
2135 static UWord stats__ocacheL2_n_nodes = 0;
2137 static void init_ocacheL2 ( void )
2139 tl_assert(!ocacheL2);
2140 tl_assert(sizeof(Word) == sizeof(Addr)); /* since OCacheLine.tag :: Addr */
2141 tl_assert(0 == offsetof(OCacheLine,tag));
2143 = VG_(OSetGen_Create)( offsetof(OCacheLine,tag),
2144 NULL, /* fast cmp */
2145 ocacheL2_malloc, "mc.ioL2", ocacheL2_free );
2146 tl_assert(ocacheL2);
2147 stats__ocacheL2_n_nodes = 0;
2150 /* Find line with the given tag in the tree, or NULL if not found. */
2151 static OCacheLine* ocacheL2_find_tag ( Addr tag )
2154 tl_assert(is_valid_oc_tag(tag));
2155 stats__ocacheL2_refs++;
2156 line = VG_(OSetGen_Lookup)( ocacheL2, &tag );
2160 /* Delete the line with the given tag from the tree, if it is present, and
2161 free up the associated memory. */
2162 static void ocacheL2_del_tag ( Addr tag )
2165 tl_assert(is_valid_oc_tag(tag));
2166 stats__ocacheL2_refs++;
2167 line = VG_(OSetGen_Remove)( ocacheL2, &tag );
2169 VG_(OSetGen_FreeNode)(ocacheL2, line);
2170 tl_assert(stats__ocacheL2_n_nodes > 0);
2171 stats__ocacheL2_n_nodes--;
2175 /* Add a copy of the given line to the tree. It must not already be
2177 static void ocacheL2_add_line ( OCacheLine* line )
2180 tl_assert(is_valid_oc_tag(line->tag));
2181 copy = VG_(OSetGen_AllocNode)( ocacheL2, sizeof(OCacheLine) );
2184 stats__ocacheL2_refs++;
2185 VG_(OSetGen_Insert)( ocacheL2, copy );
2186 stats__ocacheL2_n_nodes++;
2187 if (stats__ocacheL2_n_nodes > stats__ocacheL2_n_nodes_max)
2188 stats__ocacheL2_n_nodes_max = stats__ocacheL2_n_nodes;
2192 //////////////////////////////////////////////////////////////
2194 __attribute__((noinline))
2195 static OCacheLine* find_OCacheLine_SLOW ( Addr a )
2197 OCacheLine *victim, *inL2;
2200 UWord setno = (a >> OC_BITS_PER_LINE) & (OC_N_SETS - 1);
2201 UWord tagmask = ~((1 << OC_BITS_PER_LINE) - 1);
2202 UWord tag = a & tagmask;
2203 tl_assert(setno >= 0 && setno < OC_N_SETS);
2205 /* we already tried line == 0; skip therefore. */
2206 for (line = 1; line < OC_LINES_PER_SET; line++) {
2207 if (ocacheL1->set[setno].line[line].tag == tag) {
2209 stats_ocacheL1_found_at_1++;
2211 stats_ocacheL1_found_at_N++;
2213 if (UNLIKELY(0 == (ocacheL1_event_ctr++
2214 & ((1<<OC_MOVE_FORWARDS_EVERY_BITS)-1)))) {
2215 moveLineForwards( &ocacheL1->set[setno], line );
2218 return &ocacheL1->set[setno].line[line];
2222 /* A miss. Use the last slot. Implicitly this means we're
2223 ejecting the line in the last slot. */
2224 stats_ocacheL1_misses++;
2225 tl_assert(line == OC_LINES_PER_SET);
2227 tl_assert(line > 0);
2229 /* First, move the to-be-ejected line to the L2 cache. */
2230 victim = &ocacheL1->set[setno].line[line];
2231 c = classify_OCacheLine(victim);
2234 /* the line is empty (has invalid tag); ignore it. */
2237 /* line contains zeroes. We must ensure the backing store is
2238 updated accordingly, either by copying the line there
2239 verbatim, or by ensuring it isn't present there. We
2240 chosse the latter on the basis that it reduces the size of
2241 the backing store. */
2242 ocacheL2_del_tag( victim->tag );
2245 /* line contains at least one real, useful origin. Copy it
2246 to the backing store. */
2247 stats_ocacheL1_lossage++;
2248 inL2 = ocacheL2_find_tag( victim->tag );
2252 ocacheL2_add_line( victim );
2259 /* Now we must reload the L1 cache from the backing tree, if
2261 tl_assert(tag != victim->tag); /* stay sane */
2262 inL2 = ocacheL2_find_tag( tag );
2264 /* We're in luck. It's in the L2. */
2265 ocacheL1->set[setno].line[line] = *inL2;
2267 /* Missed at both levels of the cache hierarchy. We have to
2268 declare it as full of zeroes (unknown origins). */
2269 stats__ocacheL2_misses++;
2270 zeroise_OCacheLine( &ocacheL1->set[setno].line[line], tag );
2273 /* Move it one forwards */
2274 moveLineForwards( &ocacheL1->set[setno], line );
2277 return &ocacheL1->set[setno].line[line];
2280 static INLINE OCacheLine* find_OCacheLine ( Addr a )
2282 UWord setno = (a >> OC_BITS_PER_LINE) & (OC_N_SETS - 1);
2283 UWord tagmask = ~((1 << OC_BITS_PER_LINE) - 1);
2284 UWord tag = a & tagmask;
2286 stats_ocacheL1_find++;
2288 if (OC_ENABLE_ASSERTIONS) {
2289 tl_assert(setno >= 0 && setno < OC_N_SETS);
2290 tl_assert(0 == (tag & (4 * OC_W32S_PER_LINE - 1)));
2293 if (LIKELY(ocacheL1->set[setno].line[0].tag == tag)) {
2294 return &ocacheL1->set[setno].line[0];
2297 return find_OCacheLine_SLOW( a );
2300 static INLINE void set_aligned_word64_Origin_to_undef ( Addr a, UInt otag )
2302 //// BEGIN inlined, specialised version of MC_(helperc_b_store8)
2303 //// Set the origins for a+0 .. a+7
2305 UWord lineoff = oc_line_offset(a);
2306 if (OC_ENABLE_ASSERTIONS) {
2307 tl_assert(lineoff >= 0
2308 && lineoff < OC_W32S_PER_LINE -1/*'cos 8-aligned*/);
2310 line = find_OCacheLine( a );
2311 line->descr[lineoff+0] = 0xF;
2312 line->descr[lineoff+1] = 0xF;
2313 line->w32[lineoff+0] = otag;
2314 line->w32[lineoff+1] = otag;
2316 //// END inlined, specialised version of MC_(helperc_b_store8)
2320 /*------------------------------------------------------------*/
2321 /*--- Aligned fast case permission setters, ---*/
2322 /*--- for dealing with stacks ---*/
2323 /*------------------------------------------------------------*/
2325 /*--------------------- 32-bit ---------------------*/
2327 /* Nb: by "aligned" here we mean 4-byte aligned */
2329 static INLINE void make_aligned_word32_undefined ( Addr a )
2331 PROF_EVENT(300, "make_aligned_word32_undefined");
2333 #ifndef PERF_FAST_STACK2
2334 make_mem_undefined(a, 4);
2340 if (UNLIKELY(a > MAX_PRIMARY_ADDRESS)) {
2341 PROF_EVENT(301, "make_aligned_word32_undefined-slow1");
2342 make_mem_undefined(a, 4);
2346 sm = get_secmap_for_writing_low(a);
2348 sm->vabits8[sm_off] = VA_BITS8_UNDEFINED;
2354 void make_aligned_word32_undefined_w_otag ( Addr a, UInt otag )
2356 make_aligned_word32_undefined(a);
2357 //// BEGIN inlined, specialised version of MC_(helperc_b_store4)
2358 //// Set the origins for a+0 .. a+3
2360 UWord lineoff = oc_line_offset(a);
2361 if (OC_ENABLE_ASSERTIONS) {
2362 tl_assert(lineoff >= 0 && lineoff < OC_W32S_PER_LINE);
2364 line = find_OCacheLine( a );
2365 line->descr[lineoff] = 0xF;
2366 line->w32[lineoff] = otag;
2368 //// END inlined, specialised version of MC_(helperc_b_store4)
2372 void make_aligned_word32_noaccess ( Addr a )
2374 PROF_EVENT(310, "make_aligned_word32_noaccess");
2376 #ifndef PERF_FAST_STACK2
2377 MC_(make_mem_noaccess)(a, 4);
2383 if (UNLIKELY(a > MAX_PRIMARY_ADDRESS)) {
2384 PROF_EVENT(311, "make_aligned_word32_noaccess-slow1");
2385 MC_(make_mem_noaccess)(a, 4);
2389 sm = get_secmap_for_writing_low(a);
2391 sm->vabits8[sm_off] = VA_BITS8_NOACCESS;
2393 //// BEGIN inlined, specialised version of MC_(helperc_b_store4)
2394 //// Set the origins for a+0 .. a+3.
2395 if (UNLIKELY( MC_(clo_mc_level) == 3 )) {
2397 UWord lineoff = oc_line_offset(a);
2398 if (OC_ENABLE_ASSERTIONS) {
2399 tl_assert(lineoff >= 0 && lineoff < OC_W32S_PER_LINE);
2401 line = find_OCacheLine( a );
2402 line->descr[lineoff] = 0;
2404 //// END inlined, specialised version of MC_(helperc_b_store4)
2409 /*--------------------- 64-bit ---------------------*/
2411 /* Nb: by "aligned" here we mean 8-byte aligned */
2413 static INLINE void make_aligned_word64_undefined ( Addr a )
2415 PROF_EVENT(320, "make_aligned_word64_undefined");
2417 #ifndef PERF_FAST_STACK2
2418 make_mem_undefined(a, 8);
2424 if (UNLIKELY(a > MAX_PRIMARY_ADDRESS)) {
2425 PROF_EVENT(321, "make_aligned_word64_undefined-slow1");
2426 make_mem_undefined(a, 8);
2430 sm = get_secmap_for_writing_low(a);
2431 sm_off16 = SM_OFF_16(a);
2432 ((UShort*)(sm->vabits8))[sm_off16] = VA_BITS16_UNDEFINED;
2438 void make_aligned_word64_undefined_w_otag ( Addr a, UInt otag )
2440 make_aligned_word64_undefined(a);
2441 //// BEGIN inlined, specialised version of MC_(helperc_b_store8)
2442 //// Set the origins for a+0 .. a+7
2444 UWord lineoff = oc_line_offset(a);
2445 tl_assert(lineoff >= 0
2446 && lineoff < OC_W32S_PER_LINE -1/*'cos 8-aligned*/);
2447 line = find_OCacheLine( a );
2448 line->descr[lineoff+0] = 0xF;
2449 line->descr[lineoff+1] = 0xF;
2450 line->w32[lineoff+0] = otag;
2451 line->w32[lineoff+1] = otag;
2453 //// END inlined, specialised version of MC_(helperc_b_store8)
2457 void make_aligned_word64_noaccess ( Addr a )
2459 PROF_EVENT(330, "make_aligned_word64_noaccess");
2461 #ifndef PERF_FAST_STACK2
2462 MC_(make_mem_noaccess)(a, 8);
2468 if (UNLIKELY(a > MAX_PRIMARY_ADDRESS)) {
2469 PROF_EVENT(331, "make_aligned_word64_noaccess-slow1");
2470 MC_(make_mem_noaccess)(a, 8);
2474 sm = get_secmap_for_writing_low(a);
2475 sm_off16 = SM_OFF_16(a);
2476 ((UShort*)(sm->vabits8))[sm_off16] = VA_BITS16_NOACCESS;
2478 //// BEGIN inlined, specialised version of MC_(helperc_b_store8)
2479 //// Clear the origins for a+0 .. a+7.
2480 if (UNLIKELY( MC_(clo_mc_level) == 3 )) {
2482 UWord lineoff = oc_line_offset(a);
2483 tl_assert(lineoff >= 0
2484 && lineoff < OC_W32S_PER_LINE -1/*'cos 8-aligned*/);
2485 line = find_OCacheLine( a );
2486 line->descr[lineoff+0] = 0;
2487 line->descr[lineoff+1] = 0;
2489 //// END inlined, specialised version of MC_(helperc_b_store8)
2495 /*------------------------------------------------------------*/
2496 /*--- Stack pointer adjustment ---*/
2497 /*------------------------------------------------------------*/
2499 #ifdef PERF_FAST_STACK
2502 # define MAYBE_USED __attribute__((unused))
2505 /*--------------- adjustment by 4 bytes ---------------*/
2508 static void VG_REGPARM(2) mc_new_mem_stack_4_w_ECU(Addr new_SP, UInt ecu)
2510 UInt otag = ecu | MC_OKIND_STACK;
2511 PROF_EVENT(110, "new_mem_stack_4");
2512 if (VG_IS_4_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2513 make_aligned_word32_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP, otag );
2515 MC_(make_mem_undefined_w_otag) ( -VG_STACK_REDZONE_SZB + new_SP, 4, otag );
2520 static void VG_REGPARM(1) mc_new_mem_stack_4(Addr new_SP)
2522 PROF_EVENT(110, "new_mem_stack_4");
2523 if (VG_IS_4_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2524 make_aligned_word32_undefined ( -VG_STACK_REDZONE_SZB + new_SP );
2526 make_mem_undefined ( -VG_STACK_REDZONE_SZB + new_SP, 4 );
2531 static void VG_REGPARM(1) mc_die_mem_stack_4(Addr new_SP)
2533 PROF_EVENT(120, "die_mem_stack_4");
2534 if (VG_IS_4_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2535 make_aligned_word32_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-4 );
2537 MC_(make_mem_noaccess) ( -VG_STACK_REDZONE_SZB + new_SP-4, 4 );
2541 /*--------------- adjustment by 8 bytes ---------------*/
2544 static void VG_REGPARM(2) mc_new_mem_stack_8_w_ECU(Addr new_SP, UInt ecu)
2546 UInt otag = ecu | MC_OKIND_STACK;
2547 PROF_EVENT(111, "new_mem_stack_8");
2548 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2549 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP, otag );
2550 } else if (VG_IS_4_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2551 make_aligned_word32_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP , otag );
2552 make_aligned_word32_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+4, otag );
2554 MC_(make_mem_undefined_w_otag) ( -VG_STACK_REDZONE_SZB + new_SP, 8, otag );
2559 static void VG_REGPARM(1) mc_new_mem_stack_8(Addr new_SP)
2561 PROF_EVENT(111, "new_mem_stack_8");
2562 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2563 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP );
2564 } else if (VG_IS_4_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2565 make_aligned_word32_undefined ( -VG_STACK_REDZONE_SZB + new_SP );
2566 make_aligned_word32_undefined ( -VG_STACK_REDZONE_SZB + new_SP+4 );
2568 make_mem_undefined ( -VG_STACK_REDZONE_SZB + new_SP, 8 );
2573 static void VG_REGPARM(1) mc_die_mem_stack_8(Addr new_SP)
2575 PROF_EVENT(121, "die_mem_stack_8");
2576 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2577 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-8 );
2578 } else if (VG_IS_4_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2579 make_aligned_word32_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-8 );
2580 make_aligned_word32_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-4 );
2582 MC_(make_mem_noaccess) ( -VG_STACK_REDZONE_SZB + new_SP-8, 8 );
2586 /*--------------- adjustment by 12 bytes ---------------*/
2589 static void VG_REGPARM(2) mc_new_mem_stack_12_w_ECU(Addr new_SP, UInt ecu)
2591 UInt otag = ecu | MC_OKIND_STACK;
2592 PROF_EVENT(112, "new_mem_stack_12");
2593 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2594 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP , otag );
2595 make_aligned_word32_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+8, otag );
2596 } else if (VG_IS_4_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2597 /* from previous test we don't have 8-alignment at offset +0,
2598 hence must have 8 alignment at offsets +4/-4. Hence safe to
2599 do 4 at +0 and then 8 at +4/. */
2600 make_aligned_word32_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP , otag );
2601 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+4, otag );
2603 MC_(make_mem_undefined_w_otag) ( -VG_STACK_REDZONE_SZB + new_SP, 12, otag );
2608 static void VG_REGPARM(1) mc_new_mem_stack_12(Addr new_SP)
2610 PROF_EVENT(112, "new_mem_stack_12");
2611 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2612 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP );
2613 make_aligned_word32_undefined ( -VG_STACK_REDZONE_SZB + new_SP+8 );
2614 } else if (VG_IS_4_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2615 /* from previous test we don't have 8-alignment at offset +0,
2616 hence must have 8 alignment at offsets +4/-4. Hence safe to
2617 do 4 at +0 and then 8 at +4/. */
2618 make_aligned_word32_undefined ( -VG_STACK_REDZONE_SZB + new_SP );
2619 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+4 );
2621 make_mem_undefined ( -VG_STACK_REDZONE_SZB + new_SP, 12 );
2626 static void VG_REGPARM(1) mc_die_mem_stack_12(Addr new_SP)
2628 PROF_EVENT(122, "die_mem_stack_12");
2629 /* Note the -12 in the test */
2630 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP-12 )) {
2631 /* We have 8-alignment at -12, hence ok to do 8 at -12 and 4 at
2633 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-12 );
2634 make_aligned_word32_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-4 );
2635 } else if (VG_IS_4_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2636 /* We have 4-alignment at +0, but we don't have 8-alignment at
2637 -12. So we must have 8-alignment at -8. Hence do 4 at -12
2638 and then 8 at -8. */
2639 make_aligned_word32_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-12 );
2640 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-8 );
2642 MC_(make_mem_noaccess) ( -VG_STACK_REDZONE_SZB + new_SP-12, 12 );
2646 /*--------------- adjustment by 16 bytes ---------------*/
2649 static void VG_REGPARM(2) mc_new_mem_stack_16_w_ECU(Addr new_SP, UInt ecu)
2651 UInt otag = ecu | MC_OKIND_STACK;
2652 PROF_EVENT(113, "new_mem_stack_16");
2653 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2654 /* Have 8-alignment at +0, hence do 8 at +0 and 8 at +8. */
2655 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP , otag );
2656 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+8, otag );
2657 } else if (VG_IS_4_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2658 /* Have 4 alignment at +0 but not 8; hence 8 must be at +4.
2659 Hence do 4 at +0, 8 at +4, 4 at +12. */
2660 make_aligned_word32_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP , otag );
2661 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+4 , otag );
2662 make_aligned_word32_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+12, otag );
2664 MC_(make_mem_undefined_w_otag) ( -VG_STACK_REDZONE_SZB + new_SP, 16, otag );
2669 static void VG_REGPARM(1) mc_new_mem_stack_16(Addr new_SP)
2671 PROF_EVENT(113, "new_mem_stack_16");
2672 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2673 /* Have 8-alignment at +0, hence do 8 at +0 and 8 at +8. */
2674 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP );
2675 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+8 );
2676 } else if (VG_IS_4_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2677 /* Have 4 alignment at +0 but not 8; hence 8 must be at +4.
2678 Hence do 4 at +0, 8 at +4, 4 at +12. */
2679 make_aligned_word32_undefined ( -VG_STACK_REDZONE_SZB + new_SP );
2680 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+4 );
2681 make_aligned_word32_undefined ( -VG_STACK_REDZONE_SZB + new_SP+12 );
2683 make_mem_undefined ( -VG_STACK_REDZONE_SZB + new_SP, 16 );
2688 static void VG_REGPARM(1) mc_die_mem_stack_16(Addr new_SP)
2690 PROF_EVENT(123, "die_mem_stack_16");
2691 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2692 /* Have 8-alignment at +0, hence do 8 at -16 and 8 at -8. */
2693 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-16 );
2694 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-8 );
2695 } else if (VG_IS_4_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2696 /* 8 alignment must be at -12. Do 4 at -16, 8 at -12, 4 at -4. */
2697 make_aligned_word32_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-16 );
2698 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-12 );
2699 make_aligned_word32_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-4 );
2701 MC_(make_mem_noaccess) ( -VG_STACK_REDZONE_SZB + new_SP-16, 16 );
2705 /*--------------- adjustment by 32 bytes ---------------*/
2708 static void VG_REGPARM(2) mc_new_mem_stack_32_w_ECU(Addr new_SP, UInt ecu)
2710 UInt otag = ecu | MC_OKIND_STACK;
2711 PROF_EVENT(114, "new_mem_stack_32");
2712 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2713 /* Straightforward */
2714 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP , otag );
2715 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+8 , otag );
2716 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+16, otag );
2717 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+24, otag );
2718 } else if (VG_IS_4_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2719 /* 8 alignment must be at +4. Hence do 8 at +4,+12,+20 and 4 at
2721 make_aligned_word32_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP , otag );
2722 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+4 , otag );
2723 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+12, otag );
2724 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+20, otag );
2725 make_aligned_word32_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+28, otag );
2727 MC_(make_mem_undefined_w_otag) ( -VG_STACK_REDZONE_SZB + new_SP, 32, otag );
2732 static void VG_REGPARM(1) mc_new_mem_stack_32(Addr new_SP)
2734 PROF_EVENT(114, "new_mem_stack_32");
2735 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2736 /* Straightforward */
2737 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP );
2738 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+8 );
2739 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+16 );
2740 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+24 );
2741 } else if (VG_IS_4_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2742 /* 8 alignment must be at +4. Hence do 8 at +4,+12,+20 and 4 at
2744 make_aligned_word32_undefined ( -VG_STACK_REDZONE_SZB + new_SP );
2745 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+4 );
2746 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+12 );
2747 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+20 );
2748 make_aligned_word32_undefined ( -VG_STACK_REDZONE_SZB + new_SP+28 );
2750 make_mem_undefined ( -VG_STACK_REDZONE_SZB + new_SP, 32 );
2755 static void VG_REGPARM(1) mc_die_mem_stack_32(Addr new_SP)
2757 PROF_EVENT(124, "die_mem_stack_32");
2758 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2759 /* Straightforward */
2760 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-32 );
2761 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-24 );
2762 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-16 );
2763 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP- 8 );
2764 } else if (VG_IS_4_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2765 /* 8 alignment must be at -4 etc. Hence do 8 at -12,-20,-28 and
2767 make_aligned_word32_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-32 );
2768 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-28 );
2769 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-20 );
2770 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-12 );
2771 make_aligned_word32_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-4 );
2773 MC_(make_mem_noaccess) ( -VG_STACK_REDZONE_SZB + new_SP-32, 32 );
2777 /*--------------- adjustment by 112 bytes ---------------*/
2780 static void VG_REGPARM(2) mc_new_mem_stack_112_w_ECU(Addr new_SP, UInt ecu)
2782 UInt otag = ecu | MC_OKIND_STACK;
2783 PROF_EVENT(115, "new_mem_stack_112");
2784 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2785 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP , otag );
2786 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+8 , otag );
2787 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+16, otag );
2788 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+24, otag );
2789 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+32, otag );
2790 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+40, otag );
2791 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+48, otag );
2792 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+56, otag );
2793 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+64, otag );
2794 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+72, otag );
2795 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+80, otag );
2796 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+88, otag );
2797 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+96, otag );
2798 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+104, otag );
2800 MC_(make_mem_undefined_w_otag) ( -VG_STACK_REDZONE_SZB + new_SP, 112, otag );
2805 static void VG_REGPARM(1) mc_new_mem_stack_112(Addr new_SP)
2807 PROF_EVENT(115, "new_mem_stack_112");
2808 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2809 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP );
2810 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+8 );
2811 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+16 );
2812 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+24 );
2813 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+32 );
2814 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+40 );
2815 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+48 );
2816 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+56 );
2817 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+64 );
2818 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+72 );
2819 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+80 );
2820 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+88 );
2821 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+96 );
2822 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+104 );
2824 make_mem_undefined ( -VG_STACK_REDZONE_SZB + new_SP, 112 );
2829 static void VG_REGPARM(1) mc_die_mem_stack_112(Addr new_SP)
2831 PROF_EVENT(125, "die_mem_stack_112");
2832 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2833 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-112);
2834 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-104);
2835 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-96 );
2836 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-88 );
2837 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-80 );
2838 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-72 );
2839 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-64 );
2840 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-56 );
2841 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-48 );
2842 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-40 );
2843 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-32 );
2844 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-24 );
2845 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-16 );
2846 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP- 8 );
2848 MC_(make_mem_noaccess) ( -VG_STACK_REDZONE_SZB + new_SP-112, 112 );
2852 /*--------------- adjustment by 128 bytes ---------------*/
2855 static void VG_REGPARM(2) mc_new_mem_stack_128_w_ECU(Addr new_SP, UInt ecu)
2857 UInt otag = ecu | MC_OKIND_STACK;
2858 PROF_EVENT(116, "new_mem_stack_128");
2859 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2860 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP , otag );
2861 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+8 , otag );
2862 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+16, otag );
2863 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+24, otag );
2864 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+32, otag );
2865 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+40, otag );
2866 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+48, otag );
2867 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+56, otag );
2868 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+64, otag );
2869 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+72, otag );
2870 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+80, otag );
2871 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+88, otag );
2872 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+96, otag );
2873 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+104, otag );
2874 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+112, otag );
2875 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+120, otag );
2877 MC_(make_mem_undefined_w_otag) ( -VG_STACK_REDZONE_SZB + new_SP, 128, otag );
2882 static void VG_REGPARM(1) mc_new_mem_stack_128(Addr new_SP)
2884 PROF_EVENT(116, "new_mem_stack_128");
2885 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2886 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP );
2887 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+8 );
2888 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+16 );
2889 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+24 );
2890 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+32 );
2891 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+40 );
2892 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+48 );
2893 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+56 );
2894 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+64 );
2895 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+72 );
2896 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+80 );
2897 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+88 );
2898 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+96 );
2899 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+104 );
2900 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+112 );
2901 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+120 );
2903 make_mem_undefined ( -VG_STACK_REDZONE_SZB + new_SP, 128 );
2908 static void VG_REGPARM(1) mc_die_mem_stack_128(Addr new_SP)
2910 PROF_EVENT(126, "die_mem_stack_128");
2911 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2912 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-128);
2913 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-120);
2914 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-112);
2915 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-104);
2916 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-96 );
2917 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-88 );
2918 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-80 );
2919 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-72 );
2920 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-64 );
2921 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-56 );
2922 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-48 );
2923 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-40 );
2924 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-32 );
2925 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-24 );
2926 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-16 );
2927 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP- 8 );
2929 MC_(make_mem_noaccess) ( -VG_STACK_REDZONE_SZB + new_SP-128, 128 );
2933 /*--------------- adjustment by 144 bytes ---------------*/
2936 static void VG_REGPARM(2) mc_new_mem_stack_144_w_ECU(Addr new_SP, UInt ecu)
2938 UInt otag = ecu | MC_OKIND_STACK;
2939 PROF_EVENT(117, "new_mem_stack_144");
2940 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2941 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP, otag );
2942 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+8, otag );
2943 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+16, otag );
2944 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+24, otag );
2945 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+32, otag );
2946 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+40, otag );
2947 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+48, otag );
2948 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+56, otag );
2949 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+64, otag );
2950 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+72, otag );
2951 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+80, otag );
2952 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+88, otag );
2953 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+96, otag );
2954 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+104, otag );
2955 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+112, otag );
2956 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+120, otag );
2957 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+128, otag );
2958 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+136, otag );
2960 MC_(make_mem_undefined_w_otag) ( -VG_STACK_REDZONE_SZB + new_SP, 144, otag );
2965 static void VG_REGPARM(1) mc_new_mem_stack_144(Addr new_SP)
2967 PROF_EVENT(117, "new_mem_stack_144");
2968 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2969 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP );
2970 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+8 );
2971 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+16 );
2972 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+24 );
2973 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+32 );
2974 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+40 );
2975 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+48 );
2976 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+56 );
2977 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+64 );
2978 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+72 );
2979 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+80 );
2980 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+88 );
2981 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+96 );
2982 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+104 );
2983 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+112 );
2984 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+120 );
2985 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+128 );
2986 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+136 );
2988 make_mem_undefined ( -VG_STACK_REDZONE_SZB + new_SP, 144 );
2993 static void VG_REGPARM(1) mc_die_mem_stack_144(Addr new_SP)
2995 PROF_EVENT(127, "die_mem_stack_144");
2996 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
2997 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-144);
2998 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-136);
2999 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-128);
3000 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-120);
3001 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-112);
3002 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-104);
3003 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-96 );
3004 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-88 );
3005 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-80 );
3006 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-72 );
3007 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-64 );
3008 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-56 );
3009 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-48 );
3010 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-40 );
3011 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-32 );
3012 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-24 );
3013 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-16 );
3014 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP- 8 );
3016 MC_(make_mem_noaccess) ( -VG_STACK_REDZONE_SZB + new_SP-144, 144 );
3020 /*--------------- adjustment by 160 bytes ---------------*/
3023 static void VG_REGPARM(2) mc_new_mem_stack_160_w_ECU(Addr new_SP, UInt ecu)
3025 UInt otag = ecu | MC_OKIND_STACK;
3026 PROF_EVENT(118, "new_mem_stack_160");
3027 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
3028 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP, otag );
3029 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+8, otag );
3030 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+16, otag );
3031 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+24, otag );
3032 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+32, otag );
3033 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+40, otag );
3034 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+48, otag );
3035 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+56, otag );
3036 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+64, otag );
3037 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+72, otag );
3038 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+80, otag );
3039 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+88, otag );
3040 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+96, otag );
3041 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+104, otag );
3042 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+112, otag );
3043 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+120, otag );
3044 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+128, otag );
3045 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+136, otag );
3046 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+144, otag );
3047 make_aligned_word64_undefined_w_otag ( -VG_STACK_REDZONE_SZB + new_SP+152, otag );
3049 MC_(make_mem_undefined_w_otag) ( -VG_STACK_REDZONE_SZB + new_SP, 160, otag );
3054 static void VG_REGPARM(1) mc_new_mem_stack_160(Addr new_SP)
3056 PROF_EVENT(118, "new_mem_stack_160");
3057 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
3058 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP );
3059 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+8 );
3060 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+16 );
3061 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+24 );
3062 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+32 );
3063 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+40 );
3064 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+48 );
3065 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+56 );
3066 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+64 );
3067 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+72 );
3068 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+80 );
3069 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+88 );
3070 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+96 );
3071 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+104 );
3072 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+112 );
3073 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+120 );
3074 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+128 );
3075 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+136 );
3076 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+144 );
3077 make_aligned_word64_undefined ( -VG_STACK_REDZONE_SZB + new_SP+152 );
3079 make_mem_undefined ( -VG_STACK_REDZONE_SZB + new_SP, 160 );
3084 static void VG_REGPARM(1) mc_die_mem_stack_160(Addr new_SP)
3086 PROF_EVENT(128, "die_mem_stack_160");
3087 if (VG_IS_8_ALIGNED( -VG_STACK_REDZONE_SZB + new_SP )) {
3088 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-160);
3089 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-152);
3090 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-144);
3091 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-136);
3092 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-128);
3093 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-120);
3094 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-112);
3095 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-104);
3096 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-96 );
3097 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-88 );
3098 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-80 );
3099 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-72 );
3100 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-64 );
3101 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-56 );
3102 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-48 );
3103 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-40 );
3104 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-32 );
3105 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-24 );
3106 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP-16 );
3107 make_aligned_word64_noaccess ( -VG_STACK_REDZONE_SZB + new_SP- 8 );
3109 MC_(make_mem_noaccess) ( -VG_STACK_REDZONE_SZB + new_SP-160, 160 );
3113 /*--------------- adjustment by N bytes ---------------*/
3115 static void mc_new_mem_stack_w_ECU ( Addr a, SizeT len, UInt ecu )
3117 UInt otag = ecu | MC_OKIND_STACK;
3118 PROF_EVENT(115, "new_mem_stack_w_otag");
3119 MC_(make_mem_undefined_w_otag) ( -VG_STACK_REDZONE_SZB + a, len, otag );
3122 static void mc_new_mem_stack ( Addr a, SizeT len )
3124 PROF_EVENT(115, "new_mem_stack");
3125 make_mem_undefined ( -VG_STACK_REDZONE_SZB + a, len );
3128 static void mc_die_mem_stack ( Addr a, SizeT len )
3130 PROF_EVENT(125, "die_mem_stack");
3131 MC_(make_mem_noaccess) ( -VG_STACK_REDZONE_SZB + a, len );
3135 /* The AMD64 ABI says:
3137 "The 128-byte area beyond the location pointed to by %rsp is considered
3138 to be reserved and shall not be modified by signal or interrupt
3139 handlers. Therefore, functions may use this area for temporary data
3140 that is not needed across function calls. In particular, leaf functions
3141 may use this area for their entire stack frame, rather than adjusting
3142 the stack pointer in the prologue and epilogue. This area is known as
3145 So after any call or return we need to mark this redzone as containing
3148 Consider this: we're in function f. f calls g. g moves rsp down
3149 modestly (say 16 bytes) and writes stuff all over the red zone, making it
3150 defined. g returns. f is buggy and reads from parts of the red zone
3151 that it didn't write on. But because g filled that area in, f is going
3152 to be picking up defined V bits and so any errors from reading bits of
3153 the red zone it didn't write, will be missed. The only solution I could
3154 think of was to make the red zone undefined when g returns to f.
3156 This is in accordance with the ABI, which makes it clear the redzone
3157 is volatile across function calls.
3159 The problem occurs the other way round too: f could fill the RZ up
3160 with defined values and g could mistakenly read them. So the RZ
3161 also needs to be nuked on function calls.
3165 /* Here's a simple cache to hold nia -> ECU mappings. It could be
3166 improved so as to have a lower miss rate. */
3168 static UWord stats__nia_cache_queries = 0;
3169 static UWord stats__nia_cache_misses = 0;
3172 struct { UWord nia0; UWord ecu0; /* nia0 maps to ecu0 */
3173 UWord nia1; UWord ecu1; } /* nia1 maps to ecu1 */
3176 #define N_NIA_TO_ECU_CACHE 511
3178 static WCacheEnt nia_to_ecu_cache[N_NIA_TO_ECU_CACHE];
3180 static void init_nia_to_ecu_cache ( void )
3184 ExeContext* zero_ec;
3186 /* Fill all the slots with an entry for address zero, and the
3187 relevant otags accordingly. Hence the cache is initially filled
3189 zero_ec = VG_(make_depth_1_ExeContext_from_Addr)(zero_addr);
3191 zero_ecu = VG_(get_ECU_from_ExeContext)(zero_ec);
3192 tl_assert(VG_(is_plausible_ECU)(zero_ecu));
3193 for (i = 0; i < N_NIA_TO_ECU_CACHE; i++) {
3194 nia_to_ecu_cache[i].nia0 = zero_addr;
3195 nia_to_ecu_cache[i].ecu0 = zero_ecu;
3196 nia_to_ecu_cache[i].nia1 = zero_addr;
3197 nia_to_ecu_cache[i].ecu1 = zero_ecu;
3201 static inline UInt convert_nia_to_ecu ( Addr nia )
3207 tl_assert( sizeof(nia_to_ecu_cache[0].nia1) == sizeof(nia) );
3209 stats__nia_cache_queries++;
3210 i = nia % N_NIA_TO_ECU_CACHE;
3211 tl_assert(i >= 0 && i < N_NIA_TO_ECU_CACHE);
3213 if (LIKELY( nia_to_ecu_cache[i].nia0 == nia ))
3214 return nia_to_ecu_cache[i].ecu0;
3216 if (LIKELY( nia_to_ecu_cache[i].nia1 == nia )) {
3217 # define SWAP(_w1,_w2) { UWord _t = _w1; _w1 = _w2; _w2 = _t; }
3218 SWAP( nia_to_ecu_cache[i].nia0, nia_to_ecu_cache[i].nia1 );
3219 SWAP( nia_to_ecu_cache[i].ecu0, nia_to_ecu_cache[i].ecu1 );
3221 return nia_to_ecu_cache[i].ecu0;
3224 stats__nia_cache_misses++;
3225 ec = VG_(make_depth_1_ExeContext_from_Addr)(nia);
3227 ecu = VG_(get_ECU_from_ExeContext)(ec);
3228 tl_assert(VG_(is_plausible_ECU)(ecu));
3230 nia_to_ecu_cache[i].nia1 = nia_to_ecu_cache[i].nia0;
3231 nia_to_ecu_cache[i].ecu1 = nia_to_ecu_cache[i].ecu0;
3233 nia_to_ecu_cache[i].nia0 = nia;
3234 nia_to_ecu_cache[i].ecu0 = (UWord)ecu;
3239 /* Note that this serves both the origin-tracking and
3240 no-origin-tracking modes. We assume that calls to it are
3241 sufficiently infrequent that it isn't worth specialising for the
3242 with/without origin-tracking cases. */
3243 void MC_(helperc_MAKE_STACK_UNINIT) ( Addr base, UWord len, Addr nia )
3246 tl_assert(sizeof(UWord) == sizeof(SizeT));
3248 VG_(printf)("helperc_MAKE_STACK_UNINIT (%#lx,%lu,nia=%#lx)\n",
3251 if (UNLIKELY( MC_(clo_mc_level) == 3 )) {
3252 UInt ecu = convert_nia_to_ecu ( nia );
3253 tl_assert(VG_(is_plausible_ECU)(ecu));
3254 otag = ecu | MC_OKIND_STACK;
3256 tl_assert(nia == 0);
3261 /* Really slow version */
3262 MC_(make_mem_undefined)(base, len, otag);
3266 /* Slow(ish) version, which is fairly easily seen to be correct.
3268 if (LIKELY( VG_IS_8_ALIGNED(base) && len==128 )) {
3269 make_aligned_word64_undefined(base + 0, otag);
3270 make_aligned_word64_undefined(base + 8, otag);
3271 make_aligned_word64_undefined(base + 16, otag);
3272 make_aligned_word64_undefined(base + 24, otag);
3274 make_aligned_word64_undefined(base + 32, otag);
3275 make_aligned_word64_undefined(base + 40, otag);
3276 make_aligned_word64_undefined(base + 48, otag);
3277 make_aligned_word64_undefined(base + 56, otag);
3279 make_aligned_word64_undefined(base + 64, otag);
3280 make_aligned_word64_undefined(base + 72, otag);
3281 make_aligned_word64_undefined(base + 80, otag);
3282 make_aligned_word64_undefined(base + 88, otag);
3284 make_aligned_word64_undefined(base + 96, otag);
3285 make_aligned_word64_undefined(base + 104, otag);
3286 make_aligned_word64_undefined(base + 112, otag);
3287 make_aligned_word64_undefined(base + 120, otag);
3289 MC_(make_mem_undefined)(base, len, otag);
3293 /* Idea is: go fast when
3294 * 8-aligned and length is 128
3295 * the sm is available in the main primary map
3296 * the address range falls entirely with a single secondary map
3297 If all those conditions hold, just update the V+A bits by writing
3298 directly into the vabits array. (If the sm was distinguished, this
3299 will make a copy and then write to it.)
3302 if (LIKELY( len == 128 && VG_IS_8_ALIGNED(base) )) {
3303 /* Now we know the address range is suitably sized and aligned. */
3304 UWord a_lo = (UWord)(base);
3305 UWord a_hi = (UWord)(base + 128 - 1);
3306 tl_assert(a_lo < a_hi); // paranoia: detect overflow
3307 if (a_hi <= MAX_PRIMARY_ADDRESS) {
3308 // Now we know the entire range is within the main primary map.
3309 SecMap* sm = get_secmap_for_writing_low(a_lo);
3310 SecMap* sm_hi = get_secmap_for_writing_low(a_hi);
3311 /* Now we know that the entire address range falls within a
3312 single secondary map, and that that secondary 'lives' in
3313 the main primary map. */
3314 if (LIKELY(sm == sm_hi)) {
3315 // Finally, we know that the range is entirely within one secmap.
3316 UWord v_off = SM_OFF(a_lo);
3317 UShort* p = (UShort*)(&sm->vabits8[v_off]);
3318 p[ 0] = VA_BITS16_UNDEFINED;
3319 p[ 1] = VA_BITS16_UNDEFINED;
3320 p[ 2] = VA_BITS16_UNDEFINED;
3321 p[ 3] = VA_BITS16_UNDEFINED;
3322 p[ 4] = VA_BITS16_UNDEFINED;
3323 p[ 5] = VA_BITS16_UNDEFINED;
3324 p[ 6] = VA_BITS16_UNDEFINED;
3325 p[ 7] = VA_BITS16_UNDEFINED;
3326 p[ 8] = VA_BITS16_UNDEFINED;
3327 p[ 9] = VA_BITS16_UNDEFINED;
3328 p[10] = VA_BITS16_UNDEFINED;
3329 p[11] = VA_BITS16_UNDEFINED;
3330 p[12] = VA_BITS16_UNDEFINED;
3331 p[13] = VA_BITS16_UNDEFINED;
3332 p[14] = VA_BITS16_UNDEFINED;
3333 p[15] = VA_BITS16_UNDEFINED;
3334 if (UNLIKELY( MC_(clo_mc_level) == 3 )) {
3335 set_aligned_word64_Origin_to_undef( base + 8 * 0, otag );
3336 set_aligned_word64_Origin_to_undef( base + 8 * 1, otag );
3337 set_aligned_word64_Origin_to_undef( base + 8 * 2, otag );
3338 set_aligned_word64_Origin_to_undef( base + 8 * 3, otag );
3339 set_aligned_word64_Origin_to_undef( base + 8 * 4, otag );
3340 set_aligned_word64_Origin_to_undef( base + 8 * 5, otag );
3341 set_aligned_word64_Origin_to_undef( base + 8 * 6, otag );
3342 set_aligned_word64_Origin_to_undef( base + 8 * 7, otag );
3343 set_aligned_word64_Origin_to_undef( base + 8 * 8, otag );
3344 set_aligned_word64_Origin_to_undef( base + 8 * 9, otag );
3345 set_aligned_word64_Origin_to_undef( base + 8 * 10, otag );
3346 set_aligned_word64_Origin_to_undef( base + 8 * 11, otag );
3347 set_aligned_word64_Origin_to_undef( base + 8 * 12, otag );
3348 set_aligned_word64_Origin_to_undef( base + 8 * 13, otag );
3349 set_aligned_word64_Origin_to_undef( base + 8 * 14, otag );
3350 set_aligned_word64_Origin_to_undef( base + 8 * 15, otag );
3357 /* 288 bytes (36 ULongs) is the magic value for ELF ppc64. */
3358 if (LIKELY( len == 288 && VG_IS_8_ALIGNED(base) )) {
3359 /* Now we know the address range is suitably sized and aligned. */
3360 UWord a_lo = (UWord)(base);
3361 UWord a_hi = (UWord)(base + 288 - 1);
3362 tl_assert(a_lo < a_hi); // paranoia: detect overflow
3363 if (a_hi <= MAX_PRIMARY_ADDRESS) {
3364 // Now we know the entire range is within the main primary map.
3365 SecMap* sm = get_secmap_for_writing_low(a_lo);
3366 SecMap* sm_hi = get_secmap_for_writing_low(a_hi);
3367 /* Now we know that the entire address range falls within a
3368 single secondary map, and that that secondary 'lives' in
3369 the main primary map. */
3370 if (LIKELY(sm == sm_hi)) {
3371 // Finally, we know that the range is entirely within one secmap.
3372 UWord v_off = SM_OFF(a_lo);
3373 UShort* p = (UShort*)(&sm->vabits8[v_off]);
3374 p[ 0] = VA_BITS16_UNDEFINED;
3375 p[ 1] = VA_BITS16_UNDEFINED;
3376 p[ 2] = VA_BITS16_UNDEFINED;
3377 p[ 3] = VA_BITS16_UNDEFINED;
3378 p[ 4] = VA_BITS16_UNDEFINED;
3379 p[ 5] = VA_BITS16_UNDEFINED;
3380 p[ 6] = VA_BITS16_UNDEFINED;
3381 p[ 7] = VA_BITS16_UNDEFINED;
3382 p[ 8] = VA_BITS16_UNDEFINED;
3383 p[ 9] = VA_BITS16_UNDEFINED;
3384 p[10] = VA_BITS16_UNDEFINED;
3385 p[11] = VA_BITS16_UNDEFINED;
3386 p[12] = VA_BITS16_UNDEFINED;
3387 p[13] = VA_BITS16_UNDEFINED;
3388 p[14] = VA_BITS16_UNDEFINED;
3389 p[15] = VA_BITS16_UNDEFINED;
3390 p[16] = VA_BITS16_UNDEFINED;
3391 p[17] = VA_BITS16_UNDEFINED;
3392 p[18] = VA_BITS16_UNDEFINED;
3393 p[19] = VA_BITS16_UNDEFINED;
3394 p[20] = VA_BITS16_UNDEFINED;
3395 p[21] = VA_BITS16_UNDEFINED;
3396 p[22] = VA_BITS16_UNDEFINED;
3397 p[23] = VA_BITS16_UNDEFINED;
3398 p[24] = VA_BITS16_UNDEFINED;
3399 p[25] = VA_BITS16_UNDEFINED;
3400 p[26] = VA_BITS16_UNDEFINED;
3401 p[27] = VA_BITS16_UNDEFINED;
3402 p[28] = VA_BITS16_UNDEFINED;
3403 p[29] = VA_BITS16_UNDEFINED;
3404 p[30] = VA_BITS16_UNDEFINED;
3405 p[31] = VA_BITS16_UNDEFINED;
3406 p[32] = VA_BITS16_UNDEFINED;
3407 p[33] = VA_BITS16_UNDEFINED;
3408 p[34] = VA_BITS16_UNDEFINED;
3409 p[35] = VA_BITS16_UNDEFINED;
3410 if (UNLIKELY( MC_(clo_mc_level) == 3 )) {
3411 set_aligned_word64_Origin_to_undef( base + 8 * 0, otag );
3412 set_aligned_word64_Origin_to_undef( base + 8 * 1, otag );
3413 set_aligned_word64_Origin_to_undef( base + 8 * 2, otag );
3414 set_aligned_word64_Origin_to_undef( base + 8 * 3, otag );
3415 set_aligned_word64_Origin_to_undef( base + 8 * 4, otag );
3416 set_aligned_word64_Origin_to_undef( base + 8 * 5, otag );
3417 set_aligned_word64_Origin_to_undef( base + 8 * 6, otag );
3418 set_aligned_word64_Origin_to_undef( base + 8 * 7, otag );
3419 set_aligned_word64_Origin_to_undef( base + 8 * 8, otag );
3420 set_aligned_word64_Origin_to_undef( base + 8 * 9, otag );
3421 set_aligned_word64_Origin_to_undef( base + 8 * 10, otag );
3422 set_aligned_word64_Origin_to_undef( base + 8 * 11, otag );
3423 set_aligned_word64_Origin_to_undef( base + 8 * 12, otag );
3424 set_aligned_word64_Origin_to_undef( base + 8 * 13, otag );
3425 set_aligned_word64_Origin_to_undef( base + 8 * 14, otag );
3426 set_aligned_word64_Origin_to_undef( base + 8 * 15, otag );
3427 set_aligned_word64_Origin_to_undef( base + 8 * 16, otag );
3428 set_aligned_word64_Origin_to_undef( base + 8 * 17, otag );
3429 set_aligned_word64_Origin_to_undef( base + 8 * 18, otag );
3430 set_aligned_word64_Origin_to_undef( base + 8 * 19, otag );
3431 set_aligned_word64_Origin_to_undef( base + 8 * 20, otag );
3432 set_aligned_word64_Origin_to_undef( base + 8 * 21, otag );
3433 set_aligned_word64_Origin_to_undef( base + 8 * 22, otag );
3434 set_aligned_word64_Origin_to_undef( base + 8 * 23, otag );
3435 set_aligned_word64_Origin_to_undef( base + 8 * 24, otag );
3436 set_aligned_word64_Origin_to_undef( base + 8 * 25, otag );
3437 set_aligned_word64_Origin_to_undef( base + 8 * 26, otag );
3438 set_aligned_word64_Origin_to_undef( base + 8 * 27, otag );
3439 set_aligned_word64_Origin_to_undef( base + 8 * 28, otag );
3440 set_aligned_word64_Origin_to_undef( base + 8 * 29, otag );
3441 set_aligned_word64_Origin_to_undef( base + 8 * 30, otag );
3442 set_aligned_word64_Origin_to_undef( base + 8 * 31, otag );
3443 set_aligned_word64_Origin_to_undef( base + 8 * 32, otag );
3444 set_aligned_word64_Origin_to_undef( base + 8 * 33, otag );
3445 set_aligned_word64_Origin_to_undef( base + 8 * 34, otag );
3446 set_aligned_word64_Origin_to_undef( base + 8 * 35, otag );
3453 /* else fall into slow case */
3454 MC_(make_mem_undefined_w_otag)(base, len, otag);
3458 /*------------------------------------------------------------*/
3459 /*--- Checking memory ---*/
3460 /*------------------------------------------------------------*/
3471 /* Check permissions for address range. If inadequate permissions
3472 exist, *bad_addr is set to the offending address, so the caller can
3475 /* Returns True if [a .. a+len) is not addressible. Otherwise,
3476 returns False, and if bad_addr is non-NULL, sets *bad_addr to
3477 indicate the lowest failing address. Functions below are
3479 Bool MC_(check_mem_is_noaccess) ( Addr a, SizeT len, Addr* bad_addr )
3484 PROF_EVENT(60, "check_mem_is_noaccess");
3485 for (i = 0; i < len; i++) {
3486 PROF_EVENT(61, "check_mem_is_noaccess(loop)");
3487 vabits2 = get_vabits2(a);
3488 if (VA_BITS2_NOACCESS != vabits2) {
3489 if (bad_addr != NULL) *bad_addr = a;
3497 static Bool is_mem_addressable ( Addr a, SizeT len,
3498 /*OUT*/Addr* bad_addr )
3503 PROF_EVENT(62, "is_mem_addressable");
3504 for (i = 0; i < len; i++) {
3505 PROF_EVENT(63, "is_mem_addressable(loop)");
3506 vabits2 = get_vabits2(a);
3507 if (VA_BITS2_NOACCESS == vabits2) {
3508 if (bad_addr != NULL) *bad_addr = a;
3516 static MC_ReadResult is_mem_defined ( Addr a, SizeT len,
3517 /*OUT*/Addr* bad_addr,
3523 PROF_EVENT(64, "is_mem_defined");
3524 DEBUG("is_mem_defined\n");
3526 if (otag) *otag = 0;
3527 if (bad_addr) *bad_addr = 0;
3528 for (i = 0; i < len; i++) {
3529 PROF_EVENT(65, "is_mem_defined(loop)");
3530 vabits2 = get_vabits2(a);
3531 if (VA_BITS2_DEFINED != vabits2) {
3532 // Error! Nb: Report addressability errors in preference to
3533 // definedness errors. And don't report definedeness errors unless
3534 // --undef-value-errors=yes.
3538 if (VA_BITS2_NOACCESS == vabits2) {
3541 if (MC_(clo_mc_level) >= 2) {
3542 if (otag && MC_(clo_mc_level) == 3) {
3543 *otag = MC_(helperc_b_load1)( a );
3554 /* Check a zero-terminated ascii string. Tricky -- don't want to
3555 examine the actual bytes, to find the end, until we're sure it is
3558 static Bool mc_is_defined_asciiz ( Addr a, Addr* bad_addr, UInt* otag )
3562 PROF_EVENT(66, "mc_is_defined_asciiz");
3563 DEBUG("mc_is_defined_asciiz\n");
3565 if (otag) *otag = 0;
3566 if (bad_addr) *bad_addr = 0;
3568 PROF_EVENT(67, "mc_is_defined_asciiz(loop)");
3569 vabits2 = get_vabits2(a);
3570 if (VA_BITS2_DEFINED != vabits2) {
3571 // Error! Nb: Report addressability errors in preference to
3572 // definedness errors. And don't report definedeness errors unless
3573 // --undef-value-errors=yes.
3577 if (VA_BITS2_NOACCESS == vabits2) {
3580 if (MC_(clo_mc_level) >= 2) {
3581 if (otag && MC_(clo_mc_level) == 3) {
3582 *otag = MC_(helperc_b_load1)( a );
3587 /* Ok, a is safe to read. */
3588 if (* ((UChar*)a) == 0) {
3596 /*------------------------------------------------------------*/
3597 /*--- Memory event handlers ---*/
3598 /*------------------------------------------------------------*/
3601 void check_mem_is_addressable ( CorePart part, ThreadId tid, Char* s,
3602 Addr base, SizeT size )
3605 Bool ok = is_mem_addressable ( base, size, &bad_addr );
3609 case Vg_CoreSysCall:
3610 MC_(record_memparam_error) ( tid, bad_addr,
3611 /*isAddrErr*/True, s, 0/*otag*/ );
3615 MC_(record_core_mem_error)( tid, s );
3619 VG_(tool_panic)("check_mem_is_addressable: unexpected CorePart");
3625 void check_mem_is_defined ( CorePart part, ThreadId tid, Char* s,
3626 Addr base, SizeT size )
3630 MC_ReadResult res = is_mem_defined ( base, size, &bad_addr, &otag );
3633 Bool isAddrErr = ( MC_AddrErr == res ? True : False );
3636 case Vg_CoreSysCall:
3637 MC_(record_memparam_error) ( tid, bad_addr, isAddrErr, s,
3638 isAddrErr ? 0 : otag );
3641 case Vg_CoreSysCallArgInMem:
3642 MC_(record_regparam_error) ( tid, s, otag );
3645 /* If we're being asked to jump to a silly address, record an error
3646 message before potentially crashing the entire system. */
3647 case Vg_CoreTranslate:
3648 MC_(record_jump_error)( tid, bad_addr );
3652 VG_(tool_panic)("check_mem_is_defined: unexpected CorePart");
3658 void check_mem_is_defined_asciiz ( CorePart part, ThreadId tid,
3662 Addr bad_addr = 0; // shut GCC up
3665 tl_assert(part == Vg_CoreSysCall);
3666 res = mc_is_defined_asciiz ( (Addr)str, &bad_addr, &otag );
3668 Bool isAddrErr = ( MC_AddrErr == res ? True : False );
3669 MC_(record_memparam_error) ( tid, bad_addr, isAddrErr, s,
3670 isAddrErr ? 0 : otag );
3674 /* Handling of mmap and mprotect is not as simple as it seems.
3676 The underlying semantics are that memory obtained from mmap is
3677 always initialised, but may be inaccessible. And changes to the
3678 protection of memory do not change its contents and hence not its
3679 definedness state. Problem is we can't model
3680 inaccessible-but-with-some-definedness state; once we mark memory
3681 as inaccessible we lose all info about definedness, and so can't
3682 restore that if it is later made accessible again.
3684 One obvious thing to do is this:
3686 mmap/mprotect NONE -> noaccess
3687 mmap/mprotect other -> defined
3689 The problem case here is: taking accessible memory, writing
3690 uninitialised data to it, mprotecting it NONE and later mprotecting
3691 it back to some accessible state causes the undefinedness to be
3694 A better proposal is:
3696 (1) mmap NONE -> make noaccess
3697 (2) mmap other -> make defined
3699 (3) mprotect NONE -> # no change
3700 (4) mprotect other -> change any "noaccess" to "defined"
3702 (2) is OK because memory newly obtained from mmap really is defined
3703 (zeroed out by the kernel -- doing anything else would
3704 constitute a massive security hole.)
3706 (1) is OK because the only way to make the memory usable is via
3707 (4), in which case we also wind up correctly marking it all as
3710 (3) is the weak case. We choose not to change memory state.
3711 (presumably the range is in some mixture of "defined" and
3712 "undefined", viz, accessible but with arbitrary V bits). Doing
3713 nothing means we retain the V bits, so that if the memory is
3714 later mprotected "other", the V bits remain unchanged, so there
3715 can be no false negatives. The bad effect is that if there's
3716 an access in the area, then MC cannot warn; but at least we'll
3717 get a SEGV to show, so it's better than nothing.
3719 Consider the sequence (3) followed by (4). Any memory that was
3720 "defined" or "undefined" previously retains its state (as
3721 required). Any memory that was "noaccess" before can only have
3722 been made that way by (1), and so it's OK to change it to
3725 See https://bugs.kde.org/show_bug.cgi?id=205541
3726 and https://bugs.kde.org/show_bug.cgi?id=210268
3729 void mc_new_mem_mmap ( Addr a, SizeT len, Bool rr, Bool ww, Bool xx,
3732 if (rr || ww || xx) {
3733 /* (2) mmap/mprotect other -> defined */
3734 MC_(make_mem_defined)(a, len);
3736 /* (1) mmap/mprotect NONE -> noaccess */
3737 MC_(make_mem_noaccess)(a, len);
3742 void mc_new_mem_mprotect ( Addr a, SizeT len, Bool rr, Bool ww, Bool xx )
3744 if (rr || ww || xx) {
3745 /* (4) mprotect other -> change any "noaccess" to "defined" */
3746 make_mem_defined_if_noaccess(a, len);
3748 /* (3) mprotect NONE -> # no change */
3755 void mc_new_mem_startup( Addr a, SizeT len,
3756 Bool rr, Bool ww, Bool xx, ULong di_handle )
3758 // Because code is defined, initialised variables get put in the data
3759 // segment and are defined, and uninitialised variables get put in the
3760 // bss segment and are auto-zeroed (and so defined).
3762 // It's possible that there will be padding between global variables.
3763 // This will also be auto-zeroed, and marked as defined by Memcheck. If
3764 // a program uses it, Memcheck will not complain. This is arguably a
3765 // false negative, but it's a grey area -- the behaviour is defined (the
3766 // padding is zeroed) but it's probably not what the user intended. And
3767 // we can't avoid it.
3769 // Note: we generally ignore RWX permissions, because we can't track them
3770 // without requiring more than one A bit which would slow things down a
3771 // lot. But on Darwin the 0th page is mapped but !R and !W and !X.
3772 // So we mark any such pages as "unaddressable".
3773 DEBUG("mc_new_mem_startup(%#lx, %llu, rr=%u, ww=%u, xx=%u)\n",
3774 a, (ULong)len, rr, ww, xx);
3775 mc_new_mem_mmap(a, len, rr, ww, xx, di_handle);
3779 void mc_post_mem_write(CorePart part, ThreadId tid, Addr a, SizeT len)
3781 MC_(make_mem_defined)(a, len);
3785 /*------------------------------------------------------------*/
3786 /*--- Register event handlers ---*/
3787 /*------------------------------------------------------------*/
3789 /* Try and get a nonzero origin for the guest state section of thread
3790 tid characterised by (offset,size). Return 0 if nothing to show
3792 static UInt mb_get_origin_for_guest_offset ( ThreadId tid,
3793 Int offset, SizeT size )
3798 sh2off = MC_(get_otrack_shadow_offset)( offset, size );
3800 return 0; /* This piece of guest state is not tracked */
3801 tl_assert(sh2off >= 0);
3802 tl_assert(0 == (sh2off % 4));
3805 VG_(get_shadow_regs_area)( tid, &area[1], 2/*shadowno*/,sh2off,4 );
3806 tl_assert(area[0] == 0x31);
3807 tl_assert(area[5] == 0x27);
3808 otag = *(UInt*)&area[1];
3813 /* When some chunk of guest state is written, mark the corresponding
3814 shadow area as valid. This is used to initialise arbitrarily large
3815 chunks of guest state, hence the _SIZE value, which has to be as
3816 big as the biggest guest state.
3818 static void mc_post_reg_write ( CorePart part, ThreadId tid,
3819 PtrdiffT offset, SizeT size)
3821 # define MAX_REG_WRITE_SIZE 1664
3822 UChar area[MAX_REG_WRITE_SIZE];
3823 tl_assert(size <= MAX_REG_WRITE_SIZE);
3824 VG_(memset)(area, V_BITS8_DEFINED, size);
3825 VG_(set_shadow_regs_area)( tid, 1/*shadowNo*/,offset,size, area );
3826 # undef MAX_REG_WRITE_SIZE
3830 void mc_post_reg_write_clientcall ( ThreadId tid,
3831 PtrdiffT offset, SizeT size, Addr f)
3833 mc_post_reg_write(/*dummy*/0, tid, offset, size);
3836 /* Look at the definedness of the guest's shadow state for
3837 [offset, offset+len). If any part of that is undefined, record
3840 static void mc_pre_reg_read ( CorePart part, ThreadId tid, Char* s,
3841 PtrdiffT offset, SizeT size)
3848 tl_assert(size <= 16);
3850 VG_(get_shadow_regs_area)( tid, area, 1/*shadowNo*/,offset,size );
3853 for (i = 0; i < size; i++) {
3854 if (area[i] != V_BITS8_DEFINED) {
3863 /* We've found some undefinedness. See if we can also find an
3865 otag = mb_get_origin_for_guest_offset( tid, offset, size );
3866 MC_(record_regparam_error) ( tid, s, otag );
3870 /*------------------------------------------------------------*/
3871 /*--- Functions called directly from generated code: ---*/
3872 /*--- Load/store handlers. ---*/
3873 /*------------------------------------------------------------*/
3875 /* Types: LOADV32, LOADV16, LOADV8 are:
3877 so they return 32-bits on 32-bit machines and 64-bits on
3878 64-bit machines. Addr has the same size as a host word.
3880 LOADV64 is always ULong fn ( Addr a )
3882 Similarly for STOREV8, STOREV16, STOREV32, the supplied vbits
3883 are a UWord, and for STOREV64 they are a ULong.
3886 /* If any part of '_a' indicated by the mask is 1, either '_a' is not
3887 naturally '_sz/8'-aligned, or it exceeds the range covered by the
3888 primary map. This is all very tricky (and important!), so let's
3889 work through the maths by hand (below), *and* assert for these
3890 values at startup. */
3891 #define MASK(_szInBytes) \
3892 ( ~((0x10000UL-(_szInBytes)) | ((N_PRIMARY_MAP-1) << 16)) )
3894 /* MASK only exists so as to define this macro. */
3895 #define UNALIGNED_OR_HIGH(_a,_szInBits) \
3896 ((_a) & MASK((_szInBits>>3)))
3898 /* On a 32-bit machine:
3900 N_PRIMARY_BITS == 16, so
3901 N_PRIMARY_MAP == 0x10000, so
3902 N_PRIMARY_MAP-1 == 0xFFFF, so
3903 (N_PRIMARY_MAP-1) << 16 == 0xFFFF0000, and so
3905 MASK(1) = ~ ( (0x10000 - 1) | 0xFFFF0000 )
3906 = ~ ( 0xFFFF | 0xFFFF0000 )
3910 MASK(2) = ~ ( (0x10000 - 2) | 0xFFFF0000 )
3911 = ~ ( 0xFFFE | 0xFFFF0000 )
3915 MASK(4) = ~ ( (0x10000 - 4) | 0xFFFF0000 )
3916 = ~ ( 0xFFFC | 0xFFFF0000 )
3920 MASK(8) = ~ ( (0x10000 - 8) | 0xFFFF0000 )
3921 = ~ ( 0xFFF8 | 0xFFFF0000 )
3925 Hence in the 32-bit case, "a & MASK(1/2/4/8)" is a nonzero value
3926 precisely when a is not 1/2/4/8-bytes aligned. And obviously, for
3927 the 1-byte alignment case, it is always a zero value, since MASK(1)
3928 is zero. All as expected.
3930 On a 64-bit machine, it's more complex, since we're testing
3931 simultaneously for misalignment and for the address being at or
3934 N_PRIMARY_BITS == 19, so
3935 N_PRIMARY_MAP == 0x80000, so
3936 N_PRIMARY_MAP-1 == 0x7FFFF, so
3937 (N_PRIMARY_MAP-1) << 16 == 0x7FFFF'0000, and so
3939 MASK(1) = ~ ( (0x10000 - 1) | 0x7FFFF'0000 )
3940 = ~ ( 0xFFFF | 0x7FFFF'0000 )
3942 = 0xFFFF'FFF8'0000'0000
3944 MASK(2) = ~ ( (0x10000 - 2) | 0x7FFFF'0000 )
3945 = ~ ( 0xFFFE | 0x7FFFF'0000 )
3947 = 0xFFFF'FFF8'0000'0001
3949 MASK(4) = ~ ( (0x10000 - 4) | 0x7FFFF'0000 )
3950 = ~ ( 0xFFFC | 0x7FFFF'0000 )
3952 = 0xFFFF'FFF8'0000'0003
3954 MASK(8) = ~ ( (0x10000 - 8) | 0x7FFFF'0000 )
3955 = ~ ( 0xFFF8 | 0x7FFFF'0000 )
3957 = 0xFFFF'FFF8'0000'0007
3961 /* ------------------------ Size = 8 ------------------------ */
3964 ULong mc_LOADV64 ( Addr a, Bool isBigEndian )
3966 PROF_EVENT(200, "mc_LOADV64");
3968 #ifndef PERF_FAST_LOADV
3969 return mc_LOADVn_slow( a, 64, isBigEndian );
3972 UWord sm_off16, vabits16;
3975 if (UNLIKELY( UNALIGNED_OR_HIGH(a,64) )) {
3976 PROF_EVENT(201, "mc_LOADV64-slow1");
3977 return (ULong)mc_LOADVn_slow( a, 64, isBigEndian );
3980 sm = get_secmap_for_reading_low(a);
3981 sm_off16 = SM_OFF_16(a);
3982 vabits16 = ((UShort*)(sm->vabits8))[sm_off16];
3984 // Handle common case quickly: a is suitably aligned, is mapped, and
3986 // Convert V bits from compact memory form to expanded register form.
3987 if (LIKELY(vabits16 == VA_BITS16_DEFINED)) {
3988 return V_BITS64_DEFINED;
3989 } else if (LIKELY(vabits16 == VA_BITS16_UNDEFINED)) {
3990 return V_BITS64_UNDEFINED;
3992 /* Slow case: the 8 bytes are not all-defined or all-undefined. */
3993 PROF_EVENT(202, "mc_LOADV64-slow2");
3994 return mc_LOADVn_slow( a, 64, isBigEndian );
4000 VG_REGPARM(1) ULong MC_(helperc_LOADV64be) ( Addr a )
4002 return mc_LOADV64(a, True);
4004 VG_REGPARM(1) ULong MC_(helperc_LOADV64le) ( Addr a )
4006 return mc_LOADV64(a, False);
4011 void mc_STOREV64 ( Addr a, ULong vbits64, Bool isBigEndian )
4013 PROF_EVENT(210, "mc_STOREV64");
4015 #ifndef PERF_FAST_STOREV
4016 // XXX: this slow case seems to be marginally faster than the fast case!
4017 // Investigate further.
4018 mc_STOREVn_slow( a, 64, vbits64, isBigEndian );
4021 UWord sm_off16, vabits16;
4024 if (UNLIKELY( UNALIGNED_OR_HIGH(a,64) )) {
4025 PROF_EVENT(211, "mc_STOREV64-slow1");
4026 mc_STOREVn_slow( a, 64, vbits64, isBigEndian );
4030 sm = get_secmap_for_reading_low(a);
4031 sm_off16 = SM_OFF_16(a);
4032 vabits16 = ((UShort*)(sm->vabits8))[sm_off16];
4034 if (LIKELY( !is_distinguished_sm(sm) &&
4035 (VA_BITS16_DEFINED == vabits16 ||
4036 VA_BITS16_UNDEFINED == vabits16) ))
4038 /* Handle common case quickly: a is suitably aligned, */
4039 /* is mapped, and is addressible. */
4040 // Convert full V-bits in register to compact 2-bit form.
4041 if (V_BITS64_DEFINED == vbits64) {
4042 ((UShort*)(sm->vabits8))[sm_off16] = (UShort)VA_BITS16_DEFINED;
4043 } else if (V_BITS64_UNDEFINED == vbits64) {
4044 ((UShort*)(sm->vabits8))[sm_off16] = (UShort)VA_BITS16_UNDEFINED;
4046 /* Slow but general case -- writing partially defined bytes. */
4047 PROF_EVENT(212, "mc_STOREV64-slow2");
4048 mc_STOREVn_slow( a, 64, vbits64, isBigEndian );
4051 /* Slow but general case. */
4052 PROF_EVENT(213, "mc_STOREV64-slow3");
4053 mc_STOREVn_slow( a, 64, vbits64, isBigEndian );
4059 VG_REGPARM(1) void MC_(helperc_STOREV64be) ( Addr a, ULong vbits64 )
4061 mc_STOREV64(a, vbits64, True);
4063 VG_REGPARM(1) void MC_(helperc_STOREV64le) ( Addr a, ULong vbits64 )
4065 mc_STOREV64(a, vbits64, False);
4069 /* ------------------------ Size = 4 ------------------------ */
4072 UWord mc_LOADV32 ( Addr a, Bool isBigEndian )
4074 PROF_EVENT(220, "mc_LOADV32");
4076 #ifndef PERF_FAST_LOADV
4077 return (UWord)mc_LOADVn_slow( a, 32, isBigEndian );
4080 UWord sm_off, vabits8;
4083 if (UNLIKELY( UNALIGNED_OR_HIGH(a,32) )) {
4084 PROF_EVENT(221, "mc_LOADV32-slow1");
4085 return (UWord)mc_LOADVn_slow( a, 32, isBigEndian );
4088 sm = get_secmap_for_reading_low(a);
4090 vabits8 = sm->vabits8[sm_off];
4092 // Handle common case quickly: a is suitably aligned, is mapped, and the
4093 // entire word32 it lives in is addressible.
4094 // Convert V bits from compact memory form to expanded register form.
4095 // For 64-bit platforms, set the high 32 bits of retval to 1 (undefined).
4096 // Almost certainly not necessary, but be paranoid.
4097 if (LIKELY(vabits8 == VA_BITS8_DEFINED)) {
4098 return ((UWord)0xFFFFFFFF00000000ULL | (UWord)V_BITS32_DEFINED);
4099 } else if (LIKELY(vabits8 == VA_BITS8_UNDEFINED)) {
4100 return ((UWord)0xFFFFFFFF00000000ULL | (UWord)V_BITS32_UNDEFINED);
4102 /* Slow case: the 4 bytes are not all-defined or all-undefined. */
4103 PROF_EVENT(222, "mc_LOADV32-slow2");
4104 return (UWord)mc_LOADVn_slow( a, 32, isBigEndian );
4110 VG_REGPARM(1) UWord MC_(helperc_LOADV32be) ( Addr a )
4112 return mc_LOADV32(a, True);
4114 VG_REGPARM(1) UWord MC_(helperc_LOADV32le) ( Addr a )
4116 return mc_LOADV32(a, False);
4121 void mc_STOREV32 ( Addr a, UWord vbits32, Bool isBigEndian )
4123 PROF_EVENT(230, "mc_STOREV32");
4125 #ifndef PERF_FAST_STOREV
4126 mc_STOREVn_slow( a, 32, (ULong)vbits32, isBigEndian );
4129 UWord sm_off, vabits8;
4132 if (UNLIKELY( UNALIGNED_OR_HIGH(a,32) )) {
4133 PROF_EVENT(231, "mc_STOREV32-slow1");
4134 mc_STOREVn_slow( a, 32, (ULong)vbits32, isBigEndian );
4138 sm = get_secmap_for_reading_low(a);
4140 vabits8 = sm->vabits8[sm_off];
4142 // Cleverness: sometimes we don't have to write the shadow memory at
4143 // all, if we can tell that what we want to write is the same as what is
4144 // already there. The 64/16/8 bit cases also have cleverness at this
4145 // point, but it works a little differently to the code below.
4146 if (V_BITS32_DEFINED == vbits32) {
4147 if (vabits8 == (UInt)VA_BITS8_DEFINED) {
4149 } else if (!is_distinguished_sm(sm) && VA_BITS8_UNDEFINED == vabits8) {
4150 sm->vabits8[sm_off] = (UInt)VA_BITS8_DEFINED;
4152 // not defined/undefined, or distinguished and changing state
4153 PROF_EVENT(232, "mc_STOREV32-slow2");
4154 mc_STOREVn_slow( a, 32, (ULong)vbits32, isBigEndian );
4156 } else if (V_BITS32_UNDEFINED == vbits32) {
4157 if (vabits8 == (UInt)VA_BITS8_UNDEFINED) {
4159 } else if (!is_distinguished_sm(sm) && VA_BITS8_DEFINED == vabits8) {
4160 sm->vabits8[sm_off] = (UInt)VA_BITS8_UNDEFINED;
4162 // not defined/undefined, or distinguished and changing state
4163 PROF_EVENT(233, "mc_STOREV32-slow3");
4164 mc_STOREVn_slow( a, 32, (ULong)vbits32, isBigEndian );
4167 // Partially defined word
4168 PROF_EVENT(234, "mc_STOREV32-slow4");
4169 mc_STOREVn_slow( a, 32, (ULong)vbits32, isBigEndian );
4175 VG_REGPARM(2) void MC_(helperc_STOREV32be) ( Addr a, UWord vbits32 )
4177 mc_STOREV32(a, vbits32, True);
4179 VG_REGPARM(2) void MC_(helperc_STOREV32le) ( Addr a, UWord vbits32 )
4181 mc_STOREV32(a, vbits32, False);
4185 /* ------------------------ Size = 2 ------------------------ */
4188 UWord mc_LOADV16 ( Addr a, Bool isBigEndian )
4190 PROF_EVENT(240, "mc_LOADV16");
4192 #ifndef PERF_FAST_LOADV
4193 return (UWord)mc_LOADVn_slow( a, 16, isBigEndian );
4196 UWord sm_off, vabits8;
4199 if (UNLIKELY( UNALIGNED_OR_HIGH(a,16) )) {
4200 PROF_EVENT(241, "mc_LOADV16-slow1");
4201 return (UWord)mc_LOADVn_slow( a, 16, isBigEndian );
4204 sm = get_secmap_for_reading_low(a);
4206 vabits8 = sm->vabits8[sm_off];
4207 // Handle common case quickly: a is suitably aligned, is mapped, and is
4209 // Convert V bits from compact memory form to expanded register form
4210 if (vabits8 == VA_BITS8_DEFINED ) { return V_BITS16_DEFINED; }
4211 else if (vabits8 == VA_BITS8_UNDEFINED) { return V_BITS16_UNDEFINED; }
4213 // The 4 (yes, 4) bytes are not all-defined or all-undefined, check
4214 // the two sub-bytes.
4215 UChar vabits4 = extract_vabits4_from_vabits8(a, vabits8);
4216 if (vabits4 == VA_BITS4_DEFINED ) { return V_BITS16_DEFINED; }
4217 else if (vabits4 == VA_BITS4_UNDEFINED) { return V_BITS16_UNDEFINED; }
4219 /* Slow case: the two bytes are not all-defined or all-undefined. */
4220 PROF_EVENT(242, "mc_LOADV16-slow2");
4221 return (UWord)mc_LOADVn_slow( a, 16, isBigEndian );
4228 VG_REGPARM(1) UWord MC_(helperc_LOADV16be) ( Addr a )
4230 return mc_LOADV16(a, True);
4232 VG_REGPARM(1) UWord MC_(helperc_LOADV16le) ( Addr a )
4234 return mc_LOADV16(a, False);
4239 void mc_STOREV16 ( Addr a, UWord vbits16, Bool isBigEndian )
4241 PROF_EVENT(250, "mc_STOREV16");
4243 #ifndef PERF_FAST_STOREV
4244 mc_STOREVn_slow( a, 16, (ULong)vbits16, isBigEndian );
4247 UWord sm_off, vabits8;
4250 if (UNLIKELY( UNALIGNED_OR_HIGH(a,16) )) {
4251 PROF_EVENT(251, "mc_STOREV16-slow1");
4252 mc_STOREVn_slow( a, 16, (ULong)vbits16, isBigEndian );
4256 sm = get_secmap_for_reading_low(a);
4258 vabits8 = sm->vabits8[sm_off];
4259 if (LIKELY( !is_distinguished_sm(sm) &&
4260 (VA_BITS8_DEFINED == vabits8 ||
4261 VA_BITS8_UNDEFINED == vabits8) ))
4263 /* Handle common case quickly: a is suitably aligned, */
4264 /* is mapped, and is addressible. */
4265 // Convert full V-bits in register to compact 2-bit form.
4266 if (V_BITS16_DEFINED == vbits16) {
4267 insert_vabits4_into_vabits8( a, VA_BITS4_DEFINED ,
4268 &(sm->vabits8[sm_off]) );
4269 } else if (V_BITS16_UNDEFINED == vbits16) {
4270 insert_vabits4_into_vabits8( a, VA_BITS4_UNDEFINED,
4271 &(sm->vabits8[sm_off]) );
4273 /* Slow but general case -- writing partially defined bytes. */
4274 PROF_EVENT(252, "mc_STOREV16-slow2");
4275 mc_STOREVn_slow( a, 16, (ULong)vbits16, isBigEndian );
4278 /* Slow but general case. */
4279 PROF_EVENT(253, "mc_STOREV16-slow3");
4280 mc_STOREVn_slow( a, 16, (ULong)vbits16, isBigEndian );
4286 VG_REGPARM(2) void MC_(helperc_STOREV16be) ( Addr a, UWord vbits16 )
4288 mc_STOREV16(a, vbits16, True);
4290 VG_REGPARM(2) void MC_(helperc_STOREV16le) ( Addr a, UWord vbits16 )
4292 mc_STOREV16(a, vbits16, False);
4296 /* ------------------------ Size = 1 ------------------------ */
4297 /* Note: endianness is irrelevant for size == 1 */
4300 UWord MC_(helperc_LOADV8) ( Addr a )
4302 PROF_EVENT(260, "mc_LOADV8");
4304 #ifndef PERF_FAST_LOADV
4305 return (UWord)mc_LOADVn_slow( a, 8, False/*irrelevant*/ );
4308 UWord sm_off, vabits8;
4311 if (UNLIKELY( UNALIGNED_OR_HIGH(a,8) )) {
4312 PROF_EVENT(261, "mc_LOADV8-slow1");
4313 return (UWord)mc_LOADVn_slow( a, 8, False/*irrelevant*/ );
4316 sm = get_secmap_for_reading_low(a);
4318 vabits8 = sm->vabits8[sm_off];
4319 // Convert V bits from compact memory form to expanded register form
4320 // Handle common case quickly: a is mapped, and the entire
4321 // word32 it lives in is addressible.
4322 if (vabits8 == VA_BITS8_DEFINED ) { return V_BITS8_DEFINED; }
4323 else if (vabits8 == VA_BITS8_UNDEFINED) { return V_BITS8_UNDEFINED; }
4325 // The 4 (yes, 4) bytes are not all-defined or all-undefined, check
4327 UChar vabits2 = extract_vabits2_from_vabits8(a, vabits8);
4328 if (vabits2 == VA_BITS2_DEFINED ) { return V_BITS8_DEFINED; }
4329 else if (vabits2 == VA_BITS2_UNDEFINED) { return V_BITS8_UNDEFINED; }
4331 /* Slow case: the byte is not all-defined or all-undefined. */
4332 PROF_EVENT(262, "mc_LOADV8-slow2");
4333 return (UWord)mc_LOADVn_slow( a, 8, False/*irrelevant*/ );
4342 void MC_(helperc_STOREV8) ( Addr a, UWord vbits8 )
4344 PROF_EVENT(270, "mc_STOREV8");
4346 #ifndef PERF_FAST_STOREV
4347 mc_STOREVn_slow( a, 8, (ULong)vbits8, False/*irrelevant*/ );
4350 UWord sm_off, vabits8;
4353 if (UNLIKELY( UNALIGNED_OR_HIGH(a,8) )) {
4354 PROF_EVENT(271, "mc_STOREV8-slow1");
4355 mc_STOREVn_slow( a, 8, (ULong)vbits8, False/*irrelevant*/ );
4359 sm = get_secmap_for_reading_low(a);
4361 vabits8 = sm->vabits8[sm_off];
4363 ( !is_distinguished_sm(sm) &&
4364 ( (VA_BITS8_DEFINED == vabits8 || VA_BITS8_UNDEFINED == vabits8)
4365 || (VA_BITS2_NOACCESS != extract_vabits2_from_vabits8(a, vabits8))
4370 /* Handle common case quickly: a is mapped, the entire word32 it
4371 lives in is addressible. */
4372 // Convert full V-bits in register to compact 2-bit form.
4373 if (V_BITS8_DEFINED == vbits8) {
4374 insert_vabits2_into_vabits8( a, VA_BITS2_DEFINED,
4375 &(sm->vabits8[sm_off]) );
4376 } else if (V_BITS8_UNDEFINED == vbits8) {
4377 insert_vabits2_into_vabits8( a, VA_BITS2_UNDEFINED,
4378 &(sm->vabits8[sm_off]) );
4380 /* Slow but general case -- writing partially defined bytes. */
4381 PROF_EVENT(272, "mc_STOREV8-slow2");
4382 mc_STOREVn_slow( a, 8, (ULong)vbits8, False/*irrelevant*/ );
4385 /* Slow but general case. */
4386 PROF_EVENT(273, "mc_STOREV8-slow3");
4387 mc_STOREVn_slow( a, 8, (ULong)vbits8, False/*irrelevant*/ );
4394 /*------------------------------------------------------------*/
4395 /*--- Functions called directly from generated code: ---*/
4396 /*--- Value-check failure handlers. ---*/
4397 /*------------------------------------------------------------*/
4399 /* Call these ones when an origin is available ... */
4401 void MC_(helperc_value_check0_fail_w_o) ( UWord origin ) {
4402 MC_(record_cond_error) ( VG_(get_running_tid)(), (UInt)origin );
4406 void MC_(helperc_value_check1_fail_w_o) ( UWord origin ) {
4407 MC_(record_value_error) ( VG_(get_running_tid)(), 1, (UInt)origin );
4411 void MC_(helperc_value_check4_fail_w_o) ( UWord origin ) {
4412 MC_(record_value_error) ( VG_(get_running_tid)(), 4, (UInt)origin );
4416 void MC_(helperc_value_check8_fail_w_o) ( UWord origin ) {
4417 MC_(record_value_error) ( VG_(get_running_tid)(), 8, (UInt)origin );
4421 void MC_(helperc_value_checkN_fail_w_o) ( HWord sz, UWord origin ) {
4422 MC_(record_value_error) ( VG_(get_running_tid)(), (Int)sz, (UInt)origin );
4425 /* ... and these when an origin isn't available. */
4428 void MC_(helperc_value_check0_fail_no_o) ( void ) {
4429 MC_(record_cond_error) ( VG_(get_running_tid)(), 0/*origin*/ );
4433 void MC_(helperc_value_check1_fail_no_o) ( void ) {
4434 MC_(record_value_error) ( VG_(get_running_tid)(), 1, 0/*origin*/ );
4438 void MC_(helperc_value_check4_fail_no_o) ( void ) {
4439 MC_(record_value_error) ( VG_(get_running_tid)(), 4, 0/*origin*/ );
4443 void MC_(helperc_value_check8_fail_no_o) ( void ) {
4444 MC_(record_value_error) ( VG_(get_running_tid)(), 8, 0/*origin*/ );
4448 void MC_(helperc_value_checkN_fail_no_o) ( HWord sz ) {
4449 MC_(record_value_error) ( VG_(get_running_tid)(), (Int)sz, 0/*origin*/ );
4453 /*------------------------------------------------------------*/
4454 /*--- Metadata get/set functions, for client requests. ---*/
4455 /*------------------------------------------------------------*/
4457 // Nb: this expands the V+A bits out into register-form V bits, even though
4458 // they're in memory. This is for backward compatibility, and because it's
4459 // probably what the user wants.
4461 /* Copy Vbits from/to address 'a'. Returns: 1 == OK, 2 == alignment
4462 error [no longer used], 3 == addressing error. */
4463 /* Nb: We used to issue various definedness/addressability errors from here,
4464 but we took them out because they ranged from not-very-helpful to
4465 downright annoying, and they complicated the error data structures. */
4466 static Int mc_get_or_set_vbits_for_client (
4470 Bool setting, /* True <=> set vbits, False <=> get vbits */
4471 Bool is_client_request /* True <=> real user request
4472 False <=> internal call from gdbserver */
4479 /* Check that arrays are addressible before doing any getting/setting.
4480 vbits to be checked only for real user request. */
4481 for (i = 0; i < szB; i++) {
4482 if (VA_BITS2_NOACCESS == get_vabits2(a + i) ||
4483 (is_client_request && VA_BITS2_NOACCESS == get_vabits2(vbits + i))) {
4491 for (i = 0; i < szB; i++) {
4492 ok = set_vbits8(a + i, ((UChar*)vbits)[i]);
4497 for (i = 0; i < szB; i++) {
4498 ok = get_vbits8(a + i, &vbits8);
4500 ((UChar*)vbits)[i] = vbits8;
4502 if (is_client_request)
4503 // The bytes in vbits[] have now been set, so mark them as such.
4504 MC_(make_mem_defined)(vbits, szB);
4511 /*------------------------------------------------------------*/
4512 /*--- Detecting leaked (unreachable) malloc'd blocks. ---*/
4513 /*------------------------------------------------------------*/
4515 /* For the memory leak detector, say whether an entire 64k chunk of
4516 address space is possibly in use, or not. If in doubt return
4519 Bool MC_(is_within_valid_secondary) ( Addr a )
4521 SecMap* sm = maybe_get_secmap_for ( a );
4522 if (sm == NULL || sm == &sm_distinguished[SM_DIST_NOACCESS]
4523 || MC_(in_ignored_range)(a)) {
4524 /* Definitely not in use. */
4532 /* For the memory leak detector, say whether or not a given word
4533 address is to be regarded as valid. */
4534 Bool MC_(is_valid_aligned_word) ( Addr a )
4536 tl_assert(sizeof(UWord) == 4 || sizeof(UWord) == 8);
4537 tl_assert(VG_IS_WORD_ALIGNED(a));
4538 if (is_mem_defined( a, sizeof(UWord), NULL, NULL) == MC_Ok
4539 && !MC_(in_ignored_range)(a)) {
4547 /*------------------------------------------------------------*/
4548 /*--- Initialisation ---*/
4549 /*------------------------------------------------------------*/
4551 static void init_shadow_memory ( void )
4556 tl_assert(V_BIT_UNDEFINED == 1);
4557 tl_assert(V_BIT_DEFINED == 0);
4558 tl_assert(V_BITS8_UNDEFINED == 0xFF);
4559 tl_assert(V_BITS8_DEFINED == 0);
4561 /* Build the 3 distinguished secondaries */
4562 sm = &sm_distinguished[SM_DIST_NOACCESS];
4563 for (i = 0; i < SM_CHUNKS; i++) sm->vabits8[i] = VA_BITS8_NOACCESS;
4565 sm = &sm_distinguished[SM_DIST_UNDEFINED];
4566 for (i = 0; i < SM_CHUNKS; i++) sm->vabits8[i] = VA_BITS8_UNDEFINED;
4568 sm = &sm_distinguished[SM_DIST_DEFINED];
4569 for (i = 0; i < SM_CHUNKS; i++) sm->vabits8[i] = VA_BITS8_DEFINED;
4571 /* Set up the primary map. */
4572 /* These entries gradually get overwritten as the used address
4574 for (i = 0; i < N_PRIMARY_MAP; i++)
4575 primary_map[i] = &sm_distinguished[SM_DIST_NOACCESS];
4577 /* Auxiliary primary maps */
4578 init_auxmap_L1_L2();
4580 /* auxmap_size = auxmap_used = 0;
4581 no ... these are statically initialised */
4583 /* Secondary V bit table */
4584 secVBitTable = createSecVBitTable();
4588 /*------------------------------------------------------------*/
4589 /*--- Sanity check machinery (permanently engaged) ---*/
4590 /*------------------------------------------------------------*/
4592 static Bool mc_cheap_sanity_check ( void )
4595 PROF_EVENT(490, "cheap_sanity_check");
4596 /* Check for sane operating level */
4597 if (MC_(clo_mc_level) < 1 || MC_(clo_mc_level) > 3)
4599 /* nothing else useful we can rapidly check */
4603 static Bool mc_expensive_sanity_check ( void )
4606 Word n_secmaps_found;
4611 if (0) VG_(printf)("expensive sanity check\n");
4614 n_sanity_expensive++;
4615 PROF_EVENT(491, "expensive_sanity_check");
4617 /* Check for sane operating level */
4618 if (MC_(clo_mc_level) < 1 || MC_(clo_mc_level) > 3)
4621 /* Check that the 3 distinguished SMs are still as they should be. */
4623 /* Check noaccess DSM. */
4624 sm = &sm_distinguished[SM_DIST_NOACCESS];
4625 for (i = 0; i < SM_CHUNKS; i++)
4626 if (sm->vabits8[i] != VA_BITS8_NOACCESS)
4629 /* Check undefined DSM. */
4630 sm = &sm_distinguished[SM_DIST_UNDEFINED];
4631 for (i = 0; i < SM_CHUNKS; i++)
4632 if (sm->vabits8[i] != VA_BITS8_UNDEFINED)
4635 /* Check defined DSM. */
4636 sm = &sm_distinguished[SM_DIST_DEFINED];
4637 for (i = 0; i < SM_CHUNKS; i++)
4638 if (sm->vabits8[i] != VA_BITS8_DEFINED)
4642 VG_(printf)("memcheck expensive sanity: "
4643 "distinguished_secondaries have changed\n");
4647 /* If we're not checking for undefined value errors, the secondary V bit
4648 * table should be empty. */
4649 if (MC_(clo_mc_level) == 1) {
4650 if (0 != VG_(OSetGen_Size)(secVBitTable))
4654 /* check the auxiliary maps, very thoroughly */
4655 n_secmaps_found = 0;
4656 errmsg = check_auxmap_L1_L2_sanity( &n_secmaps_found );
4658 VG_(printf)("memcheck expensive sanity, auxmaps:\n\t%s", errmsg);
4662 /* n_secmaps_found is now the number referred to by the auxiliary
4663 primary map. Now add on the ones referred to by the main
4665 for (i = 0; i < N_PRIMARY_MAP; i++) {
4666 if (primary_map[i] == NULL) {
4669 if (!is_distinguished_sm(primary_map[i]))
4674 /* check that the number of secmaps issued matches the number that
4675 are reachable (iow, no secmap leaks) */
4676 if (n_secmaps_found != (n_issued_SMs - n_deissued_SMs))
4680 VG_(printf)("memcheck expensive sanity: "
4681 "apparent secmap leakage\n");
4686 VG_(printf)("memcheck expensive sanity: "
4687 "auxmap covers wrong address space\n");
4691 /* there is only one pointer to each secmap (expensive) */
4696 /*------------------------------------------------------------*/
4697 /*--- Command line args ---*/
4698 /*------------------------------------------------------------*/
4700 Bool MC_(clo_partial_loads_ok) = False;
4701 Long MC_(clo_freelist_vol) = 20*1000*1000LL;
4702 LeakCheckMode MC_(clo_leak_check) = LC_Summary;
4703 VgRes MC_(clo_leak_resolution) = Vg_HighRes;
4704 Bool MC_(clo_show_reachable) = False;
4705 Bool MC_(clo_show_possibly_lost) = True;
4706 Bool MC_(clo_workaround_gcc296_bugs) = False;
4707 Int MC_(clo_malloc_fill) = -1;
4708 Int MC_(clo_free_fill) = -1;
4709 Int MC_(clo_mc_level) = 2;
4711 static Bool mc_process_cmd_line_options(Char* arg)
4715 tl_assert( MC_(clo_mc_level) >= 1 && MC_(clo_mc_level) <= 3 );
4717 /* Set MC_(clo_mc_level):
4718 1 = A bit tracking only
4719 2 = A and V bit tracking, but no V bit origins
4720 3 = A and V bit tracking, and V bit origins
4722 Do this by inspecting --undef-value-errors= and
4723 --track-origins=. Reject the case --undef-value-errors=no
4724 --track-origins=yes as meaningless.
4726 if (0 == VG_(strcmp)(arg, "--undef-value-errors=no")) {
4727 if (MC_(clo_mc_level) == 3) {
4730 MC_(clo_mc_level) = 1;
4734 if (0 == VG_(strcmp)(arg, "--undef-value-errors=yes")) {
4735 if (MC_(clo_mc_level) == 1)
4736 MC_(clo_mc_level) = 2;
4739 if (0 == VG_(strcmp)(arg, "--track-origins=no")) {
4740 if (MC_(clo_mc_level) == 3)
4741 MC_(clo_mc_level) = 2;
4744 if (0 == VG_(strcmp)(arg, "--track-origins=yes")) {
4745 if (MC_(clo_mc_level) == 1) {
4748 MC_(clo_mc_level) = 3;
4753 if VG_BOOL_CLO(arg, "--partial-loads-ok", MC_(clo_partial_loads_ok)) {}
4754 else if VG_BOOL_CLO(arg, "--show-reachable", MC_(clo_show_reachable)) {}
4755 else if VG_BOOL_CLO(arg, "--show-possibly-lost",
4756 MC_(clo_show_possibly_lost)) {}
4757 else if VG_BOOL_CLO(arg, "--workaround-gcc296-bugs",
4758 MC_(clo_workaround_gcc296_bugs)) {}
4760 else if VG_BINT_CLO(arg, "--freelist-vol", MC_(clo_freelist_vol),
4761 0, 10*1000*1000*1000LL) {}
4763 else if VG_XACT_CLO(arg, "--leak-check=no",
4764 MC_(clo_leak_check), LC_Off) {}
4765 else if VG_XACT_CLO(arg, "--leak-check=summary",
4766 MC_(clo_leak_check), LC_Summary) {}
4767 else if VG_XACT_CLO(arg, "--leak-check=yes",
4768 MC_(clo_leak_check), LC_Full) {}
4769 else if VG_XACT_CLO(arg, "--leak-check=full",
4770 MC_(clo_leak_check), LC_Full) {}
4772 else if VG_XACT_CLO(arg, "--leak-resolution=low",
4773 MC_(clo_leak_resolution), Vg_LowRes) {}
4774 else if VG_XACT_CLO(arg, "--leak-resolution=med",
4775 MC_(clo_leak_resolution), Vg_MedRes) {}
4776 else if VG_XACT_CLO(arg, "--leak-resolution=high",
4777 MC_(clo_leak_resolution), Vg_HighRes) {}
4779 else if VG_STR_CLO(arg, "--ignore-ranges", tmp_str) {
4781 Bool ok = parse_ignore_ranges(tmp_str);
4784 tl_assert(ignoreRanges.used >= 0);
4785 tl_assert(ignoreRanges.used < M_IGNORE_RANGES);
4786 for (i = 0; i < ignoreRanges.used; i++) {
4787 Addr s = ignoreRanges.start[i];
4788 Addr e = ignoreRanges.end[i];
4789 Addr limit = 0x4000000; /* 64M - entirely arbitrary limit */
4791 VG_(message)(Vg_DebugMsg,
4792 "ERROR: --ignore-ranges: end <= start in range:\n");
4793 VG_(message)(Vg_DebugMsg,
4794 " 0x%lx-0x%lx\n", s, e);
4797 if (e - s > limit) {
4798 VG_(message)(Vg_DebugMsg,
4799 "ERROR: --ignore-ranges: suspiciously large range:\n");
4800 VG_(message)(Vg_DebugMsg,
4801 " 0x%lx-0x%lx (size %ld)\n", s, e, (UWord)(e-s));
4807 else if VG_BHEX_CLO(arg, "--malloc-fill", MC_(clo_malloc_fill), 0x00,0xFF) {}
4808 else if VG_BHEX_CLO(arg, "--free-fill", MC_(clo_free_fill), 0x00,0xFF) {}
4811 return VG_(replacement_malloc_process_cmd_line_option)(arg);
4817 VG_(fmsg_bad_option)(arg,
4818 "--track-origins=yes has no effect when --undef-value-errors=no.\n");
4821 static void mc_print_usage(void)
4824 " --leak-check=no|summary|full search for memory leaks at exit? [summary]\n"
4825 " --leak-resolution=low|med|high differentiation of leak stack traces [high]\n"
4826 " --show-reachable=no|yes show reachable blocks in leak check? [no]\n"
4827 " --show-possibly-lost=no|yes show possibly lost blocks in leak check?\n"
4829 " --undef-value-errors=no|yes check for undefined value errors [yes]\n"
4830 " --track-origins=no|yes show origins of undefined values? [no]\n"
4831 " --partial-loads-ok=no|yes too hard to explain here; see manual [no]\n"
4832 " --freelist-vol=<number> volume of freed blocks queue [20000000]\n"
4833 " --workaround-gcc296-bugs=no|yes self explanatory [no]\n"
4834 " --ignore-ranges=0xPP-0xQQ[,0xRR-0xSS] assume given addresses are OK\n"
4835 " --malloc-fill=<hexnumber> fill malloc'd areas with given value\n"
4836 " --free-fill=<hexnumber> fill free'd areas with given value\n"
4840 static void mc_print_debug_usage(void)
4848 /*------------------------------------------------------------*/
4849 /*--- Client blocks ---*/
4850 /*------------------------------------------------------------*/
4852 /* Client block management:
4854 This is managed as an expanding array of client block descriptors.
4855 Indices of live descriptors are issued to the client, so it can ask
4856 to free them later. Therefore we cannot slide live entries down
4857 over dead ones. Instead we must use free/inuse flags and scan for
4858 an empty slot at allocation time. This in turn means allocation is
4859 relatively expensive, so we hope this does not happen too often.
4861 An unused block has start == size == 0
4864 /* type CGenBlock is defined in mc_include.h */
4866 /* This subsystem is self-initialising. */
4867 static UWord cgb_size = 0;
4868 static UWord cgb_used = 0;
4869 static CGenBlock* cgbs = NULL;
4871 /* Stats for this subsystem. */
4872 static ULong cgb_used_MAX = 0; /* Max in use. */
4873 static ULong cgb_allocs = 0; /* Number of allocs. */
4874 static ULong cgb_discards = 0; /* Number of discards. */
4875 static ULong cgb_search = 0; /* Number of searches. */
4878 /* Get access to the client block array. */
4879 void MC_(get_ClientBlock_array)( /*OUT*/CGenBlock** blocks,
4880 /*OUT*/UWord* nBlocks )
4883 *nBlocks = cgb_used;
4888 Int alloc_client_block ( void )
4891 CGenBlock* cgbs_new;
4895 for (i = 0; i < cgb_used; i++) {
4897 if (cgbs[i].start == 0 && cgbs[i].size == 0)
4901 /* Not found. Try to allocate one at the end. */
4902 if (cgb_used < cgb_size) {
4907 /* Ok, we have to allocate a new one. */
4908 tl_assert(cgb_used == cgb_size);
4909 sz_new = (cgbs == NULL) ? 10 : (2 * cgb_size);
4911 cgbs_new = VG_(malloc)( "mc.acb.1", sz_new * sizeof(CGenBlock) );
4912 for (i = 0; i < cgb_used; i++)
4913 cgbs_new[i] = cgbs[i];
4921 if (cgb_used > cgb_used_MAX)
4922 cgb_used_MAX = cgb_used;
4927 static void show_client_block_stats ( void )
4929 VG_(message)(Vg_DebugMsg,
4930 "general CBs: %llu allocs, %llu discards, %llu maxinuse, %llu search\n",
4931 cgb_allocs, cgb_discards, cgb_used_MAX, cgb_search
4935 static void print_monitor_help ( void )
4940 "memcheck monitor commands:\n"
4941 " mc.get_vbits <addr> [<len>]\n"
4942 " returns validity bits for <len> (or 1) bytes at <addr>\n"
4943 " bit values 0 = valid, 1 = invalid, __ = unaddressable byte\n"
4944 " Example: mc.get_vbits 0x8049c78 10\n"
4945 " mc.make_memory [noaccess|undefined\n"
4946 " |defined|ifaddressabledefined] <addr> [<len>]\n"
4947 " mark <len> (or 1) bytes at <addr> with the given accessibility\n"
4948 " mc.check_memory [addressable|defined] <addr> [<len>]\n"
4949 " check that <len> (or 1) bytes at <addr> have the given accessibility\n"
4950 " and outputs a description of <addr>\n"
4951 " mc.leak_check [full*|summary]\n"
4952 " [reachable|leakpossible*|definiteleak]\n"
4954 " Examples: mc.leak_check\n"
4955 " mc.leak_check any summary\n"
4959 /* return True if request recognised, False otherwise */
4960 static Bool handle_gdb_monitor_command (ThreadId tid, Char *req)
4963 Char s[VG_(strlen(req))]; /* copy for strtok_r */
4966 VG_(strcpy) (s, req);
4968 wcmd = VG_(strtok_r) (s, " ", &ssaveptr);
4969 /* NB: if possible, avoid introducing a new command below which
4970 starts with the same 4 first letters as an already existing
4971 command. This ensures a shorter abbreviation for the user. */
4972 switch (VG_(keyword_id)
4973 ("help mc.get_vbits mc.leak_check mc.make_memory mc.check_memory",
4974 wcmd, kwd_report_duplicated_matches)) {
4975 case -2: /* multiple matches */
4977 case -1: /* not found */
4980 print_monitor_help();
4982 case 1: { /* mc.get_vbits */
4985 VG_(strtok_get_address_and_size) (&address, &szB, &ssaveptr);
4989 Int unaddressable = 0;
4990 for (i = 0; i < szB; i++) {
4991 Int res = mc_get_or_set_vbits_for_client
4992 (address+i, (Addr) &vbits, 1,
4993 False, /* get them */
4994 False /* is client request */ );
4995 if ((i % 32) == 0 && i != 0)
4996 VG_(gdb_printf) ("\n");
4997 else if ((i % 4) == 0 && i != 0)
4998 VG_(gdb_printf) (" ");
5000 VG_(gdb_printf) ("%02x", vbits);
5002 tl_assert(3 == res);
5004 VG_(gdb_printf) ("__");
5008 VG_(gdb_printf) ("\n");
5009 if (unaddressable) {
5011 ("Address %p len %ld has %d bytes unaddressable\n",
5012 (void *)address, szB, unaddressable);
5017 case 2: { /* mc.leak_check */
5019 Bool save_clo_show_reachable = MC_(clo_show_reachable);
5020 Bool save_clo_show_possibly_lost = MC_(clo_show_possibly_lost);
5025 MC_(clo_show_reachable) = False;
5028 for (kw = VG_(strtok_r) (NULL, " ", &ssaveptr);
5030 kw = VG_(strtok_r) (NULL, " ", &ssaveptr)) {
5031 switch (VG_(keyword_id)
5033 "reachable leakpossible definiteleak",
5034 kw, kwd_report_all)) {
5035 case -2: err++; break;
5036 case -1: err++; break;
5037 case 0: mode = LC_Full; break;
5038 case 1: mode = LC_Summary; break;
5039 case 2: MC_(clo_show_reachable) = True;
5040 MC_(clo_show_possibly_lost) = True; break;
5041 case 3: MC_(clo_show_reachable) = False;
5042 MC_(clo_show_possibly_lost) = True; break;
5043 case 4: MC_(clo_show_reachable) = False;
5044 MC_(clo_show_possibly_lost) = False; break;
5045 default: tl_assert (0);
5049 MC_(detect_memory_leaks)(tid, mode);
5051 MC_(clo_show_reachable) = save_clo_show_reachable;
5052 MC_(clo_show_possibly_lost) = save_clo_show_possibly_lost;
5056 case 3: { /* mc.make_memory */
5059 int kwdid = VG_(keyword_id)
5060 ("noaccess undefined defined ifaddressabledefined",
5061 VG_(strtok_r) (NULL, " ", &ssaveptr), kwd_report_all);
5062 VG_(strtok_get_address_and_size) (&address, &szB, &ssaveptr);
5063 if (address == (Addr) 0 && szB == 0) return True;
5067 case 0: MC_(make_mem_noaccess) (address, szB); break;
5068 case 1: make_mem_undefined_w_tid_and_okind ( address, szB, tid,
5069 MC_OKIND_USER ); break;
5070 case 2: MC_(make_mem_defined) ( address, szB ); break;
5071 case 3: make_mem_defined_if_addressable ( address, szB ); break;;
5072 default: tl_assert(0);
5077 case 4: { /* mc.check_memory */
5085 ExeContext* origin_ec;
5088 int kwdid = VG_(keyword_id)
5089 ("addressable defined",
5090 VG_(strtok_r) (NULL, " ", &ssaveptr), kwd_report_all);
5091 VG_(strtok_get_address_and_size) (&address, &szB, &ssaveptr);
5092 if (address == (Addr) 0 && szB == 0) return True;
5097 if (is_mem_addressable ( address, szB, &bad_addr ))
5098 VG_(gdb_printf) ("Address %p len %ld addressable\n",
5099 (void *)address, szB);
5102 ("Address %p len %ld not addressable:\nbad address %p\n",
5103 (void *)address, szB, (void *) bad_addr);
5104 MC_(pp_describe_addr) (address);
5106 case 1: res = is_mem_defined ( address, szB, &bad_addr, &otag );
5107 if (MC_AddrErr == res)
5109 ("Address %p len %ld not addressable:\nbad address %p\n",
5110 (void *)address, szB, (void *) bad_addr);
5111 else if (MC_ValueErr == res) {
5114 case MC_OKIND_STACK:
5115 src = " was created by a stack allocation"; break;
5117 src = " was created by a heap allocation"; break;
5119 src = " was created by a client request"; break;
5120 case MC_OKIND_UNKNOWN:
5122 default: tl_assert(0);
5125 ("Address %p len %ld not defined:\n"
5126 "Uninitialised value at %p%s\n",
5127 (void *)address, szB, (void *) bad_addr, src);
5129 if (VG_(is_plausible_ECU)(ecu)) {
5130 origin_ec = VG_(get_ExeContext_from_ECU)( ecu );
5131 VG_(pp_ExeContext)( origin_ec );
5135 VG_(gdb_printf) ("Address %p len %ld defined\n",
5136 (void *)address, szB);
5137 MC_(pp_describe_addr) (address);
5139 default: tl_assert(0);
5150 /*------------------------------------------------------------*/
5151 /*--- Client requests ---*/
5152 /*------------------------------------------------------------*/
5154 static Bool mc_handle_client_request ( ThreadId tid, UWord* arg, UWord* ret )
5160 if (!VG_IS_TOOL_USERREQ('M','C',arg[0])
5161 && VG_USERREQ__MALLOCLIKE_BLOCK != arg[0]
5162 && VG_USERREQ__RESIZEINPLACE_BLOCK != arg[0]
5163 && VG_USERREQ__FREELIKE_BLOCK != arg[0]
5164 && VG_USERREQ__CREATE_MEMPOOL != arg[0]
5165 && VG_USERREQ__DESTROY_MEMPOOL != arg[0]
5166 && VG_USERREQ__MEMPOOL_ALLOC != arg[0]
5167 && VG_USERREQ__MEMPOOL_FREE != arg[0]
5168 && VG_USERREQ__MEMPOOL_TRIM != arg[0]
5169 && VG_USERREQ__MOVE_MEMPOOL != arg[0]
5170 && VG_USERREQ__MEMPOOL_CHANGE != arg[0]
5171 && VG_USERREQ__MEMPOOL_EXISTS != arg[0]
5172 && VG_USERREQ__GDB_MONITOR_COMMAND != arg[0])
5176 case VG_USERREQ__CHECK_MEM_IS_ADDRESSABLE:
5177 ok = is_mem_addressable ( arg[1], arg[2], &bad_addr );
5179 MC_(record_user_error) ( tid, bad_addr, /*isAddrErr*/True, 0 );
5180 *ret = ok ? (UWord)NULL : bad_addr;
5183 case VG_USERREQ__CHECK_MEM_IS_DEFINED: {
5186 res = is_mem_defined ( arg[1], arg[2], &bad_addr, &otag );
5187 if (MC_AddrErr == res)
5188 MC_(record_user_error) ( tid, bad_addr, /*isAddrErr*/True, 0 );
5189 else if (MC_ValueErr == res)
5190 MC_(record_user_error) ( tid, bad_addr, /*isAddrErr*/False, otag );
5191 *ret = ( res==MC_Ok ? (UWord)NULL : bad_addr );
5195 case VG_USERREQ__DO_LEAK_CHECK:
5196 MC_(detect_memory_leaks)(tid, arg[1] ? LC_Summary : LC_Full);
5197 *ret = 0; /* return value is meaningless */
5200 case VG_USERREQ__MAKE_MEM_NOACCESS:
5201 MC_(make_mem_noaccess) ( arg[1], arg[2] );
5205 case VG_USERREQ__MAKE_MEM_UNDEFINED:
5206 make_mem_undefined_w_tid_and_okind ( arg[1], arg[2], tid,
5211 case VG_USERREQ__MAKE_MEM_DEFINED:
5212 MC_(make_mem_defined) ( arg[1], arg[2] );
5216 case VG_USERREQ__MAKE_MEM_DEFINED_IF_ADDRESSABLE:
5217 make_mem_defined_if_addressable ( arg[1], arg[2] );
5221 case VG_USERREQ__CREATE_BLOCK: /* describe a block */
5222 if (arg[1] != 0 && arg[2] != 0) {
5223 i = alloc_client_block();
5224 /* VG_(printf)("allocated %d %p\n", i, cgbs); */
5225 cgbs[i].start = arg[1];
5226 cgbs[i].size = arg[2];
5227 cgbs[i].desc = VG_(strdup)("mc.mhcr.1", (Char *)arg[3]);
5228 cgbs[i].where = VG_(record_ExeContext) ( tid, 0/*first_ip_delta*/ );
5234 case VG_USERREQ__DISCARD: /* discard */
5236 || arg[2] >= cgb_used ||
5237 (cgbs[arg[2]].start == 0 && cgbs[arg[2]].size == 0)) {
5240 tl_assert(arg[2] >= 0 && arg[2] < cgb_used);
5241 cgbs[arg[2]].start = cgbs[arg[2]].size = 0;
5242 VG_(free)(cgbs[arg[2]].desc);
5248 case VG_USERREQ__GET_VBITS:
5249 *ret = mc_get_or_set_vbits_for_client
5250 ( arg[1], arg[2], arg[3],
5251 False /* get them */,
5252 True /* is client request */ );
5255 case VG_USERREQ__SET_VBITS:
5256 *ret = mc_get_or_set_vbits_for_client
5257 ( arg[1], arg[2], arg[3],
5258 True /* set them */,
5259 True /* is client request */ );
5262 case VG_USERREQ__COUNT_LEAKS: { /* count leaked bytes */
5263 UWord** argp = (UWord**)arg;
5264 // MC_(bytes_leaked) et al were set by the last leak check (or zero
5265 // if no prior leak checks performed).
5266 *argp[1] = MC_(bytes_leaked) + MC_(bytes_indirect);
5267 *argp[2] = MC_(bytes_dubious);
5268 *argp[3] = MC_(bytes_reachable);
5269 *argp[4] = MC_(bytes_suppressed);
5270 // there is no argp[5]
5271 //*argp[5] = MC_(bytes_indirect);
5272 // XXX need to make *argp[1-4] defined; currently done in the
5273 // VALGRIND_COUNT_LEAKS_MACRO by initialising them to zero.
5277 case VG_USERREQ__COUNT_LEAK_BLOCKS: { /* count leaked blocks */
5278 UWord** argp = (UWord**)arg;
5279 // MC_(blocks_leaked) et al were set by the last leak check (or zero
5280 // if no prior leak checks performed).
5281 *argp[1] = MC_(blocks_leaked) + MC_(blocks_indirect);
5282 *argp[2] = MC_(blocks_dubious);
5283 *argp[3] = MC_(blocks_reachable);
5284 *argp[4] = MC_(blocks_suppressed);
5285 // there is no argp[5]
5286 //*argp[5] = MC_(blocks_indirect);
5287 // XXX need to make *argp[1-4] defined; currently done in the
5288 // VALGRIND_COUNT_LEAK_BLOCKS_MACRO by initialising them to zero.
5292 case VG_USERREQ__MALLOCLIKE_BLOCK: {
5293 Addr p = (Addr)arg[1];
5294 SizeT sizeB = arg[2];
5295 //UInt rzB = arg[3]; XXX: unused!
5296 Bool is_zeroed = (Bool)arg[4];
5298 MC_(new_block) ( tid, p, sizeB, /*ignored*/0, is_zeroed,
5299 MC_AllocCustom, MC_(malloc_list) );
5302 case VG_USERREQ__RESIZEINPLACE_BLOCK: {
5303 Addr p = (Addr)arg[1];
5304 SizeT oldSizeB = arg[2];
5305 SizeT newSizeB = arg[3];
5308 MC_(handle_resizeInPlace) ( tid, p, oldSizeB, newSizeB, rzB );
5311 case VG_USERREQ__FREELIKE_BLOCK: {
5312 Addr p = (Addr)arg[1];
5315 MC_(handle_free) ( tid, p, rzB, MC_AllocCustom );
5319 case _VG_USERREQ__MEMCHECK_RECORD_OVERLAP_ERROR: {
5320 Char* s = (Char*)arg[1];
5321 Addr dst = (Addr) arg[2];
5322 Addr src = (Addr) arg[3];
5323 SizeT len = (SizeT)arg[4];
5324 MC_(record_overlap_error)(tid, s, src, dst, len);
5328 case VG_USERREQ__CREATE_MEMPOOL: {
5329 Addr pool = (Addr)arg[1];
5331 Bool is_zeroed = (Bool)arg[3];
5333 MC_(create_mempool) ( pool, rzB, is_zeroed );
5337 case VG_USERREQ__DESTROY_MEMPOOL: {
5338 Addr pool = (Addr)arg[1];
5340 MC_(destroy_mempool) ( pool );
5344 case VG_USERREQ__MEMPOOL_ALLOC: {
5345 Addr pool = (Addr)arg[1];
5346 Addr addr = (Addr)arg[2];
5349 MC_(mempool_alloc) ( tid, pool, addr, size );
5353 case VG_USERREQ__MEMPOOL_FREE: {
5354 Addr pool = (Addr)arg[1];
5355 Addr addr = (Addr)arg[2];
5357 MC_(mempool_free) ( pool, addr );
5361 case VG_USERREQ__MEMPOOL_TRIM: {
5362 Addr pool = (Addr)arg[1];
5363 Addr addr = (Addr)arg[2];
5366 MC_(mempool_trim) ( pool, addr, size );
5370 case VG_USERREQ__MOVE_MEMPOOL: {
5371 Addr poolA = (Addr)arg[1];
5372 Addr poolB = (Addr)arg[2];
5374 MC_(move_mempool) ( poolA, poolB );
5378 case VG_USERREQ__MEMPOOL_CHANGE: {
5379 Addr pool = (Addr)arg[1];
5380 Addr addrA = (Addr)arg[2];
5381 Addr addrB = (Addr)arg[3];
5384 MC_(mempool_change) ( pool, addrA, addrB, size );
5388 case VG_USERREQ__MEMPOOL_EXISTS: {
5389 Addr pool = (Addr)arg[1];
5391 *ret = (UWord) MC_(mempool_exists) ( pool );
5395 case VG_USERREQ__GDB_MONITOR_COMMAND: {
5396 Bool handled = handle_gdb_monitor_command (tid, (Char*)arg[1]);
5407 "Warning: unknown memcheck client request code %llx\n",
5416 /*------------------------------------------------------------*/
5417 /*--- Crude profiling machinery. ---*/
5418 /*------------------------------------------------------------*/
5420 // We track a number of interesting events (using PROF_EVENT)
5421 // if MC_PROFILE_MEMORY is defined.
5423 #ifdef MC_PROFILE_MEMORY
5425 UInt MC_(event_ctr)[N_PROF_EVENTS];
5426 HChar* MC_(event_ctr_name)[N_PROF_EVENTS];
5428 static void init_prof_mem ( void )
5431 for (i = 0; i < N_PROF_EVENTS; i++) {
5432 MC_(event_ctr)[i] = 0;
5433 MC_(event_ctr_name)[i] = NULL;
5437 static void done_prof_mem ( void )
5440 Bool spaced = False;
5441 for (i = 0; i < N_PROF_EVENTS; i++) {
5442 if (!spaced && (i % 10) == 0) {
5446 if (MC_(event_ctr)[i] > 0) {
5448 VG_(printf)( "prof mem event %3d: %9d %s\n",
5449 i, MC_(event_ctr)[i],
5450 MC_(event_ctr_name)[i]
5451 ? MC_(event_ctr_name)[i] : "unnamed");
5458 static void init_prof_mem ( void ) { }
5459 static void done_prof_mem ( void ) { }
5464 /*------------------------------------------------------------*/
5465 /*--- Origin tracking stuff ---*/
5466 /*------------------------------------------------------------*/
5468 /*--------------------------------------------*/
5469 /*--- Origin tracking: load handlers ---*/
5470 /*--------------------------------------------*/
5472 static INLINE UInt merge_origins ( UInt or1, UInt or2 ) {
5473 return or1 > or2 ? or1 : or2;
5476 UWord VG_REGPARM(1) MC_(helperc_b_load1)( Addr a ) {
5479 UWord lineoff = oc_line_offset(a);
5480 UWord byteoff = a & 3; /* 0, 1, 2 or 3 */
5482 if (OC_ENABLE_ASSERTIONS) {
5483 tl_assert(lineoff >= 0 && lineoff < OC_W32S_PER_LINE);
5486 line = find_OCacheLine( a );
5488 descr = line->descr[lineoff];
5489 if (OC_ENABLE_ASSERTIONS) {
5490 tl_assert(descr < 0x10);
5493 if (LIKELY(0 == (descr & (1 << byteoff)))) {
5496 return line->w32[lineoff];
5500 UWord VG_REGPARM(1) MC_(helperc_b_load2)( Addr a ) {
5503 UWord lineoff, byteoff;
5505 if (UNLIKELY(a & 1)) {
5506 /* Handle misaligned case, slowly. */
5507 UInt oLo = (UInt)MC_(helperc_b_load1)( a + 0 );
5508 UInt oHi = (UInt)MC_(helperc_b_load1)( a + 1 );
5509 return merge_origins(oLo, oHi);
5512 lineoff = oc_line_offset(a);
5513 byteoff = a & 3; /* 0 or 2 */
5515 if (OC_ENABLE_ASSERTIONS) {
5516 tl_assert(lineoff >= 0 && lineoff < OC_W32S_PER_LINE);
5518 line = find_OCacheLine( a );
5520 descr = line->descr[lineoff];
5521 if (OC_ENABLE_ASSERTIONS) {
5522 tl_assert(descr < 0x10);
5525 if (LIKELY(0 == (descr & (3 << byteoff)))) {
5528 return line->w32[lineoff];
5532 UWord VG_REGPARM(1) MC_(helperc_b_load4)( Addr a ) {
5537 if (UNLIKELY(a & 3)) {
5538 /* Handle misaligned case, slowly. */
5539 UInt oLo = (UInt)MC_(helperc_b_load2)( a + 0 );
5540 UInt oHi = (UInt)MC_(helperc_b_load2)( a + 2 );
5541 return merge_origins(oLo, oHi);
5544 lineoff = oc_line_offset(a);
5545 if (OC_ENABLE_ASSERTIONS) {
5546 tl_assert(lineoff >= 0 && lineoff < OC_W32S_PER_LINE);
5549 line = find_OCacheLine( a );
5551 descr = line->descr[lineoff];
5552 if (OC_ENABLE_ASSERTIONS) {
5553 tl_assert(descr < 0x10);
5556 if (LIKELY(0 == descr)) {
5559 return line->w32[lineoff];
5563 UWord VG_REGPARM(1) MC_(helperc_b_load8)( Addr a ) {
5565 UChar descrLo, descrHi, descr;
5568 if (UNLIKELY(a & 7)) {
5569 /* Handle misaligned case, slowly. */
5570 UInt oLo = (UInt)MC_(helperc_b_load4)( a + 0 );
5571 UInt oHi = (UInt)MC_(helperc_b_load4)( a + 4 );
5572 return merge_origins(oLo, oHi);
5575 lineoff = oc_line_offset(a);
5576 if (OC_ENABLE_ASSERTIONS) {
5577 tl_assert(lineoff == (lineoff & 6)); /*0,2,4,6*//*since 8-aligned*/
5580 line = find_OCacheLine( a );
5582 descrLo = line->descr[lineoff + 0];
5583 descrHi = line->descr[lineoff + 1];
5584 descr = descrLo | descrHi;
5585 if (OC_ENABLE_ASSERTIONS) {
5586 tl_assert(descr < 0x10);
5589 if (LIKELY(0 == descr)) {
5590 return 0; /* both 32-bit chunks are defined */
5592 UInt oLo = descrLo == 0 ? 0 : line->w32[lineoff + 0];
5593 UInt oHi = descrHi == 0 ? 0 : line->w32[lineoff + 1];
5594 return merge_origins(oLo, oHi);
5598 UWord VG_REGPARM(1) MC_(helperc_b_load16)( Addr a ) {
5599 UInt oLo = (UInt)MC_(helperc_b_load8)( a + 0 );
5600 UInt oHi = (UInt)MC_(helperc_b_load8)( a + 8 );
5601 UInt oBoth = merge_origins(oLo, oHi);
5602 return (UWord)oBoth;
5606 /*--------------------------------------------*/
5607 /*--- Origin tracking: store handlers ---*/
5608 /*--------------------------------------------*/
5610 void VG_REGPARM(2) MC_(helperc_b_store1)( Addr a, UWord d32 ) {
5612 UWord lineoff = oc_line_offset(a);
5613 UWord byteoff = a & 3; /* 0, 1, 2 or 3 */
5615 if (OC_ENABLE_ASSERTIONS) {
5616 tl_assert(lineoff >= 0 && lineoff < OC_W32S_PER_LINE);
5619 line = find_OCacheLine( a );
5622 line->descr[lineoff] &= ~(1 << byteoff);
5624 line->descr[lineoff] |= (1 << byteoff);
5625 line->w32[lineoff] = d32;
5629 void VG_REGPARM(2) MC_(helperc_b_store2)( Addr a, UWord d32 ) {
5631 UWord lineoff, byteoff;
5633 if (UNLIKELY(a & 1)) {
5634 /* Handle misaligned case, slowly. */
5635 MC_(helperc_b_store1)( a + 0, d32 );
5636 MC_(helperc_b_store1)( a + 1, d32 );
5640 lineoff = oc_line_offset(a);
5641 byteoff = a & 3; /* 0 or 2 */
5643 if (OC_ENABLE_ASSERTIONS) {
5644 tl_assert(lineoff >= 0 && lineoff < OC_W32S_PER_LINE);
5647 line = find_OCacheLine( a );
5650 line->descr[lineoff] &= ~(3 << byteoff);
5652 line->descr[lineoff] |= (3 << byteoff);
5653 line->w32[lineoff] = d32;
5657 void VG_REGPARM(2) MC_(helperc_b_store4)( Addr a, UWord d32 ) {
5661 if (UNLIKELY(a & 3)) {
5662 /* Handle misaligned case, slowly. */
5663 MC_(helperc_b_store2)( a + 0, d32 );
5664 MC_(helperc_b_store2)( a + 2, d32 );
5668 lineoff = oc_line_offset(a);
5669 if (OC_ENABLE_ASSERTIONS) {
5670 tl_assert(lineoff >= 0 && lineoff < OC_W32S_PER_LINE);
5673 line = find_OCacheLine( a );
5676 line->descr[lineoff] = 0;
5678 line->descr[lineoff] = 0xF;
5679 line->w32[lineoff] = d32;
5683 void VG_REGPARM(2) MC_(helperc_b_store8)( Addr a, UWord d32 ) {
5687 if (UNLIKELY(a & 7)) {
5688 /* Handle misaligned case, slowly. */
5689 MC_(helperc_b_store4)( a + 0, d32 );
5690 MC_(helperc_b_store4)( a + 4, d32 );
5694 lineoff = oc_line_offset(a);
5695 if (OC_ENABLE_ASSERTIONS) {
5696 tl_assert(lineoff == (lineoff & 6)); /*0,2,4,6*//*since 8-aligned*/
5699 line = find_OCacheLine( a );
5702 line->descr[lineoff + 0] = 0;
5703 line->descr[lineoff + 1] = 0;
5705 line->descr[lineoff + 0] = 0xF;
5706 line->descr[lineoff + 1] = 0xF;
5707 line->w32[lineoff + 0] = d32;
5708 line->w32[lineoff + 1] = d32;
5712 void VG_REGPARM(2) MC_(helperc_b_store16)( Addr a, UWord d32 ) {
5713 MC_(helperc_b_store8)( a + 0, d32 );
5714 MC_(helperc_b_store8)( a + 8, d32 );
5718 /*--------------------------------------------*/
5719 /*--- Origin tracking: sarp handlers ---*/
5720 /*--------------------------------------------*/
5722 __attribute__((noinline))
5723 static void ocache_sarp_Set_Origins ( Addr a, UWord len, UInt otag ) {
5724 if ((a & 1) && len >= 1) {
5725 MC_(helperc_b_store1)( a, otag );
5729 if ((a & 2) && len >= 2) {
5730 MC_(helperc_b_store2)( a, otag );
5735 tl_assert(0 == (a & 3));
5737 MC_(helperc_b_store4)( a, otag );
5742 MC_(helperc_b_store2)( a, otag );
5747 MC_(helperc_b_store1)( a, otag );
5751 tl_assert(len == 0);
5754 __attribute__((noinline))
5755 static void ocache_sarp_Clear_Origins ( Addr a, UWord len ) {
5756 if ((a & 1) && len >= 1) {
5757 MC_(helperc_b_store1)( a, 0 );
5761 if ((a & 2) && len >= 2) {
5762 MC_(helperc_b_store2)( a, 0 );
5767 tl_assert(0 == (a & 3));
5769 MC_(helperc_b_store4)( a, 0 );
5774 MC_(helperc_b_store2)( a, 0 );
5779 MC_(helperc_b_store1)( a, 0 );
5783 tl_assert(len == 0);
5787 /*------------------------------------------------------------*/
5788 /*--- Setup and finalisation ---*/
5789 /*------------------------------------------------------------*/
5791 static void mc_post_clo_init ( void )
5793 /* If we've been asked to emit XML, mash around various other
5794 options so as to constrain the output somewhat. */
5796 /* Extract as much info as possible from the leak checker. */
5797 /* MC_(clo_show_reachable) = True; */
5798 MC_(clo_leak_check) = LC_Full;
5801 tl_assert( MC_(clo_mc_level) >= 1 && MC_(clo_mc_level) <= 3 );
5803 if (MC_(clo_mc_level) == 3) {
5804 /* We're doing origin tracking. */
5805 # ifdef PERF_FAST_STACK
5806 VG_(track_new_mem_stack_4_w_ECU) ( mc_new_mem_stack_4_w_ECU );
5807 VG_(track_new_mem_stack_8_w_ECU) ( mc_new_mem_stack_8_w_ECU );
5808 VG_(track_new_mem_stack_12_w_ECU) ( mc_new_mem_stack_12_w_ECU );
5809 VG_(track_new_mem_stack_16_w_ECU) ( mc_new_mem_stack_16_w_ECU );
5810 VG_(track_new_mem_stack_32_w_ECU) ( mc_new_mem_stack_32_w_ECU );
5811 VG_(track_new_mem_stack_112_w_ECU) ( mc_new_mem_stack_112_w_ECU );
5812 VG_(track_new_mem_stack_128_w_ECU) ( mc_new_mem_stack_128_w_ECU );
5813 VG_(track_new_mem_stack_144_w_ECU) ( mc_new_mem_stack_144_w_ECU );
5814 VG_(track_new_mem_stack_160_w_ECU) ( mc_new_mem_stack_160_w_ECU );
5816 VG_(track_new_mem_stack_w_ECU) ( mc_new_mem_stack_w_ECU );
5818 /* Not doing origin tracking */
5819 # ifdef PERF_FAST_STACK
5820 VG_(track_new_mem_stack_4) ( mc_new_mem_stack_4 );
5821 VG_(track_new_mem_stack_8) ( mc_new_mem_stack_8 );
5822 VG_(track_new_mem_stack_12) ( mc_new_mem_stack_12 );
5823 VG_(track_new_mem_stack_16) ( mc_new_mem_stack_16 );
5824 VG_(track_new_mem_stack_32) ( mc_new_mem_stack_32 );
5825 VG_(track_new_mem_stack_112) ( mc_new_mem_stack_112 );
5826 VG_(track_new_mem_stack_128) ( mc_new_mem_stack_128 );
5827 VG_(track_new_mem_stack_144) ( mc_new_mem_stack_144 );
5828 VG_(track_new_mem_stack_160) ( mc_new_mem_stack_160 );
5830 VG_(track_new_mem_stack) ( mc_new_mem_stack );
5833 /* This origin tracking cache is huge (~100M), so only initialise
5835 if (MC_(clo_mc_level) >= 3) {
5837 tl_assert(ocacheL1 != NULL);
5838 tl_assert(ocacheL2 != NULL);
5840 tl_assert(ocacheL1 == NULL);
5841 tl_assert(ocacheL2 == NULL);
5845 static void print_SM_info(char* type, int n_SMs)
5847 VG_(message)(Vg_DebugMsg,
5848 " memcheck: SMs: %s = %d (%ldk, %ldM)\n",
5851 n_SMs * sizeof(SecMap) / 1024UL,
5852 n_SMs * sizeof(SecMap) / (1024 * 1024UL) );
5855 static void mc_fini ( Int exitcode )
5857 MC_(print_malloc_stats)();
5859 if (MC_(clo_leak_check) != LC_Off) {
5860 MC_(detect_memory_leaks)(1/*bogus ThreadId*/, MC_(clo_leak_check));
5862 if (VG_(clo_verbosity) == 1 && !VG_(clo_xml)) {
5864 "For a detailed leak analysis, rerun with: --leak-check=full\n"
5870 if (VG_(clo_verbosity) == 1 && !VG_(clo_xml)) {
5871 VG_(message)(Vg_UserMsg,
5872 "For counts of detected and suppressed errors, rerun with: -v\n");
5875 if (MC_(any_value_errors) && !VG_(clo_xml) && VG_(clo_verbosity) >= 1
5876 && MC_(clo_mc_level) == 2) {
5877 VG_(message)(Vg_UserMsg,
5878 "Use --track-origins=yes to see where "
5879 "uninitialised values come from\n");
5884 if (VG_(clo_stats)) {
5885 SizeT max_secVBit_szB, max_SMs_szB, max_shmem_szB;
5887 VG_(message)(Vg_DebugMsg,
5888 " memcheck: sanity checks: %d cheap, %d expensive\n",
5889 n_sanity_cheap, n_sanity_expensive );
5890 VG_(message)(Vg_DebugMsg,
5891 " memcheck: auxmaps: %lld auxmap entries (%lldk, %lldM) in use\n",
5893 n_auxmap_L2_nodes * 64,
5894 n_auxmap_L2_nodes / 16 );
5895 VG_(message)(Vg_DebugMsg,
5896 " memcheck: auxmaps_L1: %lld searches, %lld cmps, ratio %lld:10\n",
5897 n_auxmap_L1_searches, n_auxmap_L1_cmps,
5898 (10ULL * n_auxmap_L1_cmps)
5899 / (n_auxmap_L1_searches ? n_auxmap_L1_searches : 1)
5901 VG_(message)(Vg_DebugMsg,
5902 " memcheck: auxmaps_L2: %lld searches, %lld nodes\n",
5903 n_auxmap_L2_searches, n_auxmap_L2_nodes
5906 print_SM_info("n_issued ", n_issued_SMs);
5907 print_SM_info("n_deissued ", n_deissued_SMs);
5908 print_SM_info("max_noaccess ", max_noaccess_SMs);
5909 print_SM_info("max_undefined", max_undefined_SMs);
5910 print_SM_info("max_defined ", max_defined_SMs);
5911 print_SM_info("max_non_DSM ", max_non_DSM_SMs);
5913 // Three DSMs, plus the non-DSM ones
5914 max_SMs_szB = (3 + max_non_DSM_SMs) * sizeof(SecMap);
5915 // The 3*sizeof(Word) bytes is the AVL node metadata size.
5916 // The 4*sizeof(Word) bytes is the malloc metadata size.
5917 // Hardwiring these sizes in sucks, but I don't see how else to do it.
5918 max_secVBit_szB = max_secVBit_nodes *
5919 (sizeof(SecVBitNode) + 3*sizeof(Word) + 4*sizeof(Word));
5920 max_shmem_szB = sizeof(primary_map) + max_SMs_szB + max_secVBit_szB;
5922 VG_(message)(Vg_DebugMsg,
5923 " memcheck: max sec V bit nodes: %d (%ldk, %ldM)\n",
5924 max_secVBit_nodes, max_secVBit_szB / 1024,
5925 max_secVBit_szB / (1024 * 1024));
5926 VG_(message)(Vg_DebugMsg,
5927 " memcheck: set_sec_vbits8 calls: %llu (new: %llu, updates: %llu)\n",
5928 sec_vbits_new_nodes + sec_vbits_updates,
5929 sec_vbits_new_nodes, sec_vbits_updates );
5930 VG_(message)(Vg_DebugMsg,
5931 " memcheck: max shadow mem size: %ldk, %ldM\n",
5932 max_shmem_szB / 1024, max_shmem_szB / (1024 * 1024));
5934 if (MC_(clo_mc_level) >= 3) {
5935 VG_(message)(Vg_DebugMsg,
5936 " ocacheL1: %'12lu refs %'12lu misses (%'lu lossage)\n",
5937 stats_ocacheL1_find,
5938 stats_ocacheL1_misses,
5939 stats_ocacheL1_lossage );
5940 VG_(message)(Vg_DebugMsg,
5941 " ocacheL1: %'12lu at 0 %'12lu at 1\n",
5942 stats_ocacheL1_find - stats_ocacheL1_misses
5943 - stats_ocacheL1_found_at_1
5944 - stats_ocacheL1_found_at_N,
5945 stats_ocacheL1_found_at_1 );
5946 VG_(message)(Vg_DebugMsg,
5947 " ocacheL1: %'12lu at 2+ %'12lu move-fwds\n",
5948 stats_ocacheL1_found_at_N,
5949 stats_ocacheL1_movefwds );
5950 VG_(message)(Vg_DebugMsg,
5951 " ocacheL1: %'12lu sizeB %'12u useful\n",
5952 (UWord)sizeof(OCache),
5953 4 * OC_W32S_PER_LINE * OC_LINES_PER_SET * OC_N_SETS );
5954 VG_(message)(Vg_DebugMsg,
5955 " ocacheL2: %'12lu refs %'12lu misses\n",
5956 stats__ocacheL2_refs,
5957 stats__ocacheL2_misses );
5958 VG_(message)(Vg_DebugMsg,
5959 " ocacheL2: %'9lu max nodes %'9lu curr nodes\n",
5960 stats__ocacheL2_n_nodes_max,
5961 stats__ocacheL2_n_nodes );
5962 VG_(message)(Vg_DebugMsg,
5963 " niacache: %'12lu refs %'12lu misses\n",
5964 stats__nia_cache_queries, stats__nia_cache_misses);
5966 tl_assert(ocacheL1 == NULL);
5967 tl_assert(ocacheL2 == NULL);
5972 VG_(message)(Vg_DebugMsg,
5973 "------ Valgrind's client block stats follow ---------------\n" );
5974 show_client_block_stats();
5978 /* mark the given addr/len unaddressable for watchpoint implementation
5979 The PointKind will be handled at access time */
5980 static Bool mc_mark_unaddressable_for_watchpoint (PointKind kind, Bool insert,
5981 Addr addr, SizeT len)
5983 /* GDBTD this is somewhat fishy. We might rather have to save the previous
5984 accessibility and definedness in gdbserver so as to allow restoring it
5985 properly. Currently, we assume that the user only watches things
5986 which are properly addressable and defined */
5988 MC_(make_mem_noaccess) (addr, len);
5990 MC_(make_mem_defined) (addr, len);
5994 static void mc_pre_clo_init(void)
5996 VG_(details_name) ("Memcheck");
5997 VG_(details_version) (NULL);
5998 VG_(details_description) ("a memory error detector");
5999 VG_(details_copyright_author)(
6000 "Copyright (C) 2002-2010, and GNU GPL'd, by Julian Seward et al.");
6001 VG_(details_bug_reports_to) (VG_BUGS_TO);
6002 VG_(details_avg_translation_sizeB) ( 640 );
6004 VG_(basic_tool_funcs) (mc_post_clo_init,
6008 VG_(needs_final_IR_tidy_pass) ( MC_(final_tidy) );
6011 VG_(needs_core_errors) ();
6012 VG_(needs_tool_errors) (MC_(eq_Error),
6013 MC_(before_pp_Error),
6015 True,/*show TIDs for errors*/
6016 MC_(update_Error_extra),
6017 MC_(is_recognised_suppression),
6018 MC_(read_extra_suppression_info),
6019 MC_(error_matches_suppression),
6020 MC_(get_error_name),
6021 MC_(get_extra_suppression_info));
6022 VG_(needs_libc_freeres) ();
6023 VG_(needs_command_line_options)(mc_process_cmd_line_options,
6025 mc_print_debug_usage);
6026 VG_(needs_client_requests) (mc_handle_client_request);
6027 VG_(needs_sanity_checks) (mc_cheap_sanity_check,
6028 mc_expensive_sanity_check);
6029 VG_(needs_malloc_replacement) (MC_(malloc),
6031 MC_(__builtin_vec_new),
6035 MC_(__builtin_delete),
6036 MC_(__builtin_vec_delete),
6038 MC_(malloc_usable_size),
6039 MC_MALLOC_REDZONE_SZB );
6041 VG_(needs_xml_output) ();
6043 VG_(track_new_mem_startup) ( mc_new_mem_startup );
6044 VG_(track_new_mem_stack_signal)( make_mem_undefined_w_tid );
6045 // We assume that brk()/sbrk() does not initialise new memory. Is this
6046 // accurate? John Reiser says:
6048 // 0) sbrk() can *decrease* process address space. No zero fill is done
6049 // for a decrease, not even the fragment on the high end of the last page
6050 // that is beyond the new highest address. For maximum safety and
6051 // portability, then the bytes in the last page that reside above [the
6052 // new] sbrk(0) should be considered to be uninitialized, but in practice
6053 // it is exceedingly likely that they will retain their previous
6056 // 1) If an increase is large enough to require new whole pages, then
6057 // those new whole pages (like all new pages) are zero-filled by the
6058 // operating system. So if sbrk(0) already is page aligned, then
6059 // sbrk(PAGE_SIZE) *does* zero-fill the new memory.
6061 // 2) Any increase that lies within an existing allocated page is not
6062 // changed. So if (x = sbrk(0)) is not page aligned, then
6063 // sbrk(PAGE_SIZE) yields ((PAGE_SIZE -1) & -x) bytes which keep their
6064 // existing contents, and an additional PAGE_SIZE bytes which are zeroed.
6065 // ((PAGE_SIZE -1) & x) of them are "covered" by the sbrk(), and the rest
6066 // of them come along for the ride because the operating system deals
6067 // only in whole pages. Again, for maximum safety and portability, then
6068 // anything that lives above [the new] sbrk(0) should be considered
6069 // uninitialized, but in practice will retain previous contents [zero in
6074 // A key property of sbrk/brk is that new whole pages that are supplied
6075 // by the operating system *do* get initialized to zero.
6077 // As for the portability of all this:
6079 // sbrk and brk are not POSIX. However, any system that is a derivative
6080 // of *nix has sbrk and brk because there are too many softwares (such as
6081 // the Bourne shell) which rely on the traditional memory map (.text,
6082 // .data+.bss, stack) and the existence of sbrk/brk.
6084 // So we should arguably observe all this. However:
6085 // - The current inaccuracy has caused maybe one complaint in seven years(?)
6086 // - Relying on the zeroed-ness of whole brk'd pages is pretty grotty... I
6087 // doubt most programmers know the above information.
6088 // So I'm not terribly unhappy with marking it as undefined. --njn.
6090 // [More: I think most of what John said only applies to sbrk(). It seems
6091 // that brk() always deals in whole pages. And since this event deals
6092 // directly with brk(), not with sbrk(), perhaps it would be reasonable to
6093 // just mark all memory it allocates as defined.]
6095 VG_(track_new_mem_brk) ( make_mem_undefined_w_tid );
6097 // Handling of mmap and mprotect isn't simple (well, it is simple,
6098 // but the justification isn't.) See comments above, just prior to
6100 VG_(track_new_mem_mmap) ( mc_new_mem_mmap );
6101 VG_(track_change_mem_mprotect) ( mc_new_mem_mprotect );
6103 VG_(track_copy_mem_remap) ( MC_(copy_address_range_state) );
6105 VG_(track_die_mem_stack_signal)( MC_(make_mem_noaccess) );
6106 VG_(track_die_mem_brk) ( MC_(make_mem_noaccess) );
6107 VG_(track_die_mem_munmap) ( MC_(make_mem_noaccess) );
6109 /* Defer the specification of the new_mem_stack functions to the
6110 post_clo_init function, since we need to first parse the command
6111 line before deciding which set to use. */
6113 # ifdef PERF_FAST_STACK
6114 VG_(track_die_mem_stack_4) ( mc_die_mem_stack_4 );
6115 VG_(track_die_mem_stack_8) ( mc_die_mem_stack_8 );
6116 VG_(track_die_mem_stack_12) ( mc_die_mem_stack_12 );
6117 VG_(track_die_mem_stack_16) ( mc_die_mem_stack_16 );
6118 VG_(track_die_mem_stack_32) ( mc_die_mem_stack_32 );
6119 VG_(track_die_mem_stack_112) ( mc_die_mem_stack_112 );
6120 VG_(track_die_mem_stack_128) ( mc_die_mem_stack_128 );
6121 VG_(track_die_mem_stack_144) ( mc_die_mem_stack_144 );
6122 VG_(track_die_mem_stack_160) ( mc_die_mem_stack_160 );
6124 VG_(track_die_mem_stack) ( mc_die_mem_stack );
6126 VG_(track_ban_mem_stack) ( MC_(make_mem_noaccess) );
6128 VG_(track_pre_mem_read) ( check_mem_is_defined );
6129 VG_(track_pre_mem_read_asciiz) ( check_mem_is_defined_asciiz );
6130 VG_(track_pre_mem_write) ( check_mem_is_addressable );
6131 VG_(track_post_mem_write) ( mc_post_mem_write );
6133 if (MC_(clo_mc_level) >= 2)
6134 VG_(track_pre_reg_read) ( mc_pre_reg_read );
6136 VG_(track_post_reg_write) ( mc_post_reg_write );
6137 VG_(track_post_reg_write_clientcall_return)( mc_post_reg_write_clientcall );
6139 VG_(needs_watchpoint) ( mc_mark_unaddressable_for_watchpoint );
6141 init_shadow_memory();
6142 MC_(malloc_list) = VG_(HT_construct)( "MC_(malloc_list)" );
6143 MC_(mempool_list) = VG_(HT_construct)( "MC_(mempool_list)" );
6146 tl_assert( mc_expensive_sanity_check() );
6148 // {LOADV,STOREV}[8421] will all fail horribly if this isn't true.
6149 tl_assert(sizeof(UWord) == sizeof(Addr));
6150 // Call me paranoid. I don't care.
6151 tl_assert(sizeof(void*) == sizeof(Addr));
6153 // BYTES_PER_SEC_VBIT_NODE must be a power of two.
6154 tl_assert(-1 != VG_(log2)(BYTES_PER_SEC_VBIT_NODE));
6156 /* This is small. Always initialise it. */
6157 init_nia_to_ecu_cache();
6159 /* We can't initialise ocacheL1/ocacheL2 yet, since we don't know
6160 if we need to, since the command line args haven't been
6161 processed yet. Hence defer it to mc_post_clo_init. */
6162 tl_assert(ocacheL1 == NULL);
6163 tl_assert(ocacheL2 == NULL);
6165 /* Check some important stuff. See extensive comments above
6166 re UNALIGNED_OR_HIGH for background. */
6167 # if VG_WORDSIZE == 4
6168 tl_assert(sizeof(void*) == 4);
6169 tl_assert(sizeof(Addr) == 4);
6170 tl_assert(sizeof(UWord) == 4);
6171 tl_assert(sizeof(Word) == 4);
6172 tl_assert(MAX_PRIMARY_ADDRESS == 0xFFFFFFFFUL);
6173 tl_assert(MASK(1) == 0UL);
6174 tl_assert(MASK(2) == 1UL);
6175 tl_assert(MASK(4) == 3UL);
6176 tl_assert(MASK(8) == 7UL);
6178 tl_assert(VG_WORDSIZE == 8);
6179 tl_assert(sizeof(void*) == 8);
6180 tl_assert(sizeof(Addr) == 8);
6181 tl_assert(sizeof(UWord) == 8);
6182 tl_assert(sizeof(Word) == 8);
6183 tl_assert(MAX_PRIMARY_ADDRESS == 0x7FFFFFFFFULL);
6184 tl_assert(MASK(1) == 0xFFFFFFF800000000ULL);
6185 tl_assert(MASK(2) == 0xFFFFFFF800000001ULL);
6186 tl_assert(MASK(4) == 0xFFFFFFF800000003ULL);
6187 tl_assert(MASK(8) == 0xFFFFFFF800000007ULL);
6191 VG_DETERMINE_INTERFACE_VERSION(mc_pre_clo_init)
6193 /*--------------------------------------------------------------------*/
6194 /*--- end mc_main.c ---*/
6195 /*--------------------------------------------------------------------*/