4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
23 #include <sys/types.h>
27 #include "qemu-common.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "sysemu/sysemu.h"
35 #include "hw/xen/xen.h"
36 #include "qemu/timer.h"
37 #include "qemu/config-file.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
47 #include "exec/cpu-all.h"
49 #include "exec/cputlb.h"
50 #include "translate-all.h"
52 #include "exec/memory-internal.h"
54 //#define DEBUG_SUBPAGE
56 #if !defined(CONFIG_USER_ONLY)
57 static int in_migration;
59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
67 MemoryRegion io_mem_rom, io_mem_notdirty;
68 static MemoryRegion io_mem_unassigned;
72 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
73 /* current CPU in the current thread. It is only valid inside
75 DEFINE_TLS(CPUState *, current_cpu);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
81 #if !defined(CONFIG_USER_ONLY)
83 typedef struct PhysPageEntry PhysPageEntry;
85 struct PhysPageEntry {
86 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
88 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
92 /* Size of the L2 (and L3, etc) page tables. */
93 #define ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
96 #define P_L2_SIZE (1 << P_L2_BITS)
98 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
100 typedef PhysPageEntry Node[P_L2_SIZE];
102 typedef struct PhysPageMap {
103 unsigned sections_nb;
104 unsigned sections_nb_alloc;
106 unsigned nodes_nb_alloc;
108 MemoryRegionSection *sections;
111 struct AddressSpaceDispatch {
112 /* This is a multi-level map on the physical address space.
113 * The bottom level has pointers to MemoryRegionSections.
115 PhysPageEntry phys_map;
120 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
121 typedef struct subpage_t {
125 uint16_t sub_section[TARGET_PAGE_SIZE];
128 #define PHYS_SECTION_UNASSIGNED 0
129 #define PHYS_SECTION_NOTDIRTY 1
130 #define PHYS_SECTION_ROM 2
131 #define PHYS_SECTION_WATCH 3
133 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
135 static void io_mem_init(void);
136 static void memory_map_init(void);
138 static MemoryRegion io_mem_watch;
141 #if !defined(CONFIG_USER_ONLY)
143 static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
145 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
146 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc * 2, 16);
147 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
148 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
152 static uint16_t phys_map_node_alloc(PhysPageMap *map)
157 ret = map->nodes_nb++;
158 assert(ret != PHYS_MAP_NODE_NIL);
159 assert(ret != map->nodes_nb_alloc);
160 for (i = 0; i < P_L2_SIZE; ++i) {
161 map->nodes[ret][i].skip = 1;
162 map->nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
167 static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
168 hwaddr *index, hwaddr *nb, uint16_t leaf,
173 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
175 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
176 lp->ptr = phys_map_node_alloc(map);
177 p = map->nodes[lp->ptr];
179 for (i = 0; i < P_L2_SIZE; i++) {
181 p[i].ptr = PHYS_SECTION_UNASSIGNED;
185 p = map->nodes[lp->ptr];
187 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
189 while (*nb && lp < &p[P_L2_SIZE]) {
190 if ((*index & (step - 1)) == 0 && *nb >= step) {
196 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
202 static void phys_page_set(AddressSpaceDispatch *d,
203 hwaddr index, hwaddr nb,
206 /* Wildly overreserve - it doesn't matter much. */
207 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
209 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
212 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,
213 Node *nodes, MemoryRegionSection *sections)
216 hwaddr index = addr >> TARGET_PAGE_BITS;
219 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
220 if (lp.ptr == PHYS_MAP_NODE_NIL) {
221 return §ions[PHYS_SECTION_UNASSIGNED];
224 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
226 return §ions[lp.ptr];
229 bool memory_region_is_unassigned(MemoryRegion *mr)
231 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
232 && mr != &io_mem_watch;
235 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
237 bool resolve_subpage)
239 MemoryRegionSection *section;
242 section = phys_page_find(d->phys_map, addr, d->map.nodes, d->map.sections);
243 if (resolve_subpage && section->mr->subpage) {
244 subpage = container_of(section->mr, subpage_t, iomem);
245 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
250 static MemoryRegionSection *
251 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
252 hwaddr *plen, bool resolve_subpage)
254 MemoryRegionSection *section;
257 section = address_space_lookup_region(d, addr, resolve_subpage);
258 /* Compute offset within MemoryRegionSection */
259 addr -= section->offset_within_address_space;
261 /* Compute offset within MemoryRegion */
262 *xlat = addr + section->offset_within_region;
264 diff = int128_sub(section->mr->size, int128_make64(addr));
265 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
269 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
271 if (memory_region_is_ram(mr)) {
272 return !(is_write && mr->readonly);
274 if (memory_region_is_romd(mr)) {
281 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
282 hwaddr *xlat, hwaddr *plen,
286 MemoryRegionSection *section;
291 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
294 if (!mr->iommu_ops) {
298 iotlb = mr->iommu_ops->translate(mr, addr);
299 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
300 | (addr & iotlb.addr_mask));
301 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
302 if (!(iotlb.perm & (1 << is_write))) {
303 mr = &io_mem_unassigned;
307 as = iotlb.target_as;
310 if (memory_access_is_direct(mr, is_write)) {
311 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
312 len = MIN(page, len);
320 MemoryRegionSection *
321 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
324 MemoryRegionSection *section;
325 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
327 assert(!section->mr->iommu_ops);
332 void cpu_exec_init_all(void)
334 #if !defined(CONFIG_USER_ONLY)
335 qemu_mutex_init(&ram_list.mutex);
341 #if !defined(CONFIG_USER_ONLY)
343 static int cpu_common_post_load(void *opaque, int version_id)
345 CPUState *cpu = opaque;
347 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
348 version_id is increased. */
349 cpu->interrupt_request &= ~0x01;
350 tlb_flush(cpu->env_ptr, 1);
355 const VMStateDescription vmstate_cpu_common = {
356 .name = "cpu_common",
358 .minimum_version_id = 1,
359 .minimum_version_id_old = 1,
360 .post_load = cpu_common_post_load,
361 .fields = (VMStateField []) {
362 VMSTATE_UINT32(halted, CPUState),
363 VMSTATE_UINT32(interrupt_request, CPUState),
364 VMSTATE_END_OF_LIST()
370 CPUState *qemu_get_cpu(int index)
375 if (cpu->cpu_index == index) {
383 void cpu_exec_init(CPUArchState *env)
385 CPUState *cpu = ENV_GET_CPU(env);
386 CPUClass *cc = CPU_GET_CLASS(cpu);
390 #if defined(CONFIG_USER_ONLY)
394 CPU_FOREACH(some_cpu) {
397 cpu->cpu_index = cpu_index;
399 QTAILQ_INIT(&env->breakpoints);
400 QTAILQ_INIT(&env->watchpoints);
401 #ifndef CONFIG_USER_ONLY
402 cpu->thread_id = qemu_get_thread_id();
404 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
405 #if defined(CONFIG_USER_ONLY)
408 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
409 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
411 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
412 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
413 cpu_save, cpu_load, env);
414 assert(cc->vmsd == NULL);
415 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
417 if (cc->vmsd != NULL) {
418 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
422 #if defined(TARGET_HAS_ICE)
423 #if defined(CONFIG_USER_ONLY)
424 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
426 tb_invalidate_phys_page_range(pc, pc + 1, 0);
429 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
431 hwaddr phys = cpu_get_phys_page_debug(cpu, pc);
433 tb_invalidate_phys_addr(phys | (pc & ~TARGET_PAGE_MASK));
437 #endif /* TARGET_HAS_ICE */
439 #if defined(CONFIG_USER_ONLY)
440 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
445 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
446 int flags, CPUWatchpoint **watchpoint)
451 /* Add a watchpoint. */
452 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
453 int flags, CPUWatchpoint **watchpoint)
455 target_ulong len_mask = ~(len - 1);
458 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
459 if ((len & (len - 1)) || (addr & ~len_mask) ||
460 len == 0 || len > TARGET_PAGE_SIZE) {
461 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
462 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
465 wp = g_malloc(sizeof(*wp));
468 wp->len_mask = len_mask;
471 /* keep all GDB-injected watchpoints in front */
473 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
475 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
477 tlb_flush_page(env, addr);
484 /* Remove a specific watchpoint. */
485 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
488 target_ulong len_mask = ~(len - 1);
491 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
492 if (addr == wp->vaddr && len_mask == wp->len_mask
493 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
494 cpu_watchpoint_remove_by_ref(env, wp);
501 /* Remove a specific watchpoint by reference. */
502 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
504 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
506 tlb_flush_page(env, watchpoint->vaddr);
511 /* Remove all matching watchpoints. */
512 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
514 CPUWatchpoint *wp, *next;
516 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
517 if (wp->flags & mask)
518 cpu_watchpoint_remove_by_ref(env, wp);
523 /* Add a breakpoint. */
524 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
525 CPUBreakpoint **breakpoint)
527 #if defined(TARGET_HAS_ICE)
530 bp = g_malloc(sizeof(*bp));
535 /* keep all GDB-injected breakpoints in front */
536 if (flags & BP_GDB) {
537 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
539 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
542 breakpoint_invalidate(ENV_GET_CPU(env), pc);
553 /* Remove a specific breakpoint. */
554 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
556 #if defined(TARGET_HAS_ICE)
559 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
560 if (bp->pc == pc && bp->flags == flags) {
561 cpu_breakpoint_remove_by_ref(env, bp);
571 /* Remove a specific breakpoint by reference. */
572 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
574 #if defined(TARGET_HAS_ICE)
575 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
577 breakpoint_invalidate(ENV_GET_CPU(env), breakpoint->pc);
583 /* Remove all matching breakpoints. */
584 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
586 #if defined(TARGET_HAS_ICE)
587 CPUBreakpoint *bp, *next;
589 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
590 if (bp->flags & mask)
591 cpu_breakpoint_remove_by_ref(env, bp);
596 /* enable or disable single step mode. EXCP_DEBUG is returned by the
597 CPU loop after each instruction */
598 void cpu_single_step(CPUState *cpu, int enabled)
600 #if defined(TARGET_HAS_ICE)
601 if (cpu->singlestep_enabled != enabled) {
602 cpu->singlestep_enabled = enabled;
604 kvm_update_guest_debug(cpu, 0);
606 /* must flush all the translated code to avoid inconsistencies */
607 /* XXX: only flush what is necessary */
608 CPUArchState *env = cpu->env_ptr;
615 void cpu_abort(CPUArchState *env, const char *fmt, ...)
617 CPUState *cpu = ENV_GET_CPU(env);
623 fprintf(stderr, "qemu: fatal: ");
624 vfprintf(stderr, fmt, ap);
625 fprintf(stderr, "\n");
626 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
627 if (qemu_log_enabled()) {
628 qemu_log("qemu: fatal: ");
629 qemu_log_vprintf(fmt, ap2);
631 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
637 #if defined(CONFIG_USER_ONLY)
639 struct sigaction act;
640 sigfillset(&act.sa_mask);
641 act.sa_handler = SIG_DFL;
642 sigaction(SIGABRT, &act, NULL);
648 #if !defined(CONFIG_USER_ONLY)
649 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
653 /* The list is protected by the iothread lock here. */
654 block = ram_list.mru_block;
655 if (block && addr - block->offset < block->length) {
658 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
659 if (addr - block->offset < block->length) {
664 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
668 ram_list.mru_block = block;
672 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
678 block = qemu_get_ram_block(start);
679 assert(block == qemu_get_ram_block(end - 1));
680 start1 = (uintptr_t)block->host + (start - block->offset);
681 cpu_tlb_reset_dirty_all(start1, length);
684 /* Note: start and end must be within the same ram block. */
685 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
690 start &= TARGET_PAGE_MASK;
691 end = TARGET_PAGE_ALIGN(end);
693 length = end - start;
696 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
699 tlb_reset_dirty_range_all(start, end, length);
703 static int cpu_physical_memory_set_dirty_tracking(int enable)
706 in_migration = enable;
710 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
711 MemoryRegionSection *section,
713 hwaddr paddr, hwaddr xlat,
715 target_ulong *address)
720 if (memory_region_is_ram(section->mr)) {
722 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
724 if (!section->readonly) {
725 iotlb |= PHYS_SECTION_NOTDIRTY;
727 iotlb |= PHYS_SECTION_ROM;
730 iotlb = section - address_space_memory.dispatch->map.sections;
734 /* Make accesses to pages with watchpoints go via the
735 watchpoint trap routines. */
736 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
737 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
738 /* Avoid trapping reads of pages with a write breakpoint. */
739 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
740 iotlb = PHYS_SECTION_WATCH + paddr;
741 *address |= TLB_MMIO;
749 #endif /* defined(CONFIG_USER_ONLY) */
751 #if !defined(CONFIG_USER_ONLY)
753 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
755 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
757 static void *(*phys_mem_alloc)(size_t size) = qemu_anon_ram_alloc;
760 * Set a custom physical guest memory alloator.
761 * Accelerators with unusual needs may need this. Hopefully, we can
762 * get rid of it eventually.
764 void phys_mem_set_alloc(void *(*alloc)(size_t))
766 phys_mem_alloc = alloc;
769 static uint16_t phys_section_add(PhysPageMap *map,
770 MemoryRegionSection *section)
772 /* The physical section number is ORed with a page-aligned
773 * pointer to produce the iotlb entries. Thus it should
774 * never overflow into the page-aligned value.
776 assert(map->sections_nb < TARGET_PAGE_SIZE);
778 if (map->sections_nb == map->sections_nb_alloc) {
779 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
780 map->sections = g_renew(MemoryRegionSection, map->sections,
781 map->sections_nb_alloc);
783 map->sections[map->sections_nb] = *section;
784 memory_region_ref(section->mr);
785 return map->sections_nb++;
788 static void phys_section_destroy(MemoryRegion *mr)
790 memory_region_unref(mr);
793 subpage_t *subpage = container_of(mr, subpage_t, iomem);
794 memory_region_destroy(&subpage->iomem);
799 static void phys_sections_free(PhysPageMap *map)
801 while (map->sections_nb > 0) {
802 MemoryRegionSection *section = &map->sections[--map->sections_nb];
803 phys_section_destroy(section->mr);
805 g_free(map->sections);
809 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
812 hwaddr base = section->offset_within_address_space
814 MemoryRegionSection *existing = phys_page_find(d->phys_map, base,
815 d->map.nodes, d->map.sections);
816 MemoryRegionSection subsection = {
817 .offset_within_address_space = base,
818 .size = int128_make64(TARGET_PAGE_SIZE),
822 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
824 if (!(existing->mr->subpage)) {
825 subpage = subpage_init(d->as, base);
826 subsection.mr = &subpage->iomem;
827 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
828 phys_section_add(&d->map, &subsection));
830 subpage = container_of(existing->mr, subpage_t, iomem);
832 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
833 end = start + int128_get64(section->size) - 1;
834 subpage_register(subpage, start, end,
835 phys_section_add(&d->map, section));
839 static void register_multipage(AddressSpaceDispatch *d,
840 MemoryRegionSection *section)
842 hwaddr start_addr = section->offset_within_address_space;
843 uint16_t section_index = phys_section_add(&d->map, section);
844 uint64_t num_pages = int128_get64(int128_rshift(section->size,
848 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
851 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
853 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
854 AddressSpaceDispatch *d = as->next_dispatch;
855 MemoryRegionSection now = *section, remain = *section;
856 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
858 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
859 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
860 - now.offset_within_address_space;
862 now.size = int128_min(int128_make64(left), now.size);
863 register_subpage(d, &now);
865 now.size = int128_zero();
867 while (int128_ne(remain.size, now.size)) {
868 remain.size = int128_sub(remain.size, now.size);
869 remain.offset_within_address_space += int128_get64(now.size);
870 remain.offset_within_region += int128_get64(now.size);
872 if (int128_lt(remain.size, page_size)) {
873 register_subpage(d, &now);
874 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
875 now.size = page_size;
876 register_subpage(d, &now);
878 now.size = int128_and(now.size, int128_neg(page_size));
879 register_multipage(d, &now);
884 void qemu_flush_coalesced_mmio_buffer(void)
887 kvm_flush_coalesced_mmio_buffer();
890 void qemu_mutex_lock_ramlist(void)
892 qemu_mutex_lock(&ram_list.mutex);
895 void qemu_mutex_unlock_ramlist(void)
897 qemu_mutex_unlock(&ram_list.mutex);
904 #define HUGETLBFS_MAGIC 0x958458f6
906 static long gethugepagesize(const char *path)
912 ret = statfs(path, &fs);
913 } while (ret != 0 && errno == EINTR);
920 if (fs.f_type != HUGETLBFS_MAGIC)
921 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
926 static void *file_ram_alloc(RAMBlock *block,
931 char *sanitized_name;
938 unsigned long hpagesize;
940 hpagesize = gethugepagesize(path);
945 if (memory < hpagesize) {
949 if (kvm_enabled() && !kvm_has_sync_mmu()) {
950 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
954 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
955 sanitized_name = g_strdup(block->mr->name);
956 for (c = sanitized_name; *c != '\0'; c++) {
961 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
963 g_free(sanitized_name);
965 fd = mkstemp(filename);
967 perror("unable to create backing store for hugepages");
974 memory = (memory+hpagesize-1) & ~(hpagesize-1);
977 * ftruncate is not supported by hugetlbfs in older
978 * hosts, so don't bother bailing out on errors.
979 * If anything goes wrong with it under other filesystems,
982 if (ftruncate(fd, memory))
986 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
987 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
988 * to sidestep this quirk.
990 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
991 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
993 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
995 if (area == MAP_FAILED) {
996 perror("file_ram_alloc: can't mmap RAM pages");
1004 static void *file_ram_alloc(RAMBlock *block,
1008 fprintf(stderr, "-mem-path not supported on this host\n");
1013 static ram_addr_t find_ram_offset(ram_addr_t size)
1015 RAMBlock *block, *next_block;
1016 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1018 assert(size != 0); /* it would hand out same offset multiple times */
1020 if (QTAILQ_EMPTY(&ram_list.blocks))
1023 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1024 ram_addr_t end, next = RAM_ADDR_MAX;
1026 end = block->offset + block->length;
1028 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1029 if (next_block->offset >= end) {
1030 next = MIN(next, next_block->offset);
1033 if (next - end >= size && next - end < mingap) {
1035 mingap = next - end;
1039 if (offset == RAM_ADDR_MAX) {
1040 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1048 ram_addr_t last_ram_offset(void)
1051 ram_addr_t last = 0;
1053 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1054 last = MAX(last, block->offset + block->length);
1059 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1063 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1064 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1065 "dump-guest-core", true)) {
1066 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1068 perror("qemu_madvise");
1069 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1070 "but dump_guest_core=off specified\n");
1075 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1077 RAMBlock *new_block, *block;
1080 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1081 if (block->offset == addr) {
1087 assert(!new_block->idstr[0]);
1090 char *id = qdev_get_dev_path(dev);
1092 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1096 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1098 /* This assumes the iothread lock is taken here too. */
1099 qemu_mutex_lock_ramlist();
1100 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1101 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1102 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1107 qemu_mutex_unlock_ramlist();
1110 static int memory_try_enable_merging(void *addr, size_t len)
1112 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1113 /* disabled by the user */
1117 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1120 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1123 RAMBlock *block, *new_block;
1125 size = TARGET_PAGE_ALIGN(size);
1126 new_block = g_malloc0(sizeof(*new_block));
1129 /* This assumes the iothread lock is taken here too. */
1130 qemu_mutex_lock_ramlist();
1132 new_block->offset = find_ram_offset(size);
1134 new_block->host = host;
1135 new_block->flags |= RAM_PREALLOC_MASK;
1136 } else if (xen_enabled()) {
1138 fprintf(stderr, "-mem-path not supported with Xen\n");
1141 xen_ram_alloc(new_block->offset, size, mr);
1144 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1146 * file_ram_alloc() needs to allocate just like
1147 * phys_mem_alloc, but we haven't bothered to provide
1151 "-mem-path not supported with this accelerator\n");
1154 new_block->host = file_ram_alloc(new_block, size, mem_path);
1156 if (!new_block->host) {
1157 new_block->host = phys_mem_alloc(size);
1158 if (!new_block->host) {
1159 fprintf(stderr, "Cannot set up guest memory '%s': %s\n",
1160 new_block->mr->name, strerror(errno));
1163 memory_try_enable_merging(new_block->host, size);
1166 new_block->length = size;
1168 /* Keep the list sorted from biggest to smallest block. */
1169 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1170 if (block->length < new_block->length) {
1175 QTAILQ_INSERT_BEFORE(block, new_block, next);
1177 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1179 ram_list.mru_block = NULL;
1182 qemu_mutex_unlock_ramlist();
1184 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1185 last_ram_offset() >> TARGET_PAGE_BITS);
1186 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1187 0, size >> TARGET_PAGE_BITS);
1188 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1190 qemu_ram_setup_dump(new_block->host, size);
1191 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1192 qemu_madvise(new_block->host, size, QEMU_MADV_DONTFORK);
1195 kvm_setup_guest_memory(new_block->host, size);
1197 return new_block->offset;
1200 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1202 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1205 void qemu_ram_free_from_ptr(ram_addr_t addr)
1209 /* This assumes the iothread lock is taken here too. */
1210 qemu_mutex_lock_ramlist();
1211 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1212 if (addr == block->offset) {
1213 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1214 ram_list.mru_block = NULL;
1220 qemu_mutex_unlock_ramlist();
1223 void qemu_ram_free(ram_addr_t addr)
1227 /* This assumes the iothread lock is taken here too. */
1228 qemu_mutex_lock_ramlist();
1229 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1230 if (addr == block->offset) {
1231 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1232 ram_list.mru_block = NULL;
1234 if (block->flags & RAM_PREALLOC_MASK) {
1236 } else if (xen_enabled()) {
1237 xen_invalidate_map_cache_entry(block->host);
1239 } else if (block->fd >= 0) {
1240 munmap(block->host, block->length);
1244 qemu_anon_ram_free(block->host, block->length);
1250 qemu_mutex_unlock_ramlist();
1255 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1262 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1263 offset = addr - block->offset;
1264 if (offset < block->length) {
1265 vaddr = block->host + offset;
1266 if (block->flags & RAM_PREALLOC_MASK) {
1268 } else if (xen_enabled()) {
1272 munmap(vaddr, length);
1273 if (block->fd >= 0) {
1275 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1278 flags |= MAP_PRIVATE;
1280 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1281 flags, block->fd, offset);
1284 * Remap needs to match alloc. Accelerators that
1285 * set phys_mem_alloc never remap. If they did,
1286 * we'd need a remap hook here.
1288 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1290 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1291 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1294 if (area != vaddr) {
1295 fprintf(stderr, "Could not remap addr: "
1296 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1300 memory_try_enable_merging(vaddr, length);
1301 qemu_ram_setup_dump(vaddr, length);
1307 #endif /* !_WIN32 */
1309 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1310 With the exception of the softmmu code in this file, this should
1311 only be used for local memory (e.g. video ram) that the device owns,
1312 and knows it isn't going to access beyond the end of the block.
1314 It should not be used for general purpose DMA.
1315 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1317 void *qemu_get_ram_ptr(ram_addr_t addr)
1319 RAMBlock *block = qemu_get_ram_block(addr);
1321 if (xen_enabled()) {
1322 /* We need to check if the requested address is in the RAM
1323 * because we don't want to map the entire memory in QEMU.
1324 * In that case just map until the end of the page.
1326 if (block->offset == 0) {
1327 return xen_map_cache(addr, 0, 0);
1328 } else if (block->host == NULL) {
1330 xen_map_cache(block->offset, block->length, 1);
1333 return block->host + (addr - block->offset);
1336 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1337 * but takes a size argument */
1338 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1343 if (xen_enabled()) {
1344 return xen_map_cache(addr, *size, 1);
1348 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1349 if (addr - block->offset < block->length) {
1350 if (addr - block->offset + *size > block->length)
1351 *size = block->length - addr + block->offset;
1352 return block->host + (addr - block->offset);
1356 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1361 /* Some of the softmmu routines need to translate from a host pointer
1362 (typically a TLB entry) back to a ram offset. */
1363 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1366 uint8_t *host = ptr;
1368 if (xen_enabled()) {
1369 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1370 return qemu_get_ram_block(*ram_addr)->mr;
1373 block = ram_list.mru_block;
1374 if (block && block->host && host - block->host < block->length) {
1378 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1379 /* This case append when the block is not mapped. */
1380 if (block->host == NULL) {
1383 if (host - block->host < block->length) {
1391 *ram_addr = block->offset + (host - block->host);
1395 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1396 uint64_t val, unsigned size)
1399 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1400 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1401 tb_invalidate_phys_page_fast(ram_addr, size);
1402 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1406 stb_p(qemu_get_ram_ptr(ram_addr), val);
1409 stw_p(qemu_get_ram_ptr(ram_addr), val);
1412 stl_p(qemu_get_ram_ptr(ram_addr), val);
1417 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1418 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1419 /* we remove the notdirty callback only if the code has been
1421 if (dirty_flags == 0xff) {
1422 CPUArchState *env = current_cpu->env_ptr;
1423 tlb_set_dirty(env, env->mem_io_vaddr);
1427 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1428 unsigned size, bool is_write)
1433 static const MemoryRegionOps notdirty_mem_ops = {
1434 .write = notdirty_mem_write,
1435 .valid.accepts = notdirty_mem_accepts,
1436 .endianness = DEVICE_NATIVE_ENDIAN,
1439 /* Generate a debug exception if a watchpoint has been hit. */
1440 static void check_watchpoint(int offset, int len_mask, int flags)
1442 CPUArchState *env = current_cpu->env_ptr;
1443 target_ulong pc, cs_base;
1448 if (env->watchpoint_hit) {
1449 /* We re-entered the check after replacing the TB. Now raise
1450 * the debug interrupt so that is will trigger after the
1451 * current instruction. */
1452 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1455 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1456 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1457 if ((vaddr == (wp->vaddr & len_mask) ||
1458 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1459 wp->flags |= BP_WATCHPOINT_HIT;
1460 if (!env->watchpoint_hit) {
1461 env->watchpoint_hit = wp;
1462 tb_check_watchpoint(env);
1463 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1464 env->exception_index = EXCP_DEBUG;
1467 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1468 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1469 cpu_resume_from_signal(env, NULL);
1473 wp->flags &= ~BP_WATCHPOINT_HIT;
1478 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1479 so these check for a hit then pass through to the normal out-of-line
1481 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1484 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1486 case 1: return ldub_phys(addr);
1487 case 2: return lduw_phys(addr);
1488 case 4: return ldl_phys(addr);
1493 static void watch_mem_write(void *opaque, hwaddr addr,
1494 uint64_t val, unsigned size)
1496 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1499 stb_phys(addr, val);
1502 stw_phys(addr, val);
1505 stl_phys(addr, val);
1511 static const MemoryRegionOps watch_mem_ops = {
1512 .read = watch_mem_read,
1513 .write = watch_mem_write,
1514 .endianness = DEVICE_NATIVE_ENDIAN,
1517 static uint64_t subpage_read(void *opaque, hwaddr addr,
1520 subpage_t *subpage = opaque;
1523 #if defined(DEBUG_SUBPAGE)
1524 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
1525 subpage, len, addr);
1527 address_space_read(subpage->as, addr + subpage->base, buf, len);
1540 static void subpage_write(void *opaque, hwaddr addr,
1541 uint64_t value, unsigned len)
1543 subpage_t *subpage = opaque;
1546 #if defined(DEBUG_SUBPAGE)
1547 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
1548 " value %"PRIx64"\n",
1549 __func__, subpage, len, addr, value);
1564 address_space_write(subpage->as, addr + subpage->base, buf, len);
1567 static bool subpage_accepts(void *opaque, hwaddr addr,
1568 unsigned len, bool is_write)
1570 subpage_t *subpage = opaque;
1571 #if defined(DEBUG_SUBPAGE)
1572 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
1573 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1576 return address_space_access_valid(subpage->as, addr + subpage->base,
1580 static const MemoryRegionOps subpage_ops = {
1581 .read = subpage_read,
1582 .write = subpage_write,
1583 .valid.accepts = subpage_accepts,
1584 .endianness = DEVICE_NATIVE_ENDIAN,
1587 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1592 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1594 idx = SUBPAGE_IDX(start);
1595 eidx = SUBPAGE_IDX(end);
1596 #if defined(DEBUG_SUBPAGE)
1597 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
1598 __func__, mmio, start, end, idx, eidx, section);
1600 for (; idx <= eidx; idx++) {
1601 mmio->sub_section[idx] = section;
1607 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1611 mmio = g_malloc0(sizeof(subpage_t));
1615 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1616 "subpage", TARGET_PAGE_SIZE);
1617 mmio->iomem.subpage = true;
1618 #if defined(DEBUG_SUBPAGE)
1619 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
1620 mmio, base, TARGET_PAGE_SIZE);
1622 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1627 static uint16_t dummy_section(PhysPageMap *map, MemoryRegion *mr)
1629 MemoryRegionSection section = {
1631 .offset_within_address_space = 0,
1632 .offset_within_region = 0,
1633 .size = int128_2_64(),
1636 return phys_section_add(map, §ion);
1639 MemoryRegion *iotlb_to_region(hwaddr index)
1641 return address_space_memory.dispatch->map.sections[
1642 index & ~TARGET_PAGE_MASK].mr;
1645 static void io_mem_init(void)
1647 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1648 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1649 "unassigned", UINT64_MAX);
1650 memory_region_init_io(&io_mem_notdirty, NULL, ¬dirty_mem_ops, NULL,
1651 "notdirty", UINT64_MAX);
1652 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1653 "watch", UINT64_MAX);
1656 static void mem_begin(MemoryListener *listener)
1658 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1659 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
1662 n = dummy_section(&d->map, &io_mem_unassigned);
1663 assert(n == PHYS_SECTION_UNASSIGNED);
1664 n = dummy_section(&d->map, &io_mem_notdirty);
1665 assert(n == PHYS_SECTION_NOTDIRTY);
1666 n = dummy_section(&d->map, &io_mem_rom);
1667 assert(n == PHYS_SECTION_ROM);
1668 n = dummy_section(&d->map, &io_mem_watch);
1669 assert(n == PHYS_SECTION_WATCH);
1671 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
1673 as->next_dispatch = d;
1676 static void mem_commit(MemoryListener *listener)
1678 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1679 AddressSpaceDispatch *cur = as->dispatch;
1680 AddressSpaceDispatch *next = as->next_dispatch;
1682 as->dispatch = next;
1685 phys_sections_free(&cur->map);
1690 static void tcg_commit(MemoryListener *listener)
1694 /* since each CPU stores ram addresses in its TLB cache, we must
1695 reset the modified entries */
1698 CPUArchState *env = cpu->env_ptr;
1704 static void core_log_global_start(MemoryListener *listener)
1706 cpu_physical_memory_set_dirty_tracking(1);
1709 static void core_log_global_stop(MemoryListener *listener)
1711 cpu_physical_memory_set_dirty_tracking(0);
1714 static MemoryListener core_memory_listener = {
1715 .log_global_start = core_log_global_start,
1716 .log_global_stop = core_log_global_stop,
1720 static MemoryListener tcg_memory_listener = {
1721 .commit = tcg_commit,
1724 void address_space_init_dispatch(AddressSpace *as)
1726 as->dispatch = NULL;
1727 as->dispatch_listener = (MemoryListener) {
1729 .commit = mem_commit,
1730 .region_add = mem_add,
1731 .region_nop = mem_add,
1734 memory_listener_register(&as->dispatch_listener, as);
1737 void address_space_destroy_dispatch(AddressSpace *as)
1739 AddressSpaceDispatch *d = as->dispatch;
1741 memory_listener_unregister(&as->dispatch_listener);
1743 as->dispatch = NULL;
1746 static void memory_map_init(void)
1748 system_memory = g_malloc(sizeof(*system_memory));
1750 assert(ADDR_SPACE_BITS <= 64);
1752 memory_region_init(system_memory, NULL, "system",
1753 ADDR_SPACE_BITS == 64 ?
1754 UINT64_MAX : (0x1ULL << ADDR_SPACE_BITS));
1755 address_space_init(&address_space_memory, system_memory, "memory");
1757 system_io = g_malloc(sizeof(*system_io));
1758 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
1760 address_space_init(&address_space_io, system_io, "I/O");
1762 memory_listener_register(&core_memory_listener, &address_space_memory);
1763 if (tcg_enabled()) {
1764 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1768 MemoryRegion *get_system_memory(void)
1770 return system_memory;
1773 MemoryRegion *get_system_io(void)
1778 #endif /* !defined(CONFIG_USER_ONLY) */
1780 /* physical memory access (slow version, mainly for debug) */
1781 #if defined(CONFIG_USER_ONLY)
1782 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
1783 uint8_t *buf, int len, int is_write)
1790 page = addr & TARGET_PAGE_MASK;
1791 l = (page + TARGET_PAGE_SIZE) - addr;
1794 flags = page_get_flags(page);
1795 if (!(flags & PAGE_VALID))
1798 if (!(flags & PAGE_WRITE))
1800 /* XXX: this code should not depend on lock_user */
1801 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1804 unlock_user(p, addr, l);
1806 if (!(flags & PAGE_READ))
1808 /* XXX: this code should not depend on lock_user */
1809 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1812 unlock_user(p, addr, 0);
1823 static void invalidate_and_set_dirty(hwaddr addr,
1826 if (!cpu_physical_memory_is_dirty(addr)) {
1827 /* invalidate code */
1828 tb_invalidate_phys_page_range(addr, addr + length, 0);
1830 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1832 xen_modified_memory(addr, length);
1835 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
1837 unsigned access_size_max = mr->ops->valid.max_access_size;
1839 /* Regions are assumed to support 1-4 byte accesses unless
1840 otherwise specified. */
1841 if (access_size_max == 0) {
1842 access_size_max = 4;
1845 /* Bound the maximum access by the alignment of the address. */
1846 if (!mr->ops->impl.unaligned) {
1847 unsigned align_size_max = addr & -addr;
1848 if (align_size_max != 0 && align_size_max < access_size_max) {
1849 access_size_max = align_size_max;
1853 /* Don't attempt accesses larger than the maximum. */
1854 if (l > access_size_max) {
1855 l = access_size_max;
1858 l = 1 << (qemu_fls(l) - 1);
1864 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1865 int len, bool is_write)
1876 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1879 if (!memory_access_is_direct(mr, is_write)) {
1880 l = memory_access_size(mr, l, addr1);
1881 /* XXX: could force current_cpu to NULL to avoid
1885 /* 64 bit write access */
1887 error |= io_mem_write(mr, addr1, val, 8);
1890 /* 32 bit write access */
1892 error |= io_mem_write(mr, addr1, val, 4);
1895 /* 16 bit write access */
1897 error |= io_mem_write(mr, addr1, val, 2);
1900 /* 8 bit write access */
1902 error |= io_mem_write(mr, addr1, val, 1);
1908 addr1 += memory_region_get_ram_addr(mr);
1910 ptr = qemu_get_ram_ptr(addr1);
1911 memcpy(ptr, buf, l);
1912 invalidate_and_set_dirty(addr1, l);
1915 if (!memory_access_is_direct(mr, is_write)) {
1917 l = memory_access_size(mr, l, addr1);
1920 /* 64 bit read access */
1921 error |= io_mem_read(mr, addr1, &val, 8);
1925 /* 32 bit read access */
1926 error |= io_mem_read(mr, addr1, &val, 4);
1930 /* 16 bit read access */
1931 error |= io_mem_read(mr, addr1, &val, 2);
1935 /* 8 bit read access */
1936 error |= io_mem_read(mr, addr1, &val, 1);
1944 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
1945 memcpy(buf, ptr, l);
1956 bool address_space_write(AddressSpace *as, hwaddr addr,
1957 const uint8_t *buf, int len)
1959 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
1962 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1964 return address_space_rw(as, addr, buf, len, false);
1968 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
1969 int len, int is_write)
1971 address_space_rw(&address_space_memory, addr, buf, len, is_write);
1974 /* used for ROM loading : can write in RAM and ROM */
1975 void cpu_physical_memory_write_rom(hwaddr addr,
1976 const uint8_t *buf, int len)
1985 mr = address_space_translate(&address_space_memory,
1986 addr, &addr1, &l, true);
1988 if (!(memory_region_is_ram(mr) ||
1989 memory_region_is_romd(mr))) {
1992 addr1 += memory_region_get_ram_addr(mr);
1994 ptr = qemu_get_ram_ptr(addr1);
1995 memcpy(ptr, buf, l);
1996 invalidate_and_set_dirty(addr1, l);
2011 static BounceBuffer bounce;
2013 typedef struct MapClient {
2015 void (*callback)(void *opaque);
2016 QLIST_ENTRY(MapClient) link;
2019 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2020 = QLIST_HEAD_INITIALIZER(map_client_list);
2022 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2024 MapClient *client = g_malloc(sizeof(*client));
2026 client->opaque = opaque;
2027 client->callback = callback;
2028 QLIST_INSERT_HEAD(&map_client_list, client, link);
2032 static void cpu_unregister_map_client(void *_client)
2034 MapClient *client = (MapClient *)_client;
2036 QLIST_REMOVE(client, link);
2040 static void cpu_notify_map_clients(void)
2044 while (!QLIST_EMPTY(&map_client_list)) {
2045 client = QLIST_FIRST(&map_client_list);
2046 client->callback(client->opaque);
2047 cpu_unregister_map_client(client);
2051 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2058 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2059 if (!memory_access_is_direct(mr, is_write)) {
2060 l = memory_access_size(mr, l, addr);
2061 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2072 /* Map a physical memory region into a host virtual address.
2073 * May map a subset of the requested range, given by and returned in *plen.
2074 * May return NULL if resources needed to perform the mapping are exhausted.
2075 * Use only for reads OR writes - not for read-modify-write operations.
2076 * Use cpu_register_map_client() to know when retrying the map operation is
2077 * likely to succeed.
2079 void *address_space_map(AddressSpace *as,
2086 hwaddr l, xlat, base;
2087 MemoryRegion *mr, *this_mr;
2095 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2096 if (!memory_access_is_direct(mr, is_write)) {
2097 if (bounce.buffer) {
2100 /* Avoid unbounded allocations */
2101 l = MIN(l, TARGET_PAGE_SIZE);
2102 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
2106 memory_region_ref(mr);
2109 address_space_read(as, addr, bounce.buffer, l);
2113 return bounce.buffer;
2117 raddr = memory_region_get_ram_addr(mr);
2128 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2129 if (this_mr != mr || xlat != base + done) {
2134 memory_region_ref(mr);
2136 return qemu_ram_ptr_length(raddr + base, plen);
2139 /* Unmaps a memory region previously mapped by address_space_map().
2140 * Will also mark the memory as dirty if is_write == 1. access_len gives
2141 * the amount of memory that was actually read or written by the caller.
2143 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2144 int is_write, hwaddr access_len)
2146 if (buffer != bounce.buffer) {
2150 mr = qemu_ram_addr_from_host(buffer, &addr1);
2153 while (access_len) {
2155 l = TARGET_PAGE_SIZE;
2158 invalidate_and_set_dirty(addr1, l);
2163 if (xen_enabled()) {
2164 xen_invalidate_map_cache_entry(buffer);
2166 memory_region_unref(mr);
2170 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2172 qemu_vfree(bounce.buffer);
2173 bounce.buffer = NULL;
2174 memory_region_unref(bounce.mr);
2175 cpu_notify_map_clients();
2178 void *cpu_physical_memory_map(hwaddr addr,
2182 return address_space_map(&address_space_memory, addr, plen, is_write);
2185 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2186 int is_write, hwaddr access_len)
2188 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2191 /* warning: addr must be aligned */
2192 static inline uint32_t ldl_phys_internal(hwaddr addr,
2193 enum device_endian endian)
2201 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2203 if (l < 4 || !memory_access_is_direct(mr, false)) {
2205 io_mem_read(mr, addr1, &val, 4);
2206 #if defined(TARGET_WORDS_BIGENDIAN)
2207 if (endian == DEVICE_LITTLE_ENDIAN) {
2211 if (endian == DEVICE_BIG_ENDIAN) {
2217 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2221 case DEVICE_LITTLE_ENDIAN:
2222 val = ldl_le_p(ptr);
2224 case DEVICE_BIG_ENDIAN:
2225 val = ldl_be_p(ptr);
2235 uint32_t ldl_phys(hwaddr addr)
2237 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2240 uint32_t ldl_le_phys(hwaddr addr)
2242 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2245 uint32_t ldl_be_phys(hwaddr addr)
2247 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2250 /* warning: addr must be aligned */
2251 static inline uint64_t ldq_phys_internal(hwaddr addr,
2252 enum device_endian endian)
2260 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2262 if (l < 8 || !memory_access_is_direct(mr, false)) {
2264 io_mem_read(mr, addr1, &val, 8);
2265 #if defined(TARGET_WORDS_BIGENDIAN)
2266 if (endian == DEVICE_LITTLE_ENDIAN) {
2270 if (endian == DEVICE_BIG_ENDIAN) {
2276 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2280 case DEVICE_LITTLE_ENDIAN:
2281 val = ldq_le_p(ptr);
2283 case DEVICE_BIG_ENDIAN:
2284 val = ldq_be_p(ptr);
2294 uint64_t ldq_phys(hwaddr addr)
2296 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2299 uint64_t ldq_le_phys(hwaddr addr)
2301 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2304 uint64_t ldq_be_phys(hwaddr addr)
2306 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2310 uint32_t ldub_phys(hwaddr addr)
2313 cpu_physical_memory_read(addr, &val, 1);
2317 /* warning: addr must be aligned */
2318 static inline uint32_t lduw_phys_internal(hwaddr addr,
2319 enum device_endian endian)
2327 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2329 if (l < 2 || !memory_access_is_direct(mr, false)) {
2331 io_mem_read(mr, addr1, &val, 2);
2332 #if defined(TARGET_WORDS_BIGENDIAN)
2333 if (endian == DEVICE_LITTLE_ENDIAN) {
2337 if (endian == DEVICE_BIG_ENDIAN) {
2343 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2347 case DEVICE_LITTLE_ENDIAN:
2348 val = lduw_le_p(ptr);
2350 case DEVICE_BIG_ENDIAN:
2351 val = lduw_be_p(ptr);
2361 uint32_t lduw_phys(hwaddr addr)
2363 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2366 uint32_t lduw_le_phys(hwaddr addr)
2368 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2371 uint32_t lduw_be_phys(hwaddr addr)
2373 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2376 /* warning: addr must be aligned. The ram page is not masked as dirty
2377 and the code inside is not invalidated. It is useful if the dirty
2378 bits are used to track modified PTEs */
2379 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2386 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2388 if (l < 4 || !memory_access_is_direct(mr, true)) {
2389 io_mem_write(mr, addr1, val, 4);
2391 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2392 ptr = qemu_get_ram_ptr(addr1);
2395 if (unlikely(in_migration)) {
2396 if (!cpu_physical_memory_is_dirty(addr1)) {
2397 /* invalidate code */
2398 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2400 cpu_physical_memory_set_dirty_flags(
2401 addr1, (0xff & ~CODE_DIRTY_FLAG));
2407 /* warning: addr must be aligned */
2408 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2409 enum device_endian endian)
2416 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2418 if (l < 4 || !memory_access_is_direct(mr, true)) {
2419 #if defined(TARGET_WORDS_BIGENDIAN)
2420 if (endian == DEVICE_LITTLE_ENDIAN) {
2424 if (endian == DEVICE_BIG_ENDIAN) {
2428 io_mem_write(mr, addr1, val, 4);
2431 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2432 ptr = qemu_get_ram_ptr(addr1);
2434 case DEVICE_LITTLE_ENDIAN:
2437 case DEVICE_BIG_ENDIAN:
2444 invalidate_and_set_dirty(addr1, 4);
2448 void stl_phys(hwaddr addr, uint32_t val)
2450 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2453 void stl_le_phys(hwaddr addr, uint32_t val)
2455 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2458 void stl_be_phys(hwaddr addr, uint32_t val)
2460 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2464 void stb_phys(hwaddr addr, uint32_t val)
2467 cpu_physical_memory_write(addr, &v, 1);
2470 /* warning: addr must be aligned */
2471 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2472 enum device_endian endian)
2479 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2481 if (l < 2 || !memory_access_is_direct(mr, true)) {
2482 #if defined(TARGET_WORDS_BIGENDIAN)
2483 if (endian == DEVICE_LITTLE_ENDIAN) {
2487 if (endian == DEVICE_BIG_ENDIAN) {
2491 io_mem_write(mr, addr1, val, 2);
2494 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2495 ptr = qemu_get_ram_ptr(addr1);
2497 case DEVICE_LITTLE_ENDIAN:
2500 case DEVICE_BIG_ENDIAN:
2507 invalidate_and_set_dirty(addr1, 2);
2511 void stw_phys(hwaddr addr, uint32_t val)
2513 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2516 void stw_le_phys(hwaddr addr, uint32_t val)
2518 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2521 void stw_be_phys(hwaddr addr, uint32_t val)
2523 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2527 void stq_phys(hwaddr addr, uint64_t val)
2530 cpu_physical_memory_write(addr, &val, 8);
2533 void stq_le_phys(hwaddr addr, uint64_t val)
2535 val = cpu_to_le64(val);
2536 cpu_physical_memory_write(addr, &val, 8);
2539 void stq_be_phys(hwaddr addr, uint64_t val)
2541 val = cpu_to_be64(val);
2542 cpu_physical_memory_write(addr, &val, 8);
2545 /* virtual memory access for debug (includes writing to ROM) */
2546 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2547 uint8_t *buf, int len, int is_write)
2554 page = addr & TARGET_PAGE_MASK;
2555 phys_addr = cpu_get_phys_page_debug(cpu, page);
2556 /* if no physical page mapped, return an error */
2557 if (phys_addr == -1)
2559 l = (page + TARGET_PAGE_SIZE) - addr;
2562 phys_addr += (addr & ~TARGET_PAGE_MASK);
2564 cpu_physical_memory_write_rom(phys_addr, buf, l);
2566 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2575 #if !defined(CONFIG_USER_ONLY)
2578 * A helper function for the _utterly broken_ virtio device model to find out if
2579 * it's running on a big endian machine. Don't do this at home kids!
2581 bool virtio_is_big_endian(void);
2582 bool virtio_is_big_endian(void)
2584 #if defined(TARGET_WORDS_BIGENDIAN)
2593 #ifndef CONFIG_USER_ONLY
2594 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2599 mr = address_space_translate(&address_space_memory,
2600 phys_addr, &phys_addr, &l, false);
2602 return !(memory_region_is_ram(mr) ||
2603 memory_region_is_romd(mr));
2606 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2610 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2611 func(block->host, block->offset, block->length, opaque);