arch/x86/kvm/lapic.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2
   3 /*
   4  * Local APIC virtualization
   5  *
   6  * Copyright (C) 2006 Qumranet, Inc.
   7  * Copyright (C) 2007 Novell
   8  * Copyright (C) 2007 Intel
   9  * Copyright 2009 Red Hat, Inc. and/or its affiliates.
  10  *
  11  * Authors:
  12  *   Dor Laor <dor.laor@qumranet.com>
  13  *   Gregory Haskins <ghaskins@novell.com>
  14  *   Yaozu (Eddie) Dong <eddie.dong@intel.com>
  15  *
  16  * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation.
  17  */
  18
  19 #include <linux/kvm_host.h>
  20 #include <linux/kvm.h>
  21 #include <linux/mm.h>
  22 #include <linux/highmem.h>
  23 #include <linux/smp.h>
  24 #include <linux/hrtimer.h>
  25 #include <linux/io.h>
  26 #include <linux/export.h>
  27 #include <linux/math64.h>
  28 #include <linux/slab.h>
  29 #include <asm/processor.h>
  30 #include <asm/msr.h>
  31 #include <asm/page.h>
  32 #include <asm/current.h>
  33 #include <asm/apicdef.h>
  34 #include <asm/delay.h>
  35 #include <linux/atomic.h>
  36 #include <linux/jump_label.h>
  37 #include "kvm_cache_regs.h"
  38 #include "irq.h"
  39 #include "trace.h"
  40 #include "x86.h"
  41 #include "cpuid.h"
  42 #include "hyperv.h"
  43
  44 #ifndef CONFIG_X86_64
  45 #define mod_64(x, y) ((x) - (y) * div64_u64(x, y))
  46 #else
  47 #define mod_64(x, y) ((x) % (y))
  48 #endif
  49
  50 #define PRId64 "d"
  51 #define PRIx64 "llx"
  52 #define PRIu64 "u"
  53 #define PRIo64 "o"
  54
  55 /* 14 is the version for Xeon and Pentium 8.4.8*/
  56 #define APIC_VERSION                    (0x14UL | ((KVM_APIC_LVT_NUM - 1) << 16))
  57 #define LAPIC_MMIO_LENGTH               (1 << 12)
  58 /* followed define is not in apicdef.h */
  59 #define APIC_SHORT_MASK                 0xc0000
  60 #define APIC_DEST_NOSHORT               0x0
  61 #define APIC_DEST_MASK                  0x800
  62 #define MAX_APIC_VECTOR                 256
  63 #define APIC_VECTORS_PER_REG            32
  64
  65 #define APIC_BROADCAST                  0xFF
  66 #define X2APIC_BROADCAST                0xFFFFFFFFul
  67
  68 #define LAPIC_TIMER_ADVANCE_ADJUST_DONE 100
  69 #define LAPIC_TIMER_ADVANCE_ADJUST_INIT 1000
  70 /* step-by-step approximation to mitigate fluctuation */
  71 #define LAPIC_TIMER_ADVANCE_ADJUST_STEP 8
  72
  73 static inline int apic_test_vector(int vec, void *bitmap)
  74 {
  75         return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
  76 }
  77
  78 bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector)
  79 {
  80         struct kvm_lapic *apic = vcpu->arch.apic;
  81
  82         return apic_test_vector(vector, apic->regs + APIC_ISR) ||
  83                 apic_test_vector(vector, apic->regs + APIC_IRR);
  84 }
  85
  86 static inline int __apic_test_and_set_vector(int vec, void *bitmap)
  87 {
  88         return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
  89 }
  90
  91 static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
  92 {
  93         return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
  94 }
  95
  96 struct static_key_deferred apic_hw_disabled __read_mostly;
  97 struct static_key_deferred apic_sw_disabled __read_mostly;
  98
  99 static inline int apic_enabled(struct kvm_lapic *apic)
 100 {
 101         return kvm_apic_sw_enabled(apic) &&     kvm_apic_hw_enabled(apic);
 102 }
 103
 104 #define LVT_MASK        \
 105         (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK)
 106
 107 #define LINT_MASK       \
 108         (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
 109          APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)
 110
 111 static inline u8 kvm_xapic_id(struct kvm_lapic *apic)
 112 {
 113         return kvm_lapic_get_reg(apic, APIC_ID) >> 24;
 114 }
 115
 116 static inline u32 kvm_x2apic_id(struct kvm_lapic *apic)
 117 {
 118         return apic->vcpu->vcpu_id;
 119 }
 120
 121 bool kvm_can_post_timer_interrupt(struct kvm_vcpu *vcpu)
 122 {
 123         return pi_inject_timer && kvm_vcpu_apicv_active(vcpu);
 124 }
 125 EXPORT_SYMBOL_GPL(kvm_can_post_timer_interrupt);
 126
 127 static bool kvm_use_posted_timer_interrupt(struct kvm_vcpu *vcpu)
 128 {
 129         return kvm_can_post_timer_interrupt(vcpu) && vcpu->mode == IN_GUEST_MODE;
 130 }
 131
 132 static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map,
 133                 u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) {
 134         switch (map->mode) {
 135         case KVM_APIC_MODE_X2APIC: {
 136                 u32 offset = (dest_id >> 16) * 16;
 137                 u32 max_apic_id = map->max_apic_id;
 138
 139                 if (offset <= max_apic_id) {
 140                         u8 cluster_size = min(max_apic_id - offset + 1, 16U);
 141
 142                         offset = array_index_nospec(offset, map->max_apic_id + 1);
 143                         *cluster = &map->phys_map[offset];
 144                         *mask = dest_id & (0xffff >> (16 - cluster_size));
 145                 } else {
 146                         *mask = 0;
 147                 }
 148
 149                 return true;
 150                 }
 151         case KVM_APIC_MODE_XAPIC_FLAT:
 152                 *cluster = map->xapic_flat_map;
 153                 *mask = dest_id & 0xff;
 154                 return true;
 155         case KVM_APIC_MODE_XAPIC_CLUSTER:
 156                 *cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf];
 157                 *mask = dest_id & 0xf;
 158                 return true;
 159         default:
 160                 /* Not optimized. */
 161                 return false;
 162         }
 163 }
 164
 165 static void kvm_apic_map_free(struct rcu_head *rcu)
 166 {
 167         struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu);
 168
 169         kvfree(map);
 170 }
 171
 172 static void recalculate_apic_map(struct kvm *kvm)
 173 {
 174         struct kvm_apic_map *new, *old = NULL;
 175         struct kvm_vcpu *vcpu;
 176         int i;
 177         u32 max_id = 255; /* enough space for any xAPIC ID */
 178
 179         mutex_lock(&kvm->arch.apic_map_lock);
 180
 181         kvm_for_each_vcpu(i, vcpu, kvm)
 182                 if (kvm_apic_present(vcpu))
 183                         max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic));
 184
 185         new = kvzalloc(sizeof(struct kvm_apic_map) +
 186                            sizeof(struct kvm_lapic *) * ((u64)max_id + 1),
 187                            GFP_KERNEL_ACCOUNT);
 188
 189         if (!new)
 190                 goto out;
 191
 192         new->max_apic_id = max_id;
 193
 194         kvm_for_each_vcpu(i, vcpu, kvm) {
 195                 struct kvm_lapic *apic = vcpu->arch.apic;
 196                 struct kvm_lapic **cluster;
 197                 u16 mask;
 198                 u32 ldr;
 199                 u8 xapic_id;
 200                 u32 x2apic_id;
 201
 202                 if (!kvm_apic_present(vcpu))
 203                         continue;
 204
 205                 xapic_id = kvm_xapic_id(apic);
 206                 x2apic_id = kvm_x2apic_id(apic);
 207
 208                 /* Hotplug hack: see kvm_apic_match_physical_addr(), ... */
 209                 if ((apic_x2apic_mode(apic) || x2apic_id > 0xff) &&
 210                                 x2apic_id <= new->max_apic_id)
 211                         new->phys_map[x2apic_id] = apic;
 212                 /*
 213                  * ... xAPIC ID of VCPUs with APIC ID > 0xff will wrap-around,
 214                  * prevent them from masking VCPUs with APIC ID <= 0xff.
 215                  */
 216                 if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id])
 217                         new->phys_map[xapic_id] = apic;
 218
 219                 ldr = kvm_lapic_get_reg(apic, APIC_LDR);
 220
 221                 if (apic_x2apic_mode(apic)) {
 222                         new->mode |= KVM_APIC_MODE_X2APIC;
 223                 } else if (ldr) {
 224                         ldr = GET_APIC_LOGICAL_ID(ldr);
 225                         if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT)
 226                                 new->mode |= KVM_APIC_MODE_XAPIC_FLAT;
 227                         else
 228                                 new->mode |= KVM_APIC_MODE_XAPIC_CLUSTER;
 229                 }
 230
 231                 if (!kvm_apic_map_get_logical_dest(new, ldr, &cluster, &mask))
 232                         continue;
 233
 234                 if (mask)
 235                         cluster[ffs(mask) - 1] = apic;
 236         }
 237 out:
 238         old = rcu_dereference_protected(kvm->arch.apic_map,
 239                         lockdep_is_held(&kvm->arch.apic_map_lock));
 240         rcu_assign_pointer(kvm->arch.apic_map, new);
 241         mutex_unlock(&kvm->arch.apic_map_lock);
 242
 243         if (old)
 244                 call_rcu(&old->rcu, kvm_apic_map_free);
 245
 246         kvm_make_scan_ioapic_request(kvm);
 247 }
 248
 249 static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)
 250 {
 251         bool enabled = val & APIC_SPIV_APIC_ENABLED;
 252
 253         kvm_lapic_set_reg(apic, APIC_SPIV, val);
 254
 255         if (enabled != apic->sw_enabled) {
 256                 apic->sw_enabled = enabled;
 257                 if (enabled)
 258                         static_key_slow_dec_deferred(&apic_sw_disabled);
 259                 else
 260                         static_key_slow_inc(&apic_sw_disabled.key);
 261         }
 262 }
 263
 264 static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id)
 265 {
 266         kvm_lapic_set_reg(apic, APIC_ID, id << 24);
 267         recalculate_apic_map(apic->vcpu->kvm);
 268 }
 269
 270 static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id)
 271 {
 272         kvm_lapic_set_reg(apic, APIC_LDR, id);
 273         recalculate_apic_map(apic->vcpu->kvm);
 274 }
 275
 276 static inline u32 kvm_apic_calc_x2apic_ldr(u32 id)
 277 {
 278         return ((id >> 4) << 16) | (1 << (id & 0xf));
 279 }
 280
 281 static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id)
 282 {
 283         u32 ldr = kvm_apic_calc_x2apic_ldr(id);
 284
 285         WARN_ON_ONCE(id != apic->vcpu->vcpu_id);
 286
 287         kvm_lapic_set_reg(apic, APIC_ID, id);
 288         kvm_lapic_set_reg(apic, APIC_LDR, ldr);
 289         recalculate_apic_map(apic->vcpu->kvm);
 290 }
 291
 292 static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type)
 293 {
 294         return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED);
 295 }
 296
 297 static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type)
 298 {
 299         return kvm_lapic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK;
 300 }
 301
 302 static inline int apic_lvtt_oneshot(struct kvm_lapic *apic)
 303 {
 304         return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT;
 305 }
 306
 307 static inline int apic_lvtt_period(struct kvm_lapic *apic)
 308 {
 309         return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC;
 310 }
 311
 312 static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic)
 313 {
 314         return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE;
 315 }
 316
 317 static inline int apic_lvt_nmi_mode(u32 lvt_val)
 318 {
 319         return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI;
 320 }
 321
 322 void kvm_apic_set_version(struct kvm_vcpu *vcpu)
 323 {
 324         struct kvm_lapic *apic = vcpu->arch.apic;
 325         struct kvm_cpuid_entry2 *feat;
 326         u32 v = APIC_VERSION;
 327
 328         if (!lapic_in_kernel(vcpu))
 329                 return;
 330
 331         /*
 332          * KVM emulates 82093AA datasheet (with in-kernel IOAPIC implementation)
 333          * which doesn't have EOI register; Some buggy OSes (e.g. Windows with
 334          * Hyper-V role) disable EOI broadcast in lapic not checking for IOAPIC
 335          * version first and level-triggered interrupts never get EOIed in
 336          * IOAPIC.
 337          */
 338         feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0);
 339         if (feat && (feat->ecx & (1 << (X86_FEATURE_X2APIC & 31))) &&
 340             !ioapic_in_kernel(vcpu->kvm))
 341                 v |= APIC_LVR_DIRECTED_EOI;
 342         kvm_lapic_set_reg(apic, APIC_LVR, v);
 343 }
 344
 345 static const unsigned int apic_lvt_mask[KVM_APIC_LVT_NUM] = {
 346         LVT_MASK ,      /* part LVTT mask, timer mode mask added at runtime */
 347         LVT_MASK | APIC_MODE_MASK,      /* LVTTHMR */
 348         LVT_MASK | APIC_MODE_MASK,      /* LVTPC */
 349         LINT_MASK, LINT_MASK,   /* LVT0-1 */
 350         LVT_MASK                /* LVTERR */
 351 };
 352
 353 static int find_highest_vector(void *bitmap)
 354 {
 355         int vec;
 356         u32 *reg;
 357
 358         for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG;
 359              vec >= 0; vec -= APIC_VECTORS_PER_REG) {
 360                 reg = bitmap + REG_POS(vec);
 361                 if (*reg)
 362                         return __fls(*reg) + vec;
 363         }
 364
 365         return -1;
 366 }
 367
 368 static u8 count_vectors(void *bitmap)
 369 {
 370         int vec;
 371         u32 *reg;
 372         u8 count = 0;
 373
 374         for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) {
 375                 reg = bitmap + REG_POS(vec);
 376                 count += hweight32(*reg);
 377         }
 378
 379         return count;
 380 }
 381
 382 bool __kvm_apic_update_irr(u32 *pir, void *regs, int *max_irr)
 383 {
 384         u32 i, vec;
 385         u32 pir_val, irr_val, prev_irr_val;
 386         int max_updated_irr;
 387
 388         max_updated_irr = -1;
 389         *max_irr = -1;
 390
 391         for (i = vec = 0; i <= 7; i++, vec += 32) {
 392                 pir_val = READ_ONCE(pir[i]);
 393                 irr_val = *((u32 *)(regs + APIC_IRR + i * 0x10));
 394                 if (pir_val) {
 395                         prev_irr_val = irr_val;
 396                         irr_val |= xchg(&pir[i], 0);
 397                         *((u32 *)(regs + APIC_IRR + i * 0x10)) = irr_val;
 398                         if (prev_irr_val != irr_val) {
 399                                 max_updated_irr =
 400                                         __fls(irr_val ^ prev_irr_val) + vec;
 401                         }
 402                 }
 403                 if (irr_val)
 404                         *max_irr = __fls(irr_val) + vec;
 405         }
 406
 407         return ((max_updated_irr != -1) &&
 408                 (max_updated_irr == *max_irr));
 409 }
 410 EXPORT_SYMBOL_GPL(__kvm_apic_update_irr);
 411
 412 bool kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir, int *max_irr)
 413 {
 414         struct kvm_lapic *apic = vcpu->arch.apic;
 415
 416         return __kvm_apic_update_irr(pir, apic->regs, max_irr);
 417 }
 418 EXPORT_SYMBOL_GPL(kvm_apic_update_irr);
 419
 420 static inline int apic_search_irr(struct kvm_lapic *apic)
 421 {
 422         return find_highest_vector(apic->regs + APIC_IRR);
 423 }
 424
 425 static inline int apic_find_highest_irr(struct kvm_lapic *apic)
 426 {
 427         int result;
 428
 429         /*
 430          * Note that irr_pending is just a hint. It will be always
 431          * true with virtual interrupt delivery enabled.
 432          */
 433         if (!apic->irr_pending)
 434                 return -1;
 435
 436         result = apic_search_irr(apic);
 437         ASSERT(result == -1 || result >= 16);
 438
 439         return result;
 440 }
 441
 442 static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
 443 {
 444         struct kvm_vcpu *vcpu;
 445
 446         vcpu = apic->vcpu;
 447
 448         if (unlikely(vcpu->arch.apicv_active)) {
 449                 /* need to update RVI */
 450                 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR);
 451                 kvm_x86_ops->hwapic_irr_update(vcpu,
 452                                 apic_find_highest_irr(apic));
 453         } else {
 454                 apic->irr_pending = false;
 455                 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR);
 456                 if (apic_search_irr(apic) != -1)
 457                         apic->irr_pending = true;
 458         }
 459 }
 460
 461 static inline void apic_set_isr(int vec, struct kvm_lapic *apic)
 462 {
 463         struct kvm_vcpu *vcpu;
 464
 465         if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR))
 466                 return;
 467
 468         vcpu = apic->vcpu;
 469
 470         /*
 471          * With APIC virtualization enabled, all caching is disabled
 472          * because the processor can modify ISR under the hood.  Instead
 473          * just set SVI.
 474          */
 475         if (unlikely(vcpu->arch.apicv_active))
 476                 kvm_x86_ops->hwapic_isr_update(vcpu, vec);
 477         else {
 478                 ++apic->isr_count;
 479                 BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
 480                 /*
 481                  * ISR (in service register) bit is set when injecting an interrupt.
 482                  * The highest vector is injected. Thus the latest bit set matches
 483                  * the highest bit in ISR.
 484                  */
 485                 apic->highest_isr_cache = vec;
 486         }
 487 }
 488
 489 static inline int apic_find_highest_isr(struct kvm_lapic *apic)
 490 {
 491         int result;
 492
 493         /*
 494          * Note that isr_count is always 1, and highest_isr_cache
 495          * is always -1, with APIC virtualization enabled.
 496          */
 497         if (!apic->isr_count)
 498                 return -1;
 499         if (likely(apic->highest_isr_cache != -1))
 500                 return apic->highest_isr_cache;
 501
 502         result = find_highest_vector(apic->regs + APIC_ISR);
 503         ASSERT(result == -1 || result >= 16);
 504
 505         return result;
 506 }
 507
 508 static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)
 509 {
 510         struct kvm_vcpu *vcpu;
 511         if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR))
 512                 return;
 513
 514         vcpu = apic->vcpu;
 515
 516         /*
 517          * We do get here for APIC virtualization enabled if the guest
 518          * uses the Hyper-V APIC enlightenment.  In this case we may need
 519          * to trigger a new interrupt delivery by writing the SVI field;
 520          * on the other hand isr_count and highest_isr_cache are unused
 521          * and must be left alone.
 522          */
 523         if (unlikely(vcpu->arch.apicv_active))
 524                 kvm_x86_ops->hwapic_isr_update(vcpu,
 525                                                apic_find_highest_isr(apic));
 526         else {
 527                 --apic->isr_count;
 528                 BUG_ON(apic->isr_count < 0);
 529                 apic->highest_isr_cache = -1;
 530         }
 531 }
 532
 533 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
 534 {
 535         /* This may race with setting of irr in __apic_accept_irq() and
 536          * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq
 537          * will cause vmexit immediately and the value will be recalculated
 538          * on the next vmentry.
 539          */
 540         return apic_find_highest_irr(vcpu->arch.apic);
 541 }
 542 EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr);
 543
 544 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
 545                              int vector, int level, int trig_mode,
 546                              struct dest_map *dest_map);
 547
 548 int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq,
 549                      struct dest_map *dest_map)
 550 {
 551         struct kvm_lapic *apic = vcpu->arch.apic;
 552
 553         return __apic_accept_irq(apic, irq->delivery_mode, irq->vector,
 554                         irq->level, irq->trig_mode, dest_map);
 555 }
 556
 557 int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
 558                     unsigned long ipi_bitmap_high, u32 min,
 559                     unsigned long icr, int op_64_bit)
 560 {
 561         int i;
 562         struct kvm_apic_map *map;
 563         struct kvm_vcpu *vcpu;
 564         struct kvm_lapic_irq irq = {0};
 565         int cluster_size = op_64_bit ? 64 : 32;
 566         int count = 0;
 567
 568         irq.vector = icr & APIC_VECTOR_MASK;
 569         irq.delivery_mode = icr & APIC_MODE_MASK;
 570         irq.level = (icr & APIC_INT_ASSERT) != 0;
 571         irq.trig_mode = icr & APIC_INT_LEVELTRIG;
 572
 573         if (icr & APIC_DEST_MASK)
 574                 return -KVM_EINVAL;
 575         if (icr & APIC_SHORT_MASK)
 576                 return -KVM_EINVAL;
 577
 578         rcu_read_lock();
 579         map = rcu_dereference(kvm->arch.apic_map);
 580
 581         if (unlikely(!map)) {
 582                 count = -EOPNOTSUPP;
 583                 goto out;
 584         }
 585
 586         if (min > map->max_apic_id)
 587                 goto out;
 588         /* Bits above cluster_size are masked in the caller.  */
 589         for_each_set_bit(i, &ipi_bitmap_low,
 590                 min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) {
 591                 if (map->phys_map[min + i]) {
 592                         vcpu = map->phys_map[min + i]->vcpu;
 593                         count += kvm_apic_set_irq(vcpu, &irq, NULL);
 594                 }
 595         }
 596
 597         min += cluster_size;
 598
 599         if (min > map->max_apic_id)
 600                 goto out;
 601
 602         for_each_set_bit(i, &ipi_bitmap_high,
 603                 min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) {
 604                 if (map->phys_map[min + i]) {
 605                         vcpu = map->phys_map[min + i]->vcpu;
 606                         count += kvm_apic_set_irq(vcpu, &irq, NULL);
 607                 }
 608         }
 609
 610 out:
 611         rcu_read_unlock();
 612         return count;
 613 }
 614
 615 static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val)
 616 {
 617
 618         return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val,
 619                                       sizeof(val));
 620 }
 621
 622 static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val)
 623 {
 624
 625         return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val,
 626                                       sizeof(*val));
 627 }
 628
 629 static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu)
 630 {
 631         return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED;
 632 }
 633
 634 static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu)
 635 {
 636         u8 val;
 637         if (pv_eoi_get_user(vcpu, &val) < 0)
 638                 printk(KERN_WARNING "Can't read EOI MSR value: 0x%llx\n",
 639                            (unsigned long long)vcpu->arch.pv_eoi.msr_val);
 640         return val & 0x1;
 641 }
 642
 643 static void pv_eoi_set_pending(struct kvm_vcpu *vcpu)
 644 {
 645         if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) {
 646                 printk(KERN_WARNING "Can't set EOI MSR value: 0x%llx\n",
 647                            (unsigned long long)vcpu->arch.pv_eoi.msr_val);
 648                 return;
 649         }
 650         __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
 651 }
 652
 653 static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu)
 654 {
 655         if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) {
 656                 printk(KERN_WARNING "Can't clear EOI MSR value: 0x%llx\n",
 657                            (unsigned long long)vcpu->arch.pv_eoi.msr_val);
 658                 return;
 659         }
 660         __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
 661 }
 662
 663 static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr)
 664 {
 665         int highest_irr;
 666         if (apic->vcpu->arch.apicv_active)
 667                 highest_irr = kvm_x86_ops->sync_pir_to_irr(apic->vcpu);
 668         else
 669                 highest_irr = apic_find_highest_irr(apic);
 670         if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr)
 671                 return -1;
 672         return highest_irr;
 673 }
 674
 675 static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr)
 676 {
 677         u32 tpr, isrv, ppr, old_ppr;
 678         int isr;
 679
 680         old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI);
 681         tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI);
 682         isr = apic_find_highest_isr(apic);
 683         isrv = (isr != -1) ? isr : 0;
 684
 685         if ((tpr & 0xf0) >= (isrv & 0xf0))
 686                 ppr = tpr & 0xff;
 687         else
 688                 ppr = isrv & 0xf0;
 689
 690         *new_ppr = ppr;
 691         if (old_ppr != ppr)
 692                 kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr);
 693
 694         return ppr < old_ppr;
 695 }
 696
 697 static void apic_update_ppr(struct kvm_lapic *apic)
 698 {
 699         u32 ppr;
 700
 701         if (__apic_update_ppr(apic, &ppr) &&
 702             apic_has_interrupt_for_ppr(apic, ppr) != -1)
 703                 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
 704 }
 705
 706 void kvm_apic_update_ppr(struct kvm_vcpu *vcpu)
 707 {
 708         apic_update_ppr(vcpu->arch.apic);
 709 }
 710 EXPORT_SYMBOL_GPL(kvm_apic_update_ppr);
 711
 712 static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr)
 713 {
 714         kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr);
 715         apic_update_ppr(apic);
 716 }
 717
 718 static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda)
 719 {
 720         return mda == (apic_x2apic_mode(apic) ?
 721                         X2APIC_BROADCAST : APIC_BROADCAST);
 722 }
 723
 724 static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda)
 725 {
 726         if (kvm_apic_broadcast(apic, mda))
 727                 return true;
 728
 729         if (apic_x2apic_mode(apic))
 730                 return mda == kvm_x2apic_id(apic);
 731
 732         /*
 733          * Hotplug hack: Make LAPIC in xAPIC mode also accept interrupts as if
 734          * it were in x2APIC mode.  Hotplugged VCPUs start in xAPIC mode and
 735          * this allows unique addressing of VCPUs with APIC ID over 0xff.
 736          * The 0xff condition is needed because writeable xAPIC ID.
 737          */
 738         if (kvm_x2apic_id(apic) > 0xff && mda == kvm_x2apic_id(apic))
 739                 return true;
 740
 741         return mda == kvm_xapic_id(apic);
 742 }
 743
 744 static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
 745 {
 746         u32 logical_id;
 747
 748         if (kvm_apic_broadcast(apic, mda))
 749                 return true;
 750
 751         logical_id = kvm_lapic_get_reg(apic, APIC_LDR);
 752
 753         if (apic_x2apic_mode(apic))
 754                 return ((logical_id >> 16) == (mda >> 16))
 755                        && (logical_id & mda & 0xffff) != 0;
 756
 757         logical_id = GET_APIC_LOGICAL_ID(logical_id);
 758
 759         switch (kvm_lapic_get_reg(apic, APIC_DFR)) {
 760         case APIC_DFR_FLAT:
 761                 return (logical_id & mda) != 0;
 762         case APIC_DFR_CLUSTER:
 763                 return ((logical_id >> 4) == (mda >> 4))
 764                        && (logical_id & mda & 0xf) != 0;
 765         default:
 766                 return false;
 767         }
 768 }
 769
 770 /* The KVM local APIC implementation has two quirks:
 771  *
 772  *  - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs
 773  *    in xAPIC mode if the "destination & 0xff" matches its xAPIC ID.
 774  *    KVM doesn't do that aliasing.
 775  *
 776  *  - in-kernel IOAPIC messages have to be delivered directly to
 777  *    x2APIC, because the kernel does not support interrupt remapping.
 778  *    In order to support broadcast without interrupt remapping, x2APIC
 779  *    rewrites the destination of non-IPI messages from APIC_BROADCAST
 780  *    to X2APIC_BROADCAST.
 781  *
 782  * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API.  This is
 783  * important when userspace wants to use x2APIC-format MSIs, because
 784  * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7".
 785  */
 786 static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id,
 787                 struct kvm_lapic *source, struct kvm_lapic *target)
 788 {
 789         bool ipi = source != NULL;
 790
 791         if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled &&
 792             !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target))
 793                 return X2APIC_BROADCAST;
 794
 795         return dest_id;
 796 }
 797
 798 bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
 799                            int short_hand, unsigned int dest, int dest_mode)
 800 {
 801         struct kvm_lapic *target = vcpu->arch.apic;
 802         u32 mda = kvm_apic_mda(vcpu, dest, source, target);
 803
 804         ASSERT(target);
 805         switch (short_hand) {
 806         case APIC_DEST_NOSHORT:
 807                 if (dest_mode == APIC_DEST_PHYSICAL)
 808                         return kvm_apic_match_physical_addr(target, mda);
 809                 else
 810                         return kvm_apic_match_logical_addr(target, mda);
 811         case APIC_DEST_SELF:
 812                 return target == source;
 813         case APIC_DEST_ALLINC:
 814                 return true;
 815         case APIC_DEST_ALLBUT:
 816                 return target != source;
 817         default:
 818                 return false;
 819         }
 820 }
 821 EXPORT_SYMBOL_GPL(kvm_apic_match_dest);
 822
 823 int kvm_vector_to_index(u32 vector, u32 dest_vcpus,
 824                        const unsigned long *bitmap, u32 bitmap_size)
 825 {
 826         u32 mod;
 827         int i, idx = -1;
 828
 829         mod = vector % dest_vcpus;
 830
 831         for (i = 0; i <= mod; i++) {
 832                 idx = find_next_bit(bitmap, bitmap_size, idx + 1);
 833                 BUG_ON(idx == bitmap_size);
 834         }
 835
 836         return idx;
 837 }
 838
 839 static void kvm_apic_disabled_lapic_found(struct kvm *kvm)
 840 {
 841         if (!kvm->arch.disabled_lapic_found) {
 842                 kvm->arch.disabled_lapic_found = true;
 843                 printk(KERN_INFO
 844                        "Disabled LAPIC found during irq injection\n");
 845         }
 846 }
 847
 848 static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src,
 849                 struct kvm_lapic_irq *irq, struct kvm_apic_map *map)
 850 {
 851         if (kvm->arch.x2apic_broadcast_quirk_disabled) {
 852                 if ((irq->dest_id == APIC_BROADCAST &&
 853                                 map->mode != KVM_APIC_MODE_X2APIC))
 854                         return true;
 855                 if (irq->dest_id == X2APIC_BROADCAST)
 856                         return true;
 857         } else {
 858                 bool x2apic_ipi = src && *src && apic_x2apic_mode(*src);
 859                 if (irq->dest_id == (x2apic_ipi ?
 860                                      X2APIC_BROADCAST : APIC_BROADCAST))
 861                         return true;
 862         }
 863
 864         return false;
 865 }
 866
 867 /* Return true if the interrupt can be handled by using *bitmap as index mask
 868  * for valid destinations in *dst array.
 869  * Return false if kvm_apic_map_get_dest_lapic did nothing useful.
 870  * Note: we may have zero kvm_lapic destinations when we return true, which
 871  * means that the interrupt should be dropped.  In this case, *bitmap would be
 872  * zero and *dst undefined.
 873  */
 874 static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm,
 875                 struct kvm_lapic **src, struct kvm_lapic_irq *irq,
 876                 struct kvm_apic_map *map, struct kvm_lapic ***dst,
 877                 unsigned long *bitmap)
 878 {
 879         int i, lowest;
 880
 881         if (irq->shorthand == APIC_DEST_SELF && src) {
 882                 *dst = src;
 883                 *bitmap = 1;
 884                 return true;
 885         } else if (irq->shorthand)
 886                 return false;
 887
 888         if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map))
 889                 return false;
 890
 891         if (irq->dest_mode == APIC_DEST_PHYSICAL) {
 892                 if (irq->dest_id > map->max_apic_id) {
 893                         *bitmap = 0;
 894                 } else {
 895                         u32 dest_id = array_index_nospec(irq->dest_id, map->max_apic_id + 1);
 896                         *dst = &map->phys_map[dest_id];
 897                         *bitmap = 1;
 898                 }
 899                 return true;
 900         }
 901
 902         *bitmap = 0;
 903         if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst,
 904                                 (u16 *)bitmap))
 905                 return false;
 906
 907         if (!kvm_lowest_prio_delivery(irq))
 908                 return true;
 909
 910         if (!kvm_vector_hashing_enabled()) {
 911                 lowest = -1;
 912                 for_each_set_bit(i, bitmap, 16) {
 913                         if (!(*dst)[i])
 914                                 continue;
 915                         if (lowest < 0)
 916                                 lowest = i;
 917                         else if (kvm_apic_compare_prio((*dst)[i]->vcpu,
 918                                                 (*dst)[lowest]->vcpu) < 0)
 919                                 lowest = i;
 920                 }
 921         } else {
 922                 if (!*bitmap)
 923                         return true;
 924
 925                 lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap),
 926                                 bitmap, 16);
 927
 928                 if (!(*dst)[lowest]) {
 929                         kvm_apic_disabled_lapic_found(kvm);
 930                         *bitmap = 0;
 931                         return true;
 932                 }
 933         }
 934
 935         *bitmap = (lowest >= 0) ? 1 << lowest : 0;
 936
 937         return true;
 938 }
 939
 940 bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
 941                 struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map)
 942 {
 943         struct kvm_apic_map *map;
 944         unsigned long bitmap;
 945         struct kvm_lapic **dst = NULL;
 946         int i;
 947         bool ret;
 948
 949         *r = -1;
 950
 951         if (irq->shorthand == APIC_DEST_SELF) {
 952                 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map);
 953                 return true;
 954         }
 955
 956         rcu_read_lock();
 957         map = rcu_dereference(kvm->arch.apic_map);
 958
 959         ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap);
 960         if (ret) {
 961                 *r = 0;
 962                 for_each_set_bit(i, &bitmap, 16) {
 963                         if (!dst[i])
 964                                 continue;
 965                         *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map);
 966                 }
 967         }
 968
 969         rcu_read_unlock();
 970         return ret;
 971 }
 972
 973 /*
 974  * This routine tries to handler interrupts in posted mode, here is how
 975  * it deals with different cases:
 976  * - For single-destination interrupts, handle it in posted mode
 977  * - Else if vector hashing is enabled and it is a lowest-priority
 978  *   interrupt, handle it in posted mode and use the following mechanism
 979  *   to find the destinaiton vCPU.
 980  *      1. For lowest-priority interrupts, store all the possible
 981  *         destination vCPUs in an array.
 982  *      2. Use "guest vector % max number of destination vCPUs" to find
 983  *         the right destination vCPU in the array for the lowest-priority
 984  *         interrupt.
 985  * - Otherwise, use remapped mode to inject the interrupt.
 986  */
 987 bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq,
 988                         struct kvm_vcpu **dest_vcpu)
 989 {
 990         struct kvm_apic_map *map;
 991         unsigned long bitmap;
 992         struct kvm_lapic **dst = NULL;
 993         bool ret = false;
 994
 995         if (irq->shorthand)
 996                 return false;
 997
 998         rcu_read_lock();
 999         map = rcu_dereference(kvm->arch.apic_map);
1000
1001         if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) &&
1002                         hweight16(bitmap) == 1) {
1003                 unsigned long i = find_first_bit(&bitmap, 16);
1004
1005                 if (dst[i]) {
1006                         *dest_vcpu = dst[i]->vcpu;
1007                         ret = true;
1008                 }
1009         }
1010
1011         rcu_read_unlock();
1012         return ret;
1013 }
1014
1015 /*
1016  * Add a pending IRQ into lapic.
1017  * Return 1 if successfully added and 0 if discarded.
1018  */
1019 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
1020                              int vector, int level, int trig_mode,
1021                              struct dest_map *dest_map)
1022 {
1023         int result = 0;
1024         struct kvm_vcpu *vcpu = apic->vcpu;
1025
1026         trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode,
1027                                   trig_mode, vector);
1028         switch (delivery_mode) {
1029         case APIC_DM_LOWEST:
1030                 vcpu->arch.apic_arb_prio++;
1031                 /* fall through */
1032         case APIC_DM_FIXED:
1033                 if (unlikely(trig_mode && !level))
1034                         break;
1035
1036                 /* FIXME add logic for vcpu on reset */
1037                 if (unlikely(!apic_enabled(apic)))
1038                         break;
1039
1040                 result = 1;
1041
1042                 if (dest_map) {
1043                         __set_bit(vcpu->vcpu_id, dest_map->map);
1044                         dest_map->vectors[vcpu->vcpu_id] = vector;
1045                 }
1046
1047                 if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) {
1048                         if (trig_mode)
1049                                 kvm_lapic_set_vector(vector,
1050                                                      apic->regs + APIC_TMR);
1051                         else
1052                                 kvm_lapic_clear_vector(vector,
1053                                                        apic->regs + APIC_TMR);
1054                 }
1055
1056                 if (vcpu->arch.apicv_active)
1057                         kvm_x86_ops->deliver_posted_interrupt(vcpu, vector);
1058                 else {
1059                         kvm_lapic_set_irr(vector, apic);
1060
1061                         kvm_make_request(KVM_REQ_EVENT, vcpu);
1062                         kvm_vcpu_kick(vcpu);
1063                 }
1064                 break;
1065
1066         case APIC_DM_REMRD:
1067                 result = 1;
1068                 vcpu->arch.pv.pv_unhalted = 1;
1069                 kvm_make_request(KVM_REQ_EVENT, vcpu);
1070                 kvm_vcpu_kick(vcpu);
1071                 break;
1072
1073         case APIC_DM_SMI:
1074                 result = 1;
1075                 kvm_make_request(KVM_REQ_SMI, vcpu);
1076                 kvm_vcpu_kick(vcpu);
1077                 break;
1078
1079         case APIC_DM_NMI:
1080                 result = 1;
1081                 kvm_inject_nmi(vcpu);
1082                 kvm_vcpu_kick(vcpu);
1083                 break;
1084
1085         case APIC_DM_INIT:
1086                 if (!trig_mode || level) {
1087                         result = 1;
1088                         /* assumes that there are only KVM_APIC_INIT/SIPI */
1089                         apic->pending_events = (1UL << KVM_APIC_INIT);
1090                         /* make sure pending_events is visible before sending
1091                          * the request */
1092                         smp_wmb();
1093                         kvm_make_request(KVM_REQ_EVENT, vcpu);
1094                         kvm_vcpu_kick(vcpu);
1095                 }
1096                 break;
1097
1098         case APIC_DM_STARTUP:
1099                 result = 1;
1100                 apic->sipi_vector = vector;
1101                 /* make sure sipi_vector is visible for the receiver */
1102                 smp_wmb();
1103                 set_bit(KVM_APIC_SIPI, &apic->pending_events);
1104                 kvm_make_request(KVM_REQ_EVENT, vcpu);
1105                 kvm_vcpu_kick(vcpu);
1106                 break;
1107
1108         case APIC_DM_EXTINT:
1109                 /*
1110                  * Should only be called by kvm_apic_local_deliver() with LVT0,
1111                  * before NMI watchdog was enabled. Already handled by
1112                  * kvm_apic_accept_pic_intr().
1113                  */
1114                 break;
1115
1116         default:
1117                 printk(KERN_ERR "TODO: unsupported delivery mode %x\n",
1118                        delivery_mode);
1119                 break;
1120         }
1121         return result;
1122 }
1123
1124 int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
1125 {
1126         return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio;
1127 }
1128
1129 static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector)
1130 {
1131         return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors);
1132 }
1133
1134 static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector)
1135 {
1136         int trigger_mode;
1137
1138         /* Eoi the ioapic only if the ioapic doesn't own the vector. */
1139         if (!kvm_ioapic_handles_vector(apic, vector))
1140                 return;
1141
1142         /* Request a KVM exit to inform the userspace IOAPIC. */
1143         if (irqchip_split(apic->vcpu->kvm)) {
1144                 apic->vcpu->arch.pending_ioapic_eoi = vector;
1145                 kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu);
1146                 return;
1147         }
1148
1149         if (apic_test_vector(vector, apic->regs + APIC_TMR))
1150                 trigger_mode = IOAPIC_LEVEL_TRIG;
1151         else
1152                 trigger_mode = IOAPIC_EDGE_TRIG;
1153
1154         kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode);
1155 }
1156
1157 static int apic_set_eoi(struct kvm_lapic *apic)
1158 {
1159         int vector = apic_find_highest_isr(apic);
1160
1161         trace_kvm_eoi(apic, vector);
1162
1163         /*
1164          * Not every write EOI will has corresponding ISR,
1165          * one example is when Kernel check timer on setup_IO_APIC
1166          */
1167         if (vector == -1)
1168                 return vector;
1169
1170         apic_clear_isr(vector, apic);
1171         apic_update_ppr(apic);
1172
1173         if (test_bit(vector, vcpu_to_synic(apic->vcpu)->vec_bitmap))
1174                 kvm_hv_synic_send_eoi(apic->vcpu, vector);
1175
1176         kvm_ioapic_send_eoi(apic, vector);
1177         kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
1178         return vector;
1179 }
1180
1181 /*
1182  * this interface assumes a trap-like exit, which has already finished
1183  * desired side effect including vISR and vPPR update.
1184  */
1185 void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector)
1186 {
1187         struct kvm_lapic *apic = vcpu->arch.apic;
1188
1189         trace_kvm_eoi(apic, vector);
1190
1191         kvm_ioapic_send_eoi(apic, vector);
1192         kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
1193 }
1194 EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated);
1195
1196 static void apic_send_ipi(struct kvm_lapic *apic)
1197 {
1198         u32 icr_low = kvm_lapic_get_reg(apic, APIC_ICR);
1199         u32 icr_high = kvm_lapic_get_reg(apic, APIC_ICR2);
1200         struct kvm_lapic_irq irq;
1201
1202         irq.vector = icr_low & APIC_VECTOR_MASK;
1203         irq.delivery_mode = icr_low & APIC_MODE_MASK;
1204         irq.dest_mode = icr_low & APIC_DEST_MASK;
1205         irq.level = (icr_low & APIC_INT_ASSERT) != 0;
1206         irq.trig_mode = icr_low & APIC_INT_LEVELTRIG;
1207         irq.shorthand = icr_low & APIC_SHORT_MASK;
1208         irq.msi_redir_hint = false;
1209         if (apic_x2apic_mode(apic))
1210                 irq.dest_id = icr_high;
1211         else
1212                 irq.dest_id = GET_APIC_DEST_FIELD(icr_high);
1213
1214         trace_kvm_apic_ipi(icr_low, irq.dest_id);
1215
1216         kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL);
1217 }
1218
1219 static u32 apic_get_tmcct(struct kvm_lapic *apic)
1220 {
1221         ktime_t remaining, now;
1222         s64 ns;
1223         u32 tmcct;
1224
1225         ASSERT(apic != NULL);
1226
1227         /* if initial count is 0, current count should also be 0 */
1228         if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 ||
1229                 apic->lapic_timer.period == 0)
1230                 return 0;
1231
1232         now = ktime_get();
1233         remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
1234         if (ktime_to_ns(remaining) < 0)
1235                 remaining = 0;
1236
1237         ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period);
1238         tmcct = div64_u64(ns,
1239                          (APIC_BUS_CYCLE_NS * apic->divide_count));
1240
1241         return tmcct;
1242 }
1243
1244 static void __report_tpr_access(struct kvm_lapic *apic, bool write)
1245 {
1246         struct kvm_vcpu *vcpu = apic->vcpu;
1247         struct kvm_run *run = vcpu->run;
1248
1249         kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu);
1250         run->tpr_access.rip = kvm_rip_read(vcpu);
1251         run->tpr_access.is_write = write;
1252 }
1253
1254 static inline void report_tpr_access(struct kvm_lapic *apic, bool write)
1255 {
1256         if (apic->vcpu->arch.tpr_access_reporting)
1257                 __report_tpr_access(apic, write);
1258 }
1259
1260 static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset)
1261 {
1262         u32 val = 0;
1263
1264         if (offset >= LAPIC_MMIO_LENGTH)
1265                 return 0;
1266
1267         switch (offset) {
1268         case APIC_ARBPRI:
1269                 break;
1270
1271         case APIC_TMCCT:        /* Timer CCR */
1272                 if (apic_lvtt_tscdeadline(apic))
1273                         return 0;
1274
1275                 val = apic_get_tmcct(apic);
1276                 break;
1277         case APIC_PROCPRI:
1278                 apic_update_ppr(apic);
1279                 val = kvm_lapic_get_reg(apic, offset);
1280                 break;
1281         case APIC_TASKPRI:
1282                 report_tpr_access(apic, false);
1283                 /* fall thru */
1284         default:
1285                 val = kvm_lapic_get_reg(apic, offset);
1286                 break;
1287         }
1288
1289         return val;
1290 }
1291
1292 static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev)
1293 {
1294         return container_of(dev, struct kvm_lapic, dev);
1295 }
1296
1297 #define APIC_REG_MASK(reg)      (1ull << ((reg) >> 4))
1298 #define APIC_REGS_MASK(first, count) \
1299         (APIC_REG_MASK(first) * ((1ull << (count)) - 1))
1300
1301 int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
1302                 void *data)
1303 {
1304         unsigned char alignment = offset & 0xf;
1305         u32 result;
1306         /* this bitmask has a bit cleared for each reserved register */
1307         u64 valid_reg_mask =
1308                 APIC_REG_MASK(APIC_ID) |
1309                 APIC_REG_MASK(APIC_LVR) |
1310                 APIC_REG_MASK(APIC_TASKPRI) |
1311                 APIC_REG_MASK(APIC_PROCPRI) |
1312                 APIC_REG_MASK(APIC_LDR) |
1313                 APIC_REG_MASK(APIC_DFR) |
1314                 APIC_REG_MASK(APIC_SPIV) |
1315                 APIC_REGS_MASK(APIC_ISR, APIC_ISR_NR) |
1316                 APIC_REGS_MASK(APIC_TMR, APIC_ISR_NR) |
1317                 APIC_REGS_MASK(APIC_IRR, APIC_ISR_NR) |
1318                 APIC_REG_MASK(APIC_ESR) |
1319                 APIC_REG_MASK(APIC_ICR) |
1320                 APIC_REG_MASK(APIC_ICR2) |
1321                 APIC_REG_MASK(APIC_LVTT) |
1322                 APIC_REG_MASK(APIC_LVTTHMR) |
1323                 APIC_REG_MASK(APIC_LVTPC) |
1324                 APIC_REG_MASK(APIC_LVT0) |
1325                 APIC_REG_MASK(APIC_LVT1) |
1326                 APIC_REG_MASK(APIC_LVTERR) |
1327                 APIC_REG_MASK(APIC_TMICT) |
1328                 APIC_REG_MASK(APIC_TMCCT) |
1329                 APIC_REG_MASK(APIC_TDCR);
1330
1331         /* ARBPRI is not valid on x2APIC */
1332         if (!apic_x2apic_mode(apic))
1333                 valid_reg_mask |= APIC_REG_MASK(APIC_ARBPRI);
1334
1335         if (offset > 0x3f0 || !(valid_reg_mask & APIC_REG_MASK(offset)))
1336                 return 1;
1337
1338         result = __apic_read(apic, offset & ~0xf);
1339
1340         trace_kvm_apic_read(offset, result);
1341
1342         switch (len) {
1343         case 1:
1344         case 2:
1345         case 4:
1346                 memcpy(data, (char *)&result + alignment, len);
1347                 break;
1348         default:
1349                 printk(KERN_ERR "Local APIC read with len = %x, "
1350                        "should be 1,2, or 4 instead\n", len);
1351                 break;
1352         }
1353         return 0;
1354 }
1355 EXPORT_SYMBOL_GPL(kvm_lapic_reg_read);
1356
1357 static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
1358 {
1359         return addr >= apic->base_address &&
1360                 addr < apic->base_address + LAPIC_MMIO_LENGTH;
1361 }
1362
1363 static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
1364                            gpa_t address, int len, void *data)
1365 {
1366         struct kvm_lapic *apic = to_lapic(this);
1367         u32 offset = address - apic->base_address;
1368
1369         if (!apic_mmio_in_range(apic, address))
1370                 return -EOPNOTSUPP;
1371
1372         if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) {
1373                 if (!kvm_check_has_quirk(vcpu->kvm,
1374                                          KVM_X86_QUIRK_LAPIC_MMIO_HOLE))
1375                         return -EOPNOTSUPP;
1376
1377                 memset(data, 0xff, len);
1378                 return 0;
1379         }
1380
1381         kvm_lapic_reg_read(apic, offset, len, data);
1382
1383         return 0;
1384 }
1385
1386 static void update_divide_count(struct kvm_lapic *apic)
1387 {
1388         u32 tmp1, tmp2, tdcr;
1389
1390         tdcr = kvm_lapic_get_reg(apic, APIC_TDCR);
1391         tmp1 = tdcr & 0xf;
1392         tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1;
1393         apic->divide_count = 0x1 << (tmp2 & 0x7);
1394 }
1395
1396 static void limit_periodic_timer_frequency(struct kvm_lapic *apic)
1397 {
1398         /*
1399          * Do not allow the guest to program periodic timers with small
1400          * interval, since the hrtimers are not throttled by the host
1401          * scheduler.
1402          */
1403         if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
1404                 s64 min_period = min_timer_period_us * 1000LL;
1405
1406                 if (apic->lapic_timer.period < min_period) {
1407                         pr_info_ratelimited(
1408                             "kvm: vcpu %i: requested %lld ns "
1409                             "lapic timer period limited to %lld ns\n",
1410                             apic->vcpu->vcpu_id,
1411                             apic->lapic_timer.period, min_period);
1412                         apic->lapic_timer.period = min_period;
1413                 }
1414         }
1415 }
1416
1417 static void apic_update_lvtt(struct kvm_lapic *apic)
1418 {
1419         u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) &
1420                         apic->lapic_timer.timer_mode_mask;
1421
1422         if (apic->lapic_timer.timer_mode != timer_mode) {
1423                 if (apic_lvtt_tscdeadline(apic) != (timer_mode ==
1424                                 APIC_LVT_TIMER_TSCDEADLINE)) {
1425                         hrtimer_cancel(&apic->lapic_timer.timer);
1426                         kvm_lapic_set_reg(apic, APIC_TMICT, 0);
1427                         apic->lapic_timer.period = 0;
1428                         apic->lapic_timer.tscdeadline = 0;
1429                 }
1430                 apic->lapic_timer.timer_mode = timer_mode;
1431                 limit_periodic_timer_frequency(apic);
1432         }
1433 }
1434
1435 /*
1436  * On APICv, this test will cause a busy wait
1437  * during a higher-priority task.
1438  */
1439
1440 static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu)
1441 {
1442         struct kvm_lapic *apic = vcpu->arch.apic;
1443         u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT);
1444
1445         if (kvm_apic_hw_enabled(apic)) {
1446                 int vec = reg & APIC_VECTOR_MASK;
1447                 void *bitmap = apic->regs + APIC_ISR;
1448
1449                 if (vcpu->arch.apicv_active)
1450                         bitmap = apic->regs + APIC_IRR;
1451
1452                 if (apic_test_vector(vec, bitmap))
1453                         return true;
1454         }
1455         return false;
1456 }
1457
1458 static inline void __wait_lapic_expire(struct kvm_vcpu *vcpu, u64 guest_cycles)
1459 {
1460         u64 timer_advance_ns = vcpu->arch.apic->lapic_timer.timer_advance_ns;
1461
1462         /*
1463          * If the guest TSC is running at a different ratio than the host, then
1464          * convert the delay to nanoseconds to achieve an accurate delay.  Note
1465          * that __delay() uses delay_tsc whenever the hardware has TSC, thus
1466          * always for VMX enabled hardware.
1467          */
1468         if (vcpu->arch.tsc_scaling_ratio == kvm_default_tsc_scaling_ratio) {
1469                 __delay(min(guest_cycles,
1470                         nsec_to_cycles(vcpu, timer_advance_ns)));
1471         } else {
1472                 u64 delay_ns = guest_cycles * 1000000ULL;
1473                 do_div(delay_ns, vcpu->arch.virtual_tsc_khz);
1474                 ndelay(min_t(u32, delay_ns, timer_advance_ns));
1475         }
1476 }
1477
1478 static inline void adjust_lapic_timer_advance(struct kvm_vcpu *vcpu,
1479                                               s64 advance_expire_delta)
1480 {
1481         struct kvm_lapic *apic = vcpu->arch.apic;
1482         u32 timer_advance_ns = apic->lapic_timer.timer_advance_ns;
1483         u64 ns;
1484
1485         /* too early */
1486         if (advance_expire_delta < 0) {
1487                 ns = -advance_expire_delta * 1000000ULL;
1488                 do_div(ns, vcpu->arch.virtual_tsc_khz);
1489                 timer_advance_ns -= min((u32)ns,
1490                         timer_advance_ns / LAPIC_TIMER_ADVANCE_ADJUST_STEP);
1491         } else {
1492         /* too late */
1493                 ns = advance_expire_delta * 1000000ULL;
1494                 do_div(ns, vcpu->arch.virtual_tsc_khz);
1495                 timer_advance_ns += min((u32)ns,
1496                         timer_advance_ns / LAPIC_TIMER_ADVANCE_ADJUST_STEP);
1497         }
1498
1499         if (abs(advance_expire_delta) < LAPIC_TIMER_ADVANCE_ADJUST_DONE)
1500                 apic->lapic_timer.timer_advance_adjust_done = true;
1501         if (unlikely(timer_advance_ns > 5000)) {
1502                 timer_advance_ns = LAPIC_TIMER_ADVANCE_ADJUST_INIT;
1503                 apic->lapic_timer.timer_advance_adjust_done = false;
1504         }
1505         apic->lapic_timer.timer_advance_ns = timer_advance_ns;
1506 }
1507
1508 static void __kvm_wait_lapic_expire(struct kvm_vcpu *vcpu)
1509 {
1510         struct kvm_lapic *apic = vcpu->arch.apic;
1511         u64 guest_tsc, tsc_deadline;
1512
1513         if (apic->lapic_timer.expired_tscdeadline == 0)
1514                 return;
1515
1516         tsc_deadline = apic->lapic_timer.expired_tscdeadline;
1517         apic->lapic_timer.expired_tscdeadline = 0;
1518         guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1519         apic->lapic_timer.advance_expire_delta = guest_tsc - tsc_deadline;
1520
1521         if (guest_tsc < tsc_deadline)
1522                 __wait_lapic_expire(vcpu, tsc_deadline - guest_tsc);
1523
1524         if (unlikely(!apic->lapic_timer.timer_advance_adjust_done))
1525                 adjust_lapic_timer_advance(vcpu, apic->lapic_timer.advance_expire_delta);
1526 }
1527
1528 void kvm_wait_lapic_expire(struct kvm_vcpu *vcpu)
1529 {
1530         if (lapic_timer_int_injected(vcpu))
1531                 __kvm_wait_lapic_expire(vcpu);
1532 }
1533 EXPORT_SYMBOL_GPL(kvm_wait_lapic_expire);
1534
1535 static void kvm_apic_inject_pending_timer_irqs(struct kvm_lapic *apic)
1536 {
1537         struct kvm_timer *ktimer = &apic->lapic_timer;
1538
1539         kvm_apic_local_deliver(apic, APIC_LVTT);
1540         if (apic_lvtt_tscdeadline(apic))
1541                 ktimer->tscdeadline = 0;
1542         if (apic_lvtt_oneshot(apic)) {
1543                 ktimer->tscdeadline = 0;
1544                 ktimer->target_expiration = 0;
1545         }
1546 }
1547
1548 static void apic_timer_expired(struct kvm_lapic *apic)
1549 {
1550         struct kvm_vcpu *vcpu = apic->vcpu;
1551         struct swait_queue_head *q = &vcpu->wq;
1552         struct kvm_timer *ktimer = &apic->lapic_timer;
1553
1554         if (atomic_read(&apic->lapic_timer.pending))
1555                 return;
1556
1557         if (apic_lvtt_tscdeadline(apic) || ktimer->hv_timer_in_use)
1558                 ktimer->expired_tscdeadline = ktimer->tscdeadline;
1559
1560         if (kvm_use_posted_timer_interrupt(apic->vcpu)) {
1561                 if (apic->lapic_timer.timer_advance_ns)
1562                         __kvm_wait_lapic_expire(vcpu);
1563                 kvm_apic_inject_pending_timer_irqs(apic);
1564                 return;
1565         }
1566
1567         atomic_inc(&apic->lapic_timer.pending);
1568         kvm_set_pending_timer(vcpu);
1569
1570         /*
1571          * For x86, the atomic_inc() is serialized, thus
1572          * using swait_active() is safe.
1573          */
1574         if (swait_active(q))
1575                 swake_up_one(q);
1576 }
1577
1578 static void start_sw_tscdeadline(struct kvm_lapic *apic)
1579 {
1580         struct kvm_timer *ktimer = &apic->lapic_timer;
1581         u64 guest_tsc, tscdeadline = ktimer->tscdeadline;
1582         u64 ns = 0;
1583         ktime_t expire;
1584         struct kvm_vcpu *vcpu = apic->vcpu;
1585         unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz;
1586         unsigned long flags;
1587         ktime_t now;
1588
1589         if (unlikely(!tscdeadline || !this_tsc_khz))
1590                 return;
1591
1592         local_irq_save(flags);
1593
1594         now = ktime_get();
1595         guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1596
1597         ns = (tscdeadline - guest_tsc) * 1000000ULL;
1598         do_div(ns, this_tsc_khz);
1599
1600         if (likely(tscdeadline > guest_tsc) &&
1601             likely(ns > apic->lapic_timer.timer_advance_ns)) {
1602                 expire = ktime_add_ns(now, ns);
1603                 expire = ktime_sub_ns(expire, ktimer->timer_advance_ns);
1604                 hrtimer_start(&ktimer->timer, expire, HRTIMER_MODE_ABS);
1605         } else
1606                 apic_timer_expired(apic);
1607
1608         local_irq_restore(flags);
1609 }
1610
1611 static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_divisor)
1612 {
1613         ktime_t now, remaining;
1614         u64 ns_remaining_old, ns_remaining_new;
1615
1616         apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT)
1617                 * APIC_BUS_CYCLE_NS * apic->divide_count;
1618         limit_periodic_timer_frequency(apic);
1619
1620         now = ktime_get();
1621         remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
1622         if (ktime_to_ns(remaining) < 0)
1623                 remaining = 0;
1624
1625         ns_remaining_old = ktime_to_ns(remaining);
1626         ns_remaining_new = mul_u64_u32_div(ns_remaining_old,
1627                                            apic->divide_count, old_divisor);
1628
1629         apic->lapic_timer.tscdeadline +=
1630                 nsec_to_cycles(apic->vcpu, ns_remaining_new) -
1631                 nsec_to_cycles(apic->vcpu, ns_remaining_old);
1632         apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new);
1633 }
1634
1635 static bool set_target_expiration(struct kvm_lapic *apic)
1636 {
1637         ktime_t now;
1638         u64 tscl = rdtsc();
1639
1640         now = ktime_get();
1641         apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT)
1642                 * APIC_BUS_CYCLE_NS * apic->divide_count;
1643
1644         if (!apic->lapic_timer.period) {
1645                 apic->lapic_timer.tscdeadline = 0;
1646                 return false;
1647         }
1648
1649         limit_periodic_timer_frequency(apic);
1650
1651         apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
1652                 nsec_to_cycles(apic->vcpu, apic->lapic_timer.period);
1653         apic->lapic_timer.target_expiration = ktime_add_ns(now, apic->lapic_timer.period);
1654
1655         return true;
1656 }
1657
1658 static void advance_periodic_target_expiration(struct kvm_lapic *apic)
1659 {
1660         ktime_t now = ktime_get();
1661         u64 tscl = rdtsc();
1662         ktime_t delta;
1663
1664         /*
1665          * Synchronize both deadlines to the same time source or
1666          * differences in the periods (caused by differences in the
1667          * underlying clocks or numerical approximation errors) will
1668          * cause the two to drift apart over time as the errors
1669          * accumulate.
1670          */
1671         apic->lapic_timer.target_expiration =
1672                 ktime_add_ns(apic->lapic_timer.target_expiration,
1673                                 apic->lapic_timer.period);
1674         delta = ktime_sub(apic->lapic_timer.target_expiration, now);
1675         apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
1676                 nsec_to_cycles(apic->vcpu, delta);
1677 }
1678
1679 static void start_sw_period(struct kvm_lapic *apic)
1680 {
1681         if (!apic->lapic_timer.period)
1682                 return;
1683
1684         if (ktime_after(ktime_get(),
1685                         apic->lapic_timer.target_expiration)) {
1686                 apic_timer_expired(apic);
1687
1688                 if (apic_lvtt_oneshot(apic))
1689                         return;
1690
1691                 advance_periodic_target_expiration(apic);
1692         }
1693
1694         hrtimer_start(&apic->lapic_timer.timer,
1695                 apic->lapic_timer.target_expiration,
1696                 HRTIMER_MODE_ABS);
1697 }
1698
1699 bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu)
1700 {
1701         if (!lapic_in_kernel(vcpu))
1702                 return false;
1703
1704         return vcpu->arch.apic->lapic_timer.hv_timer_in_use;
1705 }
1706 EXPORT_SYMBOL_GPL(kvm_lapic_hv_timer_in_use);
1707
1708 static void cancel_hv_timer(struct kvm_lapic *apic)
1709 {
1710         WARN_ON(preemptible());
1711         WARN_ON(!apic->lapic_timer.hv_timer_in_use);
1712         kvm_x86_ops->cancel_hv_timer(apic->vcpu);
1713         apic->lapic_timer.hv_timer_in_use = false;
1714 }
1715
1716 static bool start_hv_timer(struct kvm_lapic *apic)
1717 {
1718         struct kvm_timer *ktimer = &apic->lapic_timer;
1719         struct kvm_vcpu *vcpu = apic->vcpu;
1720         bool expired;
1721
1722         WARN_ON(preemptible());
1723         if (!kvm_x86_ops->set_hv_timer)
1724                 return false;
1725
1726         if (!ktimer->tscdeadline)
1727                 return false;
1728
1729         if (kvm_x86_ops->set_hv_timer(vcpu, ktimer->tscdeadline, &expired))
1730                 return false;
1731
1732         ktimer->hv_timer_in_use = true;
1733         hrtimer_cancel(&ktimer->timer);
1734
1735         /*
1736          * To simplify handling the periodic timer, leave the hv timer running
1737          * even if the deadline timer has expired, i.e. rely on the resulting
1738          * VM-Exit to recompute the periodic timer's target expiration.
1739          */
1740         if (!apic_lvtt_period(apic)) {
1741                 /*
1742                  * Cancel the hv timer if the sw timer fired while the hv timer
1743                  * was being programmed, or if the hv timer itself expired.
1744                  */
1745                 if (atomic_read(&ktimer->pending)) {
1746                         cancel_hv_timer(apic);
1747                 } else if (expired) {
1748                         apic_timer_expired(apic);
1749                         cancel_hv_timer(apic);
1750                 }
1751         }
1752
1753         trace_kvm_hv_timer_state(vcpu->vcpu_id, ktimer->hv_timer_in_use);
1754
1755         return true;
1756 }
1757
1758 static void start_sw_timer(struct kvm_lapic *apic)
1759 {
1760         struct kvm_timer *ktimer = &apic->lapic_timer;
1761
1762         WARN_ON(preemptible());
1763         if (apic->lapic_timer.hv_timer_in_use)
1764                 cancel_hv_timer(apic);
1765         if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending))
1766                 return;
1767
1768         if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
1769                 start_sw_period(apic);
1770         else if (apic_lvtt_tscdeadline(apic))
1771                 start_sw_tscdeadline(apic);
1772         trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false);
1773 }
1774
1775 static void restart_apic_timer(struct kvm_lapic *apic)
1776 {
1777         preempt_disable();
1778
1779         if (!apic_lvtt_period(apic) && atomic_read(&apic->lapic_timer.pending))
1780                 goto out;
1781
1782         if (!start_hv_timer(apic))
1783                 start_sw_timer(apic);
1784 out:
1785         preempt_enable();
1786 }
1787
1788 void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu)
1789 {
1790         struct kvm_lapic *apic = vcpu->arch.apic;
1791
1792         preempt_disable();
1793         /* If the preempt notifier has already run, it also called apic_timer_expired */
1794         if (!apic->lapic_timer.hv_timer_in_use)
1795                 goto out;
1796         WARN_ON(swait_active(&vcpu->wq));
1797         cancel_hv_timer(apic);
1798         apic_timer_expired(apic);
1799
1800         if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
1801                 advance_periodic_target_expiration(apic);
1802                 restart_apic_timer(apic);
1803         }
1804 out:
1805         preempt_enable();
1806 }
1807 EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer);
1808
1809 void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu)
1810 {
1811         restart_apic_timer(vcpu->arch.apic);
1812 }
1813 EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_hv_timer);
1814
1815 void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu)
1816 {
1817         struct kvm_lapic *apic = vcpu->arch.apic;
1818
1819         preempt_disable();
1820         /* Possibly the TSC deadline timer is not enabled yet */
1821         if (apic->lapic_timer.hv_timer_in_use)
1822                 start_sw_timer(apic);
1823         preempt_enable();
1824 }
1825 EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_sw_timer);
1826
1827 void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu)
1828 {
1829         struct kvm_lapic *apic = vcpu->arch.apic;
1830
1831         WARN_ON(!apic->lapic_timer.hv_timer_in_use);
1832         restart_apic_timer(apic);
1833 }
1834
1835 static void start_apic_timer(struct kvm_lapic *apic)
1836 {
1837         atomic_set(&apic->lapic_timer.pending, 0);
1838
1839         if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
1840             && !set_target_expiration(apic))
1841                 return;
1842
1843         restart_apic_timer(apic);
1844 }
1845
1846 static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
1847 {
1848         bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val);
1849
1850         if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) {
1851                 apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode;
1852                 if (lvt0_in_nmi_mode) {
1853                         atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
1854                 } else
1855                         atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
1856         }
1857 }
1858
1859 int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
1860 {
1861         int ret = 0;
1862
1863         trace_kvm_apic_write(reg, val);
1864
1865         switch (reg) {
1866         case APIC_ID:           /* Local APIC ID */
1867                 if (!apic_x2apic_mode(apic))
1868                         kvm_apic_set_xapic_id(apic, val >> 24);
1869                 else
1870                         ret = 1;
1871                 break;
1872
1873         case APIC_TASKPRI:
1874                 report_tpr_access(apic, true);
1875                 apic_set_tpr(apic, val & 0xff);
1876                 break;
1877
1878         case APIC_EOI:
1879                 apic_set_eoi(apic);
1880                 break;
1881
1882         case APIC_LDR:
1883                 if (!apic_x2apic_mode(apic))
1884                         kvm_apic_set_ldr(apic, val & APIC_LDR_MASK);
1885                 else
1886                         ret = 1;
1887                 break;
1888
1889         case APIC_DFR:
1890                 if (!apic_x2apic_mode(apic)) {
1891                         kvm_lapic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF);
1892                         recalculate_apic_map(apic->vcpu->kvm);
1893                 } else
1894                         ret = 1;
1895                 break;
1896
1897         case APIC_SPIV: {
1898                 u32 mask = 0x3ff;
1899                 if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI)
1900                         mask |= APIC_SPIV_DIRECTED_EOI;
1901                 apic_set_spiv(apic, val & mask);
1902                 if (!(val & APIC_SPIV_APIC_ENABLED)) {
1903                         int i;
1904                         u32 lvt_val;
1905
1906                         for (i = 0; i < KVM_APIC_LVT_NUM; i++) {
1907                                 lvt_val = kvm_lapic_get_reg(apic,
1908                                                        APIC_LVTT + 0x10 * i);
1909                                 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i,
1910                                              lvt_val | APIC_LVT_MASKED);
1911                         }
1912                         apic_update_lvtt(apic);
1913                         atomic_set(&apic->lapic_timer.pending, 0);
1914
1915                 }
1916                 break;
1917         }
1918         case APIC_ICR:
1919                 /* No delay here, so we always clear the pending bit */
1920                 kvm_lapic_set_reg(apic, APIC_ICR, val & ~(1 << 12));
1921                 apic_send_ipi(apic);
1922                 break;
1923
1924         case APIC_ICR2:
1925                 if (!apic_x2apic_mode(apic))
1926                         val &= 0xff000000;
1927                 kvm_lapic_set_reg(apic, APIC_ICR2, val);
1928                 break;
1929
1930         case APIC_LVT0:
1931                 apic_manage_nmi_watchdog(apic, val);
1932                 /* fall through */
1933         case APIC_LVTTHMR:
1934         case APIC_LVTPC:
1935         case APIC_LVT1:
1936         case APIC_LVTERR:
1937                 /* TODO: Check vector */
1938                 if (!kvm_apic_sw_enabled(apic))
1939                         val |= APIC_LVT_MASKED;
1940
1941                 val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4];
1942                 kvm_lapic_set_reg(apic, reg, val);
1943
1944                 break;
1945
1946         case APIC_LVTT:
1947                 if (!kvm_apic_sw_enabled(apic))
1948                         val |= APIC_LVT_MASKED;
1949                 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask);
1950                 kvm_lapic_set_reg(apic, APIC_LVTT, val);
1951                 apic_update_lvtt(apic);
1952                 break;
1953
1954         case APIC_TMICT:
1955                 if (apic_lvtt_tscdeadline(apic))
1956                         break;
1957
1958                 hrtimer_cancel(&apic->lapic_timer.timer);
1959                 kvm_lapic_set_reg(apic, APIC_TMICT, val);
1960                 start_apic_timer(apic);
1961                 break;
1962
1963         case APIC_TDCR: {
1964                 uint32_t old_divisor = apic->divide_count;
1965
1966                 kvm_lapic_set_reg(apic, APIC_TDCR, val);
1967                 update_divide_count(apic);
1968                 if (apic->divide_count != old_divisor &&
1969                                 apic->lapic_timer.period) {
1970                         hrtimer_cancel(&apic->lapic_timer.timer);
1971                         update_target_expiration(apic, old_divisor);
1972                         restart_apic_timer(apic);
1973                 }
1974                 break;
1975         }
1976         case APIC_ESR:
1977                 if (apic_x2apic_mode(apic) && val != 0)
1978                         ret = 1;
1979                 break;
1980
1981         case APIC_SELF_IPI:
1982                 if (apic_x2apic_mode(apic)) {
1983                         kvm_lapic_reg_write(apic, APIC_ICR, 0x40000 | (val & 0xff));
1984                 } else
1985                         ret = 1;
1986                 break;
1987         default:
1988                 ret = 1;
1989                 break;
1990         }
1991
1992         return ret;
1993 }
1994 EXPORT_SYMBOL_GPL(kvm_lapic_reg_write);
1995
1996 static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
1997                             gpa_t address, int len, const void *data)
1998 {
1999         struct kvm_lapic *apic = to_lapic(this);
2000         unsigned int offset = address - apic->base_address;
2001         u32 val;
2002
2003         if (!apic_mmio_in_range(apic, address))
2004                 return -EOPNOTSUPP;
2005
2006         if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) {
2007                 if (!kvm_check_has_quirk(vcpu->kvm,
2008                                          KVM_X86_QUIRK_LAPIC_MMIO_HOLE))
2009                         return -EOPNOTSUPP;
2010
2011                 return 0;
2012         }
2013
2014         /*
2015          * APIC register must be aligned on 128-bits boundary.
2016          * 32/64/128 bits registers must be accessed thru 32 bits.
2017          * Refer SDM 8.4.1
2018          */
2019         if (len != 4 || (offset & 0xf))
2020                 return 0;
2021
2022         val = *(u32*)data;
2023
2024         kvm_lapic_reg_write(apic, offset & 0xff0, val);
2025
2026         return 0;
2027 }
2028
2029 void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu)
2030 {
2031         kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0);
2032 }
2033 EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi);
2034
2035 /* emulate APIC access in a trap manner */
2036 void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset)
2037 {
2038         u32 val = 0;
2039
2040         /* hw has done the conditional check and inst decode */
2041         offset &= 0xff0;
2042
2043         kvm_lapic_reg_read(vcpu->arch.apic, offset, 4, &val);
2044
2045         /* TODO: optimize to just emulate side effect w/o one more write */
2046         kvm_lapic_reg_write(vcpu->arch.apic, offset, val);
2047 }
2048 EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode);
2049
2050 void kvm_free_lapic(struct kvm_vcpu *vcpu)
2051 {
2052         struct kvm_lapic *apic = vcpu->arch.apic;
2053
2054         if (!vcpu->arch.apic)
2055                 return;
2056
2057         hrtimer_cancel(&apic->lapic_timer.timer);
2058
2059         if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE))
2060                 static_key_slow_dec_deferred(&apic_hw_disabled);
2061
2062         if (!apic->sw_enabled)
2063                 static_key_slow_dec_deferred(&apic_sw_disabled);
2064
2065         if (apic->regs)
2066                 free_page((unsigned long)apic->regs);
2067
2068         kfree(apic);
2069 }
2070
2071 /*
2072  *----------------------------------------------------------------------
2073  * LAPIC interface
2074  *----------------------------------------------------------------------
2075  */
2076 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu)
2077 {
2078         struct kvm_lapic *apic = vcpu->arch.apic;
2079
2080         if (!lapic_in_kernel(vcpu) ||
2081                 !apic_lvtt_tscdeadline(apic))
2082                 return 0;
2083
2084         return apic->lapic_timer.tscdeadline;
2085 }
2086
2087 void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data)
2088 {
2089         struct kvm_lapic *apic = vcpu->arch.apic;
2090
2091         if (!lapic_in_kernel(vcpu) || apic_lvtt_oneshot(apic) ||
2092                         apic_lvtt_period(apic))
2093                 return;
2094
2095         hrtimer_cancel(&apic->lapic_timer.timer);
2096         apic->lapic_timer.tscdeadline = data;
2097         start_apic_timer(apic);
2098 }
2099
2100 void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8)
2101 {
2102         struct kvm_lapic *apic = vcpu->arch.apic;
2103
2104         apic_set_tpr(apic, ((cr8 & 0x0f) << 4)
2105                      | (kvm_lapic_get_reg(apic, APIC_TASKPRI) & 4));
2106 }
2107
2108 u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
2109 {
2110         u64 tpr;
2111
2112         tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI);
2113
2114         return (tpr & 0xf0) >> 4;
2115 }
2116
2117 void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
2118 {
2119         u64 old_value = vcpu->arch.apic_base;
2120         struct kvm_lapic *apic = vcpu->arch.apic;
2121
2122         if (!apic)
2123                 value |= MSR_IA32_APICBASE_BSP;
2124
2125         vcpu->arch.apic_base = value;
2126
2127         if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE)
2128                 kvm_update_cpuid(vcpu);
2129
2130         if (!apic)
2131                 return;
2132
2133         /* update jump label if enable bit changes */
2134         if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) {
2135                 if (value & MSR_IA32_APICBASE_ENABLE) {
2136                         kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
2137                         static_key_slow_dec_deferred(&apic_hw_disabled);
2138                 } else {
2139                         static_key_slow_inc(&apic_hw_disabled.key);
2140                         recalculate_apic_map(vcpu->kvm);
2141                 }
2142         }
2143
2144         if (((old_value ^ value) & X2APIC_ENABLE) && (value & X2APIC_ENABLE))
2145                 kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);
2146
2147         if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE))
2148                 kvm_x86_ops->set_virtual_apic_mode(vcpu);
2149
2150         apic->base_address = apic->vcpu->arch.apic_base &
2151                              MSR_IA32_APICBASE_BASE;
2152
2153         if ((value & MSR_IA32_APICBASE_ENABLE) &&
2154              apic->base_address != APIC_DEFAULT_PHYS_BASE)
2155                 pr_warn_once("APIC base relocation is unsupported by KVM");
2156 }
2157
2158 void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event)
2159 {
2160         struct kvm_lapic *apic = vcpu->arch.apic;
2161         int i;
2162
2163         if (!apic)
2164                 return;
2165
2166         /* Stop the timer in case it's a reset to an active apic */
2167         hrtimer_cancel(&apic->lapic_timer.timer);
2168
2169         if (!init_event) {
2170                 kvm_lapic_set_base(vcpu, APIC_DEFAULT_PHYS_BASE |
2171                                          MSR_IA32_APICBASE_ENABLE);
2172                 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
2173         }
2174         kvm_apic_set_version(apic->vcpu);
2175
2176         for (i = 0; i < KVM_APIC_LVT_NUM; i++)
2177                 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED);
2178         apic_update_lvtt(apic);
2179         if (kvm_vcpu_is_reset_bsp(vcpu) &&
2180             kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED))
2181                 kvm_lapic_set_reg(apic, APIC_LVT0,
2182                              SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT));
2183         apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
2184
2185         kvm_lapic_set_reg(apic, APIC_DFR, 0xffffffffU);
2186         apic_set_spiv(apic, 0xff);
2187         kvm_lapic_set_reg(apic, APIC_TASKPRI, 0);
2188         if (!apic_x2apic_mode(apic))
2189                 kvm_apic_set_ldr(apic, 0);
2190         kvm_lapic_set_reg(apic, APIC_ESR, 0);
2191         kvm_lapic_set_reg(apic, APIC_ICR, 0);
2192         kvm_lapic_set_reg(apic, APIC_ICR2, 0);
2193         kvm_lapic_set_reg(apic, APIC_TDCR, 0);
2194         kvm_lapic_set_reg(apic, APIC_TMICT, 0);
2195         for (i = 0; i < 8; i++) {
2196                 kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0);
2197                 kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0);
2198                 kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0);
2199         }
2200         apic->irr_pending = vcpu->arch.apicv_active;
2201         apic->isr_count = vcpu->arch.apicv_active ? 1 : 0;
2202         apic->highest_isr_cache = -1;
2203         update_divide_count(apic);
2204         atomic_set(&apic->lapic_timer.pending, 0);
2205         if (kvm_vcpu_is_bsp(vcpu))
2206                 kvm_lapic_set_base(vcpu,
2207                                 vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP);
2208         vcpu->arch.pv_eoi.msr_val = 0;
2209         apic_update_ppr(apic);
2210         if (vcpu->arch.apicv_active) {
2211                 kvm_x86_ops->apicv_post_state_restore(vcpu);
2212                 kvm_x86_ops->hwapic_irr_update(vcpu, -1);
2213                 kvm_x86_ops->hwapic_isr_update(vcpu, -1);
2214         }
2215
2216         vcpu->arch.apic_arb_prio = 0;
2217         vcpu->arch.apic_attention = 0;
2218 }
2219
2220 /*
2221  *----------------------------------------------------------------------
2222  * timer interface
2223  *----------------------------------------------------------------------
2224  */
2225
2226 static bool lapic_is_periodic(struct kvm_lapic *apic)
2227 {
2228         return apic_lvtt_period(apic);
2229 }
2230
2231 int apic_has_pending_timer(struct kvm_vcpu *vcpu)
2232 {
2233         struct kvm_lapic *apic = vcpu->arch.apic;
2234
2235         if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT))
2236                 return atomic_read(&apic->lapic_timer.pending);
2237
2238         return 0;
2239 }
2240
2241 int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type)
2242 {
2243         u32 reg = kvm_lapic_get_reg(apic, lvt_type);
2244         int vector, mode, trig_mode;
2245
2246         if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) {
2247                 vector = reg & APIC_VECTOR_MASK;
2248                 mode = reg & APIC_MODE_MASK;
2249                 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER;
2250                 return __apic_accept_irq(apic, mode, vector, 1, trig_mode,
2251                                         NULL);
2252         }
2253         return 0;
2254 }
2255
2256 void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu)
2257 {
2258         struct kvm_lapic *apic = vcpu->arch.apic;
2259
2260         if (apic)
2261                 kvm_apic_local_deliver(apic, APIC_LVT0);
2262 }
2263
2264 static const struct kvm_io_device_ops apic_mmio_ops = {
2265         .read     = apic_mmio_read,
2266         .write    = apic_mmio_write,
2267 };
2268
2269 static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
2270 {
2271         struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
2272         struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer);
2273
2274         apic_timer_expired(apic);
2275
2276         if (lapic_is_periodic(apic)) {
2277                 advance_periodic_target_expiration(apic);
2278                 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
2279                 return HRTIMER_RESTART;
2280         } else
2281                 return HRTIMER_NORESTART;
2282 }
2283
2284 int kvm_create_lapic(struct kvm_vcpu *vcpu, int timer_advance_ns)
2285 {
2286         struct kvm_lapic *apic;
2287
2288         ASSERT(vcpu != NULL);
2289
2290         apic = kzalloc(sizeof(*apic), GFP_KERNEL_ACCOUNT);
2291         if (!apic)
2292                 goto nomem;
2293
2294         vcpu->arch.apic = apic;
2295
2296         apic->regs = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
2297         if (!apic->regs) {
2298                 printk(KERN_ERR "malloc apic regs error for vcpu %x\n",
2299                        vcpu->vcpu_id);
2300                 goto nomem_free_apic;
2301         }
2302         apic->vcpu = vcpu;
2303
2304         hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
2305                      HRTIMER_MODE_ABS);
2306         apic->lapic_timer.timer.function = apic_timer_fn;
2307         if (timer_advance_ns == -1) {
2308                 apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_ADJUST_INIT;
2309                 apic->lapic_timer.timer_advance_adjust_done = false;
2310         } else {
2311                 apic->lapic_timer.timer_advance_ns = timer_advance_ns;
2312                 apic->lapic_timer.timer_advance_adjust_done = true;
2313         }
2314
2315
2316         /*
2317          * APIC is created enabled. This will prevent kvm_lapic_set_base from
2318          * thinking that APIC state has changed.
2319          */
2320         vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE;
2321         static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */
2322         kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
2323
2324         return 0;
2325 nomem_free_apic:
2326         kfree(apic);
2327         vcpu->arch.apic = NULL;
2328 nomem:
2329         return -ENOMEM;
2330 }
2331
2332 int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu)
2333 {
2334         struct kvm_lapic *apic = vcpu->arch.apic;
2335         u32 ppr;
2336
2337         if (!kvm_apic_hw_enabled(apic))
2338                 return -1;
2339
2340         __apic_update_ppr(apic, &ppr);
2341         return apic_has_interrupt_for_ppr(apic, ppr);
2342 }
2343
2344 int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu)
2345 {
2346         u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0);
2347         int r = 0;
2348
2349         if (!kvm_apic_hw_enabled(vcpu->arch.apic))
2350                 r = 1;
2351         if ((lvt0 & APIC_LVT_MASKED) == 0 &&
2352             GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT)
2353                 r = 1;
2354         return r;
2355 }
2356
2357 void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu)
2358 {
2359         struct kvm_lapic *apic = vcpu->arch.apic;
2360
2361         if (atomic_read(&apic->lapic_timer.pending) > 0) {
2362                 kvm_apic_inject_pending_timer_irqs(apic);
2363                 atomic_set(&apic->lapic_timer.pending, 0);
2364         }
2365 }
2366
2367 int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu)
2368 {
2369         int vector = kvm_apic_has_interrupt(vcpu);
2370         struct kvm_lapic *apic = vcpu->arch.apic;
2371         u32 ppr;
2372
2373         if (vector == -1)
2374                 return -1;
2375
2376         /*
2377          * We get here even with APIC virtualization enabled, if doing
2378          * nested virtualization and L1 runs with the "acknowledge interrupt
2379          * on exit" mode.  Then we cannot inject the interrupt via RVI,
2380          * because the process would deliver it through the IDT.
2381          */
2382
2383         apic_clear_irr(vector, apic);
2384         if (test_bit(vector, vcpu_to_synic(vcpu)->auto_eoi_bitmap)) {
2385                 /*
2386                  * For auto-EOI interrupts, there might be another pending
2387                  * interrupt above PPR, so check whether to raise another
2388                  * KVM_REQ_EVENT.
2389                  */
2390                 apic_update_ppr(apic);
2391         } else {
2392                 /*
2393                  * For normal interrupts, PPR has been raised and there cannot
2394                  * be a higher-priority pending interrupt---except if there was
2395                  * a concurrent interrupt injection, but that would have
2396                  * triggered KVM_REQ_EVENT already.
2397                  */
2398                 apic_set_isr(vector, apic);
2399                 __apic_update_ppr(apic, &ppr);
2400         }
2401
2402         return vector;
2403 }
2404
2405 static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu,
2406                 struct kvm_lapic_state *s, bool set)
2407 {
2408         if (apic_x2apic_mode(vcpu->arch.apic)) {
2409                 u32 *id = (u32 *)(s->regs + APIC_ID);
2410                 u32 *ldr = (u32 *)(s->regs + APIC_LDR);
2411
2412                 if (vcpu->kvm->arch.x2apic_format) {
2413                         if (*id != vcpu->vcpu_id)
2414                                 return -EINVAL;
2415                 } else {
2416                         if (set)
2417                                 *id >>= 24;
2418                         else
2419                                 *id <<= 24;
2420                 }
2421
2422                 /* In x2APIC mode, the LDR is fixed and based on the id */
2423                 if (set)
2424                         *ldr = kvm_apic_calc_x2apic_ldr(*id);
2425         }
2426
2427         return 0;
2428 }
2429
2430 int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
2431 {
2432         memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s));
2433         return kvm_apic_state_fixup(vcpu, s, false);
2434 }
2435
2436 int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
2437 {
2438         struct kvm_lapic *apic = vcpu->arch.apic;
2439         int r;
2440
2441
2442         kvm_lapic_set_base(vcpu, vcpu->arch.apic_base);
2443         /* set SPIV separately to get count of SW disabled APICs right */
2444         apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV)));
2445
2446         r = kvm_apic_state_fixup(vcpu, s, true);
2447         if (r)
2448                 return r;
2449         memcpy(vcpu->arch.apic->regs, s->regs, sizeof(*s));
2450
2451         recalculate_apic_map(vcpu->kvm);
2452         kvm_apic_set_version(vcpu);
2453
2454         apic_update_ppr(apic);
2455         hrtimer_cancel(&apic->lapic_timer.timer);
2456         apic_update_lvtt(apic);
2457         apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
2458         update_divide_count(apic);
2459         start_apic_timer(apic);
2460         apic->irr_pending = true;
2461         apic->isr_count = vcpu->arch.apicv_active ?
2462                                 1 : count_vectors(apic->regs + APIC_ISR);
2463         apic->highest_isr_cache = -1;
2464         if (vcpu->arch.apicv_active) {
2465                 kvm_x86_ops->apicv_post_state_restore(vcpu);
2466                 kvm_x86_ops->hwapic_irr_update(vcpu,
2467                                 apic_find_highest_irr(apic));
2468                 kvm_x86_ops->hwapic_isr_update(vcpu,
2469                                 apic_find_highest_isr(apic));
2470         }
2471         kvm_make_request(KVM_REQ_EVENT, vcpu);
2472         if (ioapic_in_kernel(vcpu->kvm))
2473                 kvm_rtc_eoi_tracking_restore_one(vcpu);
2474
2475         vcpu->arch.apic_arb_prio = 0;
2476
2477         return 0;
2478 }
2479
2480 void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
2481 {
2482         struct hrtimer *timer;
2483
2484         if (!lapic_in_kernel(vcpu) ||
2485                 kvm_can_post_timer_interrupt(vcpu))
2486                 return;
2487
2488         timer = &vcpu->arch.apic->lapic_timer.timer;
2489         if (hrtimer_cancel(timer))
2490                 hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
2491 }
2492
2493 /*
2494  * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt
2495  *
2496  * Detect whether guest triggered PV EOI since the
2497  * last entry. If yes, set EOI on guests's behalf.
2498  * Clear PV EOI in guest memory in any case.
2499  */
2500 static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu,
2501                                         struct kvm_lapic *apic)
2502 {
2503         bool pending;
2504         int vector;
2505         /*
2506          * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host
2507          * and KVM_PV_EOI_ENABLED in guest memory as follows:
2508          *
2509          * KVM_APIC_PV_EOI_PENDING is unset:
2510          *      -> host disabled PV EOI.
2511          * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set:
2512          *      -> host enabled PV EOI, guest did not execute EOI yet.
2513          * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset:
2514          *      -> host enabled PV EOI, guest executed EOI.
2515          */
2516         BUG_ON(!pv_eoi_enabled(vcpu));
2517         pending = pv_eoi_get_pending(vcpu);
2518         /*
2519          * Clear pending bit in any case: it will be set again on vmentry.
2520          * While this might not be ideal from performance point of view,
2521          * this makes sure pv eoi is only enabled when we know it's safe.
2522          */
2523         pv_eoi_clr_pending(vcpu);
2524         if (pending)
2525                 return;
2526         vector = apic_set_eoi(apic);
2527         trace_kvm_pv_eoi(apic, vector);
2528 }
2529
2530 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu)
2531 {
2532         u32 data;
2533
2534         if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention))
2535                 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic);
2536
2537         if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
2538                 return;
2539
2540         if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
2541                                   sizeof(u32)))
2542                 return;
2543
2544         apic_set_tpr(vcpu->arch.apic, data & 0xff);
2545 }
2546
2547 /*
2548  * apic_sync_pv_eoi_to_guest - called before vmentry
2549  *
2550  * Detect whether it's safe to enable PV EOI and
2551  * if yes do so.
2552  */
2553 static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu,
2554                                         struct kvm_lapic *apic)
2555 {
2556         if (!pv_eoi_enabled(vcpu) ||
2557             /* IRR set or many bits in ISR: could be nested. */
2558             apic->irr_pending ||
2559             /* Cache not set: could be safe but we don't bother. */
2560             apic->highest_isr_cache == -1 ||
2561             /* Need EOI to update ioapic. */
2562             kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) {
2563                 /*
2564                  * PV EOI was disabled by apic_sync_pv_eoi_from_guest
2565                  * so we need not do anything here.
2566                  */
2567                 return;
2568         }
2569
2570         pv_eoi_set_pending(apic->vcpu);
2571 }
2572
2573 void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu)
2574 {
2575         u32 data, tpr;
2576         int max_irr, max_isr;
2577         struct kvm_lapic *apic = vcpu->arch.apic;
2578
2579         apic_sync_pv_eoi_to_guest(vcpu, apic);
2580
2581         if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
2582                 return;
2583
2584         tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff;
2585         max_irr = apic_find_highest_irr(apic);
2586         if (max_irr < 0)
2587                 max_irr = 0;
2588         max_isr = apic_find_highest_isr(apic);
2589         if (max_isr < 0)
2590                 max_isr = 0;
2591         data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24);
2592
2593         kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
2594                                 sizeof(u32));
2595 }
2596
2597 int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr)
2598 {
2599         if (vapic_addr) {
2600                 if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
2601                                         &vcpu->arch.apic->vapic_cache,
2602                                         vapic_addr, sizeof(u32)))
2603                         return -EINVAL;
2604                 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
2605         } else {
2606                 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
2607         }
2608
2609         vcpu->arch.apic->vapic_addr = vapic_addr;
2610         return 0;
2611 }
2612
2613 int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data)
2614 {
2615         struct kvm_lapic *apic = vcpu->arch.apic;
2616         u32 reg = (msr - APIC_BASE_MSR) << 4;
2617
2618         if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
2619                 return 1;
2620
2621         if (reg == APIC_ICR2)
2622                 return 1;
2623
2624         /* if this is ICR write vector before command */
2625         if (reg == APIC_ICR)
2626                 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
2627         return kvm_lapic_reg_write(apic, reg, (u32)data);
2628 }
2629
2630 int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data)
2631 {
2632         struct kvm_lapic *apic = vcpu->arch.apic;
2633         u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0;
2634
2635         if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
2636                 return 1;
2637
2638         if (reg == APIC_DFR || reg == APIC_ICR2)
2639                 return 1;
2640
2641         if (kvm_lapic_reg_read(apic, reg, 4, &low))
2642                 return 1;
2643         if (reg == APIC_ICR)
2644                 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
2645
2646         *data = (((u64)high) << 32) | low;
2647
2648         return 0;
2649 }
2650
2651 int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data)
2652 {
2653         struct kvm_lapic *apic = vcpu->arch.apic;
2654
2655         if (!lapic_in_kernel(vcpu))
2656                 return 1;
2657
2658         /* if this is ICR write vector before command */
2659         if (reg == APIC_ICR)
2660                 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
2661         return kvm_lapic_reg_write(apic, reg, (u32)data);
2662 }
2663
2664 int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data)
2665 {
2666         struct kvm_lapic *apic = vcpu->arch.apic;
2667         u32 low, high = 0;
2668
2669         if (!lapic_in_kernel(vcpu))
2670                 return 1;
2671
2672         if (kvm_lapic_reg_read(apic, reg, 4, &low))
2673                 return 1;
2674         if (reg == APIC_ICR)
2675                 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
2676
2677         *data = (((u64)high) << 32) | low;
2678
2679         return 0;
2680 }
2681
2682 int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len)
2683 {
2684         u64 addr = data & ~KVM_MSR_ENABLED;
2685         struct gfn_to_hva_cache *ghc = &vcpu->arch.pv_eoi.data;
2686         unsigned long new_len;
2687
2688         if (!IS_ALIGNED(addr, 4))
2689                 return 1;
2690
2691         vcpu->arch.pv_eoi.msr_val = data;
2692         if (!pv_eoi_enabled(vcpu))
2693                 return 0;
2694
2695         if (addr == ghc->gpa && len <= ghc->len)
2696                 new_len = ghc->len;
2697         else
2698                 new_len = len;
2699
2700         return kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, addr, new_len);
2701 }
2702
2703 void kvm_apic_accept_events(struct kvm_vcpu *vcpu)
2704 {
2705         struct kvm_lapic *apic = vcpu->arch.apic;
2706         u8 sipi_vector;
2707         unsigned long pe;
2708
2709         if (!lapic_in_kernel(vcpu) || !apic->pending_events)
2710                 return;
2711
2712         /*
2713          * INITs are latched while in SMM.  Because an SMM CPU cannot
2714          * be in KVM_MP_STATE_INIT_RECEIVED state, just eat SIPIs
2715          * and delay processing of INIT until the next RSM.
2716          */
2717         if (is_smm(vcpu)) {
2718                 WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED);
2719                 if (test_bit(KVM_APIC_SIPI, &apic->pending_events))
2720                         clear_bit(KVM_APIC_SIPI, &apic->pending_events);
2721                 return;
2722         }
2723
2724         pe = xchg(&apic->pending_events, 0);
2725         if (test_bit(KVM_APIC_INIT, &pe)) {
2726                 kvm_vcpu_reset(vcpu, true);
2727                 if (kvm_vcpu_is_bsp(apic->vcpu))
2728                         vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
2729                 else
2730                         vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
2731         }
2732         if (test_bit(KVM_APIC_SIPI, &pe) &&
2733             vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
2734                 /* evaluate pending_events before reading the vector */
2735                 smp_rmb();
2736                 sipi_vector = apic->sipi_vector;
2737                 kvm_vcpu_deliver_sipi_vector(vcpu, sipi_vector);
2738                 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
2739         }
2740 }
2741
2742 void kvm_lapic_init(void)
2743 {
2744         /* do not patch jump label more than once per second */
2745         jump_label_rate_limit(&apic_hw_disabled, HZ);
2746         jump_label_rate_limit(&apic_sw_disabled, HZ);
2747 }
2748
2749 void kvm_lapic_exit(void)
2750 {
2751         static_key_deferred_flush(&apic_hw_disabled);
2752         static_key_deferred_flush(&apic_sw_disabled);
2753 }