x86-preempt-lazy.patch

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index df075d94c86a1fbbee902a41014a475af49c0f4b..92818ed43e26cd202311651db02eb5c289436595 100644 (file)
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -22,6 +22,7 @@ config X86_64
 ### Arch settings
 config X86
        def_bool y
+       select HAVE_PREEMPT_LAZY
        select ARCH_MIGHT_HAVE_ACPI_PDC if ACPI
        select ARCH_HAS_DEBUG_STRICT_USER_COPY_CHECKS
        select ARCH_HAS_FAST_MULTIPLIER

diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index 1d4e4f279a3281e094684ad29bd4b544d37af6ab..a8b06504d532a6925c3cade9150c03ac0aefcfbf 100644 (file)
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -30,6 +30,8 @@ struct thread_info {
        __u32                   status;         /* thread synchronous flags */
        __u32                   cpu;            /* current CPU */
        int                     saved_preempt_count;
+       int                     preempt_lazy_count;     /* 0 => lazy preemptable
+                                                        <0  => BUG */
        mm_segment_t            addr_limit;
        void __user             *sysenter_return;
        unsigned int            sig_on_uaccess_error:1;
@@ -71,6 +73,7 @@ struct thread_info {
 #define TIF_SYSCALL_EMU                6       /* syscall emulation active */
 #define TIF_SYSCALL_AUDIT      7       /* syscall auditing active */
 #define TIF_SECCOMP            8       /* secure computing */
+#define TIF_NEED_RESCHED_LAZY  9       /* lazy rescheduling necessary */
 #define TIF_USER_RETURN_NOTIFY 11      /* notify kernel of userspace return */
 #define TIF_UPROBE             12      /* breakpointed or singlestepping */
 #define TIF_NOTSC              16      /* TSC is not accessible in userland */
@@ -95,6 +98,7 @@ struct thread_info {
 #define _TIF_SYSCALL_EMU       (1 << TIF_SYSCALL_EMU)
 #define _TIF_SYSCALL_AUDIT     (1 << TIF_SYSCALL_AUDIT)
 #define _TIF_SECCOMP           (1 << TIF_SECCOMP)
+#define _TIF_NEED_RESCHED_LAZY (1 << TIF_NEED_RESCHED_LAZY)
 #define _TIF_USER_RETURN_NOTIFY        (1 << TIF_USER_RETURN_NOTIFY)
 #define _TIF_UPROBE            (1 << TIF_UPROBE)
 #define _TIF_NOTSC             (1 << TIF_NOTSC)
@@ -144,6 +148,8 @@ struct thread_info {
 #define _TIF_WORK_CTXSW_PREV (_TIF_WORK_CTXSW|_TIF_USER_RETURN_NOTIFY)
 #define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW)
 
+#define _TIF_NEED_RESCHED_MASK (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY)
+
 #define STACK_WARN             (THREAD_SIZE/8)
 #define KERNEL_STACK_OFFSET    (5*(BITS_PER_LONG/8))
 

diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
index 9f6b9341950f7895b247b1c320cc0f9cd75cda9d..5701b507510b9ed5f0ff7c597a766fad485ddca3 100644 (file)
--- a/arch/x86/kernel/asm-offsets.c
+++ b/arch/x86/kernel/asm-offsets.c
@@ -32,6 +32,7 @@ void common(void) {
        OFFSET(TI_flags, thread_info, flags);
        OFFSET(TI_status, thread_info, status);
        OFFSET(TI_addr_limit, thread_info, addr_limit);
+       OFFSET(TI_preempt_lazy_count, thread_info, preempt_lazy_count);
 
        BLANK();
        OFFSET(crypto_tfm_ctx_offset, crypto_tfm, __crt_ctx);
@@ -71,4 +72,5 @@ void common(void) {
 
        BLANK();
        DEFINE(PTREGS_SIZE, sizeof(struct pt_regs));
+       DEFINE(_PREEMPT_ENABLED, PREEMPT_ENABLED);
 }

diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index 31e2d5bf3e38887ca06402bff6c647b9aa9a3c5c..feb75ab591d0be151aee33b6435ddbeaee8dcdb6 100644 (file)
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -359,8 +359,24 @@ END(ret_from_exception)
 ENTRY(resume_kernel)
        DISABLE_INTERRUPTS(CLBR_ANY)
 need_resched:
+       # preempt count == 0 + NEED_RS set?
        cmpl $0,PER_CPU_VAR(__preempt_count)
+#ifndef CONFIG_PREEMPT_LAZY
        jnz restore_all
+#else
+       jz test_int_off
+
+       # atleast preempt count == 0 ?
+       cmpl $_PREEMPT_ENABLED,PER_CPU_VAR(__preempt_count)
+       jne restore_all
+
+       cmpl $0,TI_preempt_lazy_count(%ebp)     # non-zero preempt_lazy_count ?
+       jnz restore_all
+
+       testl $_TIF_NEED_RESCHED_LAZY, TI_flags(%ebp)
+       jz restore_all
+test_int_off:
+#endif
        testl $X86_EFLAGS_IF,PT_EFLAGS(%esp)    # interrupts off (exception path) ?
        jz restore_all
        call preempt_schedule_irq
@@ -591,7 +607,7 @@ ENDPROC(system_call)
        ALIGN
        RING0_PTREGS_FRAME              # can't unwind into user space anyway
 work_pending:
-       testb $_TIF_NEED_RESCHED, %cl
+       testl $_TIF_NEED_RESCHED_MASK, %ecx
        jz work_notifysig
 work_resched:
        call schedule
@@ -604,7 +620,7 @@ work_resched:
        andl $_TIF_WORK_MASK, %ecx      # is there any work to be done other
                                        # than syscall tracing?
        jz restore_all
-       testb $_TIF_NEED_RESCHED, %cl
+       testl $_TIF_NEED_RESCHED_MASK, %ecx
        jnz work_resched
 
 work_notifysig:                                # deal with pending signals and

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 070328ee6c5fc2afade381f7752a97db390059bb..b2b0927e226eb2b8c5b24b659032ffcaa549623c 100644 (file)
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -457,8 +457,8 @@ GLOBAL(int_with_check)
        /* First do a reschedule test. */
        /* edx: work, edi: workmask */
 int_careful:
-       bt $TIF_NEED_RESCHED,%edx
-       jnc  int_very_careful
+       testl $_TIF_NEED_RESCHED_MASK,%edx
+       jz  int_very_careful
        TRACE_IRQS_ON
        ENABLE_INTERRUPTS(CLBR_NONE)
        pushq_cfi %rdi
@@ -897,8 +897,8 @@ native_irq_return_ldt:
        /* edi: workmask, edx: work */
 retint_careful:
        CFI_RESTORE_STATE
-       bt    $TIF_NEED_RESCHED,%edx
-       jnc   retint_signal
+       testl $_TIF_NEED_RESCHED_MASK,%edx
+       jz   retint_signal
        TRACE_IRQS_ON
        ENABLE_INTERRUPTS(CLBR_NONE)
        pushq_cfi %rdi
@@ -930,7 +930,22 @@ retint_signal:
        /* rcx:  threadinfo. interrupts off. */
 ENTRY(retint_kernel)
        cmpl $0,PER_CPU_VAR(__preempt_count)
+#ifndef CONFIG_PREEMPT_LAZY
        jnz  retint_restore_args
+#else
+       jz  check_int_off
+
+       # atleast preempt count == 0 ?
+       cmpl $_PREEMPT_ENABLED,PER_CPU_VAR(__preempt_count)
+       jnz retint_restore_args
+
+       cmpl $0, TI_preempt_lazy_count(%rcx)
+       jnz retint_restore_args
+
+       bt $TIF_NEED_RESCHED_LAZY,TI_flags(%rcx)
+       jnc  retint_restore_args
+check_int_off:
+#endif
        bt   $9,EFLAGS-ARGOFFSET(%rsp)  /* interrupts off? */
        jnc  retint_restore_args
        call preempt_schedule_irq