Merge branch '4.0.8-rt6'

[zynq/linux.git] / mm / memcontrol.c

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index b34ef4a32a3b266768248743849e241183e112ed..ea10085f6a1060bdffbb1042e823bbace187e36d 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -60,6 +60,8 @@
 #include <net/sock.h>
 #include <net/ip.h>
 #include <net/tcp_memcontrol.h>
+#include <linux/locallock.h>
+
 #include "slab.h"
 
 #include <asm/uaccess.h>
@@ -79,6 +81,7 @@ int do_swap_account __read_mostly;
 #define do_swap_account                0
 #endif
 
+static DEFINE_LOCAL_IRQ_LOCK(event_lock);
 static const char * const mem_cgroup_stat_names[] = {
        "cache",
        "rss",
@@ -2115,14 +2118,17 @@ static void drain_local_stock(struct work_struct *dummy)
  */
 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
 {
-       struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
+       struct memcg_stock_pcp *stock;
+       int cpu = get_cpu_light();
+
+       stock = &per_cpu(memcg_stock, cpu);
 
        if (stock->cached != memcg) { /* reset if necessary */
                drain_stock(stock);
                stock->cached = memcg;
        }
        stock->nr_pages += nr_pages;
-       put_cpu_var(memcg_stock);
+       put_cpu_light();
 }
 
 /*
@@ -2138,7 +2144,7 @@ static void drain_all_stock(struct mem_cgroup *root_memcg)
                return;
        /* Notify other cpus that system-wide "drain" is running */
        get_online_cpus();
-       curcpu = get_cpu();
+       curcpu = get_cpu_light();
        for_each_online_cpu(cpu) {
                struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
                struct mem_cgroup *memcg;
@@ -2155,7 +2161,7 @@ static void drain_all_stock(struct mem_cgroup *root_memcg)
                                schedule_work_on(cpu, &stock->work);
                }
        }
-       put_cpu();
+       put_cpu_light();
        put_online_cpus();
        mutex_unlock(&percpu_charge_mutex);
 }
@@ -2853,12 +2859,12 @@ static int mem_cgroup_move_account(struct page *page,
 
        ret = 0;
 
-       local_irq_disable();
+       local_lock_irq(event_lock);
        mem_cgroup_charge_statistics(to, page, nr_pages);
        memcg_check_events(to, page);
        mem_cgroup_charge_statistics(from, page, -nr_pages);
        memcg_check_events(from, page);
-       local_irq_enable();
+       local_unlock_irq(event_lock);
 out_unlock:
        unlock_page(page);
 out:
@@ -5546,10 +5552,10 @@ void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
                VM_BUG_ON_PAGE(!PageTransHuge(page), page);
        }
 
-       local_irq_disable();
+       local_lock_irq(event_lock);
        mem_cgroup_charge_statistics(memcg, page, nr_pages);
        memcg_check_events(memcg, page);
-       local_irq_enable();
+       local_unlock_irq(event_lock);
 
        if (do_swap_account && PageSwapCache(page)) {
                swp_entry_t entry = { .val = page_private(page) };
@@ -5605,14 +5611,14 @@ static void uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout,
                memcg_oom_recover(memcg);
        }
 
-       local_irq_save(flags);
+       local_lock_irqsave(event_lock, flags);
        __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS], nr_anon);
        __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_CACHE], nr_file);
        __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], nr_huge);
        __this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT], pgpgout);
        __this_cpu_add(memcg->stat->nr_page_events, nr_pages);
        memcg_check_events(memcg, dummy_page);
-       local_irq_restore(flags);
+       local_unlock_irqrestore(event_lock, flags);
 
        if (!mem_cgroup_is_root(memcg))
                css_put_many(&memcg->css, nr_pages);
@@ -5816,6 +5822,7 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
 {
        struct mem_cgroup *memcg;
        unsigned short oldid;
+       unsigned long flags;
 
        VM_BUG_ON_PAGE(PageLRU(page), page);
        VM_BUG_ON_PAGE(page_count(page), page);
@@ -5838,11 +5845,10 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
        if (!mem_cgroup_is_root(memcg))
                page_counter_uncharge(&memcg->memory, 1);
 
-       /* XXX: caller holds IRQ-safe mapping->tree_lock */
-       VM_BUG_ON(!irqs_disabled());
-
+       local_lock_irqsave(event_lock, flags);
        mem_cgroup_charge_statistics(memcg, page, -1);
        memcg_check_events(memcg, page);
+       local_unlock_irqrestore(event_lock, flags);
 }
 
 /**