atomic_t nr_todo; /* nr left to execute */
bool executed; /* actually executed? */
int ret; /* collected return value */
- struct completion completion; /* fired if nr_todo reaches 0 */
+ struct task_struct *waiter; /* woken when nr_todo reaches 0 */
};
/* the actual stopper, one per every possible cpu, enabled on online cpus */
struct cpu_stopper {
- spinlock_t lock;
+ raw_spinlock_t lock;
bool enabled; /* is this stopper enabled? */
struct list_head works; /* list of pending works */
};
{
memset(done, 0, sizeof(*done));
atomic_set(&done->nr_todo, nr_todo);
- init_completion(&done->completion);
+ done->waiter = current;
}
/* signal completion unless @done is NULL */
if (done) {
if (executed)
done->executed = true;
- if (atomic_dec_and_test(&done->nr_todo))
- complete(&done->completion);
+ if (atomic_dec_and_test(&done->nr_todo)) {
+ wake_up_process(done->waiter);
+ done->waiter = NULL;
+ }
}
}
unsigned long flags;
- spin_lock_irqsave(&stopper->lock, flags);
+ raw_spin_lock_irqsave(&stopper->lock, flags);
if (stopper->enabled) {
list_add_tail(&work->list, &stopper->works);
} else
cpu_stop_signal_done(work->done, false);
- spin_unlock_irqrestore(&stopper->lock, flags);
+ raw_spin_unlock_irqrestore(&stopper->lock, flags);
+}
+
+static void wait_for_stop_done(struct cpu_stop_done *done)
+{
+ set_current_state(TASK_UNINTERRUPTIBLE);
+ while (atomic_read(&done->nr_todo)) {
+ schedule();
+ set_current_state(TASK_UNINTERRUPTIBLE);
+ }
+ /*
+ * We need to wait until cpu_stop_signal_done() has cleared
+ * done->waiter.
+ */
+ while (done->waiter)
+ cpu_relax();
+ set_current_state(TASK_RUNNING);
}
/**
cpu_stop_init_done(&done, 1);
cpu_stop_queue_work(cpu, &work);
- wait_for_completion(&done.completion);
+ wait_for_stop_done(&done);
return done.executed ? done.ret : -ENOENT;
}
struct irq_cpu_stop_queue_work_info call_args;
struct multi_stop_data msdata;
- preempt_disable();
+ preempt_disable_nort();
msdata = (struct multi_stop_data){
.fn = fn,
.data = arg,
* This relies on the stopper workqueues to be FIFO.
*/
if (!cpu_active(cpu1) || !cpu_active(cpu2)) {
- preempt_enable();
+ preempt_enable_nort();
return -ENOENT;
}
&irq_cpu_stop_queue_work,
&call_args, 1);
lg_local_unlock(&stop_cpus_lock);
- preempt_enable();
+ preempt_enable_nort();
- wait_for_completion(&done.completion);
+ wait_for_stop_done(&done);
return done.executed ? done.ret : -ENOENT;
}
static void queue_stop_cpus_work(const struct cpumask *cpumask,
cpu_stop_fn_t fn, void *arg,
- struct cpu_stop_done *done)
+ struct cpu_stop_done *done, bool inactive)
{
struct cpu_stop_work *work;
unsigned int cpu;
}
/*
- * Disable preemption while queueing to avoid getting
- * preempted by a stopper which might wait for other stoppers
- * to enter @fn which can lead to deadlock.
+ * Make sure that all work is queued on all cpus before
+ * any of the cpus can execute it.
*/
- lg_global_lock(&stop_cpus_lock);
+ if (!inactive)
+ lg_global_lock(&stop_cpus_lock);
+ else
+ lg_global_trylock_relax(&stop_cpus_lock);
for_each_cpu(cpu, cpumask)
cpu_stop_queue_work(cpu, &per_cpu(stop_cpus_work, cpu));
lg_global_unlock(&stop_cpus_lock);
struct cpu_stop_done done;
cpu_stop_init_done(&done, cpumask_weight(cpumask));
- queue_stop_cpus_work(cpumask, fn, arg, &done);
- wait_for_completion(&done.completion);
+ queue_stop_cpus_work(cpumask, fn, arg, &done, false);
+ wait_for_stop_done(&done);
return done.executed ? done.ret : -ENOENT;
}
unsigned long flags;
int run;
- spin_lock_irqsave(&stopper->lock, flags);
+ raw_spin_lock_irqsave(&stopper->lock, flags);
run = !list_empty(&stopper->works);
- spin_unlock_irqrestore(&stopper->lock, flags);
+ raw_spin_unlock_irqrestore(&stopper->lock, flags);
return run;
}
repeat:
work = NULL;
- spin_lock_irq(&stopper->lock);
+ raw_spin_lock_irq(&stopper->lock);
if (!list_empty(&stopper->works)) {
work = list_first_entry(&stopper->works,
struct cpu_stop_work, list);
list_del_init(&work->list);
}
- spin_unlock_irq(&stopper->lock);
+ raw_spin_unlock_irq(&stopper->lock);
if (work) {
cpu_stop_fn_t fn = work->fn;
struct cpu_stop_done *done = work->done;
char ksym_buf[KSYM_NAME_LEN] __maybe_unused;
+ /*
+ * Wait until the stopper finished scheduling on all
+ * cpus
+ */
+ lg_global_lock(&stop_cpus_lock);
+ /*
+ * Let other cpu threads continue as well
+ */
+ lg_global_unlock(&stop_cpus_lock);
+
/* cpu stop callbacks are not allowed to sleep */
preempt_disable();
kallsyms_lookup((unsigned long)fn, NULL, NULL, NULL,
ksym_buf), arg);
+ /*
+ * Make sure that the wakeup and setting done->waiter
+ * to NULL is atomic.
+ */
+ local_irq_disable();
cpu_stop_signal_done(done, true);
+ local_irq_enable();
goto repeat;
}
}
unsigned long flags;
/* drain remaining works */
- spin_lock_irqsave(&stopper->lock, flags);
+ raw_spin_lock_irqsave(&stopper->lock, flags);
list_for_each_entry(work, &stopper->works, list)
cpu_stop_signal_done(work->done, false);
stopper->enabled = false;
- spin_unlock_irqrestore(&stopper->lock, flags);
+ raw_spin_unlock_irqrestore(&stopper->lock, flags);
}
static void cpu_stop_unpark(unsigned int cpu)
{
struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
- spin_lock_irq(&stopper->lock);
+ raw_spin_lock_irq(&stopper->lock);
stopper->enabled = true;
- spin_unlock_irq(&stopper->lock);
+ raw_spin_unlock_irq(&stopper->lock);
}
static struct smp_hotplug_thread cpu_stop_threads = {
for_each_possible_cpu(cpu) {
struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
- spin_lock_init(&stopper->lock);
+ raw_spin_lock_init(&stopper->lock);
INIT_LIST_HEAD(&stopper->works);
}
+ lg_lock_init(&stop_cpus_lock, "stop_cpus_lock");
+
BUG_ON(smpboot_register_percpu_thread(&cpu_stop_threads));
stop_machine_initialized = true;
return 0;
set_state(&msdata, MULTI_STOP_PREPARE);
cpu_stop_init_done(&done, num_active_cpus());
queue_stop_cpus_work(cpu_active_mask, multi_cpu_stop, &msdata,
- &done);
+ &done, true);
ret = multi_cpu_stop(&msdata);
/* Busy wait for completion. */
- while (!completion_done(&done.completion))
+ while (atomic_read(&done.nr_todo))
cpu_relax();
mutex_unlock(&stop_cpus_mutex);
projects / zynq / linux.git / blobdiff