Branch data Line data Source code
1 : : // SPDX-License-Identifier: GPL-2.0 2 : : #include <linux/spinlock.h> 3 : : #include <linux/task_work.h> 4 : : #include <linux/tracehook.h> 5 : : 6 : : static struct callback_head work_exited; /* all we need is ->next == NULL */ 7 : : 8 : : /** 9 : : * task_work_add - ask the @task to execute @work->func() 10 : : * @task: the task which should run the callback 11 : : * @work: the callback to run 12 : : * @notify: send the notification if true 13 : : * 14 : : * Queue @work for task_work_run() below and notify the @task if @notify. 15 : : * Fails if the @task is exiting/exited and thus it can't process this @work. 16 : : * Otherwise @work->func() will be called when the @task returns from kernel 17 : : * mode or exits. 18 : : * 19 : : * This is like the signal handler which runs in kernel mode, but it doesn't 20 : : * try to wake up the @task. 21 : : * 22 : : * Note: there is no ordering guarantee on works queued here. 23 : : * 24 : : * RETURNS: 25 : : * 0 if succeeds or -ESRCH. 26 : : */ 27 : : int 28 : 19540869 : task_work_add(struct task_struct *task, struct callback_head *work, bool notify) 29 : : { 30 : : struct callback_head *head; 31 : : 32 : : do { 33 : 19539652 : head = READ_ONCE(task->task_works); 34 [ + - ]: 19539652 : if (unlikely(head == &work_exited)) 35 : : return -ESRCH; 36 : 19546704 : work->next = head; 37 [ + + ]: 19546704 : } while (cmpxchg(&task->task_works, head, work) != head); 38 : : 39 [ + + ]: 19543479 : if (notify) 40 : 19536290 : set_notify_resume(task); 41 : : return 0; 42 : : } 43 : : 44 : : /** 45 : : * task_work_cancel - cancel a pending work added by task_work_add() 46 : : * @task: the task which should execute the work 47 : : * @func: identifies the work to remove 48 : : * 49 : : * Find the last queued pending work with ->func == @func and remove 50 : : * it from queue. 51 : : * 52 : : * RETURNS: 53 : : * The found work or NULL if not found. 54 : : */ 55 : : struct callback_head * 56 : 0 : task_work_cancel(struct task_struct *task, task_work_func_t func) 57 : : { 58 : 0 : struct callback_head **pprev = &task->task_works; 59 : : struct callback_head *work; 60 : : unsigned long flags; 61 : : 62 [ # # ]: 0 : if (likely(!task->task_works)) 63 : : return NULL; 64 : : /* 65 : : * If cmpxchg() fails we continue without updating pprev. 66 : : * Either we raced with task_work_add() which added the 67 : : * new entry before this work, we will find it again. Or 68 : : * we raced with task_work_run(), *pprev == NULL/exited. 69 : : */ 70 : 0 : raw_spin_lock_irqsave(&task->pi_lock, flags); 71 [ # # ]: 0 : while ((work = READ_ONCE(*pprev))) { 72 [ # # ]: 0 : if (work->func != func) 73 : 0 : pprev = &work->next; 74 [ # # ]: 0 : else if (cmpxchg(pprev, work, work->next) == work) 75 : : break; 76 : : } 77 : 0 : raw_spin_unlock_irqrestore(&task->pi_lock, flags); 78 : : 79 : 0 : return work; 80 : : } 81 : : 82 : : /** 83 : : * task_work_run - execute the works added by task_work_add() 84 : : * 85 : : * Flush the pending works. Should be used by the core kernel code. 86 : : * Called before the task returns to the user-mode or stops, or when 87 : : * it exits. In the latter case task_work_add() can no longer add the 88 : : * new work after task_work_run() returns. 89 : : */ 90 : 18484138 : void task_work_run(void) 91 : : { 92 : 18484138 : struct task_struct *task = current; 93 : : struct callback_head *work, *head, *next; 94 : : 95 : : for (;;) { 96 : : /* 97 : : * work->func() can do task_work_add(), do not set 98 : : * work_exited unless the list is empty. 99 : : */ 100 : 36958925 : raw_spin_lock_irq(&task->pi_lock); 101 : : do { 102 : 36961209 : work = READ_ONCE(task->task_works); 103 [ + + ]: 18487092 : head = !work && (task->flags & PF_EXITING) ? 104 [ + + ]: 36961209 : &work_exited : NULL; 105 [ - + ]: 36961209 : } while (cmpxchg(&task->task_works, work, head) != work); 106 : 36956554 : raw_spin_unlock_irq(&task->pi_lock); 107 : : 108 [ + + ]: 36956619 : if (!work) 109 : : break; 110 : : 111 : : do { 112 : 19543507 : next = work->next; 113 : 19543507 : work->func(work); 114 : : work = next; 115 : 19548784 : cond_resched(); 116 [ + + ]: 19549315 : } while (work); 117 : : } 118 : 18487640 : }