Branch data     Line data    Source code

       1                 :            : // SPDX-License-Identifier: GPL-2.0
       2                 :            : #include <linux/spinlock.h>
       3                 :            : #include <linux/task_work.h>
       4                 :            : #include <linux/tracehook.h>
       5                 :            : 
       6                 :            : static struct callback_head work_exited; /* all we need is ->next == NULL */
       7                 :            : 
       8                 :            : /**
       9                 :            :  * task_work_add - ask the @task to execute @work->func()
      10                 :            :  * @task: the task which should run the callback
      11                 :            :  * @work: the callback to run
      12                 :            :  * @notify: send the notification if true
      13                 :            :  *
      14                 :            :  * Queue @work for task_work_run() below and notify the @task if @notify.
      15                 :            :  * Fails if the @task is exiting/exited and thus it can't process this @work.
      16                 :            :  * Otherwise @work->func() will be called when the @task returns from kernel
      17                 :            :  * mode or exits.
      18                 :            :  *
      19                 :            :  * This is like the signal handler which runs in kernel mode, but it doesn't
      20                 :            :  * try to wake up the @task.
      21                 :            :  *
      22                 :            :  * Note: there is no ordering guarantee on works queued here.
      23                 :            :  *
      24                 :            :  * RETURNS:
      25                 :            :  * 0 if succeeds or -ESRCH.
      26                 :            :  */
      27                 :            : int
      28                 :   35469850 : task_work_add(struct task_struct *task, struct callback_head *work, bool notify)
      29                 :            : {
      30                 :            :         struct callback_head *head;
      31                 :            : 
      32                 :            :         do {
      33                 :   35470168 :                 head = READ_ONCE(task->task_works);
      34         [ +  + ]:   35470168 :                 if (unlikely(head == &work_exited))
      35                 :            :                         return -ESRCH;
      36                 :   35469740 :                 work->next = head;
      37         [ +  + ]:   35469740 :         } while (cmpxchg(&task->task_works, head, work) != head);
      38                 :            : 
      39         [ +  + ]:   35470658 :         if (notify)
      40                 :   35465076 :                 set_notify_resume(task);
      41                 :            :         return 0;
      42                 :            : }
      43                 :            : 
      44                 :            : /**
      45                 :            :  * task_work_cancel - cancel a pending work added by task_work_add()
      46                 :            :  * @task: the task which should execute the work
      47                 :            :  * @func: identifies the work to remove
      48                 :            :  *
      49                 :            :  * Find the last queued pending work with ->func == @func and remove
      50                 :            :  * it from queue.
      51                 :            :  *
      52                 :            :  * RETURNS:
      53                 :            :  * The found work or NULL if not found.
      54                 :            :  */
      55                 :            : struct callback_head *
      56                 :          0 : task_work_cancel(struct task_struct *task, task_work_func_t func)
      57                 :            : {
      58                 :          0 :         struct callback_head **pprev = &task->task_works;
      59                 :            :         struct callback_head *work;
      60                 :            :         unsigned long flags;
      61                 :            : 
      62         [ #  # ]:          0 :         if (likely(!task->task_works))
      63                 :            :                 return NULL;
      64                 :            :         /*
      65                 :            :          * If cmpxchg() fails we continue without updating pprev.
      66                 :            :          * Either we raced with task_work_add() which added the
      67                 :            :          * new entry before this work, we will find it again. Or
      68                 :            :          * we raced with task_work_run(), *pprev == NULL/exited.
      69                 :            :          */
      70                 :          0 :         raw_spin_lock_irqsave(&task->pi_lock, flags);
      71         [ #  # ]:          0 :         while ((work = READ_ONCE(*pprev))) {
      72         [ #  # ]:          0 :                 if (work->func != func)
      73                 :          0 :                         pprev = &work->next;
      74         [ #  # ]:          0 :                 else if (cmpxchg(pprev, work, work->next) == work)
      75                 :            :                         break;
      76                 :            :         }
      77                 :          0 :         raw_spin_unlock_irqrestore(&task->pi_lock, flags);
      78                 :            : 
      79                 :          0 :         return work;
      80                 :            : }
      81                 :            : 
      82                 :            : /**
      83                 :            :  * task_work_run - execute the works added by task_work_add()
      84                 :            :  *
      85                 :            :  * Flush the pending works. Should be used by the core kernel code.
      86                 :            :  * Called before the task returns to the user-mode or stops, or when
      87                 :            :  * it exits. In the latter case task_work_add() can no longer add the
      88                 :            :  * new work after task_work_run() returns.
      89                 :            :  */
      90                 :   33519204 : void task_work_run(void)
      91                 :            : {
      92                 :   33519204 :         struct task_struct *task = current;
      93                 :            :         struct callback_head *work, *head, *next;
      94                 :            : 
      95                 :            :         for (;;) {
      96                 :            :                 /*
      97                 :            :                  * work->func() can do task_work_add(), do not set
      98                 :            :                  * work_exited unless the list is empty.
      99                 :            :                  */
     100                 :   67017910 :                 raw_spin_lock_irq(&task->pi_lock);
     101                 :            :                 do {
     102                 :   67016222 :                         work = READ_ONCE(task->task_works);
     103         [ +  + ]:   33521170 :                         head = !work && (task->flags & PF_EXITING) ?
     104         [ +  + ]:   67016222 :                                 &work_exited : NULL;
     105         [ -  + ]:   67016222 :                 } while (cmpxchg(&task->task_works, work, head) != work);
     106                 :   67020788 :                 raw_spin_unlock_irq(&task->pi_lock);
     107                 :            : 
     108         [ +  + ]:   67019314 :                 if (!work)
     109                 :            :                         break;
     110                 :            : 
     111                 :            :                 do {
     112                 :   35469322 :                         next = work->next;
     113                 :   35469322 :                         work->func(work);
     114                 :            :                         work = next;
     115                 :   35470694 :                         cond_resched();
     116         [ +  + ]:   35470786 :                 } while (work);
     117                 :            :         }
     118                 :   33522072 : }