LCOV - code coverage report
Current view: top level - drivers/dma-buf - dma-fence.c (source / functions) Hit Total Coverage
Test: Real Lines: 0 158 0.0 %
Date: 2020-10-17 15:46:43 Functions: 0 16 0.0 %
Legend: Neither, QEMU, Real, Both Branches: 0 0 -

           Branch data     Line data    Source code
       1                 :            : // SPDX-License-Identifier: GPL-2.0-only
       2                 :            : /*
       3                 :            :  * Fence mechanism for dma-buf and to allow for asynchronous dma access
       4                 :            :  *
       5                 :            :  * Copyright (C) 2012 Canonical Ltd
       6                 :            :  * Copyright (C) 2012 Texas Instruments
       7                 :            :  *
       8                 :            :  * Authors:
       9                 :            :  * Rob Clark <robdclark@gmail.com>
      10                 :            :  * Maarten Lankhorst <maarten.lankhorst@canonical.com>
      11                 :            :  */
      12                 :            : 
      13                 :            : #include <linux/slab.h>
      14                 :            : #include <linux/export.h>
      15                 :            : #include <linux/atomic.h>
      16                 :            : #include <linux/dma-fence.h>
      17                 :            : #include <linux/sched/signal.h>
      18                 :            : 
      19                 :            : #define CREATE_TRACE_POINTS
      20                 :            : #include <trace/events/dma_fence.h>
      21                 :            : 
      22                 :            : EXPORT_TRACEPOINT_SYMBOL(dma_fence_emit);
      23                 :            : EXPORT_TRACEPOINT_SYMBOL(dma_fence_enable_signal);
      24                 :            : EXPORT_TRACEPOINT_SYMBOL(dma_fence_signaled);
      25                 :            : 
      26                 :            : static DEFINE_SPINLOCK(dma_fence_stub_lock);
      27                 :            : static struct dma_fence dma_fence_stub;
      28                 :            : 
      29                 :            : /*
      30                 :            :  * fence context counter: each execution context should have its own
      31                 :            :  * fence context, this allows checking if fences belong to the same
      32                 :            :  * context or not. One device can have multiple separate contexts,
      33                 :            :  * and they're used if some engine can run independently of another.
      34                 :            :  */
      35                 :            : static atomic64_t dma_fence_context_counter = ATOMIC64_INIT(1);
      36                 :            : 
      37                 :            : /**
      38                 :            :  * DOC: DMA fences overview
      39                 :            :  *
      40                 :            :  * DMA fences, represented by &struct dma_fence, are the kernel internal
      41                 :            :  * synchronization primitive for DMA operations like GPU rendering, video
      42                 :            :  * encoding/decoding, or displaying buffers on a screen.
      43                 :            :  *
      44                 :            :  * A fence is initialized using dma_fence_init() and completed using
      45                 :            :  * dma_fence_signal(). Fences are associated with a context, allocated through
      46                 :            :  * dma_fence_context_alloc(), and all fences on the same context are
      47                 :            :  * fully ordered.
      48                 :            :  *
      49                 :            :  * Since the purposes of fences is to facilitate cross-device and
      50                 :            :  * cross-application synchronization, there's multiple ways to use one:
      51                 :            :  *
      52                 :            :  * - Individual fences can be exposed as a &sync_file, accessed as a file
      53                 :            :  *   descriptor from userspace, created by calling sync_file_create(). This is
      54                 :            :  *   called explicit fencing, since userspace passes around explicit
      55                 :            :  *   synchronization points.
      56                 :            :  *
      57                 :            :  * - Some subsystems also have their own explicit fencing primitives, like
      58                 :            :  *   &drm_syncobj. Compared to &sync_file, a &drm_syncobj allows the underlying
      59                 :            :  *   fence to be updated.
      60                 :            :  *
      61                 :            :  * - Then there's also implicit fencing, where the synchronization points are
      62                 :            :  *   implicitly passed around as part of shared &dma_buf instances. Such
      63                 :            :  *   implicit fences are stored in &struct dma_resv through the
      64                 :            :  *   &dma_buf.resv pointer.
      65                 :            :  */
      66                 :            : 
      67                 :          0 : static const char *dma_fence_stub_get_name(struct dma_fence *fence)
      68                 :            : {
      69                 :          0 :         return "stub";
      70                 :            : }
      71                 :            : 
      72                 :            : static const struct dma_fence_ops dma_fence_stub_ops = {
      73                 :            :         .get_driver_name = dma_fence_stub_get_name,
      74                 :            :         .get_timeline_name = dma_fence_stub_get_name,
      75                 :            : };
      76                 :            : 
      77                 :            : /**
      78                 :            :  * dma_fence_get_stub - return a signaled fence
      79                 :            :  *
      80                 :            :  * Return a stub fence which is already signaled.
      81                 :            :  */
      82                 :          0 : struct dma_fence *dma_fence_get_stub(void)
      83                 :            : {
      84                 :            :         spin_lock(&dma_fence_stub_lock);
      85                 :          0 :         if (!dma_fence_stub.ops) {
      86                 :          0 :                 dma_fence_init(&dma_fence_stub,
      87                 :            :                                &dma_fence_stub_ops,
      88                 :            :                                &dma_fence_stub_lock,
      89                 :            :                                0, 0);
      90                 :          0 :                 dma_fence_signal_locked(&dma_fence_stub);
      91                 :            :         }
      92                 :            :         spin_unlock(&dma_fence_stub_lock);
      93                 :            : 
      94                 :          0 :         return dma_fence_get(&dma_fence_stub);
      95                 :            : }
      96                 :            : EXPORT_SYMBOL(dma_fence_get_stub);
      97                 :            : 
      98                 :            : /**
      99                 :            :  * dma_fence_context_alloc - allocate an array of fence contexts
     100                 :            :  * @num: amount of contexts to allocate
     101                 :            :  *
     102                 :            :  * This function will return the first index of the number of fence contexts
     103                 :            :  * allocated.  The fence context is used for setting &dma_fence.context to a
     104                 :            :  * unique number by passing the context to dma_fence_init().
     105                 :            :  */
     106                 :          0 : u64 dma_fence_context_alloc(unsigned num)
     107                 :            : {
     108                 :          0 :         WARN_ON(!num);
     109                 :          0 :         return atomic64_add_return(num, &dma_fence_context_counter) - num;
     110                 :            : }
     111                 :            : EXPORT_SYMBOL(dma_fence_context_alloc);
     112                 :            : 
     113                 :            : /**
     114                 :            :  * dma_fence_signal_locked - signal completion of a fence
     115                 :            :  * @fence: the fence to signal
     116                 :            :  *
     117                 :            :  * Signal completion for software callbacks on a fence, this will unblock
     118                 :            :  * dma_fence_wait() calls and run all the callbacks added with
     119                 :            :  * dma_fence_add_callback(). Can be called multiple times, but since a fence
     120                 :            :  * can only go from the unsignaled to the signaled state and not back, it will
     121                 :            :  * only be effective the first time.
     122                 :            :  *
     123                 :            :  * Unlike dma_fence_signal(), this function must be called with &dma_fence.lock
     124                 :            :  * held.
     125                 :            :  *
     126                 :            :  * Returns 0 on success and a negative error value when @fence has been
     127                 :            :  * signalled already.
     128                 :            :  */
     129                 :          0 : int dma_fence_signal_locked(struct dma_fence *fence)
     130                 :            : {
     131                 :            :         struct dma_fence_cb *cur, *tmp;
     132                 :            :         struct list_head cb_list;
     133                 :            : 
     134                 :            :         lockdep_assert_held(fence->lock);
     135                 :            : 
     136                 :          0 :         if (unlikely(test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
     137                 :            :                                       &fence->flags)))
     138                 :            :                 return -EINVAL;
     139                 :            : 
     140                 :            :         /* Stash the cb_list before replacing it with the timestamp */
     141                 :            :         list_replace(&fence->cb_list, &cb_list);
     142                 :            : 
     143                 :          0 :         fence->timestamp = ktime_get();
     144                 :          0 :         set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
     145                 :          0 :         trace_dma_fence_signaled(fence);
     146                 :            : 
     147                 :          0 :         list_for_each_entry_safe(cur, tmp, &cb_list, node) {
     148                 :            :                 INIT_LIST_HEAD(&cur->node);
     149                 :          0 :                 cur->func(fence, cur);
     150                 :            :         }
     151                 :            : 
     152                 :            :         return 0;
     153                 :            : }
     154                 :            : EXPORT_SYMBOL(dma_fence_signal_locked);
     155                 :            : 
     156                 :            : /**
     157                 :            :  * dma_fence_signal - signal completion of a fence
     158                 :            :  * @fence: the fence to signal
     159                 :            :  *
     160                 :            :  * Signal completion for software callbacks on a fence, this will unblock
     161                 :            :  * dma_fence_wait() calls and run all the callbacks added with
     162                 :            :  * dma_fence_add_callback(). Can be called multiple times, but since a fence
     163                 :            :  * can only go from the unsignaled to the signaled state and not back, it will
     164                 :            :  * only be effective the first time.
     165                 :            :  *
     166                 :            :  * Returns 0 on success and a negative error value when @fence has been
     167                 :            :  * signalled already.
     168                 :            :  */
     169                 :          0 : int dma_fence_signal(struct dma_fence *fence)
     170                 :            : {
     171                 :            :         unsigned long flags;
     172                 :            :         int ret;
     173                 :            : 
     174                 :          0 :         if (!fence)
     175                 :            :                 return -EINVAL;
     176                 :            : 
     177                 :          0 :         spin_lock_irqsave(fence->lock, flags);
     178                 :          0 :         ret = dma_fence_signal_locked(fence);
     179                 :          0 :         spin_unlock_irqrestore(fence->lock, flags);
     180                 :            : 
     181                 :          0 :         return ret;
     182                 :            : }
     183                 :            : EXPORT_SYMBOL(dma_fence_signal);
     184                 :            : 
     185                 :            : /**
     186                 :            :  * dma_fence_wait_timeout - sleep until the fence gets signaled
     187                 :            :  * or until timeout elapses
     188                 :            :  * @fence: the fence to wait on
     189                 :            :  * @intr: if true, do an interruptible wait
     190                 :            :  * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
     191                 :            :  *
     192                 :            :  * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the
     193                 :            :  * remaining timeout in jiffies on success. Other error values may be
     194                 :            :  * returned on custom implementations.
     195                 :            :  *
     196                 :            :  * Performs a synchronous wait on this fence. It is assumed the caller
     197                 :            :  * directly or indirectly (buf-mgr between reservation and committing)
     198                 :            :  * holds a reference to the fence, otherwise the fence might be
     199                 :            :  * freed before return, resulting in undefined behavior.
     200                 :            :  *
     201                 :            :  * See also dma_fence_wait() and dma_fence_wait_any_timeout().
     202                 :            :  */
     203                 :            : signed long
     204                 :          0 : dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)
     205                 :            : {
     206                 :            :         signed long ret;
     207                 :            : 
     208                 :          0 :         if (WARN_ON(timeout < 0))
     209                 :            :                 return -EINVAL;
     210                 :            : 
     211                 :          0 :         trace_dma_fence_wait_start(fence);
     212                 :          0 :         if (fence->ops->wait)
     213                 :          0 :                 ret = fence->ops->wait(fence, intr, timeout);
     214                 :            :         else
     215                 :          0 :                 ret = dma_fence_default_wait(fence, intr, timeout);
     216                 :          0 :         trace_dma_fence_wait_end(fence);
     217                 :          0 :         return ret;
     218                 :            : }
     219                 :            : EXPORT_SYMBOL(dma_fence_wait_timeout);
     220                 :            : 
     221                 :            : /**
     222                 :            :  * dma_fence_release - default relese function for fences
     223                 :            :  * @kref: &dma_fence.recfount
     224                 :            :  *
     225                 :            :  * This is the default release functions for &dma_fence. Drivers shouldn't call
     226                 :            :  * this directly, but instead call dma_fence_put().
     227                 :            :  */
     228                 :          0 : void dma_fence_release(struct kref *kref)
     229                 :            : {
     230                 :            :         struct dma_fence *fence =
     231                 :          0 :                 container_of(kref, struct dma_fence, refcount);
     232                 :            : 
     233                 :          0 :         trace_dma_fence_destroy(fence);
     234                 :            : 
     235                 :          0 :         if (WARN(!list_empty(&fence->cb_list) &&
     236                 :            :                  !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags),
     237                 :            :                  "Fence %s:%s:%llx:%llx released with pending signals!\n",
     238                 :            :                  fence->ops->get_driver_name(fence),
     239                 :            :                  fence->ops->get_timeline_name(fence),
     240                 :            :                  fence->context, fence->seqno)) {
     241                 :            :                 unsigned long flags;
     242                 :            : 
     243                 :            :                 /*
     244                 :            :                  * Failed to signal before release, likely a refcounting issue.
     245                 :            :                  *
     246                 :            :                  * This should never happen, but if it does make sure that we
     247                 :            :                  * don't leave chains dangling. We set the error flag first
     248                 :            :                  * so that the callbacks know this signal is due to an error.
     249                 :            :                  */
     250                 :          0 :                 spin_lock_irqsave(fence->lock, flags);
     251                 :          0 :                 fence->error = -EDEADLK;
     252                 :          0 :                 dma_fence_signal_locked(fence);
     253                 :          0 :                 spin_unlock_irqrestore(fence->lock, flags);
     254                 :            :         }
     255                 :            : 
     256                 :          0 :         if (fence->ops->release)
     257                 :          0 :                 fence->ops->release(fence);
     258                 :            :         else
     259                 :            :                 dma_fence_free(fence);
     260                 :          0 : }
     261                 :            : EXPORT_SYMBOL(dma_fence_release);
     262                 :            : 
     263                 :            : /**
     264                 :            :  * dma_fence_free - default release function for &dma_fence.
     265                 :            :  * @fence: fence to release
     266                 :            :  *
     267                 :            :  * This is the default implementation for &dma_fence_ops.release. It calls
     268                 :            :  * kfree_rcu() on @fence.
     269                 :            :  */
     270                 :          0 : void dma_fence_free(struct dma_fence *fence)
     271                 :            : {
     272                 :          0 :         kfree_rcu(fence, rcu);
     273                 :          0 : }
     274                 :            : EXPORT_SYMBOL(dma_fence_free);
     275                 :            : 
     276                 :            : /**
     277                 :            :  * dma_fence_enable_sw_signaling - enable signaling on fence
     278                 :            :  * @fence: the fence to enable
     279                 :            :  *
     280                 :            :  * This will request for sw signaling to be enabled, to make the fence
     281                 :            :  * complete as soon as possible. This calls &dma_fence_ops.enable_signaling
     282                 :            :  * internally.
     283                 :            :  */
     284                 :          0 : void dma_fence_enable_sw_signaling(struct dma_fence *fence)
     285                 :            : {
     286                 :            :         unsigned long flags;
     287                 :            : 
     288                 :          0 :         if (!test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
     289                 :          0 :                               &fence->flags) &&
     290                 :          0 :             !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) &&
     291                 :          0 :             fence->ops->enable_signaling) {
     292                 :          0 :                 trace_dma_fence_enable_signal(fence);
     293                 :            : 
     294                 :          0 :                 spin_lock_irqsave(fence->lock, flags);
     295                 :            : 
     296                 :          0 :                 if (!fence->ops->enable_signaling(fence))
     297                 :          0 :                         dma_fence_signal_locked(fence);
     298                 :            : 
     299                 :          0 :                 spin_unlock_irqrestore(fence->lock, flags);
     300                 :            :         }
     301                 :          0 : }
     302                 :            : EXPORT_SYMBOL(dma_fence_enable_sw_signaling);
     303                 :            : 
     304                 :            : /**
     305                 :            :  * dma_fence_add_callback - add a callback to be called when the fence
     306                 :            :  * is signaled
     307                 :            :  * @fence: the fence to wait on
     308                 :            :  * @cb: the callback to register
     309                 :            :  * @func: the function to call
     310                 :            :  *
     311                 :            :  * @cb will be initialized by dma_fence_add_callback(), no initialization
     312                 :            :  * by the caller is required. Any number of callbacks can be registered
     313                 :            :  * to a fence, but a callback can only be registered to one fence at a time.
     314                 :            :  *
     315                 :            :  * Note that the callback can be called from an atomic context.  If
     316                 :            :  * fence is already signaled, this function will return -ENOENT (and
     317                 :            :  * *not* call the callback).
     318                 :            :  *
     319                 :            :  * Add a software callback to the fence. Same restrictions apply to
     320                 :            :  * refcount as it does to dma_fence_wait(), however the caller doesn't need to
     321                 :            :  * keep a refcount to fence afterward dma_fence_add_callback() has returned:
     322                 :            :  * when software access is enabled, the creator of the fence is required to keep
     323                 :            :  * the fence alive until after it signals with dma_fence_signal(). The callback
     324                 :            :  * itself can be called from irq context.
     325                 :            :  *
     326                 :            :  * Returns 0 in case of success, -ENOENT if the fence is already signaled
     327                 :            :  * and -EINVAL in case of error.
     328                 :            :  */
     329                 :          0 : int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb,
     330                 :            :                            dma_fence_func_t func)
     331                 :            : {
     332                 :            :         unsigned long flags;
     333                 :            :         int ret = 0;
     334                 :            :         bool was_set;
     335                 :            : 
     336                 :          0 :         if (WARN_ON(!fence || !func))
     337                 :            :                 return -EINVAL;
     338                 :            : 
     339                 :          0 :         if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
     340                 :          0 :                 INIT_LIST_HEAD(&cb->node);
     341                 :          0 :                 return -ENOENT;
     342                 :            :         }
     343                 :            : 
     344                 :          0 :         spin_lock_irqsave(fence->lock, flags);
     345                 :            : 
     346                 :          0 :         was_set = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
     347                 :            :                                    &fence->flags);
     348                 :            : 
     349                 :          0 :         if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
     350                 :            :                 ret = -ENOENT;
     351                 :          0 :         else if (!was_set && fence->ops->enable_signaling) {
     352                 :          0 :                 trace_dma_fence_enable_signal(fence);
     353                 :            : 
     354                 :          0 :                 if (!fence->ops->enable_signaling(fence)) {
     355                 :          0 :                         dma_fence_signal_locked(fence);
     356                 :            :                         ret = -ENOENT;
     357                 :            :                 }
     358                 :            :         }
     359                 :            : 
     360                 :          0 :         if (!ret) {
     361                 :          0 :                 cb->func = func;
     362                 :          0 :                 list_add_tail(&cb->node, &fence->cb_list);
     363                 :            :         } else
     364                 :          0 :                 INIT_LIST_HEAD(&cb->node);
     365                 :          0 :         spin_unlock_irqrestore(fence->lock, flags);
     366                 :            : 
     367                 :          0 :         return ret;
     368                 :            : }
     369                 :            : EXPORT_SYMBOL(dma_fence_add_callback);
     370                 :            : 
     371                 :            : /**
     372                 :            :  * dma_fence_get_status - returns the status upon completion
     373                 :            :  * @fence: the dma_fence to query
     374                 :            :  *
     375                 :            :  * This wraps dma_fence_get_status_locked() to return the error status
     376                 :            :  * condition on a signaled fence. See dma_fence_get_status_locked() for more
     377                 :            :  * details.
     378                 :            :  *
     379                 :            :  * Returns 0 if the fence has not yet been signaled, 1 if the fence has
     380                 :            :  * been signaled without an error condition, or a negative error code
     381                 :            :  * if the fence has been completed in err.
     382                 :            :  */
     383                 :          0 : int dma_fence_get_status(struct dma_fence *fence)
     384                 :            : {
     385                 :            :         unsigned long flags;
     386                 :            :         int status;
     387                 :            : 
     388                 :          0 :         spin_lock_irqsave(fence->lock, flags);
     389                 :            :         status = dma_fence_get_status_locked(fence);
     390                 :          0 :         spin_unlock_irqrestore(fence->lock, flags);
     391                 :            : 
     392                 :          0 :         return status;
     393                 :            : }
     394                 :            : EXPORT_SYMBOL(dma_fence_get_status);
     395                 :            : 
     396                 :            : /**
     397                 :            :  * dma_fence_remove_callback - remove a callback from the signaling list
     398                 :            :  * @fence: the fence to wait on
     399                 :            :  * @cb: the callback to remove
     400                 :            :  *
     401                 :            :  * Remove a previously queued callback from the fence. This function returns
     402                 :            :  * true if the callback is successfully removed, or false if the fence has
     403                 :            :  * already been signaled.
     404                 :            :  *
     405                 :            :  * *WARNING*:
     406                 :            :  * Cancelling a callback should only be done if you really know what you're
     407                 :            :  * doing, since deadlocks and race conditions could occur all too easily. For
     408                 :            :  * this reason, it should only ever be done on hardware lockup recovery,
     409                 :            :  * with a reference held to the fence.
     410                 :            :  *
     411                 :            :  * Behaviour is undefined if @cb has not been added to @fence using
     412                 :            :  * dma_fence_add_callback() beforehand.
     413                 :            :  */
     414                 :            : bool
     415                 :          0 : dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb)
     416                 :            : {
     417                 :            :         unsigned long flags;
     418                 :            :         bool ret;
     419                 :            : 
     420                 :          0 :         spin_lock_irqsave(fence->lock, flags);
     421                 :            : 
     422                 :          0 :         ret = !list_empty(&cb->node);
     423                 :          0 :         if (ret)
     424                 :            :                 list_del_init(&cb->node);
     425                 :            : 
     426                 :          0 :         spin_unlock_irqrestore(fence->lock, flags);
     427                 :            : 
     428                 :          0 :         return ret;
     429                 :            : }
     430                 :            : EXPORT_SYMBOL(dma_fence_remove_callback);
     431                 :            : 
     432                 :            : struct default_wait_cb {
     433                 :            :         struct dma_fence_cb base;
     434                 :            :         struct task_struct *task;
     435                 :            : };
     436                 :            : 
     437                 :            : static void
     438                 :          0 : dma_fence_default_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
     439                 :            : {
     440                 :            :         struct default_wait_cb *wait =
     441                 :            :                 container_of(cb, struct default_wait_cb, base);
     442                 :            : 
     443                 :          0 :         wake_up_state(wait->task, TASK_NORMAL);
     444                 :          0 : }
     445                 :            : 
     446                 :            : /**
     447                 :            :  * dma_fence_default_wait - default sleep until the fence gets signaled
     448                 :            :  * or until timeout elapses
     449                 :            :  * @fence: the fence to wait on
     450                 :            :  * @intr: if true, do an interruptible wait
     451                 :            :  * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
     452                 :            :  *
     453                 :            :  * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the
     454                 :            :  * remaining timeout in jiffies on success. If timeout is zero the value one is
     455                 :            :  * returned if the fence is already signaled for consistency with other
     456                 :            :  * functions taking a jiffies timeout.
     457                 :            :  */
     458                 :            : signed long
     459                 :          0 : dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout)
     460                 :            : {
     461                 :            :         struct default_wait_cb cb;
     462                 :            :         unsigned long flags;
     463                 :          0 :         signed long ret = timeout ? timeout : 1;
     464                 :            :         bool was_set;
     465                 :            : 
     466                 :          0 :         if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
     467                 :            :                 return ret;
     468                 :            : 
     469                 :          0 :         spin_lock_irqsave(fence->lock, flags);
     470                 :            : 
     471                 :          0 :         if (intr && signal_pending(current)) {
     472                 :            :                 ret = -ERESTARTSYS;
     473                 :            :                 goto out;
     474                 :            :         }
     475                 :            : 
     476                 :          0 :         was_set = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
     477                 :            :                                    &fence->flags);
     478                 :            : 
     479                 :          0 :         if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
     480                 :            :                 goto out;
     481                 :            : 
     482                 :          0 :         if (!was_set && fence->ops->enable_signaling) {
     483                 :          0 :                 trace_dma_fence_enable_signal(fence);
     484                 :            : 
     485                 :          0 :                 if (!fence->ops->enable_signaling(fence)) {
     486                 :          0 :                         dma_fence_signal_locked(fence);
     487                 :          0 :                         goto out;
     488                 :            :                 }
     489                 :            :         }
     490                 :            : 
     491                 :          0 :         if (!timeout) {
     492                 :            :                 ret = 0;
     493                 :            :                 goto out;
     494                 :            :         }
     495                 :            : 
     496                 :          0 :         cb.base.func = dma_fence_default_wait_cb;
     497                 :          0 :         cb.task = current;
     498                 :          0 :         list_add(&cb.base.node, &fence->cb_list);
     499                 :            : 
     500                 :          0 :         while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) {
     501                 :          0 :                 if (intr)
     502                 :          0 :                         __set_current_state(TASK_INTERRUPTIBLE);
     503                 :            :                 else
     504                 :          0 :                         __set_current_state(TASK_UNINTERRUPTIBLE);
     505                 :          0 :                 spin_unlock_irqrestore(fence->lock, flags);
     506                 :            : 
     507                 :          0 :                 ret = schedule_timeout(ret);
     508                 :            : 
     509                 :          0 :                 spin_lock_irqsave(fence->lock, flags);
     510                 :          0 :                 if (ret > 0 && intr && signal_pending(current))
     511                 :            :                         ret = -ERESTARTSYS;
     512                 :            :         }
     513                 :            : 
     514                 :          0 :         if (!list_empty(&cb.base.node))
     515                 :            :                 list_del(&cb.base.node);
     516                 :          0 :         __set_current_state(TASK_RUNNING);
     517                 :            : 
     518                 :            : out:
     519                 :          0 :         spin_unlock_irqrestore(fence->lock, flags);
     520                 :          0 :         return ret;
     521                 :            : }
     522                 :            : EXPORT_SYMBOL(dma_fence_default_wait);
     523                 :            : 
     524                 :            : static bool
     525                 :            : dma_fence_test_signaled_any(struct dma_fence **fences, uint32_t count,
     526                 :            :                             uint32_t *idx)
     527                 :            : {
     528                 :            :         int i;
     529                 :            : 
     530                 :          0 :         for (i = 0; i < count; ++i) {
     531                 :          0 :                 struct dma_fence *fence = fences[i];
     532                 :          0 :                 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
     533                 :          0 :                         if (idx)
     534                 :          0 :                                 *idx = i;
     535                 :            :                         return true;
     536                 :            :                 }
     537                 :            :         }
     538                 :            :         return false;
     539                 :            : }
     540                 :            : 
     541                 :            : /**
     542                 :            :  * dma_fence_wait_any_timeout - sleep until any fence gets signaled
     543                 :            :  * or until timeout elapses
     544                 :            :  * @fences: array of fences to wait on
     545                 :            :  * @count: number of fences to wait on
     546                 :            :  * @intr: if true, do an interruptible wait
     547                 :            :  * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
     548                 :            :  * @idx: used to store the first signaled fence index, meaningful only on
     549                 :            :  *      positive return
     550                 :            :  *
     551                 :            :  * Returns -EINVAL on custom fence wait implementation, -ERESTARTSYS if
     552                 :            :  * interrupted, 0 if the wait timed out, or the remaining timeout in jiffies
     553                 :            :  * on success.
     554                 :            :  *
     555                 :            :  * Synchronous waits for the first fence in the array to be signaled. The
     556                 :            :  * caller needs to hold a reference to all fences in the array, otherwise a
     557                 :            :  * fence might be freed before return, resulting in undefined behavior.
     558                 :            :  *
     559                 :            :  * See also dma_fence_wait() and dma_fence_wait_timeout().
     560                 :            :  */
     561                 :            : signed long
     562                 :          0 : dma_fence_wait_any_timeout(struct dma_fence **fences, uint32_t count,
     563                 :            :                            bool intr, signed long timeout, uint32_t *idx)
     564                 :            : {
     565                 :            :         struct default_wait_cb *cb;
     566                 :            :         signed long ret = timeout;
     567                 :            :         unsigned i;
     568                 :            : 
     569                 :          0 :         if (WARN_ON(!fences || !count || timeout < 0))
     570                 :            :                 return -EINVAL;
     571                 :            : 
     572                 :          0 :         if (timeout == 0) {
     573                 :          0 :                 for (i = 0; i < count; ++i)
     574                 :          0 :                         if (dma_fence_is_signaled(fences[i])) {
     575                 :          0 :                                 if (idx)
     576                 :          0 :                                         *idx = i;
     577                 :            :                                 return 1;
     578                 :            :                         }
     579                 :            : 
     580                 :            :                 return 0;
     581                 :            :         }
     582                 :            : 
     583                 :            :         cb = kcalloc(count, sizeof(struct default_wait_cb), GFP_KERNEL);
     584                 :          0 :         if (cb == NULL) {
     585                 :            :                 ret = -ENOMEM;
     586                 :            :                 goto err_free_cb;
     587                 :            :         }
     588                 :            : 
     589                 :          0 :         for (i = 0; i < count; ++i) {
     590                 :          0 :                 struct dma_fence *fence = fences[i];
     591                 :            : 
     592                 :          0 :                 cb[i].task = current;
     593                 :          0 :                 if (dma_fence_add_callback(fence, &cb[i].base,
     594                 :            :                                            dma_fence_default_wait_cb)) {
     595                 :            :                         /* This fence is already signaled */
     596                 :          0 :                         if (idx)
     597                 :          0 :                                 *idx = i;
     598                 :            :                         goto fence_rm_cb;
     599                 :            :                 }
     600                 :            :         }
     601                 :            : 
     602                 :          0 :         while (ret > 0) {
     603                 :          0 :                 if (intr)
     604                 :          0 :                         set_current_state(TASK_INTERRUPTIBLE);
     605                 :            :                 else
     606                 :          0 :                         set_current_state(TASK_UNINTERRUPTIBLE);
     607                 :            : 
     608                 :          0 :                 if (dma_fence_test_signaled_any(fences, count, idx))
     609                 :            :                         break;
     610                 :            : 
     611                 :          0 :                 ret = schedule_timeout(ret);
     612                 :            : 
     613                 :          0 :                 if (ret > 0 && intr && signal_pending(current))
     614                 :            :                         ret = -ERESTARTSYS;
     615                 :            :         }
     616                 :            : 
     617                 :          0 :         __set_current_state(TASK_RUNNING);
     618                 :            : 
     619                 :            : fence_rm_cb:
     620                 :          0 :         while (i-- > 0)
     621                 :          0 :                 dma_fence_remove_callback(fences[i], &cb[i].base);
     622                 :            : 
     623                 :            : err_free_cb:
     624                 :          0 :         kfree(cb);
     625                 :            : 
     626                 :          0 :         return ret;
     627                 :            : }
     628                 :            : EXPORT_SYMBOL(dma_fence_wait_any_timeout);
     629                 :            : 
     630                 :            : /**
     631                 :            :  * dma_fence_init - Initialize a custom fence.
     632                 :            :  * @fence: the fence to initialize
     633                 :            :  * @ops: the dma_fence_ops for operations on this fence
     634                 :            :  * @lock: the irqsafe spinlock to use for locking this fence
     635                 :            :  * @context: the execution context this fence is run on
     636                 :            :  * @seqno: a linear increasing sequence number for this context
     637                 :            :  *
     638                 :            :  * Initializes an allocated fence, the caller doesn't have to keep its
     639                 :            :  * refcount after committing with this fence, but it will need to hold a
     640                 :            :  * refcount again if &dma_fence_ops.enable_signaling gets called.
     641                 :            :  *
     642                 :            :  * context and seqno are used for easy comparison between fences, allowing
     643                 :            :  * to check which fence is later by simply using dma_fence_later().
     644                 :            :  */
     645                 :            : void
     646                 :          0 : dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
     647                 :            :                spinlock_t *lock, u64 context, u64 seqno)
     648                 :            : {
     649                 :          0 :         BUG_ON(!lock);
     650                 :          0 :         BUG_ON(!ops || !ops->get_driver_name || !ops->get_timeline_name);
     651                 :            : 
     652                 :            :         kref_init(&fence->refcount);
     653                 :          0 :         fence->ops = ops;
     654                 :          0 :         INIT_LIST_HEAD(&fence->cb_list);
     655                 :          0 :         fence->lock = lock;
     656                 :          0 :         fence->context = context;
     657                 :          0 :         fence->seqno = seqno;
     658                 :          0 :         fence->flags = 0UL;
     659                 :          0 :         fence->error = 0;
     660                 :            : 
     661                 :          0 :         trace_dma_fence_init(fence);
     662                 :          0 : }
     663                 :            : EXPORT_SYMBOL(dma_fence_init);
    

Generated by: LCOV version 1.14