Branch data     Line data    Source code

       1                 :            : // SPDX-License-Identifier: GPL-2.0-only
       2                 :            : /*
       3                 :            :  * fence-chain: chain fences together in a timeline
       4                 :            :  *
       5                 :            :  * Copyright (C) 2018 Advanced Micro Devices, Inc.
       6                 :            :  * Authors:
       7                 :            :  *      Christian König <christian.koenig@amd.com>
       8                 :            :  */
       9                 :            : 
      10                 :            : #include <linux/dma-fence-chain.h>
      11                 :            : 
      12                 :            : static bool dma_fence_chain_enable_signaling(struct dma_fence *fence);
      13                 :            : 
      14                 :            : /**
      15                 :            :  * dma_fence_chain_get_prev - use RCU to get a reference to the previous fence
      16                 :            :  * @chain: chain node to get the previous node from
      17                 :            :  *
      18                 :            :  * Use dma_fence_get_rcu_safe to get a reference to the previous fence of the
      19                 :            :  * chain node.
      20                 :            :  */
      21                 :          0 : static struct dma_fence *dma_fence_chain_get_prev(struct dma_fence_chain *chain)
      22                 :            : {
      23                 :          0 :         struct dma_fence *prev;
      24                 :            : 
      25                 :          0 :         rcu_read_lock();
      26                 :          0 :         prev = dma_fence_get_rcu_safe(&chain->prev);
      27                 :          0 :         rcu_read_unlock();
      28                 :          0 :         return prev;
      29                 :            : }
      30                 :            : 
      31                 :            : /**
      32                 :            :  * dma_fence_chain_walk - chain walking function
      33                 :            :  * @fence: current chain node
      34                 :            :  *
      35                 :            :  * Walk the chain to the next node. Returns the next fence or NULL if we are at
      36                 :            :  * the end of the chain. Garbage collects chain nodes which are already
      37                 :            :  * signaled.
      38                 :            :  */
      39                 :          0 : struct dma_fence *dma_fence_chain_walk(struct dma_fence *fence)
      40                 :            : {
      41                 :          0 :         struct dma_fence_chain *chain, *prev_chain;
      42                 :          0 :         struct dma_fence *prev, *replacement, *tmp;
      43                 :            : 
      44         [ #  # ]:          0 :         chain = to_dma_fence_chain(fence);
      45                 :          0 :         if (!chain) {
      46                 :          0 :                 dma_fence_put(fence);
      47                 :          0 :                 return NULL;
      48                 :            :         }
      49                 :            : 
      50         [ #  # ]:          0 :         while ((prev = dma_fence_chain_get_prev(chain))) {
      51                 :            : 
      52         [ #  # ]:          0 :                 prev_chain = to_dma_fence_chain(prev);
      53                 :          0 :                 if (prev_chain) {
      54         [ #  # ]:          0 :                         if (!dma_fence_is_signaled(prev_chain->fence))
      55                 :            :                                 break;
      56                 :            : 
      57                 :          0 :                         replacement = dma_fence_chain_get_prev(prev_chain);
      58                 :            :                 } else {
      59         [ #  # ]:          0 :                         if (!dma_fence_is_signaled(prev))
      60                 :            :                                 break;
      61                 :            : 
      62                 :            :                         replacement = NULL;
      63                 :            :                 }
      64                 :            : 
      65                 :          0 :                 tmp = cmpxchg((void **)&chain->prev, (void *)prev, (void *)replacement);
      66         [ #  # ]:          0 :                 if (tmp == prev)
      67                 :          0 :                         dma_fence_put(tmp);
      68                 :            :                 else
      69                 :          0 :                         dma_fence_put(replacement);
      70                 :          0 :                 dma_fence_put(prev);
      71                 :            :         }
      72                 :            : 
      73                 :          0 :         dma_fence_put(fence);
      74                 :          0 :         return prev;
      75                 :            : }
      76                 :            : EXPORT_SYMBOL(dma_fence_chain_walk);
      77                 :            : 
      78                 :            : /**
      79                 :            :  * dma_fence_chain_find_seqno - find fence chain node by seqno
      80                 :            :  * @pfence: pointer to the chain node where to start
      81                 :            :  * @seqno: the sequence number to search for
      82                 :            :  *
      83                 :            :  * Advance the fence pointer to the chain node which will signal this sequence
      84                 :            :  * number. If no sequence number is provided then this is a no-op.
      85                 :            :  *
      86                 :            :  * Returns EINVAL if the fence is not a chain node or the sequence number has
      87                 :            :  * not yet advanced far enough.
      88                 :            :  */
      89                 :          0 : int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno)
      90                 :            : {
      91                 :          0 :         struct dma_fence_chain *chain;
      92                 :            : 
      93         [ #  # ]:          0 :         if (!seqno)
      94                 :            :                 return 0;
      95                 :            : 
      96         [ #  # ]:          0 :         chain = to_dma_fence_chain(*pfence);
      97         [ #  # ]:          0 :         if (!chain || chain->base.seqno < seqno)
      98                 :            :                 return -EINVAL;
      99                 :            : 
     100         [ #  # ]:          0 :         dma_fence_chain_for_each(*pfence, &chain->base) {
     101         [ #  # ]:          0 :                 if ((*pfence)->context != chain->base.context ||
     102         [ #  # ]:          0 :                     to_dma_fence_chain(*pfence)->prev_seqno < seqno)
     103                 :            :                         break;
     104                 :            :         }
     105                 :          0 :         dma_fence_put(&chain->base);
     106                 :            : 
     107                 :          0 :         return 0;
     108                 :            : }
     109                 :            : EXPORT_SYMBOL(dma_fence_chain_find_seqno);
     110                 :            : 
     111                 :          0 : static const char *dma_fence_chain_get_driver_name(struct dma_fence *fence)
     112                 :            : {
     113                 :          0 :         return "dma_fence_chain";
     114                 :            : }
     115                 :            : 
     116                 :          0 : static const char *dma_fence_chain_get_timeline_name(struct dma_fence *fence)
     117                 :            : {
     118                 :          0 :         return "unbound";
     119                 :            : }
     120                 :            : 
     121                 :          0 : static void dma_fence_chain_irq_work(struct irq_work *work)
     122                 :            : {
     123                 :          0 :         struct dma_fence_chain *chain;
     124                 :            : 
     125                 :          0 :         chain = container_of(work, typeof(*chain), work);
     126                 :            : 
     127                 :            :         /* Try to rearm the callback */
     128         [ #  # ]:          0 :         if (!dma_fence_chain_enable_signaling(&chain->base))
     129                 :            :                 /* Ok, we are done. No more unsignaled fences left */
     130                 :          0 :                 dma_fence_signal(&chain->base);
     131                 :          0 :         dma_fence_put(&chain->base);
     132                 :          0 : }
     133                 :            : 
     134                 :          0 : static void dma_fence_chain_cb(struct dma_fence *f, struct dma_fence_cb *cb)
     135                 :            : {
     136                 :          0 :         struct dma_fence_chain *chain;
     137                 :            : 
     138                 :          0 :         chain = container_of(cb, typeof(*chain), cb);
     139                 :          0 :         irq_work_queue(&chain->work);
     140                 :          0 :         dma_fence_put(f);
     141                 :          0 : }
     142                 :            : 
     143                 :          0 : static bool dma_fence_chain_enable_signaling(struct dma_fence *fence)
     144                 :            : {
     145         [ #  # ]:          0 :         struct dma_fence_chain *head = to_dma_fence_chain(fence);
     146                 :            : 
     147         [ #  # ]:          0 :         dma_fence_get(&head->base);
     148   [ #  #  #  # ]:          0 :         dma_fence_chain_for_each(fence, &head->base) {
     149         [ #  # ]:          0 :                 struct dma_fence_chain *chain = to_dma_fence_chain(fence);
     150                 :          0 :                 struct dma_fence *f = chain ? chain->fence : fence;
     151                 :            : 
     152         [ #  # ]:          0 :                 dma_fence_get(f);
     153         [ #  # ]:          0 :                 if (!dma_fence_add_callback(f, &head->cb, dma_fence_chain_cb)) {
     154                 :          0 :                         dma_fence_put(fence);
     155                 :          0 :                         return true;
     156                 :            :                 }
     157                 :          0 :                 dma_fence_put(f);
     158                 :            :         }
     159                 :          0 :         dma_fence_put(&head->base);
     160                 :          0 :         return false;
     161                 :            : }
     162                 :            : 
     163                 :          0 : static bool dma_fence_chain_signaled(struct dma_fence *fence)
     164                 :            : {
     165   [ #  #  #  # ]:          0 :         dma_fence_chain_for_each(fence, fence) {
     166         [ #  # ]:          0 :                 struct dma_fence_chain *chain = to_dma_fence_chain(fence);
     167                 :          0 :                 struct dma_fence *f = chain ? chain->fence : fence;
     168                 :            : 
     169         [ #  # ]:          0 :                 if (!dma_fence_is_signaled(f)) {
     170                 :          0 :                         dma_fence_put(fence);
     171                 :          0 :                         return false;
     172                 :            :                 }
     173                 :            :         }
     174                 :            : 
     175                 :            :         return true;
     176                 :            : }
     177                 :            : 
     178                 :          0 : static void dma_fence_chain_release(struct dma_fence *fence)
     179                 :            : {
     180         [ #  # ]:          0 :         struct dma_fence_chain *chain = to_dma_fence_chain(fence);
     181                 :          0 :         struct dma_fence *prev;
     182                 :            : 
     183                 :            :         /* Manually unlink the chain as much as possible to avoid recursion
     184                 :            :          * and potential stack overflow.
     185                 :            :          */
     186         [ #  # ]:          0 :         while ((prev = rcu_dereference_protected(chain->prev, true))) {
     187                 :          0 :                 struct dma_fence_chain *prev_chain;
     188                 :            : 
     189         [ #  # ]:          0 :                 if (kref_read(&prev->refcount) > 1)
     190                 :            :                        break;
     191                 :            : 
     192         [ #  # ]:          0 :                 prev_chain = to_dma_fence_chain(prev);
     193                 :          0 :                 if (!prev_chain)
     194                 :            :                         break;
     195                 :            : 
     196                 :            :                 /* No need for atomic operations since we hold the last
     197                 :            :                  * reference to prev_chain.
     198                 :            :                  */
     199                 :          0 :                 chain->prev = prev_chain->prev;
     200                 :          0 :                 RCU_INIT_POINTER(prev_chain->prev, NULL);
     201                 :          0 :                 dma_fence_put(prev);
     202                 :            :         }
     203                 :          0 :         dma_fence_put(prev);
     204                 :            : 
     205                 :          0 :         dma_fence_put(chain->fence);
     206                 :          0 :         dma_fence_free(fence);
     207                 :          0 : }
     208                 :            : 
     209                 :            : const struct dma_fence_ops dma_fence_chain_ops = {
     210                 :            :         .use_64bit_seqno = true,
     211                 :            :         .get_driver_name = dma_fence_chain_get_driver_name,
     212                 :            :         .get_timeline_name = dma_fence_chain_get_timeline_name,
     213                 :            :         .enable_signaling = dma_fence_chain_enable_signaling,
     214                 :            :         .signaled = dma_fence_chain_signaled,
     215                 :            :         .release = dma_fence_chain_release,
     216                 :            : };
     217                 :            : EXPORT_SYMBOL(dma_fence_chain_ops);
     218                 :            : 
     219                 :            : /**
     220                 :            :  * dma_fence_chain_init - initialize a fence chain
     221                 :            :  * @chain: the chain node to initialize
     222                 :            :  * @prev: the previous fence
     223                 :            :  * @fence: the current fence
     224                 :            :  *
     225                 :            :  * Initialize a new chain node and either start a new chain or add the node to
     226                 :            :  * the existing chain of the previous fence.
     227                 :            :  */
     228                 :          0 : void dma_fence_chain_init(struct dma_fence_chain *chain,
     229                 :            :                           struct dma_fence *prev,
     230                 :            :                           struct dma_fence *fence,
     231                 :            :                           uint64_t seqno)
     232                 :            : {
     233         [ #  # ]:          0 :         struct dma_fence_chain *prev_chain = to_dma_fence_chain(prev);
     234                 :          0 :         uint64_t context;
     235                 :            : 
     236                 :          0 :         spin_lock_init(&chain->lock);
     237                 :          0 :         rcu_assign_pointer(chain->prev, prev);
     238                 :          0 :         chain->fence = fence;
     239                 :          0 :         chain->prev_seqno = 0;
     240                 :          0 :         init_irq_work(&chain->work, dma_fence_chain_irq_work);
     241                 :            : 
     242                 :            :         /* Try to reuse the context of the previous chain node. */
     243   [ #  #  #  #  :          0 :         if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) {
                   #  # ]
     244                 :          0 :                 context = prev->context;
     245                 :          0 :                 chain->prev_seqno = prev->seqno;
     246                 :            :         } else {
     247                 :          0 :                 context = dma_fence_context_alloc(1);
     248                 :            :                 /* Make sure that we always have a valid sequence number. */
     249         [ #  # ]:          0 :                 if (prev_chain)
     250                 :          0 :                         seqno = max(prev->seqno, seqno);
     251                 :            :         }
     252                 :            : 
     253                 :          0 :         dma_fence_init(&chain->base, &dma_fence_chain_ops,
     254                 :            :                        &chain->lock, context, seqno);
     255                 :          0 : }
     256                 :            : EXPORT_SYMBOL(dma_fence_chain_init);