Branch data     Line data    Source code

       1                 :            : // SPDX-License-Identifier: GPL-2.0-only
       2                 :            : /*
       3                 :            :  * dma-fence-array: aggregate fences to be waited together
       4                 :            :  *
       5                 :            :  * Copyright (C) 2016 Collabora Ltd
       6                 :            :  * Copyright (C) 2016 Advanced Micro Devices, Inc.
       7                 :            :  * Authors:
       8                 :            :  *      Gustavo Padovan <gustavo@padovan.org>
       9                 :            :  *      Christian König <christian.koenig@amd.com>
      10                 :            :  */
      11                 :            : 
      12                 :            : #include <linux/export.h>
      13                 :            : #include <linux/slab.h>
      14                 :            : #include <linux/dma-fence-array.h>
      15                 :            : 
      16                 :            : #define PENDING_ERROR 1
      17                 :            : 
      18                 :          0 : static const char *dma_fence_array_get_driver_name(struct dma_fence *fence)
      19                 :            : {
      20                 :          0 :         return "dma_fence_array";
      21                 :            : }
      22                 :            : 
      23                 :          0 : static const char *dma_fence_array_get_timeline_name(struct dma_fence *fence)
      24                 :            : {
      25                 :          0 :         return "unbound";
      26                 :            : }
      27                 :            : 
      28                 :          0 : static void dma_fence_array_set_pending_error(struct dma_fence_array *array,
      29                 :            :                                               int error)
      30                 :            : {
      31                 :            :         /*
      32                 :            :          * Propagate the first error reported by any of our fences, but only
      33                 :            :          * before we ourselves are signaled.
      34                 :            :          */
      35         [ #  # ]:          0 :         if (error)
      36                 :          0 :                 cmpxchg(&array->base.error, PENDING_ERROR, error);
      37                 :          0 : }
      38                 :            : 
      39                 :            : static void dma_fence_array_clear_pending_error(struct dma_fence_array *array)
      40                 :            : {
      41                 :            :         /* Clear the error flag if not actually set. */
      42                 :          0 :         cmpxchg(&array->base.error, PENDING_ERROR, 0);
      43                 :            : }
      44                 :            : 
      45                 :          0 : static void irq_dma_fence_array_work(struct irq_work *wrk)
      46                 :            : {
      47                 :            :         struct dma_fence_array *array = container_of(wrk, typeof(*array), work);
      48                 :            : 
      49                 :            :         dma_fence_array_clear_pending_error(array);
      50                 :            : 
      51                 :          0 :         dma_fence_signal(&array->base);
      52                 :            :         dma_fence_put(&array->base);
      53                 :          0 : }
      54                 :            : 
      55                 :          0 : static void dma_fence_array_cb_func(struct dma_fence *f,
      56                 :            :                                     struct dma_fence_cb *cb)
      57                 :            : {
      58                 :            :         struct dma_fence_array_cb *array_cb =
      59                 :            :                 container_of(cb, struct dma_fence_array_cb, cb);
      60                 :          0 :         struct dma_fence_array *array = array_cb->array;
      61                 :            : 
      62                 :          0 :         dma_fence_array_set_pending_error(array, f->error);
      63                 :            : 
      64         [ #  # ]:          0 :         if (atomic_dec_and_test(&array->num_pending))
      65                 :          0 :                 irq_work_queue(&array->work);
      66                 :            :         else
      67                 :          0 :                 dma_fence_put(&array->base);
      68                 :          0 : }
      69                 :            : 
      70                 :          0 : static bool dma_fence_array_enable_signaling(struct dma_fence *fence)
      71                 :            : {
      72                 :            :         struct dma_fence_array *array = to_dma_fence_array(fence);
      73                 :          0 :         struct dma_fence_array_cb *cb = (void *)(&array[1]);
      74                 :            :         unsigned i;
      75                 :            : 
      76         [ #  # ]:          0 :         for (i = 0; i < array->num_fences; ++i) {
      77                 :          0 :                 cb[i].array = array;
      78                 :            :                 /*
      79                 :            :                  * As we may report that the fence is signaled before all
      80                 :            :                  * callbacks are complete, we need to take an additional
      81                 :            :                  * reference count on the array so that we do not free it too
      82                 :            :                  * early. The core fence handling will only hold the reference
      83                 :            :                  * until we signal the array as complete (but that is now
      84                 :            :                  * insufficient).
      85                 :            :                  */
      86                 :          0 :                 dma_fence_get(&array->base);
      87         [ #  # ]:          0 :                 if (dma_fence_add_callback(array->fences[i], &cb[i].cb,
      88                 :            :                                            dma_fence_array_cb_func)) {
      89                 :          0 :                         int error = array->fences[i]->error;
      90                 :            : 
      91                 :          0 :                         dma_fence_array_set_pending_error(array, error);
      92                 :            :                         dma_fence_put(&array->base);
      93         [ #  # ]:          0 :                         if (atomic_dec_and_test(&array->num_pending)) {
      94                 :            :                                 dma_fence_array_clear_pending_error(array);
      95                 :          0 :                                 return false;
      96                 :            :                         }
      97                 :            :                 }
      98                 :            :         }
      99                 :            : 
     100                 :            :         return true;
     101                 :            : }
     102                 :            : 
     103                 :          0 : static bool dma_fence_array_signaled(struct dma_fence *fence)
     104                 :            : {
     105                 :            :         struct dma_fence_array *array = to_dma_fence_array(fence);
     106                 :            : 
     107                 :          0 :         return atomic_read(&array->num_pending) <= 0;
     108                 :            : }
     109                 :            : 
     110                 :          0 : static void dma_fence_array_release(struct dma_fence *fence)
     111                 :            : {
     112                 :            :         struct dma_fence_array *array = to_dma_fence_array(fence);
     113                 :            :         unsigned i;
     114                 :            : 
     115         [ #  # ]:          0 :         for (i = 0; i < array->num_fences; ++i)
     116                 :          0 :                 dma_fence_put(array->fences[i]);
     117                 :            : 
     118                 :          0 :         kfree(array->fences);
     119                 :          0 :         dma_fence_free(fence);
     120                 :          0 : }
     121                 :            : 
     122                 :            : const struct dma_fence_ops dma_fence_array_ops = {
     123                 :            :         .get_driver_name = dma_fence_array_get_driver_name,
     124                 :            :         .get_timeline_name = dma_fence_array_get_timeline_name,
     125                 :            :         .enable_signaling = dma_fence_array_enable_signaling,
     126                 :            :         .signaled = dma_fence_array_signaled,
     127                 :            :         .release = dma_fence_array_release,
     128                 :            : };
     129                 :            : EXPORT_SYMBOL(dma_fence_array_ops);
     130                 :            : 
     131                 :            : /**
     132                 :            :  * dma_fence_array_create - Create a custom fence array
     133                 :            :  * @num_fences:         [in]    number of fences to add in the array
     134                 :            :  * @fences:             [in]    array containing the fences
     135                 :            :  * @context:            [in]    fence context to use
     136                 :            :  * @seqno:              [in]    sequence number to use
     137                 :            :  * @signal_on_any:      [in]    signal on any fence in the array
     138                 :            :  *
     139                 :            :  * Allocate a dma_fence_array object and initialize the base fence with
     140                 :            :  * dma_fence_init().
     141                 :            :  * In case of error it returns NULL.
     142                 :            :  *
     143                 :            :  * The caller should allocate the fences array with num_fences size
     144                 :            :  * and fill it with the fences it wants to add to the object. Ownership of this
     145                 :            :  * array is taken and dma_fence_put() is used on each fence on release.
     146                 :            :  *
     147                 :            :  * If @signal_on_any is true the fence array signals if any fence in the array
     148                 :            :  * signals, otherwise it signals when all fences in the array signal.
     149                 :            :  */
     150                 :          0 : struct dma_fence_array *dma_fence_array_create(int num_fences,
     151                 :            :                                                struct dma_fence **fences,
     152                 :            :                                                u64 context, unsigned seqno,
     153                 :            :                                                bool signal_on_any)
     154                 :            : {
     155                 :            :         struct dma_fence_array *array;
     156                 :            :         size_t size = sizeof(*array);
     157                 :            : 
     158                 :            :         /* Allocate the callback structures behind the array. */
     159                 :          0 :         size += num_fences * sizeof(struct dma_fence_array_cb);
     160                 :          0 :         array = kzalloc(size, GFP_KERNEL);
     161         [ #  # ]:          0 :         if (!array)
     162                 :            :                 return NULL;
     163                 :            : 
     164                 :          0 :         spin_lock_init(&array->lock);
     165                 :          0 :         dma_fence_init(&array->base, &dma_fence_array_ops, &array->lock,
     166                 :            :                        context, seqno);
     167                 :            :         init_irq_work(&array->work, irq_dma_fence_array_work);
     168                 :            : 
     169                 :          0 :         array->num_fences = num_fences;
     170         [ #  # ]:          0 :         atomic_set(&array->num_pending, signal_on_any ? 1 : num_fences);
     171                 :          0 :         array->fences = fences;
     172                 :            : 
     173                 :          0 :         array->base.error = PENDING_ERROR;
     174                 :            : 
     175                 :          0 :         return array;
     176                 :            : }
     177                 :            : EXPORT_SYMBOL(dma_fence_array_create);
     178                 :            : 
     179                 :            : /**
     180                 :            :  * dma_fence_match_context - Check if all fences are from the given context
     181                 :            :  * @fence:              [in]    fence or fence array
     182                 :            :  * @context:            [in]    fence context to check all fences against
     183                 :            :  *
     184                 :            :  * Checks the provided fence or, for a fence array, all fences in the array
     185                 :            :  * against the given context. Returns false if any fence is from a different
     186                 :            :  * context.
     187                 :            :  */
     188                 :          0 : bool dma_fence_match_context(struct dma_fence *fence, u64 context)
     189                 :            : {
     190                 :            :         struct dma_fence_array *array = to_dma_fence_array(fence);
     191                 :            :         unsigned i;
     192                 :            : 
     193         [ #  # ]:          0 :         if (!dma_fence_is_array(fence))
     194                 :          0 :                 return fence->context == context;
     195                 :            : 
     196         [ #  # ]:          0 :         for (i = 0; i < array->num_fences; i++) {
     197         [ #  # ]:          0 :                 if (array->fences[i]->context != context)
     198                 :            :                         return false;
     199                 :            :         }
     200                 :            : 
     201                 :            :         return true;
     202                 :            : }
     203                 :            : EXPORT_SYMBOL(dma_fence_match_context);