Branch data Line data Source code
1 : : // SPDX-License-Identifier: GPL-2.0-only 2 : : /* 3 : : * fence-chain: chain fences together in a timeline 4 : : * 5 : : * Copyright (C) 2018 Advanced Micro Devices, Inc. 6 : : * Authors: 7 : : * Christian König <christian.koenig@amd.com> 8 : : */ 9 : : 10 : : #include <linux/dma-fence-chain.h> 11 : : 12 : : static bool dma_fence_chain_enable_signaling(struct dma_fence *fence); 13 : : 14 : : /** 15 : : * dma_fence_chain_get_prev - use RCU to get a reference to the previous fence 16 : : * @chain: chain node to get the previous node from 17 : : * 18 : : * Use dma_fence_get_rcu_safe to get a reference to the previous fence of the 19 : : * chain node. 20 : : */ 21 : 0 : static struct dma_fence *dma_fence_chain_get_prev(struct dma_fence_chain *chain) 22 : : { 23 : 0 : struct dma_fence *prev; 24 : : 25 : 0 : rcu_read_lock(); 26 : 0 : prev = dma_fence_get_rcu_safe(&chain->prev); 27 : 0 : rcu_read_unlock(); 28 : 0 : return prev; 29 : : } 30 : : 31 : : /** 32 : : * dma_fence_chain_walk - chain walking function 33 : : * @fence: current chain node 34 : : * 35 : : * Walk the chain to the next node. Returns the next fence or NULL if we are at 36 : : * the end of the chain. Garbage collects chain nodes which are already 37 : : * signaled. 38 : : */ 39 : 0 : struct dma_fence *dma_fence_chain_walk(struct dma_fence *fence) 40 : : { 41 : 0 : struct dma_fence_chain *chain, *prev_chain; 42 : 0 : struct dma_fence *prev, *replacement, *tmp; 43 : : 44 [ # # ]: 0 : chain = to_dma_fence_chain(fence); 45 : 0 : if (!chain) { 46 : 0 : dma_fence_put(fence); 47 : 0 : return NULL; 48 : : } 49 : : 50 [ # # ]: 0 : while ((prev = dma_fence_chain_get_prev(chain))) { 51 : : 52 [ # # ]: 0 : prev_chain = to_dma_fence_chain(prev); 53 : 0 : if (prev_chain) { 54 [ # # ]: 0 : if (!dma_fence_is_signaled(prev_chain->fence)) 55 : : break; 56 : : 57 : 0 : replacement = dma_fence_chain_get_prev(prev_chain); 58 : : } else { 59 [ # # ]: 0 : if (!dma_fence_is_signaled(prev)) 60 : : break; 61 : : 62 : : replacement = NULL; 63 : : } 64 : : 65 : 0 : tmp = cmpxchg((void **)&chain->prev, (void *)prev, (void *)replacement); 66 [ # # ]: 0 : if (tmp == prev) 67 : 0 : dma_fence_put(tmp); 68 : : else 69 : 0 : dma_fence_put(replacement); 70 : 0 : dma_fence_put(prev); 71 : : } 72 : : 73 : 0 : dma_fence_put(fence); 74 : 0 : return prev; 75 : : } 76 : : EXPORT_SYMBOL(dma_fence_chain_walk); 77 : : 78 : : /** 79 : : * dma_fence_chain_find_seqno - find fence chain node by seqno 80 : : * @pfence: pointer to the chain node where to start 81 : : * @seqno: the sequence number to search for 82 : : * 83 : : * Advance the fence pointer to the chain node which will signal this sequence 84 : : * number. If no sequence number is provided then this is a no-op. 85 : : * 86 : : * Returns EINVAL if the fence is not a chain node or the sequence number has 87 : : * not yet advanced far enough. 88 : : */ 89 : 0 : int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno) 90 : : { 91 : 0 : struct dma_fence_chain *chain; 92 : : 93 [ # # ]: 0 : if (!seqno) 94 : : return 0; 95 : : 96 [ # # ]: 0 : chain = to_dma_fence_chain(*pfence); 97 [ # # ]: 0 : if (!chain || chain->base.seqno < seqno) 98 : : return -EINVAL; 99 : : 100 [ # # ]: 0 : dma_fence_chain_for_each(*pfence, &chain->base) { 101 [ # # ]: 0 : if ((*pfence)->context != chain->base.context || 102 [ # # ]: 0 : to_dma_fence_chain(*pfence)->prev_seqno < seqno) 103 : : break; 104 : : } 105 : 0 : dma_fence_put(&chain->base); 106 : : 107 : 0 : return 0; 108 : : } 109 : : EXPORT_SYMBOL(dma_fence_chain_find_seqno); 110 : : 111 : 0 : static const char *dma_fence_chain_get_driver_name(struct dma_fence *fence) 112 : : { 113 : 0 : return "dma_fence_chain"; 114 : : } 115 : : 116 : 0 : static const char *dma_fence_chain_get_timeline_name(struct dma_fence *fence) 117 : : { 118 : 0 : return "unbound"; 119 : : } 120 : : 121 : 0 : static void dma_fence_chain_irq_work(struct irq_work *work) 122 : : { 123 : 0 : struct dma_fence_chain *chain; 124 : : 125 : 0 : chain = container_of(work, typeof(*chain), work); 126 : : 127 : : /* Try to rearm the callback */ 128 [ # # ]: 0 : if (!dma_fence_chain_enable_signaling(&chain->base)) 129 : : /* Ok, we are done. No more unsignaled fences left */ 130 : 0 : dma_fence_signal(&chain->base); 131 : 0 : dma_fence_put(&chain->base); 132 : 0 : } 133 : : 134 : 0 : static void dma_fence_chain_cb(struct dma_fence *f, struct dma_fence_cb *cb) 135 : : { 136 : 0 : struct dma_fence_chain *chain; 137 : : 138 : 0 : chain = container_of(cb, typeof(*chain), cb); 139 : 0 : irq_work_queue(&chain->work); 140 : 0 : dma_fence_put(f); 141 : 0 : } 142 : : 143 : 0 : static bool dma_fence_chain_enable_signaling(struct dma_fence *fence) 144 : : { 145 [ # # ]: 0 : struct dma_fence_chain *head = to_dma_fence_chain(fence); 146 : : 147 [ # # ]: 0 : dma_fence_get(&head->base); 148 [ # # # # ]: 0 : dma_fence_chain_for_each(fence, &head->base) { 149 [ # # ]: 0 : struct dma_fence_chain *chain = to_dma_fence_chain(fence); 150 : 0 : struct dma_fence *f = chain ? chain->fence : fence; 151 : : 152 [ # # ]: 0 : dma_fence_get(f); 153 [ # # ]: 0 : if (!dma_fence_add_callback(f, &head->cb, dma_fence_chain_cb)) { 154 : 0 : dma_fence_put(fence); 155 : 0 : return true; 156 : : } 157 : 0 : dma_fence_put(f); 158 : : } 159 : 0 : dma_fence_put(&head->base); 160 : 0 : return false; 161 : : } 162 : : 163 : 0 : static bool dma_fence_chain_signaled(struct dma_fence *fence) 164 : : { 165 [ # # # # ]: 0 : dma_fence_chain_for_each(fence, fence) { 166 [ # # ]: 0 : struct dma_fence_chain *chain = to_dma_fence_chain(fence); 167 : 0 : struct dma_fence *f = chain ? chain->fence : fence; 168 : : 169 [ # # ]: 0 : if (!dma_fence_is_signaled(f)) { 170 : 0 : dma_fence_put(fence); 171 : 0 : return false; 172 : : } 173 : : } 174 : : 175 : : return true; 176 : : } 177 : : 178 : 0 : static void dma_fence_chain_release(struct dma_fence *fence) 179 : : { 180 [ # # ]: 0 : struct dma_fence_chain *chain = to_dma_fence_chain(fence); 181 : 0 : struct dma_fence *prev; 182 : : 183 : : /* Manually unlink the chain as much as possible to avoid recursion 184 : : * and potential stack overflow. 185 : : */ 186 [ # # ]: 0 : while ((prev = rcu_dereference_protected(chain->prev, true))) { 187 : 0 : struct dma_fence_chain *prev_chain; 188 : : 189 [ # # ]: 0 : if (kref_read(&prev->refcount) > 1) 190 : : break; 191 : : 192 [ # # ]: 0 : prev_chain = to_dma_fence_chain(prev); 193 : 0 : if (!prev_chain) 194 : : break; 195 : : 196 : : /* No need for atomic operations since we hold the last 197 : : * reference to prev_chain. 198 : : */ 199 : 0 : chain->prev = prev_chain->prev; 200 : 0 : RCU_INIT_POINTER(prev_chain->prev, NULL); 201 : 0 : dma_fence_put(prev); 202 : : } 203 : 0 : dma_fence_put(prev); 204 : : 205 : 0 : dma_fence_put(chain->fence); 206 : 0 : dma_fence_free(fence); 207 : 0 : } 208 : : 209 : : const struct dma_fence_ops dma_fence_chain_ops = { 210 : : .use_64bit_seqno = true, 211 : : .get_driver_name = dma_fence_chain_get_driver_name, 212 : : .get_timeline_name = dma_fence_chain_get_timeline_name, 213 : : .enable_signaling = dma_fence_chain_enable_signaling, 214 : : .signaled = dma_fence_chain_signaled, 215 : : .release = dma_fence_chain_release, 216 : : }; 217 : : EXPORT_SYMBOL(dma_fence_chain_ops); 218 : : 219 : : /** 220 : : * dma_fence_chain_init - initialize a fence chain 221 : : * @chain: the chain node to initialize 222 : : * @prev: the previous fence 223 : : * @fence: the current fence 224 : : * 225 : : * Initialize a new chain node and either start a new chain or add the node to 226 : : * the existing chain of the previous fence. 227 : : */ 228 : 0 : void dma_fence_chain_init(struct dma_fence_chain *chain, 229 : : struct dma_fence *prev, 230 : : struct dma_fence *fence, 231 : : uint64_t seqno) 232 : : { 233 [ # # ]: 0 : struct dma_fence_chain *prev_chain = to_dma_fence_chain(prev); 234 : 0 : uint64_t context; 235 : : 236 : 0 : spin_lock_init(&chain->lock); 237 : 0 : rcu_assign_pointer(chain->prev, prev); 238 : 0 : chain->fence = fence; 239 : 0 : chain->prev_seqno = 0; 240 : 0 : init_irq_work(&chain->work, dma_fence_chain_irq_work); 241 : : 242 : : /* Try to reuse the context of the previous chain node. */ 243 [ # # # # : 0 : if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) { # # ] 244 : 0 : context = prev->context; 245 : 0 : chain->prev_seqno = prev->seqno; 246 : : } else { 247 : 0 : context = dma_fence_context_alloc(1); 248 : : /* Make sure that we always have a valid sequence number. */ 249 [ # # ]: 0 : if (prev_chain) 250 : 0 : seqno = max(prev->seqno, seqno); 251 : : } 252 : : 253 : 0 : dma_fence_init(&chain->base, &dma_fence_chain_ops, 254 : : &chain->lock, context, seqno); 255 : 0 : } 256 : : EXPORT_SYMBOL(dma_fence_chain_init);