Branch data Line data Source code
1 : : // SPDX-License-Identifier: GPL-2.0-only
2 : : /*
3 : : * Fence mechanism for dma-buf and to allow for asynchronous dma access
4 : : *
5 : : * Copyright (C) 2012 Canonical Ltd
6 : : * Copyright (C) 2012 Texas Instruments
7 : : *
8 : : * Authors:
9 : : * Rob Clark <robdclark@gmail.com>
10 : : * Maarten Lankhorst <maarten.lankhorst@canonical.com>
11 : : */
12 : :
13 : : #include <linux/slab.h>
14 : : #include <linux/export.h>
15 : : #include <linux/atomic.h>
16 : : #include <linux/dma-fence.h>
17 : : #include <linux/sched/signal.h>
18 : :
19 : : #define CREATE_TRACE_POINTS
20 : : #include <trace/events/dma_fence.h>
21 : :
22 : : EXPORT_TRACEPOINT_SYMBOL(dma_fence_emit);
23 : : EXPORT_TRACEPOINT_SYMBOL(dma_fence_enable_signal);
24 : : EXPORT_TRACEPOINT_SYMBOL(dma_fence_signaled);
25 : :
26 : : static DEFINE_SPINLOCK(dma_fence_stub_lock);
27 : : static struct dma_fence dma_fence_stub;
28 : :
29 : : /*
30 : : * fence context counter: each execution context should have its own
31 : : * fence context, this allows checking if fences belong to the same
32 : : * context or not. One device can have multiple separate contexts,
33 : : * and they're used if some engine can run independently of another.
34 : : */
35 : : static atomic64_t dma_fence_context_counter = ATOMIC64_INIT(1);
36 : :
37 : : /**
38 : : * DOC: DMA fences overview
39 : : *
40 : : * DMA fences, represented by &struct dma_fence, are the kernel internal
41 : : * synchronization primitive for DMA operations like GPU rendering, video
42 : : * encoding/decoding, or displaying buffers on a screen.
43 : : *
44 : : * A fence is initialized using dma_fence_init() and completed using
45 : : * dma_fence_signal(). Fences are associated with a context, allocated through
46 : : * dma_fence_context_alloc(), and all fences on the same context are
47 : : * fully ordered.
48 : : *
49 : : * Since the purposes of fences is to facilitate cross-device and
50 : : * cross-application synchronization, there's multiple ways to use one:
51 : : *
52 : : * - Individual fences can be exposed as a &sync_file, accessed as a file
53 : : * descriptor from userspace, created by calling sync_file_create(). This is
54 : : * called explicit fencing, since userspace passes around explicit
55 : : * synchronization points.
56 : : *
57 : : * - Some subsystems also have their own explicit fencing primitives, like
58 : : * &drm_syncobj. Compared to &sync_file, a &drm_syncobj allows the underlying
59 : : * fence to be updated.
60 : : *
61 : : * - Then there's also implicit fencing, where the synchronization points are
62 : : * implicitly passed around as part of shared &dma_buf instances. Such
63 : : * implicit fences are stored in &struct dma_resv through the
64 : : * &dma_buf.resv pointer.
65 : : */
66 : :
67 : 0 : static const char *dma_fence_stub_get_name(struct dma_fence *fence)
68 : : {
69 : 0 : return "stub";
70 : : }
71 : :
72 : : static const struct dma_fence_ops dma_fence_stub_ops = {
73 : : .get_driver_name = dma_fence_stub_get_name,
74 : : .get_timeline_name = dma_fence_stub_get_name,
75 : : };
76 : :
77 : : /**
78 : : * dma_fence_get_stub - return a signaled fence
79 : : *
80 : : * Return a stub fence which is already signaled.
81 : : */
82 : 0 : struct dma_fence *dma_fence_get_stub(void)
83 : : {
84 : 0 : spin_lock(&dma_fence_stub_lock);
85 [ # # ]: 0 : if (!dma_fence_stub.ops) {
86 : 0 : dma_fence_init(&dma_fence_stub,
87 : : &dma_fence_stub_ops,
88 : : &dma_fence_stub_lock,
89 : : 0, 0);
90 : 0 : dma_fence_signal_locked(&dma_fence_stub);
91 : : }
92 : 0 : spin_unlock(&dma_fence_stub_lock);
93 : :
94 : 0 : return dma_fence_get(&dma_fence_stub);
95 : : }
96 : : EXPORT_SYMBOL(dma_fence_get_stub);
97 : :
98 : : /**
99 : : * dma_fence_context_alloc - allocate an array of fence contexts
100 : : * @num: amount of contexts to allocate
101 : : *
102 : : * This function will return the first index of the number of fence contexts
103 : : * allocated. The fence context is used for setting &dma_fence.context to a
104 : : * unique number by passing the context to dma_fence_init().
105 : : */
106 : 0 : u64 dma_fence_context_alloc(unsigned num)
107 : : {
108 [ # # ]: 0 : WARN_ON(!num);
109 : 0 : return atomic64_add_return(num, &dma_fence_context_counter) - num;
110 : : }
111 : : EXPORT_SYMBOL(dma_fence_context_alloc);
112 : :
113 : : /**
114 : : * dma_fence_signal_locked - signal completion of a fence
115 : : * @fence: the fence to signal
116 : : *
117 : : * Signal completion for software callbacks on a fence, this will unblock
118 : : * dma_fence_wait() calls and run all the callbacks added with
119 : : * dma_fence_add_callback(). Can be called multiple times, but since a fence
120 : : * can only go from the unsignaled to the signaled state and not back, it will
121 : : * only be effective the first time.
122 : : *
123 : : * Unlike dma_fence_signal(), this function must be called with &dma_fence.lock
124 : : * held.
125 : : *
126 : : * Returns 0 on success and a negative error value when @fence has been
127 : : * signalled already.
128 : : */
129 : 0 : int dma_fence_signal_locked(struct dma_fence *fence)
130 : : {
131 : 0 : struct dma_fence_cb *cur, *tmp;
132 : 0 : struct list_head cb_list;
133 : :
134 : 0 : lockdep_assert_held(fence->lock);
135 : :
136 [ # # ]: 0 : if (unlikely(test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
137 : : &fence->flags)))
138 : : return -EINVAL;
139 : :
140 : : /* Stash the cb_list before replacing it with the timestamp */
141 : 0 : list_replace(&fence->cb_list, &cb_list);
142 : :
143 : 0 : fence->timestamp = ktime_get();
144 : 0 : set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
145 : 0 : trace_dma_fence_signaled(fence);
146 : :
147 [ # # ]: 0 : list_for_each_entry_safe(cur, tmp, &cb_list, node) {
148 : 0 : INIT_LIST_HEAD(&cur->node);
149 : 0 : cur->func(fence, cur);
150 : : }
151 : :
152 : : return 0;
153 : : }
154 : : EXPORT_SYMBOL(dma_fence_signal_locked);
155 : :
156 : : /**
157 : : * dma_fence_signal - signal completion of a fence
158 : : * @fence: the fence to signal
159 : : *
160 : : * Signal completion for software callbacks on a fence, this will unblock
161 : : * dma_fence_wait() calls and run all the callbacks added with
162 : : * dma_fence_add_callback(). Can be called multiple times, but since a fence
163 : : * can only go from the unsignaled to the signaled state and not back, it will
164 : : * only be effective the first time.
165 : : *
166 : : * Returns 0 on success and a negative error value when @fence has been
167 : : * signalled already.
168 : : */
169 : 0 : int dma_fence_signal(struct dma_fence *fence)
170 : : {
171 : 0 : unsigned long flags;
172 : 0 : int ret;
173 : :
174 [ # # ]: 0 : if (!fence)
175 : : return -EINVAL;
176 : :
177 : 0 : spin_lock_irqsave(fence->lock, flags);
178 : 0 : ret = dma_fence_signal_locked(fence);
179 : 0 : spin_unlock_irqrestore(fence->lock, flags);
180 : :
181 : 0 : return ret;
182 : : }
183 : : EXPORT_SYMBOL(dma_fence_signal);
184 : :
185 : : /**
186 : : * dma_fence_wait_timeout - sleep until the fence gets signaled
187 : : * or until timeout elapses
188 : : * @fence: the fence to wait on
189 : : * @intr: if true, do an interruptible wait
190 : : * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
191 : : *
192 : : * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the
193 : : * remaining timeout in jiffies on success. Other error values may be
194 : : * returned on custom implementations.
195 : : *
196 : : * Performs a synchronous wait on this fence. It is assumed the caller
197 : : * directly or indirectly (buf-mgr between reservation and committing)
198 : : * holds a reference to the fence, otherwise the fence might be
199 : : * freed before return, resulting in undefined behavior.
200 : : *
201 : : * See also dma_fence_wait() and dma_fence_wait_any_timeout().
202 : : */
203 : : signed long
204 : 0 : dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)
205 : : {
206 : 0 : signed long ret;
207 : :
208 [ # # # # ]: 0 : if (WARN_ON(timeout < 0))
209 : : return -EINVAL;
210 : :
211 : 0 : trace_dma_fence_wait_start(fence);
212 [ # # ]: 0 : if (fence->ops->wait)
213 : 0 : ret = fence->ops->wait(fence, intr, timeout);
214 : : else
215 : 0 : ret = dma_fence_default_wait(fence, intr, timeout);
216 : 0 : trace_dma_fence_wait_end(fence);
217 : 0 : return ret;
218 : : }
219 : : EXPORT_SYMBOL(dma_fence_wait_timeout);
220 : :
221 : : /**
222 : : * dma_fence_release - default relese function for fences
223 : : * @kref: &dma_fence.recfount
224 : : *
225 : : * This is the default release functions for &dma_fence. Drivers shouldn't call
226 : : * this directly, but instead call dma_fence_put().
227 : : */
228 : 0 : void dma_fence_release(struct kref *kref)
229 : : {
230 : 0 : struct dma_fence *fence =
231 : 0 : container_of(kref, struct dma_fence, refcount);
232 : :
233 : 0 : trace_dma_fence_destroy(fence);
234 : :
235 [ # # # # : 0 : if (WARN(!list_empty(&fence->cb_list) &&
# # # # ]
236 : : !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags),
237 : : "Fence %s:%s:%llx:%llx released with pending signals!\n",
238 : : fence->ops->get_driver_name(fence),
239 : : fence->ops->get_timeline_name(fence),
240 : : fence->context, fence->seqno)) {
241 : 0 : unsigned long flags;
242 : :
243 : : /*
244 : : * Failed to signal before release, likely a refcounting issue.
245 : : *
246 : : * This should never happen, but if it does make sure that we
247 : : * don't leave chains dangling. We set the error flag first
248 : : * so that the callbacks know this signal is due to an error.
249 : : */
250 : 0 : spin_lock_irqsave(fence->lock, flags);
251 : 0 : fence->error = -EDEADLK;
252 : 0 : dma_fence_signal_locked(fence);
253 : 0 : spin_unlock_irqrestore(fence->lock, flags);
254 : : }
255 : :
256 [ # # ]: 0 : if (fence->ops->release)
257 : 0 : fence->ops->release(fence);
258 : : else
259 : 0 : dma_fence_free(fence);
260 : 0 : }
261 : : EXPORT_SYMBOL(dma_fence_release);
262 : :
263 : : /**
264 : : * dma_fence_free - default release function for &dma_fence.
265 : : * @fence: fence to release
266 : : *
267 : : * This is the default implementation for &dma_fence_ops.release. It calls
268 : : * kfree_rcu() on @fence.
269 : : */
270 : 0 : void dma_fence_free(struct dma_fence *fence)
271 : : {
272 [ # # # # ]: 0 : kfree_rcu(fence, rcu);
273 : 0 : }
274 : : EXPORT_SYMBOL(dma_fence_free);
275 : :
276 : 0 : static bool __dma_fence_enable_signaling(struct dma_fence *fence)
277 : : {
278 : 0 : bool was_set;
279 : :
280 : 0 : lockdep_assert_held(fence->lock);
281 : :
282 : 0 : was_set = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
283 : 0 : &fence->flags);
284 : :
285 [ # # ]: 0 : if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
286 : : return false;
287 : :
288 [ # # # # ]: 0 : if (!was_set && fence->ops->enable_signaling) {
289 : 0 : trace_dma_fence_enable_signal(fence);
290 : :
291 [ # # ]: 0 : if (!fence->ops->enable_signaling(fence)) {
292 : 0 : dma_fence_signal_locked(fence);
293 : 0 : return false;
294 : : }
295 : : }
296 : :
297 : : return true;
298 : : }
299 : :
300 : : /**
301 : : * dma_fence_enable_sw_signaling - enable signaling on fence
302 : : * @fence: the fence to enable
303 : : *
304 : : * This will request for sw signaling to be enabled, to make the fence
305 : : * complete as soon as possible. This calls &dma_fence_ops.enable_signaling
306 : : * internally.
307 : : */
308 : 0 : void dma_fence_enable_sw_signaling(struct dma_fence *fence)
309 : : {
310 : 0 : unsigned long flags;
311 : :
312 [ # # ]: 0 : if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
313 : : return;
314 : :
315 : 0 : spin_lock_irqsave(fence->lock, flags);
316 : 0 : __dma_fence_enable_signaling(fence);
317 : 0 : spin_unlock_irqrestore(fence->lock, flags);
318 : : }
319 : : EXPORT_SYMBOL(dma_fence_enable_sw_signaling);
320 : :
321 : : /**
322 : : * dma_fence_add_callback - add a callback to be called when the fence
323 : : * is signaled
324 : : * @fence: the fence to wait on
325 : : * @cb: the callback to register
326 : : * @func: the function to call
327 : : *
328 : : * @cb will be initialized by dma_fence_add_callback(), no initialization
329 : : * by the caller is required. Any number of callbacks can be registered
330 : : * to a fence, but a callback can only be registered to one fence at a time.
331 : : *
332 : : * Note that the callback can be called from an atomic context. If
333 : : * fence is already signaled, this function will return -ENOENT (and
334 : : * *not* call the callback).
335 : : *
336 : : * Add a software callback to the fence. Same restrictions apply to
337 : : * refcount as it does to dma_fence_wait(), however the caller doesn't need to
338 : : * keep a refcount to fence afterward dma_fence_add_callback() has returned:
339 : : * when software access is enabled, the creator of the fence is required to keep
340 : : * the fence alive until after it signals with dma_fence_signal(). The callback
341 : : * itself can be called from irq context.
342 : : *
343 : : * Returns 0 in case of success, -ENOENT if the fence is already signaled
344 : : * and -EINVAL in case of error.
345 : : */
346 : 0 : int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb,
347 : : dma_fence_func_t func)
348 : : {
349 : 0 : unsigned long flags;
350 : 0 : int ret = 0;
351 : :
352 [ # # # # ]: 0 : if (WARN_ON(!fence || !func))
353 : : return -EINVAL;
354 : :
355 [ # # ]: 0 : if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
356 : 0 : INIT_LIST_HEAD(&cb->node);
357 : 0 : return -ENOENT;
358 : : }
359 : :
360 : 0 : spin_lock_irqsave(fence->lock, flags);
361 : :
362 [ # # ]: 0 : if (__dma_fence_enable_signaling(fence)) {
363 : 0 : cb->func = func;
364 : 0 : list_add_tail(&cb->node, &fence->cb_list);
365 : : } else {
366 : 0 : INIT_LIST_HEAD(&cb->node);
367 : 0 : ret = -ENOENT;
368 : : }
369 : :
370 : 0 : spin_unlock_irqrestore(fence->lock, flags);
371 : :
372 : 0 : return ret;
373 : : }
374 : : EXPORT_SYMBOL(dma_fence_add_callback);
375 : :
376 : : /**
377 : : * dma_fence_get_status - returns the status upon completion
378 : : * @fence: the dma_fence to query
379 : : *
380 : : * This wraps dma_fence_get_status_locked() to return the error status
381 : : * condition on a signaled fence. See dma_fence_get_status_locked() for more
382 : : * details.
383 : : *
384 : : * Returns 0 if the fence has not yet been signaled, 1 if the fence has
385 : : * been signaled without an error condition, or a negative error code
386 : : * if the fence has been completed in err.
387 : : */
388 : 0 : int dma_fence_get_status(struct dma_fence *fence)
389 : : {
390 : 0 : unsigned long flags;
391 : 0 : int status;
392 : :
393 : 0 : spin_lock_irqsave(fence->lock, flags);
394 : 0 : status = dma_fence_get_status_locked(fence);
395 : 0 : spin_unlock_irqrestore(fence->lock, flags);
396 : :
397 : 0 : return status;
398 : : }
399 : : EXPORT_SYMBOL(dma_fence_get_status);
400 : :
401 : : /**
402 : : * dma_fence_remove_callback - remove a callback from the signaling list
403 : : * @fence: the fence to wait on
404 : : * @cb: the callback to remove
405 : : *
406 : : * Remove a previously queued callback from the fence. This function returns
407 : : * true if the callback is successfully removed, or false if the fence has
408 : : * already been signaled.
409 : : *
410 : : * *WARNING*:
411 : : * Cancelling a callback should only be done if you really know what you're
412 : : * doing, since deadlocks and race conditions could occur all too easily. For
413 : : * this reason, it should only ever be done on hardware lockup recovery,
414 : : * with a reference held to the fence.
415 : : *
416 : : * Behaviour is undefined if @cb has not been added to @fence using
417 : : * dma_fence_add_callback() beforehand.
418 : : */
419 : : bool
420 : 0 : dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb)
421 : : {
422 : 0 : unsigned long flags;
423 : 0 : bool ret;
424 : :
425 : 0 : spin_lock_irqsave(fence->lock, flags);
426 : :
427 [ # # ]: 0 : ret = !list_empty(&cb->node);
428 [ # # ]: 0 : if (ret)
429 : 0 : list_del_init(&cb->node);
430 : :
431 : 0 : spin_unlock_irqrestore(fence->lock, flags);
432 : :
433 : 0 : return ret;
434 : : }
435 : : EXPORT_SYMBOL(dma_fence_remove_callback);
436 : :
437 : : struct default_wait_cb {
438 : : struct dma_fence_cb base;
439 : : struct task_struct *task;
440 : : };
441 : :
442 : : static void
443 : 0 : dma_fence_default_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
444 : : {
445 : 0 : struct default_wait_cb *wait =
446 : 0 : container_of(cb, struct default_wait_cb, base);
447 : :
448 : 0 : wake_up_state(wait->task, TASK_NORMAL);
449 : 0 : }
450 : :
451 : : /**
452 : : * dma_fence_default_wait - default sleep until the fence gets signaled
453 : : * or until timeout elapses
454 : : * @fence: the fence to wait on
455 : : * @intr: if true, do an interruptible wait
456 : : * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
457 : : *
458 : : * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the
459 : : * remaining timeout in jiffies on success. If timeout is zero the value one is
460 : : * returned if the fence is already signaled for consistency with other
461 : : * functions taking a jiffies timeout.
462 : : */
463 : : signed long
464 : 0 : dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout)
465 : : {
466 : 0 : struct default_wait_cb cb;
467 : 0 : unsigned long flags;
468 [ # # ]: 0 : signed long ret = timeout ? timeout : 1;
469 : :
470 [ # # ]: 0 : if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
471 : : return ret;
472 : :
473 : 0 : spin_lock_irqsave(fence->lock, flags);
474 : :
475 [ # # # # ]: 0 : if (intr && signal_pending(current)) {
476 : 0 : ret = -ERESTARTSYS;
477 : 0 : goto out;
478 : : }
479 : :
480 [ # # ]: 0 : if (!__dma_fence_enable_signaling(fence))
481 : 0 : goto out;
482 : :
483 [ # # ]: 0 : if (!timeout) {
484 : 0 : ret = 0;
485 : 0 : goto out;
486 : : }
487 : :
488 : 0 : cb.base.func = dma_fence_default_wait_cb;
489 : 0 : cb.task = current;
490 : 0 : list_add(&cb.base.node, &fence->cb_list);
491 : :
492 [ # # # # ]: 0 : while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) {
493 [ # # ]: 0 : if (intr)
494 : 0 : __set_current_state(TASK_INTERRUPTIBLE);
495 : : else
496 : 0 : __set_current_state(TASK_UNINTERRUPTIBLE);
497 : 0 : spin_unlock_irqrestore(fence->lock, flags);
498 : :
499 : 0 : ret = schedule_timeout(ret);
500 : :
501 : 0 : spin_lock_irqsave(fence->lock, flags);
502 [ # # # # ]: 0 : if (ret > 0 && intr && signal_pending(current))
503 : 0 : ret = -ERESTARTSYS;
504 : : }
505 : :
506 [ # # ]: 0 : if (!list_empty(&cb.base.node))
507 : 0 : list_del(&cb.base.node);
508 : 0 : __set_current_state(TASK_RUNNING);
509 : :
510 : 0 : out:
511 : 0 : spin_unlock_irqrestore(fence->lock, flags);
512 : 0 : return ret;
513 : : }
514 : : EXPORT_SYMBOL(dma_fence_default_wait);
515 : :
516 : : static bool
517 : 0 : dma_fence_test_signaled_any(struct dma_fence **fences, uint32_t count,
518 : : uint32_t *idx)
519 : : {
520 : 0 : int i;
521 : :
522 [ # # ]: 0 : for (i = 0; i < count; ++i) {
523 : 0 : struct dma_fence *fence = fences[i];
524 [ # # ]: 0 : if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
525 [ # # ]: 0 : if (idx)
526 : 0 : *idx = i;
527 : 0 : return true;
528 : : }
529 : : }
530 : : return false;
531 : : }
532 : :
533 : : /**
534 : : * dma_fence_wait_any_timeout - sleep until any fence gets signaled
535 : : * or until timeout elapses
536 : : * @fences: array of fences to wait on
537 : : * @count: number of fences to wait on
538 : : * @intr: if true, do an interruptible wait
539 : : * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
540 : : * @idx: used to store the first signaled fence index, meaningful only on
541 : : * positive return
542 : : *
543 : : * Returns -EINVAL on custom fence wait implementation, -ERESTARTSYS if
544 : : * interrupted, 0 if the wait timed out, or the remaining timeout in jiffies
545 : : * on success.
546 : : *
547 : : * Synchronous waits for the first fence in the array to be signaled. The
548 : : * caller needs to hold a reference to all fences in the array, otherwise a
549 : : * fence might be freed before return, resulting in undefined behavior.
550 : : *
551 : : * See also dma_fence_wait() and dma_fence_wait_timeout().
552 : : */
553 : : signed long
554 : 0 : dma_fence_wait_any_timeout(struct dma_fence **fences, uint32_t count,
555 : : bool intr, signed long timeout, uint32_t *idx)
556 : : {
557 : 0 : struct default_wait_cb *cb;
558 : 0 : signed long ret = timeout;
559 : 0 : unsigned i;
560 : :
561 [ # # # # : 0 : if (WARN_ON(!fences || !count || timeout < 0))
# # # # ]
562 : : return -EINVAL;
563 : :
564 [ # # ]: 0 : if (timeout == 0) {
565 [ # # ]: 0 : for (i = 0; i < count; ++i)
566 [ # # ]: 0 : if (dma_fence_is_signaled(fences[i])) {
567 [ # # ]: 0 : if (idx)
568 : 0 : *idx = i;
569 : 0 : return 1;
570 : : }
571 : :
572 : : return 0;
573 : : }
574 : :
575 : 0 : cb = kcalloc(count, sizeof(struct default_wait_cb), GFP_KERNEL);
576 [ # # ]: 0 : if (cb == NULL) {
577 : 0 : ret = -ENOMEM;
578 : 0 : goto err_free_cb;
579 : : }
580 : :
581 [ # # ]: 0 : for (i = 0; i < count; ++i) {
582 : 0 : struct dma_fence *fence = fences[i];
583 : :
584 : 0 : cb[i].task = current;
585 [ # # ]: 0 : if (dma_fence_add_callback(fence, &cb[i].base,
586 : : dma_fence_default_wait_cb)) {
587 : : /* This fence is already signaled */
588 [ # # ]: 0 : if (idx)
589 : 0 : *idx = i;
590 : 0 : goto fence_rm_cb;
591 : : }
592 : : }
593 : :
594 [ # # ]: 0 : while (ret > 0) {
595 [ # # ]: 0 : if (intr)
596 : 0 : set_current_state(TASK_INTERRUPTIBLE);
597 : : else
598 : 0 : set_current_state(TASK_UNINTERRUPTIBLE);
599 : :
600 [ # # ]: 0 : if (dma_fence_test_signaled_any(fences, count, idx))
601 : : break;
602 : :
603 : 0 : ret = schedule_timeout(ret);
604 : :
605 [ # # # # ]: 0 : if (ret > 0 && intr && signal_pending(current))
606 : : ret = -ERESTARTSYS;
607 : : }
608 : :
609 : 0 : __set_current_state(TASK_RUNNING);
610 : :
611 : 0 : fence_rm_cb:
612 [ # # ]: 0 : while (i-- > 0)
613 : 0 : dma_fence_remove_callback(fences[i], &cb[i].base);
614 : :
615 : 0 : err_free_cb:
616 : 0 : kfree(cb);
617 : :
618 : 0 : return ret;
619 : : }
620 : : EXPORT_SYMBOL(dma_fence_wait_any_timeout);
621 : :
622 : : /**
623 : : * dma_fence_init - Initialize a custom fence.
624 : : * @fence: the fence to initialize
625 : : * @ops: the dma_fence_ops for operations on this fence
626 : : * @lock: the irqsafe spinlock to use for locking this fence
627 : : * @context: the execution context this fence is run on
628 : : * @seqno: a linear increasing sequence number for this context
629 : : *
630 : : * Initializes an allocated fence, the caller doesn't have to keep its
631 : : * refcount after committing with this fence, but it will need to hold a
632 : : * refcount again if &dma_fence_ops.enable_signaling gets called.
633 : : *
634 : : * context and seqno are used for easy comparison between fences, allowing
635 : : * to check which fence is later by simply using dma_fence_later().
636 : : */
637 : : void
638 : 0 : dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
639 : : spinlock_t *lock, u64 context, u64 seqno)
640 : : {
641 [ # # ]: 0 : BUG_ON(!lock);
642 [ # # # # : 0 : BUG_ON(!ops || !ops->get_driver_name || !ops->get_timeline_name);
# # ]
643 : :
644 : 0 : kref_init(&fence->refcount);
645 : 0 : fence->ops = ops;
646 : 0 : INIT_LIST_HEAD(&fence->cb_list);
647 : 0 : fence->lock = lock;
648 : 0 : fence->context = context;
649 : 0 : fence->seqno = seqno;
650 : 0 : fence->flags = 0UL;
651 : 0 : fence->error = 0;
652 : :
653 : 0 : trace_dma_fence_init(fence);
654 : 0 : }
655 : : EXPORT_SYMBOL(dma_fence_init);
|
