Branch data Line data Source code
1 : : // SPDX-License-Identifier: GPL-2.0-only
2 : : /*
3 : : * Fence mechanism for dma-buf and to allow for asynchronous dma access
4 : : *
5 : : * Copyright (C) 2012 Canonical Ltd
6 : : * Copyright (C) 2012 Texas Instruments
7 : : *
8 : : * Authors:
9 : : * Rob Clark <robdclark@gmail.com>
10 : : * Maarten Lankhorst <maarten.lankhorst@canonical.com>
11 : : */
12 : :
13 : : #include <linux/slab.h>
14 : : #include <linux/export.h>
15 : : #include <linux/atomic.h>
16 : : #include <linux/dma-fence.h>
17 : : #include <linux/sched/signal.h>
18 : :
19 : : #define CREATE_TRACE_POINTS
20 : : #include <trace/events/dma_fence.h>
21 : :
22 : : EXPORT_TRACEPOINT_SYMBOL(dma_fence_emit);
23 : : EXPORT_TRACEPOINT_SYMBOL(dma_fence_enable_signal);
24 : : EXPORT_TRACEPOINT_SYMBOL(dma_fence_signaled);
25 : :
26 : : static DEFINE_SPINLOCK(dma_fence_stub_lock);
27 : : static struct dma_fence dma_fence_stub;
28 : :
29 : : /*
30 : : * fence context counter: each execution context should have its own
31 : : * fence context, this allows checking if fences belong to the same
32 : : * context or not. One device can have multiple separate contexts,
33 : : * and they're used if some engine can run independently of another.
34 : : */
35 : : static atomic64_t dma_fence_context_counter = ATOMIC64_INIT(1);
36 : :
37 : : /**
38 : : * DOC: DMA fences overview
39 : : *
40 : : * DMA fences, represented by &struct dma_fence, are the kernel internal
41 : : * synchronization primitive for DMA operations like GPU rendering, video
42 : : * encoding/decoding, or displaying buffers on a screen.
43 : : *
44 : : * A fence is initialized using dma_fence_init() and completed using
45 : : * dma_fence_signal(). Fences are associated with a context, allocated through
46 : : * dma_fence_context_alloc(), and all fences on the same context are
47 : : * fully ordered.
48 : : *
49 : : * Since the purposes of fences is to facilitate cross-device and
50 : : * cross-application synchronization, there's multiple ways to use one:
51 : : *
52 : : * - Individual fences can be exposed as a &sync_file, accessed as a file
53 : : * descriptor from userspace, created by calling sync_file_create(). This is
54 : : * called explicit fencing, since userspace passes around explicit
55 : : * synchronization points.
56 : : *
57 : : * - Some subsystems also have their own explicit fencing primitives, like
58 : : * &drm_syncobj. Compared to &sync_file, a &drm_syncobj allows the underlying
59 : : * fence to be updated.
60 : : *
61 : : * - Then there's also implicit fencing, where the synchronization points are
62 : : * implicitly passed around as part of shared &dma_buf instances. Such
63 : : * implicit fences are stored in &struct dma_resv through the
64 : : * &dma_buf.resv pointer.
65 : : */
66 : :
67 : 0 : static const char *dma_fence_stub_get_name(struct dma_fence *fence)
68 : : {
69 : 0 : return "stub";
70 : : }
71 : :
72 : : static const struct dma_fence_ops dma_fence_stub_ops = {
73 : : .get_driver_name = dma_fence_stub_get_name,
74 : : .get_timeline_name = dma_fence_stub_get_name,
75 : : };
76 : :
77 : : /**
78 : : * dma_fence_get_stub - return a signaled fence
79 : : *
80 : : * Return a stub fence which is already signaled.
81 : : */
82 : 0 : struct dma_fence *dma_fence_get_stub(void)
83 : : {
84 : : spin_lock(&dma_fence_stub_lock);
85 : 0 : if (!dma_fence_stub.ops) {
86 : 0 : dma_fence_init(&dma_fence_stub,
87 : : &dma_fence_stub_ops,
88 : : &dma_fence_stub_lock,
89 : : 0, 0);
90 : 0 : dma_fence_signal_locked(&dma_fence_stub);
91 : : }
92 : : spin_unlock(&dma_fence_stub_lock);
93 : :
94 : 0 : return dma_fence_get(&dma_fence_stub);
95 : : }
96 : : EXPORT_SYMBOL(dma_fence_get_stub);
97 : :
98 : : /**
99 : : * dma_fence_context_alloc - allocate an array of fence contexts
100 : : * @num: amount of contexts to allocate
101 : : *
102 : : * This function will return the first index of the number of fence contexts
103 : : * allocated. The fence context is used for setting &dma_fence.context to a
104 : : * unique number by passing the context to dma_fence_init().
105 : : */
106 : 0 : u64 dma_fence_context_alloc(unsigned num)
107 : : {
108 : 0 : WARN_ON(!num);
109 : 0 : return atomic64_add_return(num, &dma_fence_context_counter) - num;
110 : : }
111 : : EXPORT_SYMBOL(dma_fence_context_alloc);
112 : :
113 : : /**
114 : : * dma_fence_signal_locked - signal completion of a fence
115 : : * @fence: the fence to signal
116 : : *
117 : : * Signal completion for software callbacks on a fence, this will unblock
118 : : * dma_fence_wait() calls and run all the callbacks added with
119 : : * dma_fence_add_callback(). Can be called multiple times, but since a fence
120 : : * can only go from the unsignaled to the signaled state and not back, it will
121 : : * only be effective the first time.
122 : : *
123 : : * Unlike dma_fence_signal(), this function must be called with &dma_fence.lock
124 : : * held.
125 : : *
126 : : * Returns 0 on success and a negative error value when @fence has been
127 : : * signalled already.
128 : : */
129 : 0 : int dma_fence_signal_locked(struct dma_fence *fence)
130 : : {
131 : : struct dma_fence_cb *cur, *tmp;
132 : : struct list_head cb_list;
133 : :
134 : : lockdep_assert_held(fence->lock);
135 : :
136 : 0 : if (unlikely(test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
137 : : &fence->flags)))
138 : : return -EINVAL;
139 : :
140 : : /* Stash the cb_list before replacing it with the timestamp */
141 : : list_replace(&fence->cb_list, &cb_list);
142 : :
143 : 0 : fence->timestamp = ktime_get();
144 : 0 : set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
145 : 0 : trace_dma_fence_signaled(fence);
146 : :
147 : 0 : list_for_each_entry_safe(cur, tmp, &cb_list, node) {
148 : : INIT_LIST_HEAD(&cur->node);
149 : 0 : cur->func(fence, cur);
150 : : }
151 : :
152 : : return 0;
153 : : }
154 : : EXPORT_SYMBOL(dma_fence_signal_locked);
155 : :
156 : : /**
157 : : * dma_fence_signal - signal completion of a fence
158 : : * @fence: the fence to signal
159 : : *
160 : : * Signal completion for software callbacks on a fence, this will unblock
161 : : * dma_fence_wait() calls and run all the callbacks added with
162 : : * dma_fence_add_callback(). Can be called multiple times, but since a fence
163 : : * can only go from the unsignaled to the signaled state and not back, it will
164 : : * only be effective the first time.
165 : : *
166 : : * Returns 0 on success and a negative error value when @fence has been
167 : : * signalled already.
168 : : */
169 : 0 : int dma_fence_signal(struct dma_fence *fence)
170 : : {
171 : : unsigned long flags;
172 : : int ret;
173 : :
174 : 0 : if (!fence)
175 : : return -EINVAL;
176 : :
177 : 0 : spin_lock_irqsave(fence->lock, flags);
178 : 0 : ret = dma_fence_signal_locked(fence);
179 : 0 : spin_unlock_irqrestore(fence->lock, flags);
180 : :
181 : 0 : return ret;
182 : : }
183 : : EXPORT_SYMBOL(dma_fence_signal);
184 : :
185 : : /**
186 : : * dma_fence_wait_timeout - sleep until the fence gets signaled
187 : : * or until timeout elapses
188 : : * @fence: the fence to wait on
189 : : * @intr: if true, do an interruptible wait
190 : : * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
191 : : *
192 : : * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the
193 : : * remaining timeout in jiffies on success. Other error values may be
194 : : * returned on custom implementations.
195 : : *
196 : : * Performs a synchronous wait on this fence. It is assumed the caller
197 : : * directly or indirectly (buf-mgr between reservation and committing)
198 : : * holds a reference to the fence, otherwise the fence might be
199 : : * freed before return, resulting in undefined behavior.
200 : : *
201 : : * See also dma_fence_wait() and dma_fence_wait_any_timeout().
202 : : */
203 : : signed long
204 : 0 : dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)
205 : : {
206 : : signed long ret;
207 : :
208 : 0 : if (WARN_ON(timeout < 0))
209 : : return -EINVAL;
210 : :
211 : 0 : trace_dma_fence_wait_start(fence);
212 : 0 : if (fence->ops->wait)
213 : 0 : ret = fence->ops->wait(fence, intr, timeout);
214 : : else
215 : 0 : ret = dma_fence_default_wait(fence, intr, timeout);
216 : 0 : trace_dma_fence_wait_end(fence);
217 : 0 : return ret;
218 : : }
219 : : EXPORT_SYMBOL(dma_fence_wait_timeout);
220 : :
221 : : /**
222 : : * dma_fence_release - default relese function for fences
223 : : * @kref: &dma_fence.recfount
224 : : *
225 : : * This is the default release functions for &dma_fence. Drivers shouldn't call
226 : : * this directly, but instead call dma_fence_put().
227 : : */
228 : 0 : void dma_fence_release(struct kref *kref)
229 : : {
230 : : struct dma_fence *fence =
231 : 0 : container_of(kref, struct dma_fence, refcount);
232 : :
233 : 0 : trace_dma_fence_destroy(fence);
234 : :
235 : 0 : if (WARN(!list_empty(&fence->cb_list) &&
236 : : !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags),
237 : : "Fence %s:%s:%llx:%llx released with pending signals!\n",
238 : : fence->ops->get_driver_name(fence),
239 : : fence->ops->get_timeline_name(fence),
240 : : fence->context, fence->seqno)) {
241 : : unsigned long flags;
242 : :
243 : : /*
244 : : * Failed to signal before release, likely a refcounting issue.
245 : : *
246 : : * This should never happen, but if it does make sure that we
247 : : * don't leave chains dangling. We set the error flag first
248 : : * so that the callbacks know this signal is due to an error.
249 : : */
250 : 0 : spin_lock_irqsave(fence->lock, flags);
251 : 0 : fence->error = -EDEADLK;
252 : 0 : dma_fence_signal_locked(fence);
253 : 0 : spin_unlock_irqrestore(fence->lock, flags);
254 : : }
255 : :
256 : 0 : if (fence->ops->release)
257 : 0 : fence->ops->release(fence);
258 : : else
259 : : dma_fence_free(fence);
260 : 0 : }
261 : : EXPORT_SYMBOL(dma_fence_release);
262 : :
263 : : /**
264 : : * dma_fence_free - default release function for &dma_fence.
265 : : * @fence: fence to release
266 : : *
267 : : * This is the default implementation for &dma_fence_ops.release. It calls
268 : : * kfree_rcu() on @fence.
269 : : */
270 : 0 : void dma_fence_free(struct dma_fence *fence)
271 : : {
272 : 0 : kfree_rcu(fence, rcu);
273 : 0 : }
274 : : EXPORT_SYMBOL(dma_fence_free);
275 : :
276 : : /**
277 : : * dma_fence_enable_sw_signaling - enable signaling on fence
278 : : * @fence: the fence to enable
279 : : *
280 : : * This will request for sw signaling to be enabled, to make the fence
281 : : * complete as soon as possible. This calls &dma_fence_ops.enable_signaling
282 : : * internally.
283 : : */
284 : 0 : void dma_fence_enable_sw_signaling(struct dma_fence *fence)
285 : : {
286 : : unsigned long flags;
287 : :
288 : 0 : if (!test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
289 : 0 : &fence->flags) &&
290 : 0 : !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) &&
291 : 0 : fence->ops->enable_signaling) {
292 : 0 : trace_dma_fence_enable_signal(fence);
293 : :
294 : 0 : spin_lock_irqsave(fence->lock, flags);
295 : :
296 : 0 : if (!fence->ops->enable_signaling(fence))
297 : 0 : dma_fence_signal_locked(fence);
298 : :
299 : 0 : spin_unlock_irqrestore(fence->lock, flags);
300 : : }
301 : 0 : }
302 : : EXPORT_SYMBOL(dma_fence_enable_sw_signaling);
303 : :
304 : : /**
305 : : * dma_fence_add_callback - add a callback to be called when the fence
306 : : * is signaled
307 : : * @fence: the fence to wait on
308 : : * @cb: the callback to register
309 : : * @func: the function to call
310 : : *
311 : : * @cb will be initialized by dma_fence_add_callback(), no initialization
312 : : * by the caller is required. Any number of callbacks can be registered
313 : : * to a fence, but a callback can only be registered to one fence at a time.
314 : : *
315 : : * Note that the callback can be called from an atomic context. If
316 : : * fence is already signaled, this function will return -ENOENT (and
317 : : * *not* call the callback).
318 : : *
319 : : * Add a software callback to the fence. Same restrictions apply to
320 : : * refcount as it does to dma_fence_wait(), however the caller doesn't need to
321 : : * keep a refcount to fence afterward dma_fence_add_callback() has returned:
322 : : * when software access is enabled, the creator of the fence is required to keep
323 : : * the fence alive until after it signals with dma_fence_signal(). The callback
324 : : * itself can be called from irq context.
325 : : *
326 : : * Returns 0 in case of success, -ENOENT if the fence is already signaled
327 : : * and -EINVAL in case of error.
328 : : */
329 : 0 : int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb,
330 : : dma_fence_func_t func)
331 : : {
332 : : unsigned long flags;
333 : : int ret = 0;
334 : : bool was_set;
335 : :
336 : 0 : if (WARN_ON(!fence || !func))
337 : : return -EINVAL;
338 : :
339 : 0 : if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
340 : 0 : INIT_LIST_HEAD(&cb->node);
341 : 0 : return -ENOENT;
342 : : }
343 : :
344 : 0 : spin_lock_irqsave(fence->lock, flags);
345 : :
346 : 0 : was_set = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
347 : : &fence->flags);
348 : :
349 : 0 : if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
350 : : ret = -ENOENT;
351 : 0 : else if (!was_set && fence->ops->enable_signaling) {
352 : 0 : trace_dma_fence_enable_signal(fence);
353 : :
354 : 0 : if (!fence->ops->enable_signaling(fence)) {
355 : 0 : dma_fence_signal_locked(fence);
356 : : ret = -ENOENT;
357 : : }
358 : : }
359 : :
360 : 0 : if (!ret) {
361 : 0 : cb->func = func;
362 : 0 : list_add_tail(&cb->node, &fence->cb_list);
363 : : } else
364 : 0 : INIT_LIST_HEAD(&cb->node);
365 : 0 : spin_unlock_irqrestore(fence->lock, flags);
366 : :
367 : 0 : return ret;
368 : : }
369 : : EXPORT_SYMBOL(dma_fence_add_callback);
370 : :
371 : : /**
372 : : * dma_fence_get_status - returns the status upon completion
373 : : * @fence: the dma_fence to query
374 : : *
375 : : * This wraps dma_fence_get_status_locked() to return the error status
376 : : * condition on a signaled fence. See dma_fence_get_status_locked() for more
377 : : * details.
378 : : *
379 : : * Returns 0 if the fence has not yet been signaled, 1 if the fence has
380 : : * been signaled without an error condition, or a negative error code
381 : : * if the fence has been completed in err.
382 : : */
383 : 0 : int dma_fence_get_status(struct dma_fence *fence)
384 : : {
385 : : unsigned long flags;
386 : : int status;
387 : :
388 : 0 : spin_lock_irqsave(fence->lock, flags);
389 : : status = dma_fence_get_status_locked(fence);
390 : 0 : spin_unlock_irqrestore(fence->lock, flags);
391 : :
392 : 0 : return status;
393 : : }
394 : : EXPORT_SYMBOL(dma_fence_get_status);
395 : :
396 : : /**
397 : : * dma_fence_remove_callback - remove a callback from the signaling list
398 : : * @fence: the fence to wait on
399 : : * @cb: the callback to remove
400 : : *
401 : : * Remove a previously queued callback from the fence. This function returns
402 : : * true if the callback is successfully removed, or false if the fence has
403 : : * already been signaled.
404 : : *
405 : : * *WARNING*:
406 : : * Cancelling a callback should only be done if you really know what you're
407 : : * doing, since deadlocks and race conditions could occur all too easily. For
408 : : * this reason, it should only ever be done on hardware lockup recovery,
409 : : * with a reference held to the fence.
410 : : *
411 : : * Behaviour is undefined if @cb has not been added to @fence using
412 : : * dma_fence_add_callback() beforehand.
413 : : */
414 : : bool
415 : 0 : dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb)
416 : : {
417 : : unsigned long flags;
418 : : bool ret;
419 : :
420 : 0 : spin_lock_irqsave(fence->lock, flags);
421 : :
422 : 0 : ret = !list_empty(&cb->node);
423 : 0 : if (ret)
424 : : list_del_init(&cb->node);
425 : :
426 : 0 : spin_unlock_irqrestore(fence->lock, flags);
427 : :
428 : 0 : return ret;
429 : : }
430 : : EXPORT_SYMBOL(dma_fence_remove_callback);
431 : :
432 : : struct default_wait_cb {
433 : : struct dma_fence_cb base;
434 : : struct task_struct *task;
435 : : };
436 : :
437 : : static void
438 : 0 : dma_fence_default_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
439 : : {
440 : : struct default_wait_cb *wait =
441 : : container_of(cb, struct default_wait_cb, base);
442 : :
443 : 0 : wake_up_state(wait->task, TASK_NORMAL);
444 : 0 : }
445 : :
446 : : /**
447 : : * dma_fence_default_wait - default sleep until the fence gets signaled
448 : : * or until timeout elapses
449 : : * @fence: the fence to wait on
450 : : * @intr: if true, do an interruptible wait
451 : : * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
452 : : *
453 : : * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the
454 : : * remaining timeout in jiffies on success. If timeout is zero the value one is
455 : : * returned if the fence is already signaled for consistency with other
456 : : * functions taking a jiffies timeout.
457 : : */
458 : : signed long
459 : 0 : dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout)
460 : : {
461 : : struct default_wait_cb cb;
462 : : unsigned long flags;
463 : 0 : signed long ret = timeout ? timeout : 1;
464 : : bool was_set;
465 : :
466 : 0 : if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
467 : : return ret;
468 : :
469 : 0 : spin_lock_irqsave(fence->lock, flags);
470 : :
471 : 0 : if (intr && signal_pending(current)) {
472 : : ret = -ERESTARTSYS;
473 : : goto out;
474 : : }
475 : :
476 : 0 : was_set = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
477 : : &fence->flags);
478 : :
479 : 0 : if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
480 : : goto out;
481 : :
482 : 0 : if (!was_set && fence->ops->enable_signaling) {
483 : 0 : trace_dma_fence_enable_signal(fence);
484 : :
485 : 0 : if (!fence->ops->enable_signaling(fence)) {
486 : 0 : dma_fence_signal_locked(fence);
487 : 0 : goto out;
488 : : }
489 : : }
490 : :
491 : 0 : if (!timeout) {
492 : : ret = 0;
493 : : goto out;
494 : : }
495 : :
496 : 0 : cb.base.func = dma_fence_default_wait_cb;
497 : 0 : cb.task = current;
498 : 0 : list_add(&cb.base.node, &fence->cb_list);
499 : :
500 : 0 : while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) {
501 : 0 : if (intr)
502 : 0 : __set_current_state(TASK_INTERRUPTIBLE);
503 : : else
504 : 0 : __set_current_state(TASK_UNINTERRUPTIBLE);
505 : 0 : spin_unlock_irqrestore(fence->lock, flags);
506 : :
507 : 0 : ret = schedule_timeout(ret);
508 : :
509 : 0 : spin_lock_irqsave(fence->lock, flags);
510 : 0 : if (ret > 0 && intr && signal_pending(current))
511 : : ret = -ERESTARTSYS;
512 : : }
513 : :
514 : 0 : if (!list_empty(&cb.base.node))
515 : : list_del(&cb.base.node);
516 : 0 : __set_current_state(TASK_RUNNING);
517 : :
518 : : out:
519 : 0 : spin_unlock_irqrestore(fence->lock, flags);
520 : 0 : return ret;
521 : : }
522 : : EXPORT_SYMBOL(dma_fence_default_wait);
523 : :
524 : : static bool
525 : : dma_fence_test_signaled_any(struct dma_fence **fences, uint32_t count,
526 : : uint32_t *idx)
527 : : {
528 : : int i;
529 : :
530 : 0 : for (i = 0; i < count; ++i) {
531 : 0 : struct dma_fence *fence = fences[i];
532 : 0 : if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
533 : 0 : if (idx)
534 : 0 : *idx = i;
535 : : return true;
536 : : }
537 : : }
538 : : return false;
539 : : }
540 : :
541 : : /**
542 : : * dma_fence_wait_any_timeout - sleep until any fence gets signaled
543 : : * or until timeout elapses
544 : : * @fences: array of fences to wait on
545 : : * @count: number of fences to wait on
546 : : * @intr: if true, do an interruptible wait
547 : : * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
548 : : * @idx: used to store the first signaled fence index, meaningful only on
549 : : * positive return
550 : : *
551 : : * Returns -EINVAL on custom fence wait implementation, -ERESTARTSYS if
552 : : * interrupted, 0 if the wait timed out, or the remaining timeout in jiffies
553 : : * on success.
554 : : *
555 : : * Synchronous waits for the first fence in the array to be signaled. The
556 : : * caller needs to hold a reference to all fences in the array, otherwise a
557 : : * fence might be freed before return, resulting in undefined behavior.
558 : : *
559 : : * See also dma_fence_wait() and dma_fence_wait_timeout().
560 : : */
561 : : signed long
562 : 0 : dma_fence_wait_any_timeout(struct dma_fence **fences, uint32_t count,
563 : : bool intr, signed long timeout, uint32_t *idx)
564 : : {
565 : : struct default_wait_cb *cb;
566 : : signed long ret = timeout;
567 : : unsigned i;
568 : :
569 : 0 : if (WARN_ON(!fences || !count || timeout < 0))
570 : : return -EINVAL;
571 : :
572 : 0 : if (timeout == 0) {
573 : 0 : for (i = 0; i < count; ++i)
574 : 0 : if (dma_fence_is_signaled(fences[i])) {
575 : 0 : if (idx)
576 : 0 : *idx = i;
577 : : return 1;
578 : : }
579 : :
580 : : return 0;
581 : : }
582 : :
583 : : cb = kcalloc(count, sizeof(struct default_wait_cb), GFP_KERNEL);
584 : 0 : if (cb == NULL) {
585 : : ret = -ENOMEM;
586 : : goto err_free_cb;
587 : : }
588 : :
589 : 0 : for (i = 0; i < count; ++i) {
590 : 0 : struct dma_fence *fence = fences[i];
591 : :
592 : 0 : cb[i].task = current;
593 : 0 : if (dma_fence_add_callback(fence, &cb[i].base,
594 : : dma_fence_default_wait_cb)) {
595 : : /* This fence is already signaled */
596 : 0 : if (idx)
597 : 0 : *idx = i;
598 : : goto fence_rm_cb;
599 : : }
600 : : }
601 : :
602 : 0 : while (ret > 0) {
603 : 0 : if (intr)
604 : 0 : set_current_state(TASK_INTERRUPTIBLE);
605 : : else
606 : 0 : set_current_state(TASK_UNINTERRUPTIBLE);
607 : :
608 : 0 : if (dma_fence_test_signaled_any(fences, count, idx))
609 : : break;
610 : :
611 : 0 : ret = schedule_timeout(ret);
612 : :
613 : 0 : if (ret > 0 && intr && signal_pending(current))
614 : : ret = -ERESTARTSYS;
615 : : }
616 : :
617 : 0 : __set_current_state(TASK_RUNNING);
618 : :
619 : : fence_rm_cb:
620 : 0 : while (i-- > 0)
621 : 0 : dma_fence_remove_callback(fences[i], &cb[i].base);
622 : :
623 : : err_free_cb:
624 : 0 : kfree(cb);
625 : :
626 : 0 : return ret;
627 : : }
628 : : EXPORT_SYMBOL(dma_fence_wait_any_timeout);
629 : :
630 : : /**
631 : : * dma_fence_init - Initialize a custom fence.
632 : : * @fence: the fence to initialize
633 : : * @ops: the dma_fence_ops for operations on this fence
634 : : * @lock: the irqsafe spinlock to use for locking this fence
635 : : * @context: the execution context this fence is run on
636 : : * @seqno: a linear increasing sequence number for this context
637 : : *
638 : : * Initializes an allocated fence, the caller doesn't have to keep its
639 : : * refcount after committing with this fence, but it will need to hold a
640 : : * refcount again if &dma_fence_ops.enable_signaling gets called.
641 : : *
642 : : * context and seqno are used for easy comparison between fences, allowing
643 : : * to check which fence is later by simply using dma_fence_later().
644 : : */
645 : : void
646 : 0 : dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
647 : : spinlock_t *lock, u64 context, u64 seqno)
648 : : {
649 : 0 : BUG_ON(!lock);
650 : 0 : BUG_ON(!ops || !ops->get_driver_name || !ops->get_timeline_name);
651 : :
652 : : kref_init(&fence->refcount);
653 : 0 : fence->ops = ops;
654 : 0 : INIT_LIST_HEAD(&fence->cb_list);
655 : 0 : fence->lock = lock;
656 : 0 : fence->context = context;
657 : 0 : fence->seqno = seqno;
658 : 0 : fence->flags = 0UL;
659 : 0 : fence->error = 0;
660 : :
661 : 0 : trace_dma_fence_init(fence);
662 : 0 : }
663 : : EXPORT_SYMBOL(dma_fence_init);
|
