Branch data Line data Source code
1 : : /*
2 : : * Copyright © 2015 Intel Corporation
3 : : *
4 : : * Permission is hereby granted, free of charge, to any person obtaining a
5 : : * copy of this software and associated documentation files (the "Software"),
6 : : * to deal in the Software without restriction, including without limitation
7 : : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 : : * and/or sell copies of the Software, and to permit persons to whom the
9 : : * Software is furnished to do so, subject to the following conditions:
10 : : *
11 : : * The above copyright notice and this permission notice (including the next
12 : : * paragraph) shall be included in all copies or substantial portions of the
13 : : * Software.
14 : : *
15 : : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 : : * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 : : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 : : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 : : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 : : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 : : * IN THE SOFTWARE.
22 : : *
23 : : */
24 : :
25 : : #include <linux/kthread.h>
26 : : #include <trace/events/dma_fence.h>
27 : : #include <uapi/linux/sched/types.h>
28 : :
29 : : #include "i915_drv.h"
30 : : #include "i915_trace.h"
31 : : #include "intel_gt_pm.h"
32 : : #include "intel_gt_requests.h"
33 : :
34 : 0 : static void irq_enable(struct intel_engine_cs *engine)
35 : : {
36 [ # # ]: 0 : if (!engine->irq_enable)
37 : : return;
38 : :
39 : : /* Caller disables interrupts */
40 : 0 : spin_lock(&engine->gt->irq_lock);
41 : 0 : engine->irq_enable(engine);
42 : 0 : spin_unlock(&engine->gt->irq_lock);
43 : : }
44 : :
45 : 0 : static void irq_disable(struct intel_engine_cs *engine)
46 : : {
47 [ # # ]: 0 : if (!engine->irq_disable)
48 : : return;
49 : :
50 : : /* Caller disables interrupts */
51 : 0 : spin_lock(&engine->gt->irq_lock);
52 : 0 : engine->irq_disable(engine);
53 : 0 : spin_unlock(&engine->gt->irq_lock);
54 : : }
55 : :
56 : 0 : static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)
57 : : {
58 : 0 : struct intel_engine_cs *engine =
59 : 0 : container_of(b, struct intel_engine_cs, breadcrumbs);
60 : :
61 : 0 : lockdep_assert_held(&b->irq_lock);
62 : :
63 : 0 : GEM_BUG_ON(!b->irq_enabled);
64 [ # # ]: 0 : if (!--b->irq_enabled)
65 : 0 : irq_disable(engine);
66 : :
67 : 0 : b->irq_armed = false;
68 : 0 : intel_gt_pm_put_async(engine->gt);
69 : 0 : }
70 : :
71 : 0 : void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
72 : : {
73 : 0 : struct intel_breadcrumbs *b = &engine->breadcrumbs;
74 : 0 : unsigned long flags;
75 : :
76 [ # # ]: 0 : if (!b->irq_armed)
77 : : return;
78 : :
79 : 0 : spin_lock_irqsave(&b->irq_lock, flags);
80 [ # # ]: 0 : if (b->irq_armed)
81 : 0 : __intel_breadcrumbs_disarm_irq(b);
82 : 0 : spin_unlock_irqrestore(&b->irq_lock, flags);
83 : : }
84 : :
85 : 0 : static inline bool __request_completed(const struct i915_request *rq)
86 : : {
87 : 0 : return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno);
88 : : }
89 : :
90 : : __maybe_unused static bool
91 : : check_signal_order(struct intel_context *ce, struct i915_request *rq)
92 : : {
93 : : if (!list_is_last(&rq->signal_link, &ce->signals) &&
94 : : i915_seqno_passed(rq->fence.seqno,
95 : : list_next_entry(rq, signal_link)->fence.seqno))
96 : : return false;
97 : :
98 : : if (!list_is_first(&rq->signal_link, &ce->signals) &&
99 : : i915_seqno_passed(list_prev_entry(rq, signal_link)->fence.seqno,
100 : : rq->fence.seqno))
101 : : return false;
102 : :
103 : : return true;
104 : : }
105 : :
106 : : static bool
107 : 0 : __dma_fence_signal(struct dma_fence *fence)
108 : : {
109 : 0 : return !test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags);
110 : : }
111 : :
112 : : static void
113 : 0 : __dma_fence_signal__timestamp(struct dma_fence *fence, ktime_t timestamp)
114 : : {
115 : 0 : fence->timestamp = timestamp;
116 : 0 : set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
117 : 0 : trace_dma_fence_signaled(fence);
118 : : }
119 : :
120 : : static void
121 : 0 : __dma_fence_signal__notify(struct dma_fence *fence,
122 : : const struct list_head *list)
123 : : {
124 : 0 : struct dma_fence_cb *cur, *tmp;
125 : :
126 : 0 : lockdep_assert_held(fence->lock);
127 : :
128 [ # # ]: 0 : list_for_each_entry_safe(cur, tmp, list, node) {
129 : 0 : INIT_LIST_HEAD(&cur->node);
130 : 0 : cur->func(fence, cur);
131 : : }
132 : : }
133 : :
134 : 0 : static void add_retire(struct intel_breadcrumbs *b, struct intel_timeline *tl)
135 : : {
136 : 0 : struct intel_engine_cs *engine =
137 : 0 : container_of(b, struct intel_engine_cs, breadcrumbs);
138 : :
139 [ # # ]: 0 : if (unlikely(intel_engine_is_virtual(engine)))
140 : 0 : engine = intel_virtual_engine_get_sibling(engine, 0);
141 : :
142 : 0 : intel_engine_add_retire(engine, tl);
143 : 0 : }
144 : :
145 : 0 : static void signal_irq_work(struct irq_work *work)
146 : : {
147 : 0 : struct intel_breadcrumbs *b = container_of(work, typeof(*b), irq_work);
148 : 0 : const ktime_t timestamp = ktime_get();
149 : 0 : struct intel_context *ce, *cn;
150 : 0 : struct list_head *pos, *next;
151 : 0 : LIST_HEAD(signal);
152 : :
153 : 0 : spin_lock(&b->irq_lock);
154 : :
155 [ # # # # ]: 0 : if (b->irq_armed && list_empty(&b->signalers))
156 : 0 : __intel_breadcrumbs_disarm_irq(b);
157 : :
158 [ # # ]: 0 : list_for_each_entry_safe(ce, cn, &b->signalers, signal_link) {
159 : 0 : GEM_BUG_ON(list_empty(&ce->signals));
160 : :
161 [ # # ]: 0 : list_for_each_safe(pos, next, &ce->signals) {
162 : 0 : struct i915_request *rq =
163 : 0 : list_entry(pos, typeof(*rq), signal_link);
164 : :
165 : 0 : GEM_BUG_ON(!check_signal_order(ce, rq));
166 : :
167 [ # # ]: 0 : if (!__request_completed(rq))
168 : : break;
169 : :
170 : 0 : GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_SIGNAL,
171 : : &rq->fence.flags));
172 : 0 : clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
173 : :
174 [ # # ]: 0 : if (!__dma_fence_signal(&rq->fence))
175 : 0 : continue;
176 : :
177 : : /*
178 : : * Queue for execution after dropping the signaling
179 : : * spinlock as the callback chain may end up adding
180 : : * more signalers to the same context or engine.
181 : : */
182 [ # # ]: 0 : i915_request_get(rq);
183 : 0 : list_add_tail(&rq->signal_link, &signal);
184 : : }
185 : :
186 : : /*
187 : : * We process the list deletion in bulk, only using a list_add
188 : : * (not list_move) above but keeping the status of
189 : : * rq->signal_link known with the I915_FENCE_FLAG_SIGNAL bit.
190 : : */
191 [ # # ]: 0 : if (!list_is_first(pos, &ce->signals)) {
192 : : /* Advance the list to the first incomplete request */
193 [ # # ]: 0 : __list_del_many(&ce->signals, pos);
194 [ # # ]: 0 : if (&ce->signals == pos) { /* now empty */
195 : 0 : list_del_init(&ce->signal_link);
196 : 0 : add_retire(b, ce->timeline);
197 : : }
198 : : }
199 : : }
200 : :
201 : 0 : spin_unlock(&b->irq_lock);
202 : :
203 [ # # ]: 0 : list_for_each_safe(pos, next, &signal) {
204 : 0 : struct i915_request *rq =
205 : 0 : list_entry(pos, typeof(*rq), signal_link);
206 : 0 : struct list_head cb_list;
207 : :
208 : 0 : spin_lock(&rq->lock);
209 : 0 : list_replace(&rq->fence.cb_list, &cb_list);
210 : 0 : __dma_fence_signal__timestamp(&rq->fence, timestamp);
211 : 0 : __dma_fence_signal__notify(&rq->fence, &cb_list);
212 : 0 : spin_unlock(&rq->lock);
213 : :
214 : 0 : i915_request_put(rq);
215 : : }
216 : 0 : }
217 : :
218 : 0 : static bool __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)
219 : : {
220 : 0 : struct intel_engine_cs *engine =
221 : 0 : container_of(b, struct intel_engine_cs, breadcrumbs);
222 : :
223 : 0 : lockdep_assert_held(&b->irq_lock);
224 [ # # ]: 0 : if (b->irq_armed)
225 : : return true;
226 : :
227 [ # # ]: 0 : if (!intel_gt_pm_get_if_awake(engine->gt))
228 : : return false;
229 : :
230 : : /*
231 : : * The breadcrumb irq will be disarmed on the interrupt after the
232 : : * waiters are signaled. This gives us a single interrupt window in
233 : : * which we can add a new waiter and avoid the cost of re-enabling
234 : : * the irq.
235 : : */
236 : 0 : b->irq_armed = true;
237 : :
238 : : /*
239 : : * Since we are waiting on a request, the GPU should be busy
240 : : * and should have its own rpm reference. This is tracked
241 : : * by i915->gt.awake, we can forgo holding our own wakref
242 : : * for the interrupt as before i915->gt.awake is released (when
243 : : * the driver is idle) we disarm the breadcrumbs.
244 : : */
245 : :
246 [ # # ]: 0 : if (!b->irq_enabled++)
247 : 0 : irq_enable(engine);
248 : :
249 : : return true;
250 : : }
251 : :
252 : 0 : void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine)
253 : : {
254 : 0 : struct intel_breadcrumbs *b = &engine->breadcrumbs;
255 : :
256 : 0 : spin_lock_init(&b->irq_lock);
257 : 0 : INIT_LIST_HEAD(&b->signalers);
258 : :
259 : 0 : init_irq_work(&b->irq_work, signal_irq_work);
260 : 0 : }
261 : :
262 : 0 : void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine)
263 : : {
264 : 0 : struct intel_breadcrumbs *b = &engine->breadcrumbs;
265 : 0 : unsigned long flags;
266 : :
267 : 0 : spin_lock_irqsave(&b->irq_lock, flags);
268 : :
269 [ # # ]: 0 : if (b->irq_enabled)
270 : 0 : irq_enable(engine);
271 : : else
272 : 0 : irq_disable(engine);
273 : :
274 : 0 : spin_unlock_irqrestore(&b->irq_lock, flags);
275 : 0 : }
276 : :
277 : 0 : void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
278 : : {
279 : 0 : }
280 : :
281 : 0 : bool i915_request_enable_breadcrumb(struct i915_request *rq)
282 : : {
283 : 0 : lockdep_assert_held(&rq->lock);
284 : :
285 [ # # ]: 0 : if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) {
286 : 0 : struct intel_breadcrumbs *b = &rq->engine->breadcrumbs;
287 : 0 : struct intel_context *ce = rq->context;
288 : 0 : struct list_head *pos;
289 : :
290 : 0 : spin_lock(&b->irq_lock);
291 : 0 : GEM_BUG_ON(test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags));
292 : :
293 [ # # ]: 0 : if (!__intel_breadcrumbs_arm_irq(b))
294 : 0 : goto unlock;
295 : :
296 : : /*
297 : : * We keep the seqno in retirement order, so we can break
298 : : * inside intel_engine_signal_breadcrumbs as soon as we've
299 : : * passed the last completed request (or seen a request that
300 : : * hasn't event started). We could walk the timeline->requests,
301 : : * but keeping a separate signalers_list has the advantage of
302 : : * hopefully being much smaller than the full list and so
303 : : * provides faster iteration and detection when there are no
304 : : * more interrupts required for this context.
305 : : *
306 : : * We typically expect to add new signalers in order, so we
307 : : * start looking for our insertion point from the tail of
308 : : * the list.
309 : : */
310 [ # # ]: 0 : list_for_each_prev(pos, &ce->signals) {
311 : 0 : struct i915_request *it =
312 : 0 : list_entry(pos, typeof(*it), signal_link);
313 : :
314 [ # # ]: 0 : if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno))
315 : : break;
316 : : }
317 [ # # ]: 0 : list_add(&rq->signal_link, pos);
318 [ # # ]: 0 : if (pos == &ce->signals) /* catch transitions from empty list */
319 : 0 : list_move_tail(&ce->signal_link, &b->signalers);
320 : 0 : GEM_BUG_ON(!check_signal_order(ce, rq));
321 : :
322 : 0 : set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
323 : 0 : unlock:
324 : 0 : spin_unlock(&b->irq_lock);
325 : : }
326 : :
327 : 0 : return !__request_completed(rq);
328 : : }
329 : :
330 : 0 : void i915_request_cancel_breadcrumb(struct i915_request *rq)
331 : : {
332 : 0 : struct intel_breadcrumbs *b = &rq->engine->breadcrumbs;
333 : :
334 : 0 : lockdep_assert_held(&rq->lock);
335 : :
336 : : /*
337 : : * We must wait for b->irq_lock so that we know the interrupt handler
338 : : * has released its reference to the intel_context and has completed
339 : : * the DMA_FENCE_FLAG_SIGNALED_BIT/I915_FENCE_FLAG_SIGNAL dance (if
340 : : * required).
341 : : */
342 : 0 : spin_lock(&b->irq_lock);
343 [ # # ]: 0 : if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) {
344 : 0 : struct intel_context *ce = rq->context;
345 : :
346 [ # # ]: 0 : list_del(&rq->signal_link);
347 [ # # ]: 0 : if (list_empty(&ce->signals))
348 : 0 : list_del_init(&ce->signal_link);
349 : :
350 : 0 : clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
351 : : }
352 : 0 : spin_unlock(&b->irq_lock);
353 : 0 : }
354 : :
355 : 0 : void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine,
356 : : struct drm_printer *p)
357 : : {
358 : 0 : struct intel_breadcrumbs *b = &engine->breadcrumbs;
359 : 0 : struct intel_context *ce;
360 : 0 : struct i915_request *rq;
361 : :
362 [ # # ]: 0 : if (list_empty(&b->signalers))
363 : : return;
364 : :
365 : 0 : drm_printf(p, "Signals:\n");
366 : :
367 : 0 : spin_lock_irq(&b->irq_lock);
368 [ # # ]: 0 : list_for_each_entry(ce, &b->signalers, signal_link) {
369 [ # # ]: 0 : list_for_each_entry(rq, &ce->signals, signal_link) {
370 [ # # ]: 0 : drm_printf(p, "\t[%llx:%llx%s] @ %dms\n",
371 : : rq->fence.context, rq->fence.seqno,
372 : 0 : i915_request_completed(rq) ? "!" :
373 [ # # ]: 0 : i915_request_started(rq) ? "*" :
374 : : "",
375 : 0 : jiffies_to_msecs(jiffies - rq->emitted_jiffies));
376 : : }
377 : : }
378 : 0 : spin_unlock_irq(&b->irq_lock);
379 : : }
|
