LCOV - code coverage report
Current view: top level - drivers/gpu/drm/i915/gt - intel_breadcrumbs.c (source / functions) Hit Total Coverage
Test: combined.info Lines: 0 162 0.0 %
Date: 2022-03-28 13:20:08 Functions: 0 13 0.0 %
Branches: 0 72 0.0 %

           Branch data     Line data    Source code
       1                 :            : /*
       2                 :            :  * Copyright © 2015 Intel Corporation
       3                 :            :  *
       4                 :            :  * Permission is hereby granted, free of charge, to any person obtaining a
       5                 :            :  * copy of this software and associated documentation files (the "Software"),
       6                 :            :  * to deal in the Software without restriction, including without limitation
       7                 :            :  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
       8                 :            :  * and/or sell copies of the Software, and to permit persons to whom the
       9                 :            :  * Software is furnished to do so, subject to the following conditions:
      10                 :            :  *
      11                 :            :  * The above copyright notice and this permission notice (including the next
      12                 :            :  * paragraph) shall be included in all copies or substantial portions of the
      13                 :            :  * Software.
      14                 :            :  *
      15                 :            :  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
      16                 :            :  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
      17                 :            :  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
      18                 :            :  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
      19                 :            :  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
      20                 :            :  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
      21                 :            :  * IN THE SOFTWARE.
      22                 :            :  *
      23                 :            :  */
      24                 :            : 
      25                 :            : #include <linux/kthread.h>
      26                 :            : #include <trace/events/dma_fence.h>
      27                 :            : #include <uapi/linux/sched/types.h>
      28                 :            : 
      29                 :            : #include "i915_drv.h"
      30                 :            : #include "i915_trace.h"
      31                 :            : #include "intel_gt_pm.h"
      32                 :            : #include "intel_gt_requests.h"
      33                 :            : 
      34                 :          0 : static void irq_enable(struct intel_engine_cs *engine)
      35                 :            : {
      36         [ #  # ]:          0 :         if (!engine->irq_enable)
      37                 :            :                 return;
      38                 :            : 
      39                 :            :         /* Caller disables interrupts */
      40                 :          0 :         spin_lock(&engine->gt->irq_lock);
      41                 :          0 :         engine->irq_enable(engine);
      42                 :          0 :         spin_unlock(&engine->gt->irq_lock);
      43                 :            : }
      44                 :            : 
      45                 :          0 : static void irq_disable(struct intel_engine_cs *engine)
      46                 :            : {
      47         [ #  # ]:          0 :         if (!engine->irq_disable)
      48                 :            :                 return;
      49                 :            : 
      50                 :            :         /* Caller disables interrupts */
      51                 :          0 :         spin_lock(&engine->gt->irq_lock);
      52                 :          0 :         engine->irq_disable(engine);
      53                 :          0 :         spin_unlock(&engine->gt->irq_lock);
      54                 :            : }
      55                 :            : 
      56                 :          0 : static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)
      57                 :            : {
      58                 :          0 :         struct intel_engine_cs *engine =
      59                 :          0 :                 container_of(b, struct intel_engine_cs, breadcrumbs);
      60                 :            : 
      61                 :          0 :         lockdep_assert_held(&b->irq_lock);
      62                 :            : 
      63                 :          0 :         GEM_BUG_ON(!b->irq_enabled);
      64         [ #  # ]:          0 :         if (!--b->irq_enabled)
      65                 :          0 :                 irq_disable(engine);
      66                 :            : 
      67                 :          0 :         b->irq_armed = false;
      68                 :          0 :         intel_gt_pm_put_async(engine->gt);
      69                 :          0 : }
      70                 :            : 
      71                 :          0 : void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
      72                 :            : {
      73                 :          0 :         struct intel_breadcrumbs *b = &engine->breadcrumbs;
      74                 :          0 :         unsigned long flags;
      75                 :            : 
      76         [ #  # ]:          0 :         if (!b->irq_armed)
      77                 :            :                 return;
      78                 :            : 
      79                 :          0 :         spin_lock_irqsave(&b->irq_lock, flags);
      80         [ #  # ]:          0 :         if (b->irq_armed)
      81                 :          0 :                 __intel_breadcrumbs_disarm_irq(b);
      82                 :          0 :         spin_unlock_irqrestore(&b->irq_lock, flags);
      83                 :            : }
      84                 :            : 
      85                 :          0 : static inline bool __request_completed(const struct i915_request *rq)
      86                 :            : {
      87                 :          0 :         return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno);
      88                 :            : }
      89                 :            : 
      90                 :            : __maybe_unused static bool
      91                 :            : check_signal_order(struct intel_context *ce, struct i915_request *rq)
      92                 :            : {
      93                 :            :         if (!list_is_last(&rq->signal_link, &ce->signals) &&
      94                 :            :             i915_seqno_passed(rq->fence.seqno,
      95                 :            :                               list_next_entry(rq, signal_link)->fence.seqno))
      96                 :            :                 return false;
      97                 :            : 
      98                 :            :         if (!list_is_first(&rq->signal_link, &ce->signals) &&
      99                 :            :             i915_seqno_passed(list_prev_entry(rq, signal_link)->fence.seqno,
     100                 :            :                               rq->fence.seqno))
     101                 :            :                 return false;
     102                 :            : 
     103                 :            :         return true;
     104                 :            : }
     105                 :            : 
     106                 :            : static bool
     107                 :          0 : __dma_fence_signal(struct dma_fence *fence)
     108                 :            : {
     109                 :          0 :         return !test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags);
     110                 :            : }
     111                 :            : 
     112                 :            : static void
     113                 :          0 : __dma_fence_signal__timestamp(struct dma_fence *fence, ktime_t timestamp)
     114                 :            : {
     115                 :          0 :         fence->timestamp = timestamp;
     116                 :          0 :         set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
     117                 :          0 :         trace_dma_fence_signaled(fence);
     118                 :            : }
     119                 :            : 
     120                 :            : static void
     121                 :          0 : __dma_fence_signal__notify(struct dma_fence *fence,
     122                 :            :                            const struct list_head *list)
     123                 :            : {
     124                 :          0 :         struct dma_fence_cb *cur, *tmp;
     125                 :            : 
     126                 :          0 :         lockdep_assert_held(fence->lock);
     127                 :            : 
     128         [ #  # ]:          0 :         list_for_each_entry_safe(cur, tmp, list, node) {
     129                 :          0 :                 INIT_LIST_HEAD(&cur->node);
     130                 :          0 :                 cur->func(fence, cur);
     131                 :            :         }
     132                 :            : }
     133                 :            : 
     134                 :          0 : static void add_retire(struct intel_breadcrumbs *b, struct intel_timeline *tl)
     135                 :            : {
     136                 :          0 :         struct intel_engine_cs *engine =
     137                 :          0 :                 container_of(b, struct intel_engine_cs, breadcrumbs);
     138                 :            : 
     139         [ #  # ]:          0 :         if (unlikely(intel_engine_is_virtual(engine)))
     140                 :          0 :                 engine = intel_virtual_engine_get_sibling(engine, 0);
     141                 :            : 
     142                 :          0 :         intel_engine_add_retire(engine, tl);
     143                 :          0 : }
     144                 :            : 
     145                 :          0 : static void signal_irq_work(struct irq_work *work)
     146                 :            : {
     147                 :          0 :         struct intel_breadcrumbs *b = container_of(work, typeof(*b), irq_work);
     148                 :          0 :         const ktime_t timestamp = ktime_get();
     149                 :          0 :         struct intel_context *ce, *cn;
     150                 :          0 :         struct list_head *pos, *next;
     151                 :          0 :         LIST_HEAD(signal);
     152                 :            : 
     153                 :          0 :         spin_lock(&b->irq_lock);
     154                 :            : 
     155   [ #  #  #  # ]:          0 :         if (b->irq_armed && list_empty(&b->signalers))
     156                 :          0 :                 __intel_breadcrumbs_disarm_irq(b);
     157                 :            : 
     158         [ #  # ]:          0 :         list_for_each_entry_safe(ce, cn, &b->signalers, signal_link) {
     159                 :          0 :                 GEM_BUG_ON(list_empty(&ce->signals));
     160                 :            : 
     161         [ #  # ]:          0 :                 list_for_each_safe(pos, next, &ce->signals) {
     162                 :          0 :                         struct i915_request *rq =
     163                 :          0 :                                 list_entry(pos, typeof(*rq), signal_link);
     164                 :            : 
     165                 :          0 :                         GEM_BUG_ON(!check_signal_order(ce, rq));
     166                 :            : 
     167         [ #  # ]:          0 :                         if (!__request_completed(rq))
     168                 :            :                                 break;
     169                 :            : 
     170                 :          0 :                         GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_SIGNAL,
     171                 :            :                                              &rq->fence.flags));
     172                 :          0 :                         clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
     173                 :            : 
     174         [ #  # ]:          0 :                         if (!__dma_fence_signal(&rq->fence))
     175                 :          0 :                                 continue;
     176                 :            : 
     177                 :            :                         /*
     178                 :            :                          * Queue for execution after dropping the signaling
     179                 :            :                          * spinlock as the callback chain may end up adding
     180                 :            :                          * more signalers to the same context or engine.
     181                 :            :                          */
     182         [ #  # ]:          0 :                         i915_request_get(rq);
     183                 :          0 :                         list_add_tail(&rq->signal_link, &signal);
     184                 :            :                 }
     185                 :            : 
     186                 :            :                 /*
     187                 :            :                  * We process the list deletion in bulk, only using a list_add
     188                 :            :                  * (not list_move) above but keeping the status of
     189                 :            :                  * rq->signal_link known with the I915_FENCE_FLAG_SIGNAL bit.
     190                 :            :                  */
     191         [ #  # ]:          0 :                 if (!list_is_first(pos, &ce->signals)) {
     192                 :            :                         /* Advance the list to the first incomplete request */
     193         [ #  # ]:          0 :                         __list_del_many(&ce->signals, pos);
     194         [ #  # ]:          0 :                         if (&ce->signals == pos) { /* now empty */
     195                 :          0 :                                 list_del_init(&ce->signal_link);
     196                 :          0 :                                 add_retire(b, ce->timeline);
     197                 :            :                         }
     198                 :            :                 }
     199                 :            :         }
     200                 :            : 
     201                 :          0 :         spin_unlock(&b->irq_lock);
     202                 :            : 
     203         [ #  # ]:          0 :         list_for_each_safe(pos, next, &signal) {
     204                 :          0 :                 struct i915_request *rq =
     205                 :          0 :                         list_entry(pos, typeof(*rq), signal_link);
     206                 :          0 :                 struct list_head cb_list;
     207                 :            : 
     208                 :          0 :                 spin_lock(&rq->lock);
     209                 :          0 :                 list_replace(&rq->fence.cb_list, &cb_list);
     210                 :          0 :                 __dma_fence_signal__timestamp(&rq->fence, timestamp);
     211                 :          0 :                 __dma_fence_signal__notify(&rq->fence, &cb_list);
     212                 :          0 :                 spin_unlock(&rq->lock);
     213                 :            : 
     214                 :          0 :                 i915_request_put(rq);
     215                 :            :         }
     216                 :          0 : }
     217                 :            : 
     218                 :          0 : static bool __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)
     219                 :            : {
     220                 :          0 :         struct intel_engine_cs *engine =
     221                 :          0 :                 container_of(b, struct intel_engine_cs, breadcrumbs);
     222                 :            : 
     223                 :          0 :         lockdep_assert_held(&b->irq_lock);
     224         [ #  # ]:          0 :         if (b->irq_armed)
     225                 :            :                 return true;
     226                 :            : 
     227         [ #  # ]:          0 :         if (!intel_gt_pm_get_if_awake(engine->gt))
     228                 :            :                 return false;
     229                 :            : 
     230                 :            :         /*
     231                 :            :          * The breadcrumb irq will be disarmed on the interrupt after the
     232                 :            :          * waiters are signaled. This gives us a single interrupt window in
     233                 :            :          * which we can add a new waiter and avoid the cost of re-enabling
     234                 :            :          * the irq.
     235                 :            :          */
     236                 :          0 :         b->irq_armed = true;
     237                 :            : 
     238                 :            :         /*
     239                 :            :          * Since we are waiting on a request, the GPU should be busy
     240                 :            :          * and should have its own rpm reference. This is tracked
     241                 :            :          * by i915->gt.awake, we can forgo holding our own wakref
     242                 :            :          * for the interrupt as before i915->gt.awake is released (when
     243                 :            :          * the driver is idle) we disarm the breadcrumbs.
     244                 :            :          */
     245                 :            : 
     246         [ #  # ]:          0 :         if (!b->irq_enabled++)
     247                 :          0 :                 irq_enable(engine);
     248                 :            : 
     249                 :            :         return true;
     250                 :            : }
     251                 :            : 
     252                 :          0 : void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine)
     253                 :            : {
     254                 :          0 :         struct intel_breadcrumbs *b = &engine->breadcrumbs;
     255                 :            : 
     256                 :          0 :         spin_lock_init(&b->irq_lock);
     257                 :          0 :         INIT_LIST_HEAD(&b->signalers);
     258                 :            : 
     259                 :          0 :         init_irq_work(&b->irq_work, signal_irq_work);
     260                 :          0 : }
     261                 :            : 
     262                 :          0 : void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine)
     263                 :            : {
     264                 :          0 :         struct intel_breadcrumbs *b = &engine->breadcrumbs;
     265                 :          0 :         unsigned long flags;
     266                 :            : 
     267                 :          0 :         spin_lock_irqsave(&b->irq_lock, flags);
     268                 :            : 
     269         [ #  # ]:          0 :         if (b->irq_enabled)
     270                 :          0 :                 irq_enable(engine);
     271                 :            :         else
     272                 :          0 :                 irq_disable(engine);
     273                 :            : 
     274                 :          0 :         spin_unlock_irqrestore(&b->irq_lock, flags);
     275                 :          0 : }
     276                 :            : 
     277                 :          0 : void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
     278                 :            : {
     279                 :          0 : }
     280                 :            : 
     281                 :          0 : bool i915_request_enable_breadcrumb(struct i915_request *rq)
     282                 :            : {
     283                 :          0 :         lockdep_assert_held(&rq->lock);
     284                 :            : 
     285         [ #  # ]:          0 :         if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) {
     286                 :          0 :                 struct intel_breadcrumbs *b = &rq->engine->breadcrumbs;
     287                 :          0 :                 struct intel_context *ce = rq->context;
     288                 :          0 :                 struct list_head *pos;
     289                 :            : 
     290                 :          0 :                 spin_lock(&b->irq_lock);
     291                 :          0 :                 GEM_BUG_ON(test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags));
     292                 :            : 
     293         [ #  # ]:          0 :                 if (!__intel_breadcrumbs_arm_irq(b))
     294                 :          0 :                         goto unlock;
     295                 :            : 
     296                 :            :                 /*
     297                 :            :                  * We keep the seqno in retirement order, so we can break
     298                 :            :                  * inside intel_engine_signal_breadcrumbs as soon as we've
     299                 :            :                  * passed the last completed request (or seen a request that
     300                 :            :                  * hasn't event started). We could walk the timeline->requests,
     301                 :            :                  * but keeping a separate signalers_list has the advantage of
     302                 :            :                  * hopefully being much smaller than the full list and so
     303                 :            :                  * provides faster iteration and detection when there are no
     304                 :            :                  * more interrupts required for this context.
     305                 :            :                  *
     306                 :            :                  * We typically expect to add new signalers in order, so we
     307                 :            :                  * start looking for our insertion point from the tail of
     308                 :            :                  * the list.
     309                 :            :                  */
     310         [ #  # ]:          0 :                 list_for_each_prev(pos, &ce->signals) {
     311                 :          0 :                         struct i915_request *it =
     312                 :          0 :                                 list_entry(pos, typeof(*it), signal_link);
     313                 :            : 
     314         [ #  # ]:          0 :                         if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno))
     315                 :            :                                 break;
     316                 :            :                 }
     317         [ #  # ]:          0 :                 list_add(&rq->signal_link, pos);
     318         [ #  # ]:          0 :                 if (pos == &ce->signals) /* catch transitions from empty list */
     319                 :          0 :                         list_move_tail(&ce->signal_link, &b->signalers);
     320                 :          0 :                 GEM_BUG_ON(!check_signal_order(ce, rq));
     321                 :            : 
     322                 :          0 :                 set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
     323                 :          0 : unlock:
     324                 :          0 :                 spin_unlock(&b->irq_lock);
     325                 :            :         }
     326                 :            : 
     327                 :          0 :         return !__request_completed(rq);
     328                 :            : }
     329                 :            : 
     330                 :          0 : void i915_request_cancel_breadcrumb(struct i915_request *rq)
     331                 :            : {
     332                 :          0 :         struct intel_breadcrumbs *b = &rq->engine->breadcrumbs;
     333                 :            : 
     334                 :          0 :         lockdep_assert_held(&rq->lock);
     335                 :            : 
     336                 :            :         /*
     337                 :            :          * We must wait for b->irq_lock so that we know the interrupt handler
     338                 :            :          * has released its reference to the intel_context and has completed
     339                 :            :          * the DMA_FENCE_FLAG_SIGNALED_BIT/I915_FENCE_FLAG_SIGNAL dance (if
     340                 :            :          * required).
     341                 :            :          */
     342                 :          0 :         spin_lock(&b->irq_lock);
     343         [ #  # ]:          0 :         if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) {
     344                 :          0 :                 struct intel_context *ce = rq->context;
     345                 :            : 
     346         [ #  # ]:          0 :                 list_del(&rq->signal_link);
     347         [ #  # ]:          0 :                 if (list_empty(&ce->signals))
     348                 :          0 :                         list_del_init(&ce->signal_link);
     349                 :            : 
     350                 :          0 :                 clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
     351                 :            :         }
     352                 :          0 :         spin_unlock(&b->irq_lock);
     353                 :          0 : }
     354                 :            : 
     355                 :          0 : void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine,
     356                 :            :                                     struct drm_printer *p)
     357                 :            : {
     358                 :          0 :         struct intel_breadcrumbs *b = &engine->breadcrumbs;
     359                 :          0 :         struct intel_context *ce;
     360                 :          0 :         struct i915_request *rq;
     361                 :            : 
     362         [ #  # ]:          0 :         if (list_empty(&b->signalers))
     363                 :            :                 return;
     364                 :            : 
     365                 :          0 :         drm_printf(p, "Signals:\n");
     366                 :            : 
     367                 :          0 :         spin_lock_irq(&b->irq_lock);
     368         [ #  # ]:          0 :         list_for_each_entry(ce, &b->signalers, signal_link) {
     369         [ #  # ]:          0 :                 list_for_each_entry(rq, &ce->signals, signal_link) {
     370         [ #  # ]:          0 :                         drm_printf(p, "\t[%llx:%llx%s] @ %dms\n",
     371                 :            :                                    rq->fence.context, rq->fence.seqno,
     372                 :          0 :                                    i915_request_completed(rq) ? "!" :
     373         [ #  # ]:          0 :                                    i915_request_started(rq) ? "*" :
     374                 :            :                                    "",
     375                 :          0 :                                    jiffies_to_msecs(jiffies - rq->emitted_jiffies));
     376                 :            :                 }
     377                 :            :         }
     378                 :          0 :         spin_unlock_irq(&b->irq_lock);
     379                 :            : }

Generated by: LCOV version 1.14