LCOV - code coverage report
Current view: top level - drivers/usb/host/dwc_otg - dwc_otg_fiq_fsm.c (source / functions) Hit Total Coverage
Test: gcov_data_raspi2_real_modules_combined.info Lines: 131 461 28.4 %
Date: 2020-09-30 20:25:40 Functions: 9 18 50.0 %
Branches: 74 327 22.6 %

           Branch data     Line data    Source code
       1                 :            : /*
       2                 :            :  * dwc_otg_fiq_fsm.c - The finite state machine FIQ
       3                 :            :  *
       4                 :            :  * Copyright (c) 2013 Raspberry Pi Foundation
       5                 :            :  *
       6                 :            :  * Author: Jonathan Bell <jonathan@raspberrypi.org>
       7                 :            :  * All rights reserved.
       8                 :            :  *
       9                 :            :  * Redistribution and use in source and binary forms, with or without
      10                 :            :  * modification, are permitted provided that the following conditions are met:
      11                 :            :  *      * Redistributions of source code must retain the above copyright
      12                 :            :  *        notice, this list of conditions and the following disclaimer.
      13                 :            :  *      * Redistributions in binary form must reproduce the above copyright
      14                 :            :  *        notice, this list of conditions and the following disclaimer in the
      15                 :            :  *        documentation and/or other materials provided with the distribution.
      16                 :            :  *      * Neither the name of Raspberry Pi nor the
      17                 :            :  *        names of its contributors may be used to endorse or promote products
      18                 :            :  *        derived from this software without specific prior written permission.
      19                 :            :  *
      20                 :            :  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
      21                 :            :  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
      22                 :            :  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
      23                 :            :  * DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
      24                 :            :  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
      25                 :            :  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
      26                 :            :  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
      27                 :            :  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
      28                 :            :  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
      29                 :            :  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
      30                 :            :  *
      31                 :            :  * This FIQ implements functionality that performs split transactions on
      32                 :            :  * the dwc_otg hardware without any outside intervention. A split transaction
      33                 :            :  * is "queued" by nominating a specific host channel to perform the entirety
      34                 :            :  * of a split transaction. This FIQ will then perform the microframe-precise
      35                 :            :  * scheduling required in each phase of the transaction until completion.
      36                 :            :  *
      37                 :            :  * The FIQ functionality is glued into the Synopsys driver via the entry point
      38                 :            :  * in the FSM enqueue function, and at the exit point in handling a HC interrupt
      39                 :            :  * for a FSM-enabled channel.
      40                 :            :  *
      41                 :            :  * NB: Large parts of this implementation have architecture-specific code.
      42                 :            :  * For porting this functionality to other ARM machines, the minimum is required:
      43                 :            :  * - An interrupt controller allowing the top-level dwc USB interrupt to be routed
      44                 :            :  *   to the FIQ
      45                 :            :  * - A method of forcing a software generated interrupt from FIQ mode that then
      46                 :            :  *   triggers an IRQ entry (with the dwc USB handler called by this IRQ number)
      47                 :            :  * - Guaranteed interrupt routing such that both the FIQ and SGI occur on the same
      48                 :            :  *   processor core - there is no locking between the FIQ and IRQ (aside from
      49                 :            :  *   local_fiq_disable)
      50                 :            :  *
      51                 :            :  */
      52                 :            : 
      53                 :            : #include "dwc_otg_fiq_fsm.h"
      54                 :            : 
      55                 :            : 
      56                 :            : char buffer[1000*16];
      57                 :            : int wptr;
      58                 :          0 : void notrace _fiq_print(enum fiq_debug_level dbg_lvl, volatile struct fiq_state *state, char *fmt, ...)
      59                 :            : {
      60                 :            :         enum fiq_debug_level dbg_lvl_req = FIQDBG_ERR;
      61                 :            :         va_list args;
      62                 :            :         char text[17];
      63                 :          0 :         hfnum_data_t hfnum = { .d32 = FIQ_READ(state->dwc_regs_base + 0x408) };
      64                 :            : 
      65   [ #  #  #  # ]:          0 :         if((dbg_lvl & dbg_lvl_req) || dbg_lvl == FIQDBG_ERR)
      66                 :            :         {
      67                 :          0 :                 snprintf(text, 9, " %4d:%1u  ", hfnum.b.frnum/8, hfnum.b.frnum & 7);
      68                 :          0 :                 va_start(args, fmt);
      69                 :          0 :                 vsnprintf(text+8, 9, fmt, args);
      70                 :          0 :                 va_end(args);
      71                 :            : 
      72                 :          0 :                 memcpy(buffer + wptr, text, 16);
      73                 :          0 :                 wptr = (wptr + 16) % sizeof(buffer);
      74                 :            :         }
      75                 :          0 : }
      76                 :            : 
      77                 :            : 
      78                 :            : #ifdef CONFIG_ARM64
      79                 :            : 
      80                 :            : inline void fiq_fsm_spin_lock(fiq_lock_t *lock)
      81                 :            : {
      82                 :            :         spin_lock((spinlock_t *)lock);
      83                 :            : }
      84                 :            : 
      85                 :            : inline void fiq_fsm_spin_unlock(fiq_lock_t *lock)
      86                 :            : {
      87                 :            :         spin_unlock((spinlock_t *)lock);
      88                 :            : }
      89                 :            : 
      90                 :            : #else
      91                 :            : 
      92                 :            : /**
      93                 :            :  * fiq_fsm_spin_lock() - ARMv6+ bare bones spinlock
      94                 :            :  * Must be called with local interrupts and FIQ disabled.
      95                 :            :  */
      96                 :            : #if defined(CONFIG_ARCH_BCM2835) && defined(CONFIG_SMP)
      97                 :  253205490 : inline void fiq_fsm_spin_lock(fiq_lock_t *lock)
      98                 :            : {
      99                 :            :         unsigned long tmp;
     100                 :            :         uint32_t newval;
     101                 :            :         fiq_lock_t lockval;
     102                 :            :         /* Nested locking, yay. If we are on the same CPU as the fiq, then the disable
     103                 :            :          * will be sufficient. If we are on a different CPU, then the lock protects us. */
     104                 :  253205490 :         prefetchw(&lock->slock);
     105                 :  253205654 :         asm volatile (
     106                 :            :         "1:     ldrex   %0, [%3]\n"
     107                 :            :         "       add     %1, %0, %4\n"
     108                 :            :         "       strex   %2, %1, [%3]\n"
     109                 :            :         "       teq     %2, #0\n"
     110                 :            :         "       bne     1b"
     111                 :            :         : "=&r" (lockval), "=&r" (newval), "=&r" (tmp)
     112                 :            :         : "r" (&lock->slock), "I" (1 << 16)
     113                 :            :         : "cc");
     114                 :            : 
     115         [ +  + ]:  509322617 :         while (lockval.tickets.next != lockval.tickets.owner) {
     116                 :    2869792 :                 wfe();
     117                 :            :                 lockval.tickets.owner = READ_ONCE(lock->tickets.owner);
     118                 :            :         }
     119                 :  253226412 :         smp_mb();
     120                 :  253226412 : }
     121                 :            : #else
     122                 :            : inline void fiq_fsm_spin_lock(fiq_lock_t *lock) { }
     123                 :            : #endif
     124                 :            : 
     125                 :            : /**
     126                 :            :  * fiq_fsm_spin_unlock() - ARMv6+ bare bones spinunlock
     127                 :            :  */
     128                 :            : #if defined(CONFIG_ARCH_BCM2835) && defined(CONFIG_SMP)
     129                 :   59681057 : inline void fiq_fsm_spin_unlock(fiq_lock_t *lock)
     130                 :            : {
     131                 :  253226017 :         smp_mb();
     132                 :  253226017 :         lock->tickets.owner++;
     133                 :            :         dsb_sev();
     134                 :   59681057 : }
     135                 :            : #else
     136                 :            : inline void fiq_fsm_spin_unlock(fiq_lock_t *lock) { }
     137                 :            : #endif
     138                 :            : 
     139                 :            : #endif
     140                 :            : 
     141                 :            : /**
     142                 :            :  * fiq_fsm_restart_channel() - Poke channel enable bit for a split transaction
     143                 :            :  * @channel: channel to re-enable
     144                 :            :  */
     145                 :    8031549 : static void fiq_fsm_restart_channel(struct fiq_state *st, int n, int force)
     146                 :            : {
     147                 :    8031549 :         hcchar_data_t hcchar = { .d32 = FIQ_READ(st->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCCHAR) };
     148                 :            : 
     149                 :    8031549 :         hcchar.b.chen = 0;
     150         [ -  + ]:    8031549 :         if (st->channel[n].hcchar_copy.b.eptype & 0x1) {
     151                 :          0 :                 hfnum_data_t hfnum = { .d32 = FIQ_READ(st->dwc_regs_base + HFNUM) };
     152                 :            :                 /* Hardware bug workaround: update the ssplit index */
     153         [ #  # ]:          0 :                 if (st->channel[n].hcsplt_copy.b.spltena)
     154                 :          0 :                         st->channel[n].expected_uframe = (hfnum.b.frnum + 1) & 0x3FFF;
     155                 :            : 
     156                 :          0 :                 hcchar.b.oddfrm = (hfnum.b.frnum & 0x1) ? 0 : 1;
     157                 :            :         }
     158                 :            : 
     159                 :    8031549 :         FIQ_WRITE(st->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCCHAR, hcchar.d32);
     160                 :    8031549 :         hcchar.d32 = FIQ_READ(st->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCCHAR);
     161                 :    8031549 :         hcchar.b.chen = 1;
     162                 :            : 
     163                 :    8031549 :         FIQ_WRITE(st->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCCHAR, hcchar.d32);
     164                 :            :         fiq_print(FIQDBG_INT, st, "HCGO %01d %01d", n, force);
     165                 :    8031549 : }
     166                 :            : 
     167                 :            : /**
     168                 :            :  * fiq_fsm_setup_csplit() - Prepare a host channel for a CSplit transaction stage
     169                 :            :  * @st: Pointer to the channel's state
     170                 :            :  * @n : channel number
     171                 :            :  *
     172                 :            :  * Change host channel registers to perform a complete-split transaction. Being mindful of the
     173                 :            :  * endpoint direction, set control regs up correctly.
     174                 :            :  */
     175                 :    4336503 : static void notrace fiq_fsm_setup_csplit(struct fiq_state *st, int n)
     176                 :            : {
     177                 :    4336503 :         hcsplt_data_t hcsplt = { .d32 = FIQ_READ(st->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCSPLT) };
     178                 :    4336503 :         hctsiz_data_t hctsiz = { .d32 = FIQ_READ(st->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCTSIZ) };
     179                 :            : 
     180                 :    4336503 :         hcsplt.b.compsplt = 1;
     181         [ +  + ]:    4336503 :         if (st->channel[n].hcchar_copy.b.epdir == 1) {
     182                 :            :                 // If IN, the CSPLIT result contains the data or a hub handshake. hctsiz = maxpacket.
     183                 :    4326324 :                 hctsiz.b.xfersize = st->channel[n].hctsiz_copy.b.xfersize;
     184                 :            :         } else {
     185                 :            :                 // If OUT, the CSPLIT result contains handshake only.
     186                 :      10179 :                 hctsiz.b.xfersize = 0;
     187                 :            :         }
     188                 :    4336503 :         FIQ_WRITE(st->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCSPLT, hcsplt.d32);
     189                 :    4336503 :         FIQ_WRITE(st->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCTSIZ, hctsiz.d32);
     190                 :    4336503 :         mb();
     191                 :    4336503 : }
     192                 :            : 
     193                 :            : /**
     194                 :            :  * fiq_fsm_restart_np_pending() - Restart a single non-periodic contended transfer
     195                 :            :  * @st: Pointer to the channel's state
     196                 :            :  * @num_channels: Total number of host channels
     197                 :            :  * @orig_channel: Channel index of completed transfer
     198                 :            :  *
     199                 :            :  * In the case where an IN and OUT transfer are simultaneously scheduled to the
     200                 :            :  * same device/EP, inadequate hub implementations will misbehave. Once the first
     201                 :            :  * transfer is complete, a pending non-periodic split can then be issued.
     202                 :            :  */
     203                 :    4336495 : static void notrace fiq_fsm_restart_np_pending(struct fiq_state *st, int num_channels, int orig_channel)
     204                 :            : {
     205                 :            :         int i;
     206                 :    4336495 :         int dev_addr = st->channel[orig_channel].hcchar_copy.b.devaddr;
     207                 :    4336495 :         int ep_num = st->channel[orig_channel].hcchar_copy.b.epnum;
     208         [ +  + ]:   39028455 :         for (i = 0; i < num_channels; i++) {
     209   [ -  +  #  # ]:   34691960 :                 if (st->channel[i].fsm == FIQ_NP_SSPLIT_PENDING &&
     210         [ #  # ]:          0 :                         st->channel[i].hcchar_copy.b.devaddr == dev_addr &&
     211                 :          0 :                         st->channel[i].hcchar_copy.b.epnum == ep_num) {
     212                 :          0 :                         st->channel[i].fsm = FIQ_NP_SSPLIT_STARTED;
     213                 :          0 :                         fiq_fsm_restart_channel(st, i, 0);
     214                 :          0 :                         break;
     215                 :            :                 }
     216                 :            :         }
     217                 :    4336495 : }
     218                 :            : 
     219                 :            : static inline int notrace fiq_get_xfer_len(struct fiq_state *st, int n)
     220                 :            : {
     221                 :            :         /* The xfersize register is a bit wonky. For IN transfers, it decrements by the packet size. */
     222                 :          0 :         hctsiz_data_t hctsiz = { .d32 = FIQ_READ(st->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCTSIZ) };
     223                 :            : 
     224   [ #  #  #  # ]:          0 :         if (st->channel[n].hcchar_copy.b.epdir == 0) {
     225                 :          0 :                 return st->channel[n].hctsiz_copy.b.xfersize;
     226                 :            :         } else {
     227                 :          0 :                 return st->channel[n].hctsiz_copy.b.xfersize - hctsiz.b.xfersize;
     228                 :            :         }
     229                 :            : 
     230                 :            : }
     231                 :            : 
     232                 :            : 
     233                 :            : /**
     234                 :            :  * fiq_increment_dma_buf() - update DMA address for bounce buffers after a CSPLIT
     235                 :            :  *
     236                 :            :  * Of use only for IN periodic transfers.
     237                 :            :  */
     238                 :          0 : static int notrace fiq_increment_dma_buf(struct fiq_state *st, int num_channels, int n)
     239                 :            : {
     240                 :            :         hcdma_data_t hcdma;
     241                 :          0 :         int i = st->channel[n].dma_info.index;
     242                 :            :         int len;
     243                 :          0 :         struct fiq_dma_blob *blob = (struct fiq_dma_blob *) st->dma_base;
     244                 :            : 
     245                 :            :         len = fiq_get_xfer_len(st, n);
     246                 :            :         fiq_print(FIQDBG_INT, st, "LEN: %03d", len);
     247                 :          0 :         st->channel[n].dma_info.slot_len[i] = len;
     248                 :          0 :         i++;
     249         [ #  # ]:          0 :         if (i > 6)
     250                 :          0 :                 BUG();
     251                 :            : 
     252                 :          0 :         hcdma.d32 = (dma_addr_t) &blob->channel[n].index[i].buf[0];
     253                 :          0 :         FIQ_WRITE(st->dwc_regs_base + HC_START + (HC_OFFSET * n) + HC_DMA, hcdma.d32);
     254                 :          0 :         st->channel[n].dma_info.index = i;
     255                 :          0 :         return 0;
     256                 :            : }
     257                 :            : 
     258                 :            : /**
     259                 :            :  * fiq_reload_hctsiz() - for IN transactions, reset HCTSIZ
     260                 :            :  */
     261                 :            : static void notrace fiq_fsm_reload_hctsiz(struct fiq_state *st, int n)
     262                 :            : {
     263                 :          0 :         hctsiz_data_t hctsiz = { .d32 = FIQ_READ(st->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCTSIZ) };
     264                 :          0 :         hctsiz.b.xfersize = st->channel[n].hctsiz_copy.b.xfersize;
     265                 :          0 :         hctsiz.b.pktcnt = 1;
     266                 :          0 :         FIQ_WRITE(st->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCTSIZ, hctsiz.d32);
     267                 :            : }
     268                 :            : 
     269                 :            : /**
     270                 :            :  * fiq_fsm_reload_hcdma() - for OUT transactions, rewind DMA pointer
     271                 :            :  */
     272                 :            : static void notrace fiq_fsm_reload_hcdma(struct fiq_state *st, int n)
     273                 :            : {
     274                 :          0 :         hcdma_data_t hcdma = st->channel[n].hcdma_copy;
     275                 :          0 :         FIQ_WRITE(st->dwc_regs_base + HC_START + (HC_OFFSET * n) + HC_DMA, hcdma.d32);
     276                 :            : }
     277                 :            : 
     278                 :            : /**
     279                 :            :  * fiq_iso_out_advance() - update DMA address and split position bits
     280                 :            :  * for isochronous OUT transactions.
     281                 :            :  *
     282                 :            :  * Returns 1 if this is the last packet queued, 0 otherwise. Split-ALL and
     283                 :            :  * Split-BEGIN states are not handled - this is done when the transaction was queued.
     284                 :            :  *
     285                 :            :  * This function must only be called from the FIQ_ISO_OUT_ACTIVE state.
     286                 :            :  */
     287                 :          0 : static int notrace fiq_iso_out_advance(struct fiq_state *st, int num_channels, int n)
     288                 :            : {
     289                 :            :         hcsplt_data_t hcsplt;
     290                 :            :         hctsiz_data_t hctsiz;
     291                 :            :         hcdma_data_t hcdma;
     292                 :          0 :         struct fiq_dma_blob *blob = (struct fiq_dma_blob *) st->dma_base;
     293                 :            :         int last = 0;
     294                 :          0 :         int i = st->channel[n].dma_info.index;
     295                 :            : 
     296                 :            :         fiq_print(FIQDBG_INT, st, "ADV %01d %01d ", n, i);
     297                 :          0 :         i++;
     298         [ #  # ]:          0 :         if (i == 4)
     299                 :            :                 last = 1;
     300         [ #  # ]:          0 :         if (st->channel[n].dma_info.slot_len[i+1] == 255)
     301                 :            :                 last = 1;
     302                 :            : 
     303                 :            :         /* New DMA address - address of bounce buffer referred to in index */
     304                 :          0 :         hcdma.d32 = (dma_addr_t) blob->channel[n].index[i].buf;
     305                 :            :         //hcdma.d32 = FIQ_READ(st->dwc_regs_base + HC_START + (HC_OFFSET * n) + HC_DMA);
     306                 :            :         //hcdma.d32 += st->channel[n].dma_info.slot_len[i];
     307                 :            :         fiq_print(FIQDBG_INT, st, "LAST: %01d ", last);
     308                 :            :         fiq_print(FIQDBG_INT, st, "LEN: %03d", st->channel[n].dma_info.slot_len[i]);
     309                 :          0 :         hcsplt.d32 = FIQ_READ(st->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCSPLT);
     310                 :          0 :         hctsiz.d32 = FIQ_READ(st->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCTSIZ);
     311                 :          0 :         hcsplt.b.xactpos = (last) ? ISOC_XACTPOS_END : ISOC_XACTPOS_MID;
     312                 :            :         /* Set up new packet length */
     313                 :          0 :         hctsiz.b.pktcnt = 1;
     314                 :          0 :         hctsiz.b.xfersize = st->channel[n].dma_info.slot_len[i];
     315                 :            :         fiq_print(FIQDBG_INT, st, "%08x", hctsiz.d32);
     316                 :            : 
     317                 :          0 :         st->channel[n].dma_info.index++;
     318                 :          0 :         FIQ_WRITE(st->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCSPLT, hcsplt.d32);
     319                 :          0 :         FIQ_WRITE(st->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCTSIZ, hctsiz.d32);
     320                 :          0 :         FIQ_WRITE(st->dwc_regs_base + HC_START + (HC_OFFSET * n) + HC_DMA, hcdma.d32);
     321                 :          0 :         return last;
     322                 :            : }
     323                 :            : 
     324                 :            : /**
     325                 :            :  * fiq_fsm_tt_next_isoc() - queue next pending isochronous out start-split on a TT
     326                 :            :  *
     327                 :            :  * Despite the limitations of the DWC core, we can force a microframe pipeline of
     328                 :            :  * isochronous OUT start-split transactions while waiting for a corresponding other-type
     329                 :            :  * of endpoint to finish its CSPLITs. TTs have big periodic buffers therefore it
     330                 :            :  * is very unlikely that filling the start-split FIFO will cause data loss.
     331                 :            :  * This allows much better interleaving of transactions in an order-independent way-
     332                 :            :  * there is no requirement to prioritise isochronous, just a state-space search has
     333                 :            :  * to be performed on each periodic start-split complete interrupt.
     334                 :            :  */
     335                 :          0 : static int notrace fiq_fsm_tt_next_isoc(struct fiq_state *st, int num_channels, int n)
     336                 :            : {
     337                 :          0 :         int hub_addr = st->channel[n].hub_addr;
     338                 :          0 :         int port_addr = st->channel[n].port_addr;
     339                 :            :         int i, poked = 0;
     340         [ #  # ]:          0 :         for (i = 0; i < num_channels; i++) {
     341   [ #  #  #  # ]:          0 :                 if (i == n || st->channel[i].fsm == FIQ_PASSTHROUGH)
     342                 :          0 :                         continue;
     343   [ #  #  #  # ]:          0 :                 if (st->channel[i].hub_addr == hub_addr &&
     344                 :          0 :                         st->channel[i].port_addr == port_addr) {
     345         [ #  # ]:          0 :                         switch (st->channel[i].fsm) {
     346                 :            :                         case FIQ_PER_ISO_OUT_PENDING:
     347         [ #  # ]:          0 :                                 if (st->channel[i].nrpackets == 1) {
     348                 :          0 :                                         st->channel[i].fsm = FIQ_PER_ISO_OUT_LAST;
     349                 :            :                                 } else {
     350                 :          0 :                                         st->channel[i].fsm = FIQ_PER_ISO_OUT_ACTIVE;
     351                 :            :                                 }
     352                 :          0 :                                 fiq_fsm_restart_channel(st, i, 0);
     353                 :            :                                 poked = 1;
     354                 :          0 :                                 break;
     355                 :            : 
     356                 :            :                         default:
     357                 :            :                                 break;
     358                 :            :                         }
     359                 :            :                 }
     360         [ #  # ]:          0 :                 if (poked)
     361                 :            :                         break;
     362                 :            :         }
     363                 :          0 :         return poked;
     364                 :            : }
     365                 :            : 
     366                 :            : /**
     367                 :            :  * fiq_fsm_tt_in_use() - search for host channels using this TT
     368                 :            :  * @n: Channel to use as reference
     369                 :            :  *
     370                 :            :  */
     371                 :          0 : int notrace noinline fiq_fsm_tt_in_use(struct fiq_state *st, int num_channels, int n)
     372                 :            : {
     373                 :          0 :         int hub_addr = st->channel[n].hub_addr;
     374                 :          0 :         int port_addr = st->channel[n].port_addr;
     375                 :            :         int i, in_use = 0;
     376         [ #  # ]:          0 :         for (i = 0; i < num_channels; i++) {
     377   [ #  #  #  # ]:          0 :                 if (i == n || st->channel[i].fsm == FIQ_PASSTHROUGH)
     378                 :          0 :                         continue;
     379         [ #  # ]:          0 :                 switch (st->channel[i].fsm) {
     380                 :            :                 /* TT is reserved for channels that are in the middle of a periodic
     381                 :            :                  * split transaction.
     382                 :            :                  */
     383                 :            :                 case FIQ_PER_SSPLIT_STARTED:
     384                 :            :                 case FIQ_PER_CSPLIT_WAIT:
     385                 :            :                 case FIQ_PER_CSPLIT_NYET1:
     386                 :            :                 //case FIQ_PER_CSPLIT_POLL:
     387                 :            :                 case FIQ_PER_ISO_OUT_ACTIVE:
     388                 :            :                 case FIQ_PER_ISO_OUT_LAST:
     389   [ #  #  #  # ]:          0 :                         if (st->channel[i].hub_addr == hub_addr &&
     390                 :          0 :                                 st->channel[i].port_addr == port_addr) {
     391                 :            :                                 in_use = 1;
     392                 :            :                         }
     393                 :            :                         break;
     394                 :            :                 default:
     395                 :            :                         break;
     396                 :            :                 }
     397         [ #  # ]:          0 :                 if (in_use)
     398                 :            :                         break;
     399                 :            :         }
     400                 :          0 :         return in_use;
     401                 :            : }
     402                 :            : 
     403                 :            : /**
     404                 :            :  * fiq_fsm_more_csplits() - determine whether additional CSPLITs need
     405                 :            :  *                      to be issued for this IN transaction.
     406                 :            :  *
     407                 :            :  * We cannot tell the inbound PID of a data packet due to hardware limitations.
     408                 :            :  * we need to make an educated guess as to whether we need to queue another CSPLIT
     409                 :            :  * or not. A no-brainer is when we have received enough data to fill the endpoint
     410                 :            :  * size, but for endpoints that give variable-length data then we have to resort
     411                 :            :  * to heuristics.
     412                 :            :  *
     413                 :            :  * We also return whether this is the last CSPLIT to be queued, again based on
     414                 :            :  * heuristics. This is to allow a 1-uframe overlap of periodic split transactions.
     415                 :            :  * Note: requires at least 1 CSPLIT to have been performed prior to being called.
     416                 :            :  */
     417                 :            : 
     418                 :            : /*
     419                 :            :  * We need some way of guaranteeing if a returned periodic packet of size X
     420                 :            :  * has a DATA0 PID.
     421                 :            :  * The heuristic value of 144 bytes assumes that the received data has maximal
     422                 :            :  * bit-stuffing and the clock frequency of the transmitting device is at the lowest
     423                 :            :  * permissible limit. If the transfer length results in a final packet size
     424                 :            :  * 144 < p <= 188, then an erroneous CSPLIT will be issued.
     425                 :            :  * Also used to ensure that an endpoint will nominally only return a single
     426                 :            :  * complete-split worth of data.
     427                 :            :  */
     428                 :            : #define DATA0_PID_HEURISTIC 144
     429                 :            : 
     430                 :          0 : static int notrace noinline fiq_fsm_more_csplits(struct fiq_state *state, int n, int *probably_last)
     431                 :            : {
     432                 :            : 
     433                 :            :         int i;
     434                 :            :         int total_len = 0;
     435                 :            :         int more_needed = 1;
     436                 :            :         struct fiq_channel_state *st = &state->channel[n];
     437                 :            : 
     438         [ #  # ]:          0 :         for (i = 0; i < st->dma_info.index; i++) {
     439                 :          0 :                         total_len += st->dma_info.slot_len[i];
     440                 :            :         }
     441                 :            : 
     442                 :          0 :         *probably_last = 0;
     443                 :            : 
     444         [ #  # ]:          0 :         if (st->hcchar_copy.b.eptype == 0x3) {
     445                 :            :                 /*
     446                 :            :                  * An interrupt endpoint will take max 2 CSPLITs. if we are receiving data
     447                 :            :                  * then this is definitely the last CSPLIT.
     448                 :            :                  */
     449                 :          0 :                 *probably_last = 1;
     450                 :            :         } else {
     451                 :            :                 /* Isoc IN. This is a bit risky if we are the first transaction:
     452                 :            :                  * we may have been held off slightly. */
     453   [ #  #  #  # ]:          0 :                 if (i > 1 && st->dma_info.slot_len[st->dma_info.index-1] <= DATA0_PID_HEURISTIC) {
     454                 :            :                         more_needed = 0;
     455                 :            :                 }
     456                 :            :                 /* If in the next uframe we will receive enough data to fill the endpoint,
     457                 :            :                  * then only issue 1 more csplit.
     458                 :            :                  */
     459         [ #  # ]:          0 :                 if (st->hctsiz_copy.b.xfersize - total_len <= DATA0_PID_HEURISTIC)
     460                 :          0 :                         *probably_last = 1;
     461                 :            :         }
     462                 :            : 
     463         [ #  # ]:          0 :         if (total_len >= st->hctsiz_copy.b.xfersize ||
     464         [ #  # ]:          0 :                 i == 6 || total_len == 0)
     465                 :            :                 /* Note: due to bit stuffing it is possible to have > 6 CSPLITs for
     466                 :            :                  * a single endpoint. Accepting more would completely break our scheduling mechanism though
     467                 :            :                  * - in these extreme cases we will pass through a truncated packet.
     468                 :            :                  */
     469                 :            :                 more_needed = 0;
     470                 :            : 
     471                 :          0 :         return more_needed;
     472                 :            : }
     473                 :            : 
     474                 :            : /**
     475                 :            :  * fiq_fsm_too_late() - Test transaction for lateness
     476                 :            :  *
     477                 :            :  * If a SSPLIT for a large IN transaction is issued too late in a frame,
     478                 :            :  * the hub will disable the port to the device and respond with ERR handshakes.
     479                 :            :  * The hub status endpoint will not reflect this change.
     480                 :            :  * Returns 1 if we will issue a SSPLIT that will result in a device babble.
     481                 :            :  */
     482                 :    4116629 : int notrace fiq_fsm_too_late(struct fiq_state *st, int n)
     483                 :            : {
     484                 :            :         int uframe;
     485                 :    4116629 :         hfnum_data_t hfnum = { .d32 = FIQ_READ(st->dwc_regs_base + HFNUM) };
     486                 :    4116629 :         uframe = hfnum.b.frnum & 0x7;
     487   [ #  #  #  #  :    4116629 :         if ((uframe < 6) && (st->channel[n].nrpackets + 1 + uframe > 7)) {
          #  #  #  #  +  
                +  +  - ]
     488                 :            :                 return 1;
     489                 :            :         } else {
     490                 :    4116629 :                 return 0;
     491                 :            :         }
     492                 :            : }
     493                 :            : 
     494                 :            : 
     495                 :            : /**
     496                 :            :  * fiq_fsm_start_next_periodic() - A half-arsed attempt at a microframe pipeline
     497                 :            :  *
     498                 :            :  * Search pending transactions in the start-split pending state and queue them.
     499                 :            :  * Don't queue packets in uframe .5 (comes out in .6) (USB2.0 11.18.4).
     500                 :            :  * Note: we specifically don't do isochronous OUT transactions first because better
     501                 :            :  * use of the TT's start-split fifo can be achieved by pipelining an IN before an OUT.
     502                 :            :  */
     503                 :          0 : static void notrace noinline fiq_fsm_start_next_periodic(struct fiq_state *st, int num_channels)
     504                 :            : {
     505                 :            :         int n;
     506                 :          0 :         hfnum_data_t hfnum = { .d32 = FIQ_READ(st->dwc_regs_base + HFNUM) };
     507         [ #  # ]:          0 :         if ((hfnum.b.frnum & 0x7) == 5)
     508                 :          0 :                 return;
     509         [ #  # ]:          0 :         for (n = 0; n < num_channels; n++) {
     510         [ #  # ]:          0 :                 if (st->channel[n].fsm == FIQ_PER_SSPLIT_QUEUED) {
     511                 :            :                         /* Check to see if any other transactions are using this TT */
     512         [ #  # ]:          0 :                         if(!fiq_fsm_tt_in_use(st, num_channels, n)) {
     513         [ #  # ]:          0 :                                 if (!fiq_fsm_too_late(st, n)) {
     514                 :          0 :                                         st->channel[n].fsm = FIQ_PER_SSPLIT_STARTED;
     515                 :            :                                         fiq_print(FIQDBG_INT, st, "NEXTPER ");
     516                 :          0 :                                         fiq_fsm_restart_channel(st, n, 0);
     517                 :            :                                 } else {
     518                 :          0 :                                         st->channel[n].fsm = FIQ_PER_SPLIT_TIMEOUT;
     519                 :            :                                 }
     520                 :            :                                 break;
     521                 :            :                         }
     522                 :            :                 }
     523                 :            :         }
     524         [ #  # ]:          0 :         for (n = 0; n < num_channels; n++) {
     525         [ #  # ]:          0 :                 if (st->channel[n].fsm == FIQ_PER_ISO_OUT_PENDING) {
     526         [ #  # ]:          0 :                         if (!fiq_fsm_tt_in_use(st, num_channels, n)) {
     527                 :            :                                 fiq_print(FIQDBG_INT, st, "NEXTISO ");
     528         [ #  # ]:          0 :                                 if (st->channel[n].nrpackets == 1)
     529                 :          0 :                                         st->channel[n].fsm = FIQ_PER_ISO_OUT_LAST;
     530                 :            :                                 else
     531                 :          0 :                                         st->channel[n].fsm = FIQ_PER_ISO_OUT_ACTIVE;
     532                 :          0 :                                 fiq_fsm_restart_channel(st, n, 0);
     533                 :          0 :                                 break;
     534                 :            :                         }
     535                 :            :                 }
     536                 :            :         }
     537                 :            : }
     538                 :            : 
     539                 :            : /**
     540                 :            :  * fiq_fsm_update_hs_isoc() - update isochronous frame and transfer data
     541                 :            :  * @state:      Pointer to fiq_state
     542                 :            :  * @n:          Channel transaction is active on
     543                 :            :  * @hcint:      Copy of host channel interrupt register
     544                 :            :  *
     545                 :            :  * Returns 0 if there are no more transactions for this HC to do, 1
     546                 :            :  * otherwise.
     547                 :            :  */
     548                 :          0 : static int notrace noinline fiq_fsm_update_hs_isoc(struct fiq_state *state, int n, hcint_data_t hcint)
     549                 :            : {
     550                 :            :         struct fiq_channel_state *st = &state->channel[n];
     551                 :            :         int xfer_len = 0, nrpackets = 0;
     552                 :            :         hcdma_data_t hcdma;
     553                 :            :         fiq_print(FIQDBG_INT, state, "HSISO %02d", n);
     554                 :            : 
     555                 :            :         xfer_len = fiq_get_xfer_len(state, n);
     556                 :          0 :         st->hs_isoc_info.iso_desc[st->hs_isoc_info.index].actual_length = xfer_len;
     557                 :            : 
     558                 :          0 :         st->hs_isoc_info.iso_desc[st->hs_isoc_info.index].status = hcint.d32;
     559                 :            : 
     560                 :          0 :         st->hs_isoc_info.index++;
     561         [ #  # ]:          0 :         if (st->hs_isoc_info.index == st->hs_isoc_info.nrframes) {
     562                 :            :                 return 0;
     563                 :            :         }
     564                 :            : 
     565                 :            :         /* grab the next DMA address offset from the array */
     566                 :          0 :         hcdma.d32 = st->hcdma_copy.d32 + st->hs_isoc_info.iso_desc[st->hs_isoc_info.index].offset;
     567                 :          0 :         FIQ_WRITE(state->dwc_regs_base + HC_START + (HC_OFFSET * n) + HC_DMA, hcdma.d32);
     568                 :            : 
     569                 :            :         /* We need to set multi_count. This is a bit tricky - has to be set per-transaction as
     570                 :            :          * the core needs to be told to send the correct number. Caution: for IN transfers,
     571                 :            :          * this is always set to the maximum size of the endpoint. */
     572                 :          0 :         xfer_len = st->hs_isoc_info.iso_desc[st->hs_isoc_info.index].length;
     573                 :            :         /* Integer divide in a FIQ: fun. FIXME: make this not suck */
     574                 :          0 :         nrpackets = (xfer_len + st->hcchar_copy.b.mps - 1) / st->hcchar_copy.b.mps;
     575         [ #  # ]:          0 :         if (nrpackets == 0)
     576                 :            :                 nrpackets = 1;
     577                 :          0 :         st->hcchar_copy.b.multicnt = nrpackets;
     578                 :          0 :         st->hctsiz_copy.b.pktcnt = nrpackets;
     579                 :            : 
     580                 :            :         /* Initial PID also needs to be set */
     581         [ #  # ]:          0 :         if (st->hcchar_copy.b.epdir == 0) {
     582                 :          0 :                 st->hctsiz_copy.b.xfersize = xfer_len;
     583      [ #  #  # ]:          0 :                 switch (st->hcchar_copy.b.multicnt) {
     584                 :            :                 case 1:
     585                 :          0 :                         st->hctsiz_copy.b.pid = DWC_PID_DATA0;
     586                 :          0 :                         break;
     587                 :            :                 case 2:
     588                 :            :                 case 3:
     589                 :          0 :                         st->hctsiz_copy.b.pid = DWC_PID_MDATA;
     590                 :          0 :                         break;
     591                 :            :                 }
     592                 :            : 
     593                 :            :         } else {
     594                 :          0 :                 st->hctsiz_copy.b.xfersize = nrpackets * st->hcchar_copy.b.mps;
     595   [ #  #  #  # ]:          0 :                 switch (st->hcchar_copy.b.multicnt) {
     596                 :            :                 case 1:
     597                 :          0 :                         st->hctsiz_copy.b.pid = DWC_PID_DATA0;
     598                 :          0 :                         break;
     599                 :            :                 case 2:
     600                 :          0 :                         st->hctsiz_copy.b.pid = DWC_PID_DATA1;
     601                 :          0 :                         break;
     602                 :            :                 case 3:
     603                 :          0 :                         st->hctsiz_copy.b.pid = DWC_PID_DATA2;
     604                 :          0 :                         break;
     605                 :            :                 }
     606                 :            :         }
     607                 :          0 :         FIQ_WRITE(state->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCTSIZ, st->hctsiz_copy.d32);
     608                 :          0 :         FIQ_WRITE(state->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCCHAR, st->hcchar_copy.d32);
     609                 :            :         /* Channel is enabled on hcint handler exit */
     610                 :            :         fiq_print(FIQDBG_INT, state, "HSISOOUT");
     611                 :          0 :         return 1;
     612                 :            : }
     613                 :            : 
     614                 :            : 
     615                 :            : /**
     616                 :            :  * fiq_fsm_do_sof() - FSM start-of-frame interrupt handler
     617                 :            :  * @state:      Pointer to the state struct passed from banked FIQ mode registers.
     618                 :            :  * @num_channels:       set according to the DWC hardware configuration
     619                 :            :  *
     620                 :            :  * The SOF handler in FSM mode has two functions
     621                 :            :  * 1. Hold off SOF from causing schedule advancement in IRQ context if there's
     622                 :            :  *    nothing to do
     623                 :            :  * 2. Advance certain FSM states that require either a microframe delay, or a microframe
     624                 :            :  *    of holdoff.
     625                 :            :  *
     626                 :            :  * The second part is architecture-specific to mach-bcm2835 -
     627                 :            :  * a sane interrupt controller would have a mask register for ARM interrupt sources
     628                 :            :  * to be promoted to the nFIQ line, but it doesn't. Instead a single interrupt
     629                 :            :  * number (USB) can be enabled. This means that certain parts of the USB specification
     630                 :            :  * that require "wait a little while, then issue another packet" cannot be fulfilled with
     631                 :            :  * the timing granularity required to achieve optimal throughout. The workaround is to use
     632                 :            :  * the SOF "timer" (125uS) to perform this task.
     633                 :            :  */
     634                 :  178515087 : static int notrace noinline fiq_fsm_do_sof(struct fiq_state *state, int num_channels)
     635                 :            : {
     636                 :  178515087 :         hfnum_data_t hfnum = { .d32 = FIQ_READ(state->dwc_regs_base + HFNUM) };
     637                 :            :         int n;
     638                 :            :         int kick_irq = 0;
     639                 :            : 
     640         [ +  + ]:  178515087 :         if ((hfnum.b.frnum & 0x7) == 1) {
     641                 :            :                 /* We cannot issue csplits for transactions in the last frame past (n+1).1
     642                 :            :                  * Check to see if there are any transactions that are stale.
     643                 :            :                  * Boot them out.
     644                 :            :                  */
     645         [ +  + ]:  178515856 :                 for (n = 0; n < num_channels; n++) {
     646         [ -  + ]:  178515856 :                         switch (state->channel[n].fsm) {
     647                 :            :                         case FIQ_PER_CSPLIT_WAIT:
     648                 :            :                         case FIQ_PER_CSPLIT_NYET1:
     649                 :            :                         case FIQ_PER_CSPLIT_POLL:
     650                 :            :                         case FIQ_PER_CSPLIT_LAST:
     651                 :            :                                 /* Check if we are no longer in the same full-speed frame. */
     652         [ #  # ]:          0 :                                 if (((state->channel[n].expected_uframe & 0x3FFF) & ~0x7) <
     653                 :            :                                                 (hfnum.b.frnum & ~0x7))
     654                 :          0 :                                         state->channel[n].fsm = FIQ_PER_SPLIT_TIMEOUT;
     655                 :            :                                 break;
     656                 :            :                         default:
     657                 :            :                                 break;
     658                 :            :                         }
     659                 :            :                 }
     660                 :            :         }
     661                 :            : 
     662         [ +  + ]: 1428120696 :         for (n = 0; n < num_channels; n++) {
     663   [ +  -  -  -  : 1428120696 :                 switch (state->channel[n].fsm) {
                -  -  + ]
     664                 :            : 
     665                 :            :                 case FIQ_NP_SSPLIT_RETRY:
     666                 :            :                 case FIQ_NP_IN_CSPLIT_RETRY:
     667                 :            :                 case FIQ_NP_OUT_CSPLIT_RETRY:
     668                 :    8031535 :                         fiq_fsm_restart_channel(state, n, 0);
     669                 :    8031535 :                         break;
     670                 :            : 
     671                 :            :                 case FIQ_HS_ISOC_SLEEPING:
     672                 :            :                         /* Is it time to wake this channel yet? */
     673         [ #  # ]:          0 :                         if (--state->channel[n].uframe_sleeps == 0) {
     674                 :          0 :                                 state->channel[n].fsm = FIQ_HS_ISOC_TURBO;
     675                 :          0 :                                 fiq_fsm_restart_channel(state, n, 0);
     676                 :            :                         }
     677                 :            :                         break;
     678                 :            : 
     679                 :            :                 case FIQ_PER_SSPLIT_QUEUED:
     680         [ #  # ]:          0 :                         if ((hfnum.b.frnum & 0x7) == 5)
     681                 :            :                                 break;
     682         [ #  # ]:          0 :                         if(!fiq_fsm_tt_in_use(state, num_channels, n)) {
     683         [ #  # ]:          0 :                                 if (!fiq_fsm_too_late(state, n)) {
     684                 :            :                                         fiq_print(FIQDBG_INT, state, "SOF GO %01d", n);
     685                 :          0 :                                         fiq_fsm_restart_channel(state, n, 0);
     686                 :          0 :                                         state->channel[n].fsm = FIQ_PER_SSPLIT_STARTED;
     687                 :            :                                 } else {
     688                 :            :                                         /* Transaction cannot be started without risking a device babble error */
     689                 :          0 :                                         state->channel[n].fsm = FIQ_PER_SPLIT_TIMEOUT;
     690                 :          0 :                                         state->haintmsk_saved.b2.chint &= ~(1 << n);
     691                 :          0 :                                         FIQ_WRITE(state->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCINTMSK, 0);
     692                 :            :                                         kick_irq |= 1;
     693                 :            :                                 }
     694                 :            :                         }
     695                 :            :                         break;
     696                 :            : 
     697                 :            :                 case FIQ_PER_ISO_OUT_PENDING:
     698                 :            :                         /* Ordinarily, this should be poked after the SSPLIT
     699                 :            :                          * complete interrupt for a competing transfer on the same
     700                 :            :                          * TT. Doesn't happen for aborted transactions though.
     701                 :            :                          */
     702         [ #  # ]:          0 :                         if ((hfnum.b.frnum & 0x7) >= 5)
     703                 :            :                                 break;
     704         [ #  # ]:          0 :                         if (!fiq_fsm_tt_in_use(state, num_channels, n)) {
     705                 :            :                                 /* Hardware bug. SOF can sometimes occur after the channel halt interrupt
     706                 :            :                                  * that caused this.
     707                 :            :                                  */
     708                 :          0 :                                         fiq_fsm_restart_channel(state, n, 0);
     709                 :            :                                         fiq_print(FIQDBG_INT, state, "SOF ISOC");
     710         [ #  # ]:          0 :                                         if (state->channel[n].nrpackets == 1) {
     711                 :          0 :                                                 state->channel[n].fsm = FIQ_PER_ISO_OUT_LAST;
     712                 :            :                                         } else {
     713                 :          0 :                                                 state->channel[n].fsm = FIQ_PER_ISO_OUT_ACTIVE;
     714                 :            :                                         }
     715                 :            :                         }
     716                 :            :                         break;
     717                 :            : 
     718                 :            :                 case FIQ_PER_CSPLIT_WAIT:
     719                 :            :                         /* we are guaranteed to be in this state if and only if the SSPLIT interrupt
     720                 :            :                          * occurred when the bus transaction occurred. The SOF interrupt reversal bug
     721                 :            :                          * will utterly bugger this up though.
     722                 :            :                          */
     723         [ #  # ]:          0 :                         if (hfnum.b.frnum != state->channel[n].expected_uframe) {
     724                 :            :                                 fiq_print(FIQDBG_INT, state, "SOFCS %d ", n);
     725                 :          0 :                                 state->channel[n].fsm = FIQ_PER_CSPLIT_POLL;
     726                 :          0 :                                 fiq_fsm_restart_channel(state, n, 0);
     727                 :          0 :                                 fiq_fsm_start_next_periodic(state, num_channels);
     728                 :            : 
     729                 :            :                         }
     730                 :            :                         break;
     731                 :            : 
     732                 :            :                 case FIQ_PER_SPLIT_TIMEOUT:
     733                 :            :                 case FIQ_DEQUEUE_ISSUED:
     734                 :            :                         /* Ugly: we have to force a HCD interrupt.
     735                 :            :                          * Poke the mask for the channel in question.
     736                 :            :                          * We will take a fake SOF because of this, but
     737                 :            :                          * that's OK.
     738                 :            :                          */
     739                 :          0 :                         state->haintmsk_saved.b2.chint &= ~(1 << n);
     740                 :          0 :                         FIQ_WRITE(state->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCINTMSK, 0);
     741                 :            :                         kick_irq |= 1;
     742                 :          0 :                         break;
     743                 :            : 
     744                 :            :                 default:
     745                 :            :                         break;
     746                 :            :                 }
     747                 :            :         }
     748                 :            : 
     749   [ +  +  +  + ]:  357029583 :         if (state->kick_np_queues ||
     750                 :  178514496 :                         dwc_frame_num_le(state->next_sched_frame, hfnum.b.frnum))
     751                 :            :                 kick_irq |= 1;
     752                 :            : 
     753                 :  178515087 :         return !kick_irq;
     754                 :            : }
     755                 :            : 
     756                 :            : 
     757                 :            : /**
     758                 :            :  * fiq_fsm_do_hcintr() - FSM host channel interrupt handler
     759                 :            :  * @state: Pointer to the FIQ state struct
     760                 :            :  * @num_channels: Number of channels as per hardware config
     761                 :            :  * @n: channel for which HAINT(i) was raised
     762                 :            :  *
     763                 :            :  * An important property is that only the CHHLT interrupt is unmasked. Unfortunately, AHBerr is as well.
     764                 :            :  */
     765                 :   15119422 : static int notrace noinline fiq_fsm_do_hcintr(struct fiq_state *state, int num_channels, int n)
     766                 :            : {
     767                 :            :         hcint_data_t hcint;
     768                 :            :         hcintmsk_data_t hcintmsk;
     769                 :            :         hcint_data_t hcint_probe;
     770                 :            :         hcchar_data_t hcchar;
     771                 :            :         int handled = 0;
     772                 :            :         int restart = 0;
     773                 :   15119422 :         int last_csplit = 0;
     774                 :            :         int start_next_periodic = 0;
     775                 :            :         struct fiq_channel_state *st = &state->channel[n];
     776                 :            :         hfnum_data_t hfnum;
     777                 :            : 
     778                 :   15119422 :         hcint.d32 = FIQ_READ(state->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCINT);
     779                 :   15119422 :         hcintmsk.d32 = FIQ_READ(state->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCINTMSK);
     780                 :   15119422 :         hcint_probe.d32 = hcint.d32 & hcintmsk.d32;
     781                 :            : 
     782                 :            :         if (st->fsm != FIQ_PASSTHROUGH) {
     783                 :            :                 fiq_print(FIQDBG_INT, state, "HC%01d ST%02d", n, st->fsm);
     784                 :            :                 fiq_print(FIQDBG_INT, state, "%08x", hcint.d32);
     785                 :            :         }
     786                 :            : 
     787   [ -  +  +  +  :   15119422 :         switch (st->fsm) {
          -  -  -  -  -  
             -  -  -  + ]
     788                 :            : 
     789                 :            :         case FIQ_PASSTHROUGH:
     790                 :            :         case FIQ_DEQUEUE_ISSUED:
     791                 :            :                 /* doesn't belong to us, kick it upstairs */
     792                 :            :                 break;
     793                 :            : 
     794                 :            :         case FIQ_PASSTHROUGH_ERRORSTATE:
     795                 :            :                 /* We are here to emulate the error recovery mechanism of the dwc HCD.
     796                 :            :                  * Several interrupts are unmasked if a previous transaction failed - it's
     797                 :            :                  * death for the FIQ to attempt to handle them as the channel isn't halted.
     798                 :            :                  * Emulate what the HCD does in this situation: mask and continue.
     799                 :            :                  * The FSM has no other state setup so this has to be handled out-of-band.
     800                 :            :                  */
     801                 :            :                 fiq_print(FIQDBG_ERR, state, "ERRST %02d", n);
     802         [ #  # ]:          0 :                 if (hcint_probe.b.nak || hcint_probe.b.ack || hcint_probe.b.datatglerr) {
     803                 :            :                         fiq_print(FIQDBG_ERR, state, "RESET %02d", n);
     804                 :            :                         /* In some random cases we can get a NAK interrupt coincident with a Xacterr
     805                 :            :                          * interrupt, after the device has disappeared.
     806                 :            :                          */
     807         [ #  # ]:          0 :                         if (!hcint.b.xacterr)
     808                 :          0 :                                 st->nr_errors = 0;
     809                 :          0 :                         hcintmsk.b.nak = 0;
     810                 :          0 :                         hcintmsk.b.ack = 0;
     811                 :          0 :                         hcintmsk.b.datatglerr = 0;
     812                 :          0 :                         FIQ_WRITE(state->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCINTMSK, hcintmsk.d32);
     813                 :          0 :                         return 1;
     814                 :            :                 }
     815         [ #  # ]:          0 :                 if (hcint_probe.b.chhltd) {
     816                 :            :                         fiq_print(FIQDBG_ERR, state, "CHHLT %02d", n);
     817                 :            :                         fiq_print(FIQDBG_ERR, state, "%08x", hcint.d32);
     818                 :            :                         return 0;
     819                 :            :                 }
     820                 :            :                 break;
     821                 :            : 
     822                 :            :         /* Non-periodic state groups */
     823                 :            :         case FIQ_NP_SSPLIT_STARTED:
     824                 :            :         case FIQ_NP_SSPLIT_RETRY:
     825                 :            :                 /* Got a HCINT for a NP SSPLIT. Expected ACK / NAK / fail */
     826         [ +  - ]:    4336496 :                 if (hcint.b.ack) {
     827                 :            :                         /* SSPLIT complete. For OUT, the data has been sent. For IN, the LS transaction
     828                 :            :                          * will start shortly. SOF needs to kick the transaction to prevent a NYET flood.
     829                 :            :                          */
     830         [ +  + ]:    4336496 :                         if(st->hcchar_copy.b.epdir == 1)
     831                 :    4326317 :                                 st->fsm = FIQ_NP_IN_CSPLIT_RETRY;
     832                 :            :                         else
     833                 :      10179 :                                 st->fsm = FIQ_NP_OUT_CSPLIT_RETRY;
     834                 :    4336496 :                         st->nr_errors = 0;
     835                 :            :                         handled = 1;
     836                 :    4336496 :                         fiq_fsm_setup_csplit(state, n);
     837         [ #  # ]:          0 :                 } else if (hcint.b.nak) {
     838                 :            :                         // No buffer space in TT. Retry on a uframe boundary.
     839                 :            :                         fiq_fsm_reload_hcdma(state, n);
     840                 :          0 :                         st->fsm = FIQ_NP_SSPLIT_RETRY;
     841                 :            :                         handled = 1;
     842         [ #  # ]:          0 :                 } else if (hcint.b.xacterr) {
     843                 :            :                         // The only other one we care about is xacterr. This implies HS bus error - retry.
     844                 :          0 :                         st->nr_errors++;
     845         [ #  # ]:          0 :                         if(st->hcchar_copy.b.epdir == 0)
     846                 :            :                                 fiq_fsm_reload_hcdma(state, n);
     847                 :          0 :                         st->fsm = FIQ_NP_SSPLIT_RETRY;
     848         [ #  # ]:          0 :                         if (st->nr_errors >= 3) {
     849                 :          0 :                                 st->fsm = FIQ_NP_SPLIT_HS_ABORTED;
     850                 :            :                         } else {
     851                 :            :                                 handled = 1;
     852                 :            :                                 restart = 1;
     853                 :            :                         }
     854                 :            :                 } else {
     855                 :          0 :                         st->fsm = FIQ_NP_SPLIT_LS_ABORTED;
     856                 :            :                         handled = 0;
     857                 :            :                         restart = 0;
     858                 :            :                 }
     859                 :            :                 break;
     860                 :            : 
     861                 :            :         case FIQ_NP_IN_CSPLIT_RETRY:
     862                 :            :                 /* Received a CSPLIT done interrupt.
     863                 :            :                  * Expected Data/NAK/STALL/NYET for IN.
     864                 :            :                  */
     865         [ +  + ]:    8015092 :                 if (hcint.b.xfercomp) {
     866                 :            :                         /* For IN, data is present. */
     867                 :      17390 :                         st->fsm = FIQ_NP_SPLIT_DONE;
     868         [ +  + ]:    7997702 :                 } else if (hcint.b.nak) {
     869                 :            :                         /* no endpoint data. Punt it upstairs */
     870                 :    4308918 :                         st->fsm = FIQ_NP_SPLIT_DONE;
     871         [ +  - ]:    3688784 :                 } else if (hcint.b.nyet) {
     872                 :            :                         /* CSPLIT NYET - retry on a uframe boundary. */
     873                 :            :                         handled = 1;
     874                 :    3688784 :                         st->nr_errors = 0;
     875         [ #  # ]:          0 :                 } else if (hcint.b.datatglerr) {
     876                 :            :                         /* data toggle errors do not set the xfercomp bit. */
     877                 :          0 :                         st->fsm = FIQ_NP_SPLIT_LS_ABORTED;
     878         [ #  # ]:          0 :                 } else if (hcint.b.xacterr) {
     879                 :            :                         /* HS error. Retry immediate */
     880                 :          0 :                         st->fsm = FIQ_NP_IN_CSPLIT_RETRY;
     881                 :          0 :                         st->nr_errors++;
     882         [ #  # ]:          0 :                         if (st->nr_errors >= 3) {
     883                 :          0 :                                 st->fsm = FIQ_NP_SPLIT_HS_ABORTED;
     884                 :            :                         } else {
     885                 :            :                                 handled = 1;
     886                 :            :                                 restart = 1;
     887                 :            :                         }
     888         [ #  # ]:          0 :                 } else if (hcint.b.stall || hcint.b.bblerr) {
     889                 :            :                         /* A STALL implies either a LS bus error or a genuine STALL. */
     890                 :          0 :                         st->fsm = FIQ_NP_SPLIT_LS_ABORTED;
     891                 :            :                 } else {
     892                 :            :                         /*  Hardware bug. It's possible in some cases to
     893                 :            :                          *  get a channel halt with nothing else set when
     894                 :            :                          *  the response was a NYET. Treat as local 3-strikes retry.
     895                 :            :                          */
     896                 :          0 :                         hcint_data_t hcint_test = hcint;
     897                 :          0 :                         hcint_test.b.chhltd = 0;
     898         [ #  # ]:          0 :                         if (!hcint_test.d32) {
     899                 :          0 :                                 st->nr_errors++;
     900         [ #  # ]:          0 :                                 if (st->nr_errors >= 3) {
     901                 :          0 :                                         st->fsm = FIQ_NP_SPLIT_HS_ABORTED;
     902                 :            :                                 } else {
     903                 :            :                                         handled = 1;
     904                 :            :                                 }
     905                 :            :                         } else {
     906                 :            :                                 /* Bail out if something unexpected happened */
     907                 :          0 :                                 st->fsm = FIQ_NP_SPLIT_HS_ABORTED;
     908                 :            :                         }
     909                 :            :                 }
     910         [ +  + ]:    8015092 :                 if (st->fsm != FIQ_NP_IN_CSPLIT_RETRY) {
     911                 :    4326308 :                         fiq_fsm_restart_np_pending(state, num_channels, n);
     912                 :            :                 }
     913                 :            :                 break;
     914                 :            : 
     915                 :            :         case FIQ_NP_OUT_CSPLIT_RETRY:
     916                 :            :                 /* Received a CSPLIT done interrupt.
     917                 :            :                  * Expected ACK/NAK/STALL/NYET/XFERCOMP for OUT.*/
     918         [ +  + ]:      16438 :                 if (hcint.b.xfercomp) {
     919                 :      10143 :                         st->fsm = FIQ_NP_SPLIT_DONE;
     920         [ +  + ]:       6295 :                 } else if (hcint.b.nak) {
     921                 :            :                         // The HCD will implement the holdoff on frame boundaries.
     922                 :         36 :                         st->fsm = FIQ_NP_SPLIT_DONE;
     923         [ +  - ]:       6259 :                 } else if (hcint.b.nyet) {
     924                 :            :                         // Hub still processing.
     925                 :       6259 :                         st->fsm = FIQ_NP_OUT_CSPLIT_RETRY;
     926                 :            :                         handled = 1;
     927                 :       6259 :                         st->nr_errors = 0;
     928                 :            :                         //restart = 1;
     929         [ #  # ]:          0 :                 } else if (hcint.b.xacterr) {
     930                 :            :                         /* HS error. retry immediate */
     931                 :          0 :                         st->fsm = FIQ_NP_OUT_CSPLIT_RETRY;
     932                 :          0 :                         st->nr_errors++;
     933         [ #  # ]:          0 :                         if (st->nr_errors >= 3) {
     934                 :          0 :                                 st->fsm = FIQ_NP_SPLIT_HS_ABORTED;
     935                 :            :                         } else {
     936                 :            :                                 handled = 1;
     937                 :            :                                 restart = 1;
     938                 :            :                         }
     939         [ #  # ]:          0 :                 } else if (hcint.b.stall) {
     940                 :            :                         /* LS bus error or genuine stall */
     941                 :          0 :                         st->fsm = FIQ_NP_SPLIT_LS_ABORTED;
     942                 :            :                 } else {
     943                 :            :                         /*
     944                 :            :                          * Hardware bug. It's possible in some cases to get a
     945                 :            :                          * channel halt with nothing else set when the response was a NYET.
     946                 :            :                          * Treat as local 3-strikes retry.
     947                 :            :                          */
     948                 :          0 :                         hcint_data_t hcint_test = hcint;
     949                 :          0 :                         hcint_test.b.chhltd = 0;
     950         [ #  # ]:          0 :                         if (!hcint_test.d32) {
     951                 :          0 :                                 st->nr_errors++;
     952         [ #  # ]:          0 :                                 if (st->nr_errors >= 3) {
     953                 :          0 :                                         st->fsm = FIQ_NP_SPLIT_HS_ABORTED;
     954                 :            :                                 } else {
     955                 :            :                                         handled = 1;
     956                 :            :                                 }
     957                 :            :                         } else {
     958                 :            :                                 // Something unexpected happened. AHBerror or babble perhaps. Let the IRQ deal with it.
     959                 :          0 :                                 st->fsm = FIQ_NP_SPLIT_HS_ABORTED;
     960                 :            :                         }
     961                 :            :                 }
     962         [ +  + ]:      16438 :                 if (st->fsm != FIQ_NP_OUT_CSPLIT_RETRY) {
     963                 :      10179 :                         fiq_fsm_restart_np_pending(state, num_channels, n);
     964                 :            :                 }
     965                 :            :                 break;
     966                 :            : 
     967                 :            :         /* Periodic split states (except isoc out) */
     968                 :            :         case FIQ_PER_SSPLIT_STARTED:
     969                 :            :                 /* Expect an ACK or failure for SSPLIT */
     970         [ #  # ]:          0 :                 if (hcint.b.ack) {
     971                 :            :                         /*
     972                 :            :                          * SSPLIT transfer complete interrupt - the generation of this interrupt is fraught with bugs.
     973                 :            :                          * For a packet queued in microframe n-3 to appear in n-2, if the channel is enabled near the EOF1
     974                 :            :                          * point for microframe n-3, the packet will not appear on the bus until microframe n.
     975                 :            :                          * Additionally, the generation of the actual interrupt is dodgy. For a packet appearing on the bus
     976                 :            :                          * in microframe n, sometimes the interrupt is generated immediately. Sometimes, it appears in n+1
     977                 :            :                          * coincident with SOF for n+1.
     978                 :            :                          * SOF is also buggy. It can sometimes be raised AFTER the first bus transaction has taken place.
     979                 :            :                          * These appear to be caused by timing/clock crossing bugs within the core itself.
     980                 :            :                          * State machine workaround.
     981                 :            :                          */
     982                 :          0 :                         hfnum.d32 = FIQ_READ(state->dwc_regs_base + HFNUM);
     983                 :          0 :                         hcchar.d32 = FIQ_READ(state->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCCHAR);
     984                 :          0 :                         fiq_fsm_setup_csplit(state, n);
     985                 :            :                         /* Poke the oddfrm bit. If we are equivalent, we received the interrupt at the correct
     986                 :            :                          * time. If not, then we're in the next SOF.
     987                 :            :                          */
     988         [ #  # ]:          0 :                         if ((hfnum.b.frnum & 0x1) == hcchar.b.oddfrm) {
     989                 :            :                                 fiq_print(FIQDBG_INT, state, "CSWAIT %01d", n);
     990                 :          0 :                                 st->expected_uframe = hfnum.b.frnum;
     991                 :          0 :                                 st->fsm = FIQ_PER_CSPLIT_WAIT;
     992                 :            :                         } else {
     993                 :            :                                 fiq_print(FIQDBG_INT, state, "CSPOL  %01d", n);
     994                 :            :                                 /* For isochronous IN endpoints,
     995                 :            :                                  * we need to hold off if we are expecting a lot of data */
     996         [ #  # ]:          0 :                                 if (st->hcchar_copy.b.mps < DATA0_PID_HEURISTIC) {
     997                 :            :                                         start_next_periodic = 1;
     998                 :            :                                 }
     999                 :            :                                 /* Danger will robinson: we are in a broken state. If our first interrupt after
    1000                 :            :                                  * this is a NYET, it will be delayed by 1 uframe and result in an unrecoverable
    1001                 :            :                                  * lag. Unmask the NYET interrupt.
    1002                 :            :                                  */
    1003                 :          0 :                                 st->expected_uframe = (hfnum.b.frnum + 1) & 0x3FFF;
    1004                 :          0 :                                 st->fsm = FIQ_PER_CSPLIT_BROKEN_NYET1;
    1005                 :            :                                 restart = 1;
    1006                 :            :                         }
    1007                 :            :                         handled = 1;
    1008         [ #  # ]:          0 :                 } else if (hcint.b.xacterr) {
    1009                 :            :                         /* 3-strikes retry is enabled, we have hit our max nr_errors */
    1010                 :          0 :                         st->fsm = FIQ_PER_SPLIT_HS_ABORTED;
    1011                 :            :                         start_next_periodic = 1;
    1012                 :            :                 } else {
    1013                 :          0 :                         st->fsm = FIQ_PER_SPLIT_HS_ABORTED;
    1014                 :            :                         start_next_periodic = 1;
    1015                 :            :                 }
    1016                 :            :                 /* We can now queue the next isochronous OUT transaction, if one is pending. */
    1017                 :          0 :                 if(fiq_fsm_tt_next_isoc(state, num_channels, n)) {
    1018                 :            :                         fiq_print(FIQDBG_INT, state, "NEXTISO ");
    1019                 :            :                 }
    1020                 :          0 :                 break;
    1021                 :            : 
    1022                 :            :         case FIQ_PER_CSPLIT_NYET1:
    1023                 :            :                 /* First CSPLIT attempt was a NYET. If we get a subsequent NYET,
    1024                 :            :                  * we are too late and the TT has dropped its CSPLIT fifo.
    1025                 :            :                  */
    1026                 :          0 :                 hfnum.d32 = FIQ_READ(state->dwc_regs_base + HFNUM);
    1027                 :          0 :                 hcchar.d32 = FIQ_READ(state->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCCHAR);
    1028                 :            :                 start_next_periodic = 1;
    1029         [ #  # ]:          0 :                 if (hcint.b.nak) {
    1030                 :          0 :                         st->fsm = FIQ_PER_SPLIT_DONE;
    1031         [ #  # ]:          0 :                 } else if (hcint.b.xfercomp) {
    1032                 :          0 :                         fiq_increment_dma_buf(state, num_channels, n);
    1033                 :          0 :                         st->fsm = FIQ_PER_CSPLIT_POLL;
    1034                 :          0 :                         st->nr_errors = 0;
    1035         [ #  # ]:          0 :                         if (fiq_fsm_more_csplits(state, n, &last_csplit)) {
    1036                 :            :                                 handled = 1;
    1037                 :            :                                 restart = 1;
    1038         [ #  # ]:          0 :                                 if (!last_csplit)
    1039                 :            :                                         start_next_periodic = 0;
    1040                 :            :                         } else {
    1041                 :          0 :                                 st->fsm = FIQ_PER_SPLIT_DONE;
    1042                 :            :                         }
    1043         [ #  # ]:          0 :                 } else if (hcint.b.nyet) {
    1044                 :            :                         /* Doh. Data lost. */
    1045                 :          0 :                         st->fsm = FIQ_PER_SPLIT_NYET_ABORTED;
    1046         [ #  # ]:          0 :                 } else if (hcint.b.xacterr || hcint.b.stall || hcint.b.bblerr) {
    1047                 :          0 :                         st->fsm = FIQ_PER_SPLIT_LS_ABORTED;
    1048                 :            :                 } else {
    1049                 :          0 :                         st->fsm = FIQ_PER_SPLIT_HS_ABORTED;
    1050                 :            :                 }
    1051                 :            :                 break;
    1052                 :            : 
    1053                 :            :         case FIQ_PER_CSPLIT_BROKEN_NYET1:
    1054                 :            :                 /*
    1055                 :            :                  * we got here because our host channel is in the delayed-interrupt
    1056                 :            :                  * state and we cannot take a NYET interrupt any later than when it
    1057                 :            :                  * occurred. Disable then re-enable the channel if this happens to force
    1058                 :            :                  * CSPLITs to occur at the right time.
    1059                 :            :                  */
    1060                 :          0 :                 hfnum.d32 = FIQ_READ(state->dwc_regs_base + HFNUM);
    1061                 :          0 :                 hcchar.d32 = FIQ_READ(state->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCCHAR);
    1062                 :            :                 fiq_print(FIQDBG_INT, state, "BROK: %01d ", n);
    1063         [ #  # ]:          0 :                 if (hcint.b.nak) {
    1064                 :          0 :                         st->fsm = FIQ_PER_SPLIT_DONE;
    1065                 :            :                         start_next_periodic = 1;
    1066         [ #  # ]:          0 :                 } else if (hcint.b.xfercomp) {
    1067                 :          0 :                         fiq_increment_dma_buf(state, num_channels, n);
    1068         [ #  # ]:          0 :                         if (fiq_fsm_more_csplits(state, n, &last_csplit)) {
    1069                 :          0 :                                 st->fsm = FIQ_PER_CSPLIT_POLL;
    1070                 :            :                                 handled = 1;
    1071                 :            :                                 restart = 1;
    1072                 :            :                                 start_next_periodic = 1;
    1073                 :            :                                 /* Reload HCTSIZ for the next transfer */
    1074                 :            :                                 fiq_fsm_reload_hctsiz(state, n);
    1075         [ #  # ]:          0 :                                 if (!last_csplit)
    1076                 :            :                                         start_next_periodic = 0;
    1077                 :            :                         } else {
    1078                 :          0 :                                 st->fsm = FIQ_PER_SPLIT_DONE;
    1079                 :            :                         }
    1080         [ #  # ]:          0 :                 } else if (hcint.b.nyet) {
    1081                 :          0 :                         st->fsm = FIQ_PER_SPLIT_NYET_ABORTED;
    1082                 :            :                         start_next_periodic = 1;
    1083         [ #  # ]:          0 :                 } else if (hcint.b.xacterr || hcint.b.stall || hcint.b.bblerr) {
    1084                 :            :                         /* Local 3-strikes retry is handled by the core. This is a ERR response.*/
    1085                 :          0 :                         st->fsm = FIQ_PER_SPLIT_LS_ABORTED;
    1086                 :            :                 } else {
    1087                 :          0 :                         st->fsm = FIQ_PER_SPLIT_HS_ABORTED;
    1088                 :            :                 }
    1089                 :            :                 break;
    1090                 :            : 
    1091                 :            :         case FIQ_PER_CSPLIT_POLL:
    1092                 :          0 :                 hfnum.d32 = FIQ_READ(state->dwc_regs_base + HFNUM);
    1093                 :          0 :                 hcchar.d32 = FIQ_READ(state->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCCHAR);
    1094                 :            :                 start_next_periodic = 1;
    1095         [ #  # ]:          0 :                 if (hcint.b.nak) {
    1096                 :          0 :                         st->fsm = FIQ_PER_SPLIT_DONE;
    1097         [ #  # ]:          0 :                 } else if (hcint.b.xfercomp) {
    1098                 :          0 :                         fiq_increment_dma_buf(state, num_channels, n);
    1099         [ #  # ]:          0 :                         if (fiq_fsm_more_csplits(state, n, &last_csplit)) {
    1100                 :            :                                 handled = 1;
    1101                 :            :                                 restart = 1;
    1102                 :            :                                 /* Reload HCTSIZ for the next transfer */
    1103                 :            :                                 fiq_fsm_reload_hctsiz(state, n);
    1104         [ #  # ]:          0 :                                 if (!last_csplit)
    1105                 :            :                                         start_next_periodic = 0;
    1106                 :            :                         } else {
    1107                 :          0 :                                 st->fsm = FIQ_PER_SPLIT_DONE;
    1108                 :            :                         }
    1109         [ #  # ]:          0 :                 } else if (hcint.b.nyet) {
    1110                 :            :                         /* Are we a NYET after the first data packet? */
    1111         [ #  # ]:          0 :                         if (st->nrpackets == 0) {
    1112                 :          0 :                                 st->fsm = FIQ_PER_CSPLIT_NYET1;
    1113                 :            :                                 handled = 1;
    1114                 :            :                                 restart = 1;
    1115                 :            :                         } else {
    1116                 :            :                                 /* We got a NYET when polling CSPLITs. Can happen
    1117                 :            :                                  * if our heuristic fails, or if someone disables us
    1118                 :            :                                  * for any significant length of time.
    1119                 :            :                                  */
    1120         [ #  # ]:          0 :                                 if (st->nr_errors >= 3) {
    1121                 :          0 :                                         st->fsm = FIQ_PER_SPLIT_NYET_ABORTED;
    1122                 :            :                                 } else {
    1123                 :          0 :                                         st->fsm = FIQ_PER_SPLIT_DONE;
    1124                 :            :                                 }
    1125                 :            :                         }
    1126         [ #  # ]:          0 :                 } else if (hcint.b.xacterr || hcint.b.stall || hcint.b.bblerr) {
    1127                 :            :                         /* For xacterr, Local 3-strikes retry is handled by the core. This is a ERR response.*/
    1128                 :          0 :                         st->fsm = FIQ_PER_SPLIT_LS_ABORTED;
    1129                 :            :                 } else {
    1130                 :          0 :                         st->fsm = FIQ_PER_SPLIT_HS_ABORTED;
    1131                 :            :                 }
    1132                 :            :                 break;
    1133                 :            : 
    1134                 :            :         case FIQ_HS_ISOC_TURBO:
    1135         [ #  # ]:          0 :                 if (fiq_fsm_update_hs_isoc(state, n, hcint)) {
    1136                 :            :                         /* more transactions to come */
    1137                 :            :                         handled = 1;
    1138                 :            :                         fiq_print(FIQDBG_INT, state, "HSISO M ");
    1139                 :            :                         /* For strided transfers, put ourselves to sleep */
    1140         [ #  # ]:          0 :                         if (st->hs_isoc_info.stride > 1) {
    1141                 :          0 :                                 st->uframe_sleeps = st->hs_isoc_info.stride - 1;
    1142                 :          0 :                                 st->fsm = FIQ_HS_ISOC_SLEEPING;
    1143                 :            :                         } else {
    1144                 :            :                                 restart = 1;
    1145                 :            :                         }
    1146                 :            :                 } else {
    1147                 :          0 :                         st->fsm = FIQ_HS_ISOC_DONE;
    1148                 :            :                         fiq_print(FIQDBG_INT, state, "HSISO F ");
    1149                 :            :                 }
    1150                 :            :                 break;
    1151                 :            : 
    1152                 :            :         case FIQ_HS_ISOC_ABORTED:
    1153                 :            :                 /* This abort is called by the driver rewriting the state mid-transaction
    1154                 :            :                  * which allows the dequeue mechanism to work more effectively.
    1155                 :            :                  */
    1156                 :            :                 break;
    1157                 :            : 
    1158                 :            :         case FIQ_PER_ISO_OUT_ACTIVE:
    1159         [ #  # ]:          0 :                 if (hcint.b.ack) {
    1160         [ #  # ]:          0 :                         if(fiq_iso_out_advance(state, num_channels, n)) {
    1161                 :            :                                 /* last OUT transfer */
    1162                 :          0 :                                 st->fsm = FIQ_PER_ISO_OUT_LAST;
    1163                 :            :                                 /*
    1164                 :            :                                  * Assuming the periodic FIFO in the dwc core
    1165                 :            :                                  * actually does its job properly, we can queue
    1166                 :            :                                  * the next ssplit now and in theory, the wire
    1167                 :            :                                  * transactions will be in-order.
    1168                 :            :                                  */
    1169                 :            :                                 // No it doesn't. It appears to process requests in host channel order.
    1170                 :            :                                 //start_next_periodic = 1;
    1171                 :            :                         }
    1172                 :            :                         handled = 1;
    1173                 :            :                         restart = 1;
    1174                 :            :                 } else {
    1175                 :            :                         /*
    1176                 :            :                          * Isochronous transactions carry on regardless. Log the error
    1177                 :            :                          * and continue.
    1178                 :            :                          */
    1179                 :            :                         //explode += 1;
    1180                 :          0 :                         st->nr_errors++;
    1181         [ #  # ]:          0 :                         if(fiq_iso_out_advance(state, num_channels, n)) {
    1182                 :          0 :                                 st->fsm = FIQ_PER_ISO_OUT_LAST;
    1183                 :            :                                 //start_next_periodic = 1;
    1184                 :            :                         }
    1185                 :            :                         handled = 1;
    1186                 :            :                         restart = 1;
    1187                 :            :                 }
    1188                 :            :                 break;
    1189                 :            : 
    1190                 :            :         case FIQ_PER_ISO_OUT_LAST:
    1191         [ #  # ]:          0 :                 if (hcint.b.ack) {
    1192                 :            :                         /* All done here */
    1193                 :          0 :                         st->fsm = FIQ_PER_ISO_OUT_DONE;
    1194                 :            :                 } else {
    1195                 :          0 :                         st->fsm = FIQ_PER_ISO_OUT_DONE;
    1196                 :          0 :                         st->nr_errors++;
    1197                 :            :                 }
    1198                 :            :                 start_next_periodic = 1;
    1199                 :            :                 break;
    1200                 :            : 
    1201                 :            :         case FIQ_PER_SPLIT_TIMEOUT:
    1202                 :            :                 /* SOF kicked us because we overran. */
    1203                 :            :                 start_next_periodic = 1;
    1204                 :          0 :                 break;
    1205                 :            : 
    1206                 :            :         default:
    1207                 :            :                 break;
    1208                 :            :         }
    1209                 :            : 
    1210         [ +  + ]:   15119422 :         if (handled) {
    1211                 :    8031539 :                 FIQ_WRITE(state->dwc_regs_base + HC_START + (HC_OFFSET * n) + HCINT, hcint.d32);
    1212                 :            :         } else {
    1213                 :            :                 /* Copy the regs into the state so the IRQ knows what to do */
    1214                 :    7087883 :                 st->hcint_copy.d32 = hcint.d32;
    1215                 :            :         }
    1216                 :            : 
    1217         [ -  + ]:   15119422 :         if (restart) {
    1218                 :            :                 /* Restart always implies handled. */
    1219                 :            :                 if (restart == 2) {
    1220                 :            :                         /* For complete-split INs, the show must go on.
    1221                 :            :                          * Force a channel restart */
    1222                 :            :                         fiq_fsm_restart_channel(state, n, 1);
    1223                 :            :                 } else {
    1224                 :          0 :                         fiq_fsm_restart_channel(state, n, 0);
    1225                 :            :                 }
    1226                 :            :         }
    1227         [ -  + ]:   15119422 :         if (start_next_periodic) {
    1228                 :          0 :                 fiq_fsm_start_next_periodic(state, num_channels);
    1229                 :            :         }
    1230                 :            :         if (st->fsm != FIQ_PASSTHROUGH)
    1231                 :            :                 fiq_print(FIQDBG_INT, state, "FSMOUT%02d", st->fsm);
    1232                 :            : 
    1233                 :   15119422 :         return handled;
    1234                 :            : }
    1235                 :            : 
    1236                 :            : 
    1237                 :            : /**
    1238                 :            :  * dwc_otg_fiq_fsm() - Flying State Machine (monster) FIQ
    1239                 :            :  * @state:              pointer to state struct passed from the banked FIQ mode registers.
    1240                 :            :  * @num_channels:       set according to the DWC hardware configuration
    1241                 :            :  * @dma:                pointer to DMA bounce buffers for split transaction slots
    1242                 :            :  *
    1243                 :            :  * The FSM FIQ performs the low-level tasks that normally would be performed by the microcode
    1244                 :            :  * inside an EHCI or similar host controller regarding split transactions. The DWC core
    1245                 :            :  * interrupts each and every time a split transaction packet is received or sent successfully.
    1246                 :            :  * This results in either an interrupt storm when everything is working "properly", or
    1247                 :            :  * the interrupt latency of the system in general breaks time-sensitive periodic split
    1248                 :            :  * transactions. Pushing the low-level, but relatively easy state machine work into the FIQ
    1249                 :            :  * solves these problems.
    1250                 :            :  *
    1251                 :            :  * Return: void
    1252                 :            :  */
    1253                 :  193544961 : void notrace dwc_otg_fiq_fsm(struct fiq_state *state, int num_channels)
    1254                 :            : {
    1255                 :            :         gintsts_data_t gintsts, gintsts_handled;
    1256                 :            :         gintmsk_data_t gintmsk;
    1257                 :            :         //hfnum_data_t hfnum;
    1258                 :            :         haint_data_t haint, haint_handled;
    1259                 :            :         haintmsk_data_t haintmsk;
    1260                 :            :         int kick_irq = 0;
    1261                 :            : 
    1262                 :            :         /* Ensure peripheral reads issued prior to FIQ entry are complete */
    1263                 :  193544961 :         dsb(sy);
    1264                 :            : 
    1265                 :  193544961 :         gintsts_handled.d32 = 0;
    1266                 :  193544961 :         haint_handled.d32 = 0;
    1267                 :            : 
    1268                 :  193544961 :         fiq_fsm_spin_lock(&state->lock);
    1269                 :  193544961 :         gintsts.d32 = FIQ_READ(state->dwc_regs_base + GINTSTS);
    1270                 :  193544961 :         gintmsk.d32 = FIQ_READ(state->dwc_regs_base + GINTMSK);
    1271                 :  193544961 :         gintsts.d32 &= gintmsk.d32;
    1272                 :            : 
    1273         [ +  + ]:  193544961 :         if (gintsts.b.sofintr) {
    1274                 :            :                 /* For FSM mode, SOF is required to keep the state machine advance for
    1275                 :            :                  * certain stages of the periodic pipeline. It's death to mask this
    1276                 :            :                  * interrupt in that case.
    1277                 :            :                  */
    1278                 :            : 
    1279         [ +  + ]:  178515034 :                 if (!fiq_fsm_do_sof(state, num_channels)) {
    1280                 :            :                         /* Kick IRQ once. Queue advancement means that all pending transactions
    1281                 :            :                          * will get serviced when the IRQ finally executes.
    1282                 :            :                          */
    1283         [ +  + ]:    7024701 :                         if (state->gintmsk_saved.b.sofintr == 1)
    1284                 :            :                                 kick_irq |= 1;
    1285                 :    7024701 :                         state->gintmsk_saved.b.sofintr = 0;
    1286                 :            :                 }
    1287                 :  178515033 :                 gintsts_handled.b.sofintr = 1;
    1288                 :            :         }
    1289                 :            : 
    1290         [ +  + ]:  193544960 :         if (gintsts.b.hcintr) {
    1291                 :            :                 int i;
    1292                 :   15079195 :                 haint.d32 = FIQ_READ(state->dwc_regs_base + HAINT);
    1293                 :   15079195 :                 haintmsk.d32 = FIQ_READ(state->dwc_regs_base + HAINTMSK);
    1294                 :   15079195 :                 haint.d32 &= haintmsk.d32;
    1295                 :            :                 haint_handled.d32 = 0;
    1296         [ +  + ]:  135712755 :                 for (i=0; i<num_channels; i++) {
    1297         [ +  + ]:  120633560 :                         if (haint.b2.chint & (1 << i)) {
    1298         [ +  + ]:   15119422 :                                 if(!fiq_fsm_do_hcintr(state, num_channels, i)) {
    1299                 :            :                                         /* HCINT was not handled in FIQ
    1300                 :            :                                          * HAINT is level-sensitive, leading to level-sensitive ginststs.b.hcint bit.
    1301                 :            :                                          * Mask HAINT(i) but keep top-level hcint unmasked.
    1302                 :            :                                          */
    1303                 :    7087883 :                                         state->haintmsk_saved.b2.chint &= ~(1 << i);
    1304                 :            :                                 } else {
    1305                 :            :                                         /* do_hcintr cleaned up after itself, but clear haint */
    1306                 :    8031539 :                                         haint_handled.b2.chint |= (1 << i);
    1307                 :            :                                 }
    1308                 :            :                         }
    1309                 :            :                 }
    1310                 :            : 
    1311         [ +  + ]:   15079195 :                 if (haint_handled.b2.chint) {
    1312                 :    7903202 :                         FIQ_WRITE(state->dwc_regs_base + HAINT, haint_handled.d32);
    1313                 :            :                 }
    1314                 :            : 
    1315         [ +  + ]:   15079195 :                 if (haintmsk.d32 != (haintmsk.d32 & state->haintmsk_saved.d32)) {
    1316                 :            :                         /*
    1317                 :            :                          * This is necessary to avoid multiple retriggers of the MPHI in the case
    1318                 :            :                          * where interrupts are held off and HCINTs start to pile up.
    1319                 :            :                          * Only wake up the IRQ if a new interrupt came in, was not handled and was
    1320                 :            :                          * masked.
    1321                 :            :                          */
    1322                 :            :                         haintmsk.d32 &= state->haintmsk_saved.d32;
    1323                 :    7068672 :                         FIQ_WRITE(state->dwc_regs_base + HAINTMSK, haintmsk.d32);
    1324                 :            :                         kick_irq |= 1;
    1325                 :            :                 }
    1326                 :            :                 /* Top-Level interrupt - always handled because it's level-sensitive */
    1327                 :   15079195 :                 gintsts_handled.b.hcintr = 1;
    1328                 :            :         }
    1329                 :            : 
    1330                 :            : 
    1331                 :            :         /* Clear the bits in the saved register that were not handled but were triggered. */
    1332                 :  193544960 :         state->gintmsk_saved.d32 &= ~(gintsts.d32 & ~gintsts_handled.d32);
    1333                 :            : 
    1334                 :            :         /* FIQ didn't handle something - mask has changed - write new mask */
    1335         [ +  + ]:  193544960 :         if (gintmsk.d32 != (gintmsk.d32 & state->gintmsk_saved.d32)) {
    1336                 :    7039377 :                 gintmsk.d32 &= state->gintmsk_saved.d32;
    1337                 :    7039377 :                 gintmsk.b.sofintr = 1;
    1338                 :    7039377 :                 FIQ_WRITE(state->dwc_regs_base + GINTMSK, gintmsk.d32);
    1339                 :            : //              fiq_print(FIQDBG_INT, state, "KICKGINT");
    1340                 :            : //              fiq_print(FIQDBG_INT, state, "%08x", gintmsk.d32);
    1341                 :            : //              fiq_print(FIQDBG_INT, state, "%08x", state->gintmsk_saved.d32);
    1342                 :            :                 kick_irq |= 1;
    1343                 :            :         }
    1344                 :            : 
    1345         [ +  + ]:  193544960 :         if (gintsts_handled.d32) {
    1346                 :            :                 /* Only applies to edge-sensitive bits in GINTSTS */
    1347                 :  193412273 :                 FIQ_WRITE(state->dwc_regs_base + GINTSTS, gintsts_handled.d32);
    1348                 :            :         }
    1349                 :            : 
    1350                 :            :         /* We got an interrupt, didn't handle it. */
    1351         [ +  + ]:  193544960 :         if (kick_irq) {
    1352                 :   14100928 :                 state->mphi_int_count++;
    1353         [ -  + ]:   14100928 :                 if (state->mphi_regs.swirq_set) {
    1354                 :          0 :                         FIQ_WRITE(state->mphi_regs.swirq_set, 1);
    1355                 :            :                 } else {
    1356                 :   14100928 :                         FIQ_WRITE(state->mphi_regs.outdda, state->dummy_send_dma);
    1357                 :   14100928 :                         FIQ_WRITE(state->mphi_regs.outddb, (1<<29));
    1358                 :            :                 }
    1359                 :            : 
    1360                 :            :         }
    1361                 :  193544960 :         state->fiq_done++;
    1362                 :  193544960 :         mb();
    1363                 :            :         fiq_fsm_spin_unlock(&state->lock);
    1364                 :  193544960 : }
    1365                 :            : 
    1366                 :            : 
    1367                 :            : /**
    1368                 :            :  * dwc_otg_fiq_nop() - FIQ "lite"
    1369                 :            :  * @state:      pointer to state struct passed from the banked FIQ mode registers.
    1370                 :            :  *
    1371                 :            :  * The "nop" handler does not intervene on any interrupts other than SOF.
    1372                 :            :  * It is limited in scope to deciding at each SOF if the IRQ SOF handler (which deals
    1373                 :            :  * with non-periodic/periodic queues) needs to be kicked.
    1374                 :            :  *
    1375                 :            :  * This is done to hold off the SOF interrupt, which occurs at a rate of 8000 per second.
    1376                 :            :  *
    1377                 :            :  * Return: void
    1378                 :            :  */
    1379                 :          0 : void notrace dwc_otg_fiq_nop(struct fiq_state *state)
    1380                 :            : {
    1381                 :            :         gintsts_data_t gintsts, gintsts_handled;
    1382                 :            :         gintmsk_data_t gintmsk;
    1383                 :            :         hfnum_data_t hfnum;
    1384                 :            : 
    1385                 :            :         /* Ensure peripheral reads issued prior to FIQ entry are complete */
    1386                 :          0 :         dsb(sy);
    1387                 :            : 
    1388                 :          0 :         fiq_fsm_spin_lock(&state->lock);
    1389                 :          0 :         hfnum.d32 = FIQ_READ(state->dwc_regs_base + HFNUM);
    1390                 :          0 :         gintsts.d32 = FIQ_READ(state->dwc_regs_base + GINTSTS);
    1391                 :          0 :         gintmsk.d32 = FIQ_READ(state->dwc_regs_base + GINTMSK);
    1392                 :          0 :         gintsts.d32 &= gintmsk.d32;
    1393                 :          0 :         gintsts_handled.d32 = 0;
    1394                 :            : 
    1395         [ #  # ]:          0 :         if (gintsts.b.sofintr) {
    1396   [ #  #  #  # ]:          0 :                 if (!state->kick_np_queues &&
    1397                 :          0 :                                 dwc_frame_num_gt(state->next_sched_frame, hfnum.b.frnum)) {
    1398                 :            :                         /* SOF handled, no work to do, just ACK interrupt */
    1399                 :          0 :                         gintsts_handled.b.sofintr = 1;
    1400                 :            :                 } else {
    1401                 :            :                         /* Kick IRQ */
    1402                 :          0 :                         state->gintmsk_saved.b.sofintr = 0;
    1403                 :            :                 }
    1404                 :            :         }
    1405                 :            : 
    1406                 :            :         /* Reset handled interrupts */
    1407         [ #  # ]:          0 :         if(gintsts_handled.d32) {
    1408                 :          0 :                 FIQ_WRITE(state->dwc_regs_base + GINTSTS, gintsts_handled.d32);
    1409                 :            :         }
    1410                 :            : 
    1411                 :            :         /* Clear the bits in the saved register that were not handled but were triggered. */
    1412                 :          0 :         state->gintmsk_saved.d32 &= ~(gintsts.d32 & ~gintsts_handled.d32);
    1413                 :            : 
    1414                 :            :         /* We got an interrupt, didn't handle it and want to mask it */
    1415         [ #  # ]:          0 :         if (~(state->gintmsk_saved.d32)) {
    1416                 :          0 :                 state->mphi_int_count++;
    1417                 :          0 :                 gintmsk.d32 &= state->gintmsk_saved.d32;
    1418                 :          0 :                 FIQ_WRITE(state->dwc_regs_base + GINTMSK, gintmsk.d32);
    1419         [ #  # ]:          0 :                 if (state->mphi_regs.swirq_set) {
    1420                 :          0 :                         FIQ_WRITE(state->mphi_regs.swirq_set, 1);
    1421                 :            :                 } else {
    1422                 :            :                         /* Force a clear before another dummy send */
    1423                 :          0 :                         FIQ_WRITE(state->mphi_regs.intstat, (1<<29));
    1424                 :          0 :                         FIQ_WRITE(state->mphi_regs.outdda, state->dummy_send_dma);
    1425                 :          0 :                         FIQ_WRITE(state->mphi_regs.outddb, (1<<29));
    1426                 :            :                 }
    1427                 :            :         }
    1428                 :          0 :         state->fiq_done++;
    1429                 :          0 :         mb();
    1430                 :            :         fiq_fsm_spin_unlock(&state->lock);
    1431                 :          0 : }

Generated by: LCOV version 1.14